Duplicity

[0.8] VerifyTest fails with AttributeError ('bytes' object has no attribute 'encode'")

Bug #1833685 reported by Sebastien Bacher
12
This bug affects 2 people
Affects Status Importance Assigned to Milestone
Duplicity
Fix Released
Medium
Unassigned
Duplicity 0.8.01

Bug Description

Using 0.8 r1377

$ cd /tmp
$ bzr branch lp:duplicity
$ cd duplicity
$ python3.7 setup.py build --force
$ PYTHONPATH=/tmp/duplicity/build/lib.linux-i686-3.7/ python3.7 ./setup.py test
(that's on an i386 chroot, path to adapt for amd64)

The tests have some errors, including that one

___________ VerifyTest.test_verify_compare_data_changed_source_file ____________

self = <testing.functional.test_verify.VerifyTest testMethod=test_verify_compare_data_changed_source_file>

    def test_verify_compare_data_changed_source_file(self):
        u"""Test verify with --compare-data gives an error if a source file is changed"""
> self.backup(u"full", u"testfiles/various_file_types", options=[])

functional/test_verify.py:83:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
functional/__init__.py:182: in backup
    result = self.run_duplicity(options=options, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <testing.functional.test_verify.VerifyTest testMethod=test_verify_compare_data_changed_source_file>
options = ['full', 'testfiles/various_file_types', 'file://testfiles/output', '--volsize', '1']
current_time = None, fail = None, passphrase_input = []

    def run_duplicity(self, options=[], current_time=None, fail=None,
                      passphrase_input=[]):
        u"""
        Run duplicity binary with given arguments and options
        """
        # We run under setsid and take input from /dev/null (below) because
        # this way we force a failure if duplicity tries to read from the
        # console unexpectedly (like for gpg password or such).

        # Check all string inputs are unicode -- we will convert to system encoding before running the command
        for item in options:
            if sys.version_info.major == 2:
                assert not isinstance(item, str), u"item " + unicode(item) + u" in options is not unicode"

        for item in passphrase_input:
            assert isinstance(item, u"".__class__), u"item " + unicode(item) + u" in passphrase_input is not unicode"

        if platform.platform().startswith(u'Linux'):
            cmd_list = [u'setsid']
            if self._setsid_w:
                cmd_list.extend([u"-w"])
        else:
            cmd_list = []
        cmd_list.extend([u"../bin/duplicity"])
        cmd_list.extend(options)
        cmd_list.extend([u"-v0"])
        cmd_list.extend([u"--no-print-statistics"])
        cmd_list.extend([u"--allow-source-mismatch"])
        cmd_list.extend([u"--archive-dir=testfiles/cache"])
        if current_time:
            cmd_list.extend([u"--current-time", current_time])
        cmd_list.extend(self.class_args)
        if fail:
            cmd_list.extend([u"--fail", u"".__class__(fail)])
        cmdline = u" ".join([u'"%s"' % x for x in cmd_list])

        if not passphrase_input:
            cmdline += u" < /dev/null"

        # The immediately following block is the nicer way to execute pexpect with
        # unicode strings, but we need to have the pre-4.0 version for some time yet,
        # so for now this is commented out so tests execute the same way on all systems.

        # if parse_version(pexpect.__version__) >= parse_version("4.0"):
        # # pexpect.spawn only supports unicode from version 4.0
        # # there was a separate pexpect.spawnu in 3.x, but it has an error on readline
        # child = pexpect.spawn(u'/bin/sh', [u'-c', cmdline], timeout=None, encoding=sys.getfilesystemencoding())
        #
        # for passphrase in passphrase_input:
        # child.expect(u'passphrase.*:')
        # child.sendline(passphrase)
        # else:

        # Manually encode to filesystem encoding and send to spawn as bytes
        # ToDo: Remove this once we no longer have to support systems with pexpect < 4.0
        if sys.version_info.major > 2:
            child = pexpect.spawn(u'/bin/sh', [u'-c', cmdline], timeout=None)
        else:
            child = pexpect.spawn(b'/bin/sh', [b'-c', cmdline.encode(sys.getfilesystemencoding(),
                                                                 u'replace')], timeout=None)

        for passphrase in passphrase_input:
            child.expect(b'passphrase.*:')
            child.sendline(passphrase)

        # if the command fails, we need to clear its output
        # so it will terminate cleanly.
        child.expect_exact(pexpect.EOF)
        lines = child.before.splitlines()
        child.wait()
        child.ptyproc.delayafterclose = 0.0
        return_val = child.exitstatus

        if fail:
            self.assertEqual(30, return_val)
        elif return_val:
            print(u"\n...command:", cmdline, file=sys.stderr)
            print(u"...cwd:", os.getcwd(), file=sys.stderr)
            print(u"...output:", file=sys.stderr)
            for line in lines:
                line = line.rstrip()
                if line:
                    print(line, file=sys.stderr)
            print(u"...return_val:", return_val, file=sys.stderr)
> raise CmdError(return_val)
E testing.functional.CmdError: 30

functional/__init__.py:165: CmdError
----------------------------- Captured stderr call -----------------------------

...command: "setsid" "-w" "../bin/duplicity" "full" "testfiles/various_file_types" "file://testfiles/output" "--volsize" "1" "-v0" "--no-print-statistics" "--allow-source-mismatch" "--archive-dir=testfiles/cache" < /dev/null
...cwd: /build/duplicity/testing
...output:
b'Traceback (innermost last):'
b' File "../bin/duplicity", line 1706, in <module>'
b' with_tempdir(main)'
b' File "../bin/duplicity", line 1692, in with_tempdir'
b' fn()'
b' File "../bin/duplicity", line 1538, in main'
b' do_backup(action)'
b' File "../bin/duplicity", line 1662, in do_backup'
b' full_backup(col_stats)'
b' File "../bin/duplicity", line 568, in full_backup'
b' globals.backend)'
b' File "../bin/duplicity", line 425, in write_multivol'
b' globals.volsize)'
b' File "/build/duplicity/duplicity/gpg.py", line 393, in GPGWriteFile'
b' data = block_iter.__next__().data'
b' File "/build/duplicity/duplicity/diffdir.py", line 543, in __next__'
b' result = self.process(next(self.input_iter)) # pylint: disable=assignment-from-no-return'
b' File "/build/duplicity/duplicity/diffdir.py", line 235, in get_delta_iter'
b' (new_path, sig_path, sigTarFile))'
b' File "/build/duplicity/duplicity/robust.py", line 41, in check_common_error'
b' return function(*args)'
b' File "/build/duplicity/duplicity/diffdir.py", line 160, in get_delta_path'
b' sigTarFile.addfile(ti)'
b' File "/usr/lib/python3.7/tarfile.py", line 1966, in addfile'
b' buf = tarinfo.tobuf(self.format, self.encoding, self.errors)'
b' File "/usr/lib/python3.7/tarfile.py", line 816, in tobuf'
b' return self.create_gnu_header(info, encoding, errors)'
b' File "/usr/lib/python3.7/tarfile.py", line 847, in create_gnu_header'
b' return buf + self._create_header(info, GNU_FORMAT, encoding, errors)'
b' File "/usr/lib/python3.7/tarfile.py", line 937, in _create_header'
b' stn(info.get("gname", ""), 32, encoding, errors),'
b' File "/usr/lib/python3.7/tarfile.py", line 161, in stn'
b' s = s.encode(encoding, errors)'
b" AttributeError: 'bytes' object has no attribute 'encode'"
...return_val: 30

See original description
Sebastien Bacher (seb128)
summary: - [0.8] VerifyTest fails with AttributeError
+ [0.8] VerifyTest fails with AttributeError ('bytes' object has no
+ attribute 'encode'")
Revision history for this message
Kenneth Loafman (kenneth-loafman) wrote :

Wrong Python version. You ran under py2.3, not 2.7. Python 2.7 is required for 0.8.00.

Changed in duplicity:
status: New → Invalid
Revision history for this message
Sebastien Bacher (seb128) wrote :

Reopening, sorry it was a typo in the description, that's using python3.7

Changed in duplicity:
status: Invalid → New
description: updated
Revision history for this message
Sebastien Bacher (seb128) wrote :

That's still an issue in current trunk

Kenneth Loafman (kenneth-loafman)
Changed in duplicity:
status: New → Fix Committed
importance: Undecided → Medium
milestone: none → 0.8.01
Kenneth Loafman (kenneth-loafman)
Changed in duplicity:
status: Fix Committed → Fix Released
Revision history for this message
Aaron Whitehouse (aaron-whitehouse) wrote :

@Kenneth, do you remember off the top of your head which commit/merge fixed this? It isn't jumping out at me in the changelog and I'm keen to know how it broke a build but we missed it. Was it Python 3.7-specific? (If so, that makes sense, given we have only just added 3.7 to the test environments.)

Revision history for this message
Kenneth Loafman (kenneth-loafman) wrote :

r1382 was the last to change test_verify.py. Lots of changes since then.

$ bzr log -l20 -v | less

will let you review the commit messages and files changed.

Python 3.7 passes all the tests. It's 3.6 that's problematic.

Revision history for this message
Sebastien Bacher (seb128) wrote :

those errors are still there with 8.01 for me (build with python 3.7.3 on Ubuntu Disco)

Changed in duplicity:
status: Fix Released → New
Revision history for this message
Aaron Whitehouse (aaron-whitehouse) wrote :

Hello Sebastien,

I am struggling to replicate this. Do you still have a failure in your environment if you use tox -e py37 ?

If I use our standard testing docker container, based on 18.04, the tests all pass fine for Python 3.7 (3 warnings). If I modify the testing docker to be based on 19.04 (and remove the Python 3.6 dependencies -- see attached), again the Python 3.7 tests pass with 3 warnings.

Could there be something else in your environment causing this? Or missing dependencies that are in the Dockerfile?

===

bzr branch lp:duplicity
cd duplicity/testing/infrastructure/
./build-duplicity_test.sh
./setup.sh
tox -e py37

=============================== warnings summary ===============================
.tox/py37/lib/python3.7/site-packages/past/types/oldstr.py:5
  /home/test/duplicity/.tox/py37/lib/python3.7/site-packages/past/types/oldstr.py:5: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working
    from collections import Iterable

.tox/py37/lib/python3.7/site-packages/past/builtins/misc.py:4
  /home/test/duplicity/.tox/py37/lib/python3.7/site-packages/past/builtins/misc.py:4: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working
    from collections import Mapping

testing/unit/test_globmatch.py:100
  /home/test/duplicity/testing/unit/test_globmatch.py:100: DeprecationWarning: invalid escape sequence \/
    u"\/usr\/[^/]*\/bin\/")

-- Docs: https://docs.pytest.org/en/latest/warnings.html
=================== 428 passed, 3 warnings in 709.06 seconds ===================
___________________________________ summary ____________________________________
  py37: commands succeeded

# Change Dockerfile to 19.04 and remove python3.6 and python3.6-dev
./build-duplicity_test.sh
./setup.sh
tox -e py37
=============================== warnings summary ===============================
.tox/py37/lib/python3.7/site-packages/past/types/oldstr.py:5
  /home/test/duplicity/.tox/py37/lib/python3.7/site-packages/past/types/oldstr.py:5: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working
    from collections import Iterable

.tox/py37/lib/python3.7/site-packages/past/builtins/misc.py:4
  /home/test/duplicity/.tox/py37/lib/python3.7/site-packages/past/builtins/misc.py:4: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working
    from collections import Mapping

testing/unit/test_globmatch.py:100
  /home/test/duplicity/testing/unit/test_globmatch.py:100: DeprecationWarning: invalid escape sequence \/
    u"\/usr\/[^/]*\/bin\/")

-- Docs: https://docs.pytest.org/en/latest/warnings.html
=================== 428 passed, 3 warnings in 739.91 seconds ===================
___________________________________ summary ____________________________________
  py37: commands succeeded

Revision history for this message
Sebastien Bacher (seb128) wrote :

Ok, you are right, something is confusing those in the chroot/pbuilder environment, it works on a builder, sorry for the noise

Changed in duplicity:
status: New → Fix Released
Revision history for this message
Kenneth Loafman (kenneth-loafman) wrote : Re: [Bug 1833685] Re: [0.8] VerifyTest fails with AttributeError ('bytes' object has no attribute 'encode'")
Download full text (9.0 KiB)

One thing to watch out for in testing is the fact that the tests run
duplicity as a subprocess using the default python. That's why we use tox
for testing, a controlled environment. The command 'setup.py test' is
coming into disfavor, probably due to its reliance on a possibly wonky
environment. We're encountering that on the Launchpad build environment as
well. I'm thinking of making setup.py test run tox instead, but it just
seems like overkill.

On Tue, Jul 16, 2019 at 8:45 AM Sebastien Bacher <email address hidden> wrote:

> Ok, you are right, something is confusing those in the chroot/pbuilder
> environment, it works on a builder, sorry for the noise
>
> ** Changed in: duplicity
> Status: New => Fix Released
>
> --
> You received this bug notification because you are subscribed to
> Duplicity.
> https://bugs.launchpad.net/bugs/1833685
>
> Title:
> [0.8] VerifyTest fails with AttributeError ('bytes' object has no
> attribute 'encode'")
>
> Status in Duplicity:
> Fix Released
>
> Bug description:
> Using 0.8 r1377
>
> $ cd /tmp
> $ bzr branch lp:duplicity
> $ cd duplicity
> $ python3.7 setup.py build --force
> $ PYTHONPATH=/tmp/duplicity/build/lib.linux-i686-3.7/ python3.7
> ./setup.py test
> (that's on an i386 chroot, path to adapt for amd64)
>
> The tests have some errors, including that one
>
> ___________ VerifyTest.test_verify_compare_data_changed_source_file
> ____________
>
> self = <testing.functional.test_verify.VerifyTest
> testMethod=test_verify_compare_data_changed_source_file>
>
> def test_verify_compare_data_changed_source_file(self):
> u"""Test verify with --compare-data gives an error if a source
> file is changed"""
> > self.backup(u"full", u"testfiles/various_file_types", options=[])
>
> functional/test_verify.py:83:
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> _ _ _ _
> functional/__init__.py:182: in backup
> result = self.run_duplicity(options=options, **kwargs)
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> _ _ _ _
>
> self = <testing.functional.test_verify.VerifyTest
> testMethod=test_verify_compare_data_changed_source_file>
> options = ['full', 'testfiles/various_file_types',
> 'file://testfiles/output', '--volsize', '1']
> current_time = None, fail = None, passphrase_input = []
>
> def run_duplicity(self, options=[], current_time=None, fail=None,
> passphrase_input=[]):
> u"""
> Run duplicity binary with given arguments and options
> """
> # We run under setsid and take input from /dev/null (below)
> because
> # this way we force a failure if duplicity tries to read from the
> # console unexpectedly (like for gpg password or such).
>
> # Check all string inputs are unicode -- we will convert to
> system encoding before running the command
> for item in options:
> if sys.version_info.major == 2:
> assert not isinstance(item, str), u"item " +
> unicode(item) + u" in options is not unicode"
>
> for item in passphrase_input:
> ...

Read more...

Revision history for this message
Sebastien Bacher (seb128) wrote :

> One thing to watch out for in testing is the fact that the tests run
> duplicity as a subprocess using the default python.

Right, I did hit that problem, I ended up following the Debian maintainer to build it on Ubuntu
https://salsa.debian.org/debian/duplicity/blob/debian/debian/patches/06-noenv

Revision history for this message
Charles Celerier (chckyn) wrote :

I'm not sure this it the same bug, but I get the same AttributeError when I run duplicity with expired AWS credentials. I just ran `python setup.py tests` and saw all but one of the `testing/unit/test_diffdir.py` tests fail. I am running from revision 1401. Here is one of the failed tests.

```
____________________________________________________________ DDTest.test_diff ____________________________________________________________

self = <testing.unit.test_diffdir.DDTest testMethod=test_diff>

    def test_diff(self):
        u"""Test making a diff"""
        sel1 = selection.Select(Path(u"testfiles/dir1"))
        diffdir.write_block_iter(diffdir.SigTarBlockIter(sel1.set_iter()),
> u"testfiles/output/dir1.sigtar")

unit/test_diffdir.py:113:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../duplicity/diffdir.py:747: in write_block_iter
    for block in block_iter:
../duplicity/diffdir.py:542: in __next__
    result = self.process(next(self.input_iter)) # pylint: disable=assignment-from-no-return
../duplicity/diffdir.py:644: in process
    return self.tarinfo2tarblock(path.index, ti)
../duplicity/diffdir.py:503: in tarinfo2tarblock
    headers = tarinfo.tobuf(errors=u'replace', encoding=globals.fsencoding)
/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/tarfile.py:816: in tobuf
    return self.create_gnu_header(info, encoding, errors)
/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/tarfile.py:847: in create_gnu_header
    return buf + self._create_header(info, GNU_FORMAT, encoding, errors)
/usr/local/Cellar/python/3.7.3/Frameworks/Python.framework/Versions/3.7/lib/python3.7/tarfile.py:937: in _create_header
    stn(info.get("gname", ""), 32, encoding, errors),
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

s = b'', length = 32, encoding = 'utf-8', errors = 'replace'

    def stn(s, length, encoding, errors):
        """Convert a string to a null-terminated bytes object.
        """
> s = s.encode(encoding, errors)
 E AttributeError: 'bytes' object has no attribute 'encode'
```

Not really what to make of this other than to ensure my AWS credentials don't expire before I run my backups.

To post a comment you must log in.
This report contains Public information  
Everyone can see this information.
Subscribing...

Other bug subscribers

Remote bug watches

Bug watches keep track of this bug in other bug trackers.