diff -Nru python-swiftclient-2.4.0/AUTHORS python-swiftclient-2.6.0/AUTHORS --- python-swiftclient-2.4.0/AUTHORS 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-2.6.0/AUTHORS 2015-09-07 15:20:14.000000000 +0000 @@ -0,0 +1,103 @@ +Christian Berendt (berendt@b1-systems.de) +Luis de Bethencourt (luis@debethencourt.com) +Darrell Bishop (darrell@swiftstack.com) +Fabien Boucher (fabien.boucher@enovance.com) +Chmouel Boudjnah (chmouel@enovance.com) +Clark Boylan (clark.boylan@gmail.com) +Chris Buccella (chris.buccella@antallagon.com) +Tim Burke (tim.burke@gmail.com) +Clint Byrum (clint@fewbar.com) +Tristan Cacqueray (tristan.cacqueray@enovance.com) +Sergio Cazzolato (sergio.j.cazzolato@intel.com) +Mahati Chamarthy (mahati.chamarthy@gmail.com) +Ray Chen (oldsharp@163.com) +Taurus Cheung (Taurus.Cheung@harmonicinc.com) +Alistair Coles (alistair.coles@hp.com) +Ian Cordasco (ian.cordasco@rackspace.com) +Nick Craig-Wood (nick@craig-wood.com) +Sean Dague (sean@dague.net) +Zack M. Davis (zdavis@swiftstack.com) +John Dickinson (me@not.mn) +EdLeafe (ed@leafe.com) +Sahid Orentino Ferdjaoui (sahid.ferdjaoui@cloudwatt.com) +Flaper Fesp (flaper87@gmail.com) +Florent Flament (florent.flament-ext@cloudwatt.com) +Josh Gachnang (josh@pcsforeducation.com) +Alex Gaynor (alex.gaynor@gmail.com) +Martin Geisler (martin@geisler.net) +Anne Gentle (anne@openstack.org) +Clay Gerrard (clay.gerrard@gmail.com) +David Goetz (david.goetz@rackspace.com) +Thomas Goirand (thomas@goirand.fr) +Davide Guerri (davide.guerri@hp.com) +Romain Hardouin (romain_hardouin@yahoo.fr) +Steven Hardy (shardy@redhat.com) +Doug Hellmann (doug.hellmann@dreamhost.com) +Greg Holt (gholt@rackspace.com) +Charles Hsu (charles0126@gmail.com) +Kun Huang (gareth@unitedstack.com) +Matthieu Huin (mhu@enovance.com) +Andreas Jaeger (aj@suse.de) +OpenStack Jenkins (jenkins@openstack.org) +Vasyl Khomenko (vasiliyk@yahoo-inc.com) +Leah Klearman (lklrmn@gmail.com) +Jaivish Kothari (jaivish.kothari@nectechnologies.in) +Jakub Krajcovic (jakub.krajcovic@gmail.com) +David Kranz (david.kranz@qrclab.com) +Sushil Kumar (sushil.kumar2@globallogic.com) +Greg Lange (greglange@gmail.com) +Alexis Lee (alexisl@hp.com) +Tong Li (litong01@us.ibm.com) +Feng Liu (mefengliu23@gmail.com) +Jing Liuqing (jing.liuqing@99cloud.net) +Hemanth Makkapati (hemanth.makkapati@mailtrust.com) +Steve Martinelli (stevemar@ca.ibm.com) +Juan J. Martinez (juan@memset.com) +Donagh McCabe (donagh.mccabe@hp.com) +Ben McCann (ben@benmccann.com) +Andy McCrae (andy.mccrae@gmail.com) +Stuart McLaren (stuart.mclaren@hp.com) +Samuel Merritt (sam@swiftstack.com) +Jola Mirecka (jola.mirecka@hp.com) +Hiroshi Miura (miurahr@nttdata.co.jp) +Sam Morrison (sorrison@gmail.com) +Dirk Mueller (dirk@dmllr.de) +Zhenguo Niu (zhenguo@unitedstack.com) +Ondrej Novy (ondrej.novy@firma.seznam.cz) +Alessandro Pilotti (apilotti@cloudbasesolutions.com) +Alessandro Pilotti (ap@pilotti.it) +Stanislaw Pitucha (stanislaw.pitucha@hp.com) +Dan Prince (dprince@redhat.com) +Li Riqiang (lrqrun@gmail.com) +Hirokazu Sakata (h.sakata@staff.east.ntt.co.jp) +Christian Schwede (cschwede@redhat.com) +Mark Seger (Mark.Seger@hp.com) +Mark Seger (mark.seger@hp.com) +Chuck Short (chuck.short@canonical.com) +David Shrewsbury (shrewsbury.dave@gmail.com) +Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) +Jeremy Stanley (fungi@yuggoth.org) +Victor Stinner (victor.stinner@enovance.com) +Jiří Suchomel (jsuchome@suse.cz) +YUZAWA Takahiko (yuzawataka@intellilink.co.jp) +Monty Taylor (mordred@inaugust.com) +TheSriram (sriram@klusterkloud.com) +Tihomir Trifonov (t.trifonov@gmail.com) +Dean Troyer (dtroyer@gmail.com) +Stanislav Vitkovskiy (stas.vitkovsky@gmail.com) +Daniel Wakefield (daniel.wakefield@hp.com) +Shane Wang (shane.wang@intel.com) +Mark Washenberger (mark.washenberger@rackspace.com) +Wu Wenxiang (wu.wenxiang@99cloud.net) +Mike Widman (mwidman@endurancewindpower.com) +Joel Wright (joel.wright@sohonet.com) +You Yamagata (bi.yamagata@gmail.com) +YangLei (yanglyy@cn.ibm.com) +Pete Zaitcev (zaitcev@kotori.zaitcev.us) +Jian Zhang (jian.zhang@intel.com) +Yuan Zhou (yuan.zhou@intel.com) +groqez (groqez@yopmail.net) +tanlin (lin.tan@intel.com) +yangxurong (yangxurong@huawei.com) +yuxcer (yuxcer@126.com) +zhang-jinnan (ben.os@99cloud.net) diff -Nru python-swiftclient-2.4.0/ChangeLog python-swiftclient-2.6.0/ChangeLog --- python-swiftclient-2.4.0/ChangeLog 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-2.6.0/ChangeLog 2015-09-07 15:20:14.000000000 +0000 @@ -0,0 +1,423 @@ +2.6.0 +----- + +* Several CLI options have learned short options. The usage strings have + been updated to reflect this. + +* Added --no-shuffle option to the CLI download command. + +* Added --absolute option for CLI TempURL generation and the corresponding + parameter to utils.generate_temp_url(). This allows for an exact, specific + time to be used for the TempURL expiry time. + +* CLI arguments are now always decoded as UTF-8. + +* Stop Connection class modifying os_options parameter. + +* Reduce memory usage for download/delete. + +* The swift service API now logs and reports the traceback + on failed operations. + +* Increase httplib._MAXHEADERS to 256 to work around header limits in recent + Python releases. + +* Added minimal working service token support to client.py. + +* Various other minor bug fixes and improvements. + + +2.5.0 +----- + +* The CLI learned an "auth" subcommand which returns bash environment + snippets for auth credentials. + +* The CLI --version option is now more explicit by calling itself + "python-swiftclient" rather than the name of the binary. + +* Now validates the checksum of each chunk of a large object as it is + uploaded. + +* Fixes uploading an object with a relative path. + +* Added the ability to download objects to a particular folder. + +* Now correctly removes all old segments of an object when replacing a + Dynamic Large Object (DLO). + +* The --skip-identical option now works properly when downloading + large objects. + +* The client.get_object() response learned a .read([length]) method. + +* Fixed an issue where an intermediate caching/proxy service could cause + object content to be improperly decoded. + +* Added a timeout parameter to HTTPConnection objects for socket-level + read timeouts. + +* Removed a dependency on simplejson. + +* Various other minor bug fixes and improvements. + +2.4.0 +----- + +* Mention --segment-size option after 413 response +* Add improvements to MD5 validation +* Unindent a chunk of st_list +* Release connection after consuming the content +* Verify MD5 of uploaded objects +* Fix crash with -l, -d /, and pseudo folders +* add functional tox target +* Fix crash when stat'ing objects with non-ascii names +* Add help message for " --help" +* Fix missing ca-certificate parameter to get_auth +* Fix deleting SLO segments on overwrite +* This patch fixes downloading files to stdout +* Fix environment sanitization for TestServiceUtils +* Fix cross account upload using --os-storage-url +* Change tests to use CaptureOutput class +* Print info message about incorrect --totals usage when neither -l nor --lh is provided. Added test coverage for --totals +* Make preauth params work +* Fix misplaced check for None in SwiftUploadObject +* Fix misnamed dictionary key +* Change tests to use new CaptureOutput class +* Workflow documentation is now in infra-manual +* Show warning when auth_version >= 2 and keystoneclient is missing +* Capture test output better +* Suppress 'No handlers...' message from keystoneclient logger +* Add unit tests for _encode_meta_headers +* Fix misnamed variable in SwiftReader +* Check that content_type header exists before using +* Adds user friendly message when --segment-size is a non-integer +* Make swift post output an error message when failing +* Replaces Stacktraces with useful error messages +* Fix KeyError raised from client Connection +* Fix race in shell when testing for errors to raise SysExit +* Fix race between container create jobs during upload +* Fix the info command with --insecure +* Allow segment size to be specified in a human readable way +* Use skipTest from testtools instead of inherited Exception +* Add tests for account listing using --lh switch +* Do not crash with "swift list --lh" for Ceph RadosGW + +2.3.1 +----- + +* Remove a debugging print statement +* Fix unit tests failing when OS_ env vars are set +* Fix bug with some OS options not being passed to client +* Add per policy container count to account stat output +* Stop creating extraneous directories + +2.3.0 +----- + +* Work toward Python 3.4 support and testing +* Add importable SwiftService incorporating shell.py logic +* Adds console script entry point +* Do not create an empty directory 'pseudo/' +* fixed unit tests when env vars are set +* Fix crash when downloading a pseudo-directory +* Clean up raw policy stats in account stat +* Update theme for docs +* Add a tox job for generating docs +* Add keystone v3 auth support + +2.2.0 +----- + +* Fix context sensitive help for info and tempurl +* Allow to specify storage policy when uploading objects +* Adding Swift Temporary URL support +* Add CONTRIBUTING.md +* Add context sensitive help +* Relax requirement for tenant_name in get_auth() +* replace string format arguments with function parameters +* Removed now unnecesary workaround for PyPy +* Use Emacs-friendly coding line +* Remove extra double quote from docstring +* Fix wrong assertions in unit tests +* fixed several pep8 issues + +2.1.0 +----- + +* Fix Python3 bugs +* Remove testtools.main() call from tests +* Move test_shell.py under tests/unit/ +* Mark swiftclient as being a universal wheel +* change assert_ to assertTrue +* change assertEquals to assertEqual +* Provide a link to the documentation to the README +* fixed typos found by RETF rules +* Fix running the unittests under py3 +* Add "." for help strings +* Declare that we support Python 3 +* Make the function tests Python3-import friendly +* Only encode metadata for user customed headers +* Add functional tests for python-swiftclient +* Removed a duplicate word in a dostring +* Mock auth_end_time in test_shell.test_download +* Don't utf8 encode urls +* Fixed several shell tests on Python3 +* Fix up StringIO use in tests for py3 +* Updated test_shell for Python3 +* Fix test_raw_upload test +* Remove validate_headers +* Use quote/unquote from six module for py3 +* Makes use of requests.Session +* Fix test_multithreading on Python 3 +* Add tests for bin/swift +* Fix swiftclient.client.quote() for Python 3 +* Add requests related unit-tests +* Update help message to specify unit of --segment-size option +* Python 3: fix tests on HTTP headers +* Updated from global requirements +* Use the standard library's copy of mock when it's available +* Replaced print statements with print function +* Removed usage of tuple unpacking in parameters +* don't use mutable defaults in kwargs +* set user-agent header +* Python 3: Get compatible types from six +* Python 3: Fix module names in import +* Python 3: Add six dependency +* Replace dict.iteritems() with dict.items() +* Python 3: Replace iter.next() with six.next(iter) +* Make bin/swift testable part 2 +* Make bin/swift testable part 1 +* Python 3: Fix tests using temporary text files +* Python 3: cast map() result to list +* Fix temporary pypy gate issue with setuptools +* Decode HTTP responses, fixes bug #1282861 +* Copy Swift's .mailmap to swiftclient repo +* Improve help strings +* TCP port is appended two time in ClientException +* add "info" as an alias to "capabilities" +* Use six.StringIO instead of StringIO.StringIO + +2.0.3 +----- + +* Help string format persistent +* Make the help strings constant +* Add LengthWrapper in put_object to honor content_length param +* Updated from global requirements +* Remove useless statement +* swift.1 manpage fix for groff warnings + +2.0.2 +----- + +* Remove multipart/form-data file upload + +2.0.1 +----- + +* Fix --insecure option on auth +* Only run flake8 on swiftclient code + +2.0 +--- + + +1.9.0 +----- + +* Remove extraneous vim configuration comments +* Rename Openstack to OpenStack +* Port to python-requests +* Add option to skip downloading/uploading identical files +* Remove tox locale overrides +* Fix swiftclient help +* Fix misspellings in python swiftclient +* changed things because reasons +* Add missing backslash +* match hacking rules in swift +* Updated from global requirements +* Install manpage in share/man/man1 instead of man/man1 +* assertEquals is deprecated, use assertEqual +* Add capabilities option +* Install swiftclient manpage +* Replace xrange in for loop with range +* Add --object-name +* retry on ratelimit +* Fix help of some optional arguments +* Updates tox.ini to use new features +* Fix Sphinx version issue +* Enable usage of proxies defined in environment (http(s)_proxy) +* Don't crash when header is value of None +* Fix download bandwidth for swift command +* Updates .gitignore +* Allow custom headers when using swift download (CLI) +* Replaced two references to Cloud Files with Swift +* Fix a typo in help text: "downlad" +* Add close to swiftclient.client.Connection +* enhance swiftclient logging +* Clarify main help for post subcommand +* Fixes python-swiftclient debugging message + +1.8.0 +----- + +* Make pbr only a build-time dependency +* Add verbose output to all stat commands +* assertEquals is deprecated, use assertEqual (H602) +* Skip sniffing and reseting if retry is disabled +* user defined headers added to swift post queries + +1.7.0 +----- + +* Sync with global requirements +* fix bug with replace old *LOs +* Extend usage message for `swift download` + +1.6.0 +----- + +* Added support for running the tests under PyPy with tox +* Remove redundant unit suffix +* Reformat help outputs +* Add a NullHandler when setting up library logging +* Assignment to reserved built-in symbol "file" +* Added headers argument support to get_object() +* Move multi-threading code to a library +* fix(gitignore) : Ignore *.egg files +* python3: Start of adding basic python3 support +* Added log statements in swift client +* Update docstring for swiftclient.Connection.__init__ +* Refuse carriage return in header value +* Adds max-backoff for retries in Connection +* Allow setting # of retries in the binary + +1.5.0 +----- + +* Note '-V 2' is necessary for auth 2.0 +* Allow storage url override for both auth vers +* Add *.swp into .gitignore +* Add -p option to download command +* add -t for totals to list command and --lh to stat +* add optional 'response_dict' parameters to many calls into which they'll return a dictionary of the response status, reason and headers +* Fixes re-auth flow with expired tokens +* Remove explicit distribute depend +* Add -l and --lh switches to swift 'list' command +* Changed the call to set_tunnel to work in python 2.6 or python 2.7 since its name changed between versions +* Add option to disable SSL compression +* python3: Introduce py33 to tox.ini +* Rename requires files to standard names +* remove busy-wait so that swift client won't use up all CPU cycles +* log get_auth request url instead of x-storage-url +* Update the man page +* Add .coveragerc file to show correct code coverage +* do not warn about etag for slo +* Eradicate eventlet and fix bug lp:959221 +* Add end_marker and path query parameters +* Switch to pbr for setup +* Switch to flake8 +* Improve Python 3.x compatibility +* Confirm we have auth creds before clearing preauth + +1.4.0 +----- + +* Improve auth option help +* Static large object support +* Fixed pep8 errors in test directory +* Allow user to specify headers at the command line +* Enhance put_object to inform when chunk is ignored +* Allow v2 to use storage_url/storage_token directly +* Add client man page swift.1 +* Allow to specify segment container +* Added "/" check when list containers +* Print useful message when keystoneclient is not installed +* Fix reporting version + +1.3.0 +----- + +* Use testr instead of nose +* Update to latest oslo version/setup +* Add generated files to .gitignore +* Add env[SWIFTCLIENT_INSECURE] +* Fix debug feature and add --debug to swift +* Use testtools as base class for test cases +* Add --os-cacert +* Add --insecure option to fix bug #1077869 +* Don't segment objects smaller than --segment-size +* Don't add trailing slash to auth URL +* Adding segment size as another x-object-manifest component +* Stop loss of precision when writing 'x-object-meta-mtime' +* Remove unused json_request +* fixed inconsistencies in parameter descriptions +* tell nose to explicity test the 'tests' directory +* Fixes setup compatibility issue on Windows +* Force utf-8 encode of HTTPConnection params +* swiftclient Connection : default optional arguments to None +* Add OpenStack trove classifier for PyPI +* Resolves issue with empty os_options for swift-bench & swift-dispersion-report +* Catch authorization failures +* Do not use dictionaries as default parameters + +1.2.0 +----- + +* Add region_name support +* Allow endpoint type to be specified +* PEP8 cleanup +* PEP8 issues fixed +* Add ability to download without writing to disk +* Fix PEP8 issues +* Change '_' to '-' in options +* Fix swiftclient 400 error when OS_AUTH_URL is set +* Add nosehtmloutput as a test dependency +* Shuffle download order (of containers and objects) +* Add timing stats to verbose download output +* Ensure Content-Length header when PUT/POST a container +* Make python-keystoneclient optional +* Fix container delete throughput and 409 retries +* Consume version info from pkg_resources +* Use keystoneclient for authentication +* Removes the title "Swift Web" from landing page + +1.1.1 +----- + +* Now url encodes/decodes x-object-manifest values +* Configurable concurrency for swift client +* Allow specify tenant:user in user +* Make swift exit on ctrl-c +* Add post-tag versioning +* Don't suppress openstack auth options +* Make swift not hang on error +* Fix pep8 errors w/pep8==1.3 +* Add missing test/tools files to the tarball +* Add build_sphinx options +* Make CLI exit nonzero on error +* Add doc and version in swiftclient.__init__.py +* Raise ClientException for invalid auth version +* Version bump after pypi release + +1.1.0 +----- + +* Removed now-unused .cache.bundle references +* Added setup.cfg for verbose test output +* Add run_tests.sh script here +* Adding fake_http_connect to test.utils +* Add openstack project infrastructure +* Add logging +* Defined version to 1.0 +* Add CHANGELOG LICENSE and MANIFEST.in +* Delete old test_client and add a gitignore +* Rename client to swiftclient +* Fix links +* Import script from swift to run unittests +* Add test_client from original swift repository +* Add AUTHORS file +* Make sure we get a header StorageURL with 1.0 +* Allow specify the tenant in user +* First commit diff -Nru python-swiftclient-2.4.0/debian/changelog python-swiftclient-2.6.0/debian/changelog --- python-swiftclient-2.4.0/debian/changelog 2015-07-15 07:59:12.000000000 +0000 +++ python-swiftclient-2.6.0/debian/changelog 2015-09-28 07:54:59.000000000 +0000 @@ -1,3 +1,18 @@ +python-swiftclient (1:2.6.0-1~ubuntu15.10.1~ppa201509280854) wily; urgency=medium + + * No-change backport to wily + + -- James Page Mon, 28 Sep 2015 08:54:59 +0100 + +python-swiftclient (1:2.6.0-1) experimental; urgency=medium + + * New upstream release. + * d/control: Align dependencies and versions with upstream. + * d/patches: Dropped. Remaining patch fixed upstream. + * d/control: Update uploaders. + + -- Corey Bryant Thu, 24 Sep 2015 14:54:41 -0400 + python-swiftclient (1:2.4.0-5) experimental; urgency=medium * Team upload. diff -Nru python-swiftclient-2.4.0/debian/control python-swiftclient-2.6.0/debian/control --- python-swiftclient-2.4.0/debian/control 2015-07-15 07:59:12.000000000 +0000 +++ python-swiftclient-2.6.0/debian/control 2015-09-26 07:34:04.000000000 +0000 @@ -3,6 +3,7 @@ Priority: extra Maintainer: PKG OpenStack Uploaders: Thomas Goirand , + Corey Bryant , Build-Depends: debhelper (>= 9), dh-python, openstack-pkg-tools, @@ -15,17 +16,16 @@ python3-setuptools, Build-Depends-Indep: python-concurrent.futures, python-coverage, - python-hacking, + python-hacking (>= 0.10.0), python-keystoneclient, - python-mock, + python-mock (>= 1.2), python-oslosphinx, python-requests, - python-simplejson, python-six, python-testtools, - python3-hacking, + python3-hacking (>= 0.10.0), python3-keystoneclient, - python3-mock, + python3-mock (>= 1.2), python3-requests, python3-simplejson, python3-six, diff -Nru python-swiftclient-2.4.0/debian/patches/multithreading-fixes.patch python-swiftclient-2.6.0/debian/patches/multithreading-fixes.patch --- python-swiftclient-2.4.0/debian/patches/multithreading-fixes.patch 2015-07-15 07:59:12.000000000 +0000 +++ python-swiftclient-2.6.0/debian/patches/multithreading-fixes.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,401 +0,0 @@ -From a4fb70ece189aff85f234ab6b3f275b69e936c03 Mon Sep 17 00:00:00 2001 -From: Tim Burke -Date: Tue, 3 Mar 2015 12:35:03 -0800 -Subject: [PATCH] Compare each chunk of large objects when uploading - -Previously, we compared the ETag from Swift against the MD5 of the -entire large object. However, the ETag for large objects is generally -the MD5 of the concatenation of the ETags for each segment, unless the -object is a DLO whose segments span more than one page of a container -listing. Rather than worry about ETags, just compare each chunk of the -segmented file. This allows the use of --skip-identical when uploading -SLOs and DLOs. - -Additionally, there are several test-related improvements: - * The default arguments for OutputManager are now evaluated on - construction, rather than on definition, so that - TestOutputManager.test_instantiation will succeed when using nosetest - as a test runner. (See also: bug 1251507) - * An account_username option is now available in the functional tests - config file for auth systems that do not follow the account:username - format. - * CaptureOutput no longer writes to the captured stream, and - MockHttpTest now captures output. These were polluting test output - unnecessarily. (See also: bug 1201376) - -Change-Id: Ic484e9a0c186c9283c4012c6a2fa77b96b8edf8a -Closes-Bug: #1201376 -Closes-Bug: #1379252 -Related-Bug: #1251507 ---- - swiftclient/multithreading.py | 7 +- - swiftclient/service.py | 100 ++++++++++++++++++--------- - tests/functional/test_swiftclient.py | 10 ++- - tests/unit/test_service.py | 128 +++++++++++++++++++++++++++++++++++ - tests/unit/utils.py | 9 ++- - 5 files changed, 214 insertions(+), 40 deletions(-) - ---- a/swiftclient/multithreading.py -+++ b/swiftclient/multithreading.py -@@ -45,7 +45,7 @@ class OutputManager(object): - """ - DEFAULT_OFFSET = 14 - -- def __init__(self, print_stream=sys.stdout, error_stream=sys.stderr): -+ def __init__(self, print_stream=None, error_stream=None): - """ - :param print_stream: The stream to which :meth:`print_msg` sends - formatted messages. -@@ -54,9 +54,10 @@ class OutputManager(object): - - On Python 2, Unicode messages are encoded to utf8. - """ -- self.print_stream = print_stream -+ self.print_stream = print_stream or sys.stdout - self.print_pool = ThreadPoolExecutor(max_workers=1) -- self.error_stream = error_stream -+ -+ self.error_stream = error_stream or sys.stderr - self.error_print_pool = ThreadPoolExecutor(max_workers=1) - self.error_count = 0 - ---- a/swiftclient/service.py -+++ b/swiftclient/service.py -@@ -1160,7 +1160,7 @@ class SwiftService(object): - 'headers': [], - 'segment_size': None, - 'use_slo': False, -- 'segment_container: None, -+ 'segment_container': None, - 'leave_segments': False, - 'changed': None, - 'skip_identical': False, -@@ -1505,6 +1505,51 @@ class SwiftService(object): - results_queue.put(res) - return res - -+ def _get_chunk_data(self, conn, container, obj, headers): -+ chunks = [] -+ if 'x-object-manifest' in headers: -+ scontainer, sprefix = headers['x-object-manifest'].split('/', 1) -+ for part in self.list(scontainer, {'prefix': sprefix}): -+ if part["success"]: -+ chunks.extend(part["listing"]) -+ else: -+ raise part["error"] -+ elif config_true_value(headers.get('x-static-large-object')): -+ _, manifest_data = conn.get_object( -+ container, obj, query_string='multipart-manifest=get') -+ for chunk in json.loads(manifest_data): -+ if chunk.get('sub_slo'): -+ scont, sobj = chunk['name'].lstrip('/').split('/', 1) -+ chunks.extend(self._get_chunk_data( -+ conn, scont, sobj, {'x-static-large-object': True})) -+ else: -+ chunks.append(chunk) -+ else: -+ chunks.append({'hash': headers.get('etag').strip('"'), -+ 'bytes': int(headers.get('content-length'))}) -+ return chunks -+ -+ def _is_identical(self, chunk_data, path): -+ try: -+ fp = open(path, 'rb') -+ except IOError: -+ return False -+ -+ with fp: -+ for chunk in chunk_data: -+ to_read = chunk['bytes'] -+ md5sum = md5() -+ while to_read: -+ data = fp.read(min(65536, to_read)) -+ if not data: -+ return False -+ md5sum.update(data) -+ to_read -= len(data) -+ if md5sum.hexdigest() != chunk['hash']: -+ return False -+ # Each chunk is verified; check that we're at the end of the file -+ return not fp.read(1) -+ - def _upload_object_job(self, conn, container, source, obj, options, - results_queue=None): - res = { -@@ -1536,32 +1581,27 @@ class SwiftService(object): - old_manifest = None - old_slo_manifest_paths = [] - new_slo_manifest_paths = set() -+ segment_size = int(0 if options['segment_size'] is None -+ else options['segment_size']) - if (options['changed'] or options['skip_identical'] - or not options['leave_segments']): -- checksum = None -- if options['skip_identical']: -- try: -- fp = open(path, 'rb') -- except IOError: -- pass -- else: -- with fp: -- md5sum = md5() -- while True: -- data = fp.read(65536) -- if not data: -- break -- md5sum.update(data) -- checksum = md5sum.hexdigest() - try: - headers = conn.head_object(container, obj) -- if options['skip_identical'] and checksum is not None: -- if checksum == headers.get('etag'): -- res.update({ -- 'success': True, -- 'status': 'skipped-identical' -- }) -- return res -+ is_slo = config_true_value( -+ headers.get('x-static-large-object')) -+ -+ if options['skip_identical'] or ( -+ is_slo and not options['leave_segments']): -+ chunk_data = self._get_chunk_data( -+ conn, container, obj, headers) -+ -+ if options['skip_identical'] and self._is_identical( -+ chunk_data, path): -+ res.update({ -+ 'success': True, -+ 'status': 'skipped-identical' -+ }) -+ return res - - cl = int(headers.get('content-length')) - mt = headers.get('x-object-meta-mtime') -@@ -1575,13 +1615,8 @@ class SwiftService(object): - return res - if not options['leave_segments']: - old_manifest = headers.get('x-object-manifest') -- if config_true_value( -- headers.get('x-static-large-object')): -- headers, manifest_data = conn.get_object( -- container, obj, -- query_string='multipart-manifest=get' -- ) -- for old_seg in json.loads(manifest_data): -+ if is_slo: -+ for old_seg in chunk_data: - seg_path = old_seg['name'].lstrip('/') - if isinstance(seg_path, text_type): - seg_path = seg_path.encode('utf-8') -@@ -1601,8 +1636,8 @@ class SwiftService(object): - # a segment job if we're reading from a stream - we may fail if we - # go over the single object limit, but this gives us a nice way - # to create objects from memory -- if (path is not None and options['segment_size'] -- and (getsize(path) > int(options['segment_size']))): -+ if (path is not None and segment_size -+ and (getsize(path) > segment_size)): - res['large_object'] = True - seg_container = container + '_segments' - if options['segment_container']: -@@ -1615,7 +1650,6 @@ class SwiftService(object): - segment_start = 0 - - while segment_start < full_size: -- segment_size = int(options['segment_size']) - if segment_start + segment_size > full_size: - segment_size = full_size - segment_start - if options['use_slo']: ---- a/tests/functional/test_swiftclient.py -+++ b/tests/functional/test_swiftclient.py -@@ -51,8 +51,13 @@ class TestFunctional(testtools.TestCase) - auth_ssl = config.getboolean('func_test', 'auth_ssl') - auth_prefix = config.get('func_test', 'auth_prefix') - self.auth_version = config.get('func_test', 'auth_version') -- self.account = config.get('func_test', 'account') -- self.username = config.get('func_test', 'username') -+ try: -+ self.account_username = config.get('func_test', -+ 'account_username') -+ except configparser.NoOptionError: -+ account = config.get('func_test', 'account') -+ username = config.get('func_test', 'username') -+ self.account_username = "%s:%s" % (account, username) - self.password = config.get('func_test', 'password') - self.auth_url = "" - if auth_ssl: -@@ -62,7 +67,6 @@ class TestFunctional(testtools.TestCase) - self.auth_url += "%s:%s%s" % (auth_host, auth_port, auth_prefix) - if self.auth_version == "1": - self.auth_url += 'v1.0' -- self.account_username = "%s:%s" % (self.account, self.username) - - else: - self.skip_tests = True ---- a/tests/unit/test_service.py -+++ b/tests/unit/test_service.py -@@ -860,3 +860,131 @@ class TestServiceUpload(testtools.TestCa - - contents = mock_conn.put_object.call_args[0][2] - self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest()) -+ -+ def test_upload_object_job_identical_etag(self): -+ with tempfile.NamedTemporaryFile() as f: -+ f.write(b'a' * 30) -+ f.flush() -+ -+ mock_conn = mock.Mock() -+ mock_conn.head_object.return_value = { -+ 'content-length': 30, -+ 'etag': md5(b'a' * 30).hexdigest()} -+ type(mock_conn).attempts = mock.PropertyMock(return_value=2) -+ -+ s = SwiftService() -+ r = s._upload_object_job(conn=mock_conn, -+ container='test_c', -+ source=f.name, -+ obj='test_o', -+ options={'changed': False, -+ 'skip_identical': True, -+ 'leave_segments': True, -+ 'header': '', -+ 'segment_size': 0}) -+ -+ self.assertTrue(r['success']) -+ self.assertIn('status', r) -+ self.assertEqual(r['status'], 'skipped-identical') -+ self.assertEqual(mock_conn.put_object.call_count, 0) -+ self.assertEqual(mock_conn.head_object.call_count, 1) -+ mock_conn.head_object.assert_called_with('test_c', 'test_o') -+ -+ def test_upload_object_job_identical_slo_with_nesting(self): -+ with tempfile.NamedTemporaryFile() as f: -+ f.write(b'a' * 30) -+ f.flush() -+ seg_etag = md5(b'a' * 10).hexdigest() -+ submanifest = "[%s]" % ",".join( -+ ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) -+ submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() -+ manifest = "[%s]" % ",".join([ -+ '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' -+ '"bytes":20,"hash":"%s"}' % submanifest_etag, -+ '{"bytes":10,"hash":"%s"}' % seg_etag]) -+ -+ mock_conn = mock.Mock() -+ mock_conn.head_object.return_value = { -+ 'x-static-large-object': True, -+ 'content-length': 30, -+ 'etag': md5(submanifest_etag.encode('ascii') + -+ seg_etag.encode('ascii')).hexdigest()} -+ mock_conn.get_object.side_effect = [ -+ (None, manifest), -+ (None, submanifest)] -+ type(mock_conn).attempts = mock.PropertyMock(return_value=2) -+ -+ s = SwiftService() -+ r = s._upload_object_job(conn=mock_conn, -+ container='test_c', -+ source=f.name, -+ obj='test_o', -+ options={'changed': False, -+ 'skip_identical': True, -+ 'leave_segments': True, -+ 'header': '', -+ 'segment_size': 10}) -+ -+ self.assertIsNone(r.get('error')) -+ self.assertTrue(r['success']) -+ self.assertEqual('skipped-identical', r.get('status')) -+ self.assertEqual(0, mock_conn.put_object.call_count) -+ self.assertEqual([mock.call('test_c', 'test_o')], -+ mock_conn.head_object.mock_calls) -+ self.assertEqual([ -+ mock.call('test_c', 'test_o', -+ query_string='multipart-manifest=get'), -+ mock.call('test_c_segments', 'test_sub_slo', -+ query_string='multipart-manifest=get'), -+ ], mock_conn.get_object.mock_calls) -+ -+ def test_upload_object_job_identical_dlo(self): -+ with tempfile.NamedTemporaryFile() as f: -+ f.write(b'a' * 30) -+ f.flush() -+ segment_etag = md5(b'a' * 10).hexdigest() -+ -+ mock_conn = mock.Mock() -+ mock_conn.head_object.return_value = { -+ 'x-object-manifest': 'test_c_segments/test_o/prefix', -+ 'content-length': 30, -+ 'etag': md5(segment_etag.encode('ascii') * 3).hexdigest()} -+ mock_conn.get_container.side_effect = [ -+ (None, [{"bytes": 10, "hash": segment_etag, -+ "name": "test_o/prefix/00"}, -+ {"bytes": 10, "hash": segment_etag, -+ "name": "test_o/prefix/01"}]), -+ (None, [{"bytes": 10, "hash": segment_etag, -+ "name": "test_o/prefix/02"}]), -+ (None, {})] -+ type(mock_conn).attempts = mock.PropertyMock(return_value=2) -+ -+ s = SwiftService() -+ with mock.patch('swiftclient.service.get_conn', -+ return_value=mock_conn): -+ r = s._upload_object_job(conn=mock_conn, -+ container='test_c', -+ source=f.name, -+ obj='test_o', -+ options={'changed': False, -+ 'skip_identical': True, -+ 'leave_segments': True, -+ 'header': '', -+ 'segment_size': 10}) -+ -+ self.assertIsNone(r.get('error')) -+ self.assertTrue(r['success']) -+ self.assertEqual('skipped-identical', r.get('status')) -+ self.assertEqual(0, mock_conn.put_object.call_count) -+ self.assertEqual(1, mock_conn.head_object.call_count) -+ self.assertEqual(3, mock_conn.get_container.call_count) -+ mock_conn.head_object.assert_called_with('test_c', 'test_o') -+ expected = [ -+ mock.call('test_c_segments', prefix='test_o/prefix', -+ marker='', delimiter=None), -+ mock.call('test_c_segments', prefix='test_o/prefix', -+ marker="test_o/prefix/01", delimiter=None), -+ mock.call('test_c_segments', prefix='test_o/prefix', -+ marker="test_o/prefix/02", delimiter=None), -+ ] -+ mock_conn.get_container.assert_has_calls(expected) ---- a/tests/unit/utils.py -+++ b/tests/unit/utils.py -@@ -208,6 +208,12 @@ class MockHttpTest(testtools.TestCase): - self.fake_connect = None - self.request_log = [] - -+ # Capture output, since the test-runner stdout/stderr moneky-patching -+ # won't cover the references to sys.stdout/sys.stderr in -+ # swiftclient.multithreading -+ self.capture_output = CaptureOutput() -+ self.capture_output.__enter__() -+ - def fake_http_connection(*args, **kwargs): - self.validateMockedRequestsConsumed() - self.request_log = [] -@@ -368,6 +374,7 @@ class MockHttpTest(testtools.TestCase): - # un-hygienic mocking on the swiftclient.client module; which may lead - # to some unfortunate test order dependency bugs by way of the broken - # window theory if any other modules are similarly patched -+ self.capture_output.__exit__() - reload_module(c) - - -@@ -393,7 +400,7 @@ class CaptureStream(object): - self.stream = stream - self._capture = six.StringIO() - self._buffer = CaptureStreamBuffer(self) -- self.streams = [self.stream, self._capture] -+ self.streams = [self._capture] - - @property - def buffer(self): diff -Nru python-swiftclient-2.4.0/debian/patches/series python-swiftclient-2.6.0/debian/patches/series --- python-swiftclient-2.4.0/debian/patches/series 2015-07-15 07:59:12.000000000 +0000 +++ python-swiftclient-2.6.0/debian/patches/series 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -multithreading-fixes.patch diff -Nru python-swiftclient-2.4.0/doc/manpages/swift.1 python-swiftclient-2.6.0/doc/manpages/swift.1 --- python-swiftclient-2.4.0/doc/manpages/swift.1 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/doc/manpages/swift.1 2015-09-07 15:20:14.000000000 +0000 @@ -104,6 +104,16 @@ proxy-url. .RE +\fBtempurl\fR \fImethod\fR \fIseconds\fR \fIpath\fR \fIkey\fR [\fI--absolute\fR] +.RS 4 +Generates a temporary URL allowing unauthenticated access to the Swift object +at the given path, using the given HTTP method, for the given number of +seconds, using the given TempURL key. If optional --absolute argument is +provided, seconds is instead interpreted as a Unix timestamp at which the URL +should expire. \fBExample\fR: tempurl GET $(date -d "Jan 1 2016" +%s) +/v1/AUTH_foo/bar_container/quux.md my_secret_tempurl_key --absolute +.RE + .SH OPTIONS .PD 0 .IP "--version Show program's version number and exit" diff -Nru python-swiftclient-2.4.0/.gitignore python-swiftclient-2.6.0/.gitignore --- python-swiftclient-2.4.0/.gitignore 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/.gitignore 2015-09-07 15:20:14.000000000 +0000 @@ -1,5 +1,3 @@ -AUTHORS -ChangeLog *.sw? dist/ .tox diff -Nru python-swiftclient-2.4.0/.mailmap python-swiftclient-2.6.0/.mailmap --- python-swiftclient-2.4.0/.mailmap 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/.mailmap 2015-09-07 15:20:14.000000000 +0000 @@ -10,6 +10,7 @@ Michael Barton Mike Barton Clay Gerrard Clay Gerrard +Clay Gerrard Clay Gerrard clayg David Goetz David Goetz @@ -50,4 +51,30 @@ Sascha Peilicke Sascha Peilicke Zhenguo Niu Peter Portante -Christian Schwede +Christian Schwede +Christian Schwede +Constantine Peresypkin +Madhuri Kumari madhuri +Morgan Fainberg +Hua Zhang +Yummy Bian +Alistair Coles +Tong Li +Paul Luse +Yuan Zhou +Jola Mirecka +Ning Zhang +Mauro Stettler +Pawel Palucki +Guang Yee +Jing Liuqing +Lorcan Browne +Eohyung Lee +Harshit Chitalia +Richard Hawkins +Sarvesh Ranjan +Minwoo Bae Minwoo B +Jaivish Kothari +Michael Matur +Kazuhiro Miyahara +Alexandra Settle diff -Nru python-swiftclient-2.4.0/requirements.txt python-swiftclient-2.6.0/requirements.txt --- python-swiftclient-2.4.0/requirements.txt 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/requirements.txt 2015-09-07 15:20:14.000000000 +0000 @@ -1,4 +1,3 @@ futures>=2.1.3 requests>=1.1 -simplejson>=2.0.9 six>=1.5.2 diff -Nru python-swiftclient-2.4.0/setup.cfg python-swiftclient-2.6.0/setup.cfg --- python-swiftclient-2.4.0/setup.cfg 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/setup.cfg 2015-09-07 15:20:14.000000000 +0000 @@ -46,3 +46,7 @@ [wheel] universal = 1 + +[pbr] +skip_authors = True +skip_changelog = True diff -Nru python-swiftclient-2.4.0/swiftclient/client.py python-swiftclient-2.6.0/swiftclient/client.py --- python-swiftclient-2.4.0/swiftclient/client.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/swiftclient/client.py 2015-09-07 15:20:14.000000000 +0000 @@ -16,18 +16,14 @@ """ OpenStack Swift client library used internally """ - import socket import requests import logging import warnings -try: - from simplejson import loads as json_loads -except ImportError: - from json import loads as json_loads from distutils.version import StrictVersion from requests.exceptions import RequestException, SSLError +from six.moves import http_client from six.moves.urllib.parse import quote as _quote from six.moves.urllib.parse import urlparse, urlunparse from time import sleep, time @@ -35,7 +31,11 @@ from swiftclient import version as swiftclient_version from swiftclient.exceptions import ClientException -from swiftclient.utils import LengthWrapper, ReadableToIterable +from swiftclient.utils import ( + LengthWrapper, ReadableToIterable, parse_api_response) + +# Defautl is 100, increase to 256 +http_client._MAXHEADERS = 256 AUTH_VERSIONS_V1 = ('1.0', '1', 1) AUTH_VERSIONS_V2 = ('2.0', '2', 2) @@ -138,9 +138,40 @@ return ret +class _ObjectBody(object): + """ + Readable and iterable object body response wrapper. + """ + + def __init__(self, resp, chunk_size): + """ + Wrap the underlying response + + :param resp: the response to wrap + :param chunk_size: number of bytes to return each iteration/next call + """ + self.resp = resp + self.chunk_size = chunk_size + + def read(self, length=None): + return self.resp.read(length) + + def __iter__(self): + return self + + def next(self): + buf = self.resp.read(self.chunk_size) + if not buf: + raise StopIteration() + return buf + + def __next__(self): + return self.next() + + class HTTPConnection(object): def __init__(self, url, proxy=None, cacert=None, insecure=False, - ssl_compression=False, default_user_agent=None): + ssl_compression=False, default_user_agent=None, timeout=None): """ Make an HTTPConnection or HTTPSConnection @@ -160,6 +191,8 @@ may be overridden on a per-request basis by explicitly setting the user-agent header on a call to request(). + :param timeout: socket read timeout value, passed directly to + the requests library. :raises ClientException: Unable to handle protocol scheme """ self.url = url @@ -168,8 +201,11 @@ self.port = self.parsed_url.port self.requests_args = {} self.request_session = requests.Session() + # Don't use requests's default headers + self.request_session.headers = None if self.parsed_url.scheme not in ('http', 'https'): - raise ClientException("Unsupported scheme") + raise ClientException('Unsupported scheme "%s" in url "%s"' + % (self.parsed_url.scheme, url)) self.requests_args['verify'] = not insecure if cacert and not insecure: # verify requests parameter is used to pass the CA_BUNDLE file @@ -189,6 +225,8 @@ default_user_agent = \ 'python-swiftclient-%s' % swiftclient_version.version_string self.default_user_agent = default_user_agent + if timeout: + self.requests_args['timeout'] = timeout def _request(self, *arg, **kwarg): """ Final wrapper before requests call, to be patched in tests """ @@ -233,7 +271,6 @@ return old_getheader(k.lower(), v) def releasing_read(*args, **kwargs): - kwargs['decode_content'] = True chunk = self.resp.raw.read(*args, **kwargs) if not chunk: # NOTE(sigmavirus24): Release the connection back to the @@ -260,7 +297,9 @@ def get_auth_1_0(url, user, key, snet, **kwargs): cacert = kwargs.get('cacert', None) insecure = kwargs.get('insecure', False) - parsed, conn = http_connection(url, cacert=cacert, insecure=insecure) + timeout = kwargs.get('timeout', None) + parsed, conn = http_connection(url, cacert=cacert, insecure=insecure, + timeout=timeout) method = 'GET' conn.request(method, parsed.path, '', {'X-Auth-User': user, 'X-Auth-Key': key}) @@ -320,6 +359,7 @@ """ insecure = kwargs.get('insecure', False) + timeout = kwargs.get('timeout', None) auth_version = kwargs.get('auth_version', '2.0') debug = logger.isEnabledFor(logging.DEBUG) and True or False @@ -340,7 +380,7 @@ project_domain_id=os_options.get('project_domain_id'), debug=debug, cacert=kwargs.get('cacert'), - auth_url=auth_url, insecure=insecure) + auth_url=auth_url, insecure=insecure, timeout=timeout) except exceptions.Unauthorized: msg = 'Unauthorized. Check username, password and tenant name/id.' if auth_version in AUTH_VERSIONS_V3: @@ -388,13 +428,15 @@ storage_url, token = None, None cacert = kwargs.get('cacert', None) insecure = kwargs.get('insecure', False) + timeout = kwargs.get('timeout', None) if auth_version in AUTH_VERSIONS_V1: storage_url, token = get_auth_1_0(auth_url, user, key, kwargs.get('snet'), cacert=cacert, - insecure=insecure) + insecure=insecure, + timeout=timeout) elif auth_version in AUTH_VERSIONS_V2 + AUTH_VERSIONS_V3: # We are handling a special use case here where the user argument # specifies both the user name and tenant name in the form tenant:user @@ -417,6 +459,7 @@ key, os_options, cacert=cacert, insecure=insecure, + timeout=timeout, auth_version=auth_version) else: raise ClientException('Unknown auth_version %s specified.' @@ -449,7 +492,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None, - end_marker=None, http_conn=None, full_listing=False): + end_marker=None, http_conn=None, full_listing=False, + service_token=None): """ Get a listing of containers for the account. @@ -463,6 +507,7 @@ conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings + :param service_token: service auth token :returns: a tuple of (response headers, a list of containers) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -492,6 +537,8 @@ qs += '&end_marker=%s' % quote(end_marker) full_path = '%s?%s' % (parsed.path, qs) headers = {'X-Auth-Token': token} + if service_token: + headers['X-Service-Token'] = service_token method = 'GET' conn.request(method, full_path, '', headers) resp = conn.getresponse() @@ -509,10 +556,10 @@ http_response_content=body) if resp.status == 204: return resp_headers, [] - return resp_headers, json_loads(body) + return resp_headers, parse_api_response(resp_headers, body) -def head_account(url, token, http_conn=None): +def head_account(url, token, http_conn=None, service_token=None): """ Get account stats. @@ -520,6 +567,7 @@ :param token: auth token :param http_conn: HTTP connection object (If None, it will create the conn object) + :param service_token: service auth token :returns: a dict containing the response's headers (all header names will be lowercase) :raises ClientException: HTTP HEAD request failed @@ -530,6 +578,8 @@ parsed, conn = http_connection(url) method = "HEAD" headers = {'X-Auth-Token': token} + if service_token: + headers['X-Service-Token'] = service_token conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() @@ -545,7 +595,8 @@ return resp_headers -def post_account(url, token, headers, http_conn=None, response_dict=None): +def post_account(url, token, headers, http_conn=None, response_dict=None, + service_token=None): """ Update an account's metadata. @@ -556,6 +607,7 @@ conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP POST request failed """ if http_conn: @@ -564,6 +616,8 @@ parsed, conn = http_connection(url) method = 'POST' headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() @@ -584,7 +638,7 @@ def get_container(url, token, container, marker=None, limit=None, prefix=None, delimiter=None, end_marker=None, path=None, http_conn=None, - full_listing=False): + full_listing=False, service_token=None): """ Get a listing of objects for the container. @@ -601,6 +655,7 @@ conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings + :param service_token: service auth token :returns: a tuple of (response headers, a list of objects) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -609,7 +664,8 @@ http_conn = http_connection(url) if full_listing: rv = get_container(url, token, container, marker, limit, prefix, - delimiter, end_marker, path, http_conn) + delimiter, end_marker, path, http_conn, + service_token) listing = rv[1] while listing: if not delimiter: @@ -618,7 +674,7 @@ marker = listing[-1].get('name', listing[-1].get('subdir')) listing = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, - http_conn)[1] + http_conn, service_token)[1] if listing: rv[1].extend(listing) return rv @@ -638,6 +694,8 @@ if path: qs += '&path=%s' % quote(path) headers = {'X-Auth-Token': token} + if service_token: + headers['X-Service-Token'] = service_token method = 'GET' conn.request(method, '%s?%s' % (cont_path, qs), '', headers) resp = conn.getresponse() @@ -659,10 +717,11 @@ resp_headers[header.lower()] = value if resp.status == 204: return resp_headers, [] - return resp_headers, json_loads(body) + return resp_headers, parse_api_response(resp_headers, body) -def head_container(url, token, container, http_conn=None, headers=None): +def head_container(url, token, container, http_conn=None, headers=None, + service_token=None): """ Get container stats. @@ -671,6 +730,7 @@ :param container: container name to get stats for :param http_conn: HTTP connection object (If None, it will create the conn object) + :param service_token: service auth token :returns: a dict containing the response's headers (all header names will be lowercase) :raises ClientException: HTTP HEAD request failed @@ -682,6 +742,8 @@ path = '%s/%s' % (parsed.path, quote(container)) method = 'HEAD' req_headers = {'X-Auth-Token': token} + if service_token: + req_headers['X-Service-Token'] = service_token if headers: req_headers.update(headers) conn.request(method, path, '', req_headers) @@ -703,7 +765,7 @@ def put_container(url, token, container, headers=None, http_conn=None, - response_dict=None): + response_dict=None, service_token=None): """ Create a container @@ -715,6 +777,7 @@ conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP PUT request failed """ if http_conn: @@ -726,6 +789,8 @@ if not headers: headers = {} headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token if 'content-length' not in (k.lower() for k in headers): headers['Content-Length'] = '0' conn.request(method, path, '', headers) @@ -745,7 +810,7 @@ def post_container(url, token, container, headers, http_conn=None, - response_dict=None): + response_dict=None, service_token=None): """ Update a container's metadata. @@ -757,6 +822,7 @@ conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP POST request failed """ if http_conn: @@ -766,6 +832,8 @@ path = '%s/%s' % (parsed.path, quote(container)) method = 'POST' headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token if 'content-length' not in (k.lower() for k in headers): headers['Content-Length'] = '0' conn.request(method, path, '', headers) @@ -785,7 +853,7 @@ def delete_container(url, token, container, http_conn=None, - response_dict=None): + response_dict=None, service_token=None): """ Delete a container @@ -796,6 +864,7 @@ conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP DELETE request failed """ if http_conn: @@ -804,6 +873,8 @@ parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) headers = {'X-Auth-Token': token} + if service_token: + headers['X-Service-Token'] = service_token method = 'DELETE' conn.request(method, path, '', headers) resp = conn.getresponse() @@ -823,7 +894,7 @@ def get_object(url, token, container, name, http_conn=None, resp_chunk_size=None, query_string=None, - response_dict=None, headers=None): + response_dict=None, headers=None, service_token=None): """ Get an object @@ -842,6 +913,7 @@ the response - status, reason and headers :param headers: an optional dictionary with additional headers to include in the request + :param service_token: service auth token :returns: a tuple of (response headers, the object's contents) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -856,6 +928,8 @@ method = 'GET' headers = headers.copy() if headers else {} headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token conn.request(method, path, '', headers) resp = conn.getresponse() @@ -874,13 +948,7 @@ http_reason=resp.reason, http_response_content=body) if resp_chunk_size: - - def _object_body(): - buf = resp.read(resp_chunk_size) - while buf: - yield buf - buf = resp.read(resp_chunk_size) - object_body = _object_body() + object_body = _ObjectBody(resp, resp_chunk_size) else: object_body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), @@ -889,7 +957,8 @@ return parsed_response['headers'], object_body -def head_object(url, token, container, name, http_conn=None): +def head_object(url, token, container, name, http_conn=None, + service_token=None): """ Get object info @@ -899,6 +968,7 @@ :param name: object name to get info for :param http_conn: HTTP connection object (If None, it will create the conn object) + :param service_token: service auth token :returns: a dict containing the response's headers (all header names will be lowercase) :raises ClientException: HTTP HEAD request failed @@ -910,6 +980,8 @@ path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) method = 'HEAD' headers = {'X-Auth-Token': token} + if service_token: + headers['X-Service-Token'] = service_token conn.request(method, path, '', headers) resp = conn.getresponse() body = resp.read() @@ -929,7 +1001,7 @@ def put_object(url, token=None, container=None, name=None, contents=None, content_length=None, etag=None, chunk_size=None, content_type=None, headers=None, http_conn=None, proxy=None, - query_string=None, response_dict=None): + query_string=None, response_dict=None, service_token=None): """ Put an object @@ -960,6 +1032,7 @@ :param query_string: if set will be appended with '?' to generated path :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :returns: etag :raises ClientException: HTTP PUT request failed """ @@ -980,6 +1053,8 @@ headers = {} if token: headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token if etag: headers['ETag'] = etag.strip('"') if content_length is not None: @@ -1033,7 +1108,7 @@ def post_object(url, token, container, name, headers, http_conn=None, - response_dict=None): + response_dict=None, service_token=None): """ Update object metadata @@ -1046,6 +1121,7 @@ conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP POST request failed """ if http_conn: @@ -1054,6 +1130,8 @@ parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token conn.request('POST', path, '', headers) resp = conn.getresponse() body = resp.read() @@ -1071,7 +1149,7 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None, headers=None, proxy=None, query_string=None, - response_dict=None): + response_dict=None, service_token=None): """ Delete object @@ -1089,6 +1167,7 @@ :param query_string: if set will be appended with '?' to generated path :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP DELETE request failed """ if http_conn: @@ -1108,6 +1187,8 @@ headers = {} if token: headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token conn.request('DELETE', path, '', headers) resp = conn.getresponse() body = resp.read() @@ -1143,18 +1224,33 @@ http_host=conn.host, http_path=parsed.path, http_status=resp.status, http_reason=resp.reason, http_response_content=body) - return json_loads(body) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return parse_api_response(resp_headers, body) class Connection(object): - """Convenience class to make requests that will also retry the request""" + + """ + Convenience class to make requests that will also retry the request + + Requests will have an X-Auth-Token header whose value is either + the preauthtoken or a token obtained from the auth service using + the user credentials provided as args to the constructor. If + os_options includes a service_username then requests will also have + an X-Service-Token header whose value is a token obtained from the + auth service using the service credentials. In this case the request + url will be set to the storage_url obtained from the auth service + for the service user, unless this is overridden by a preauthurl. + """ def __init__(self, authurl=None, user=None, key=None, retries=5, preauthurl=None, preauthtoken=None, snet=False, starting_backoff=1, max_backoff=64, tenant_name=None, os_options=None, auth_version="1", cacert=None, insecure=False, ssl_compression=True, - retry_on_ratelimit=False): + retry_on_ratelimit=False, timeout=None): """ :param authurl: authentication URL :param user: user name to authenticate as @@ -1172,7 +1268,8 @@ to an auth 2.0 system. :param os_options: The OpenStack options which can have tenant_id, auth_token, service_type, endpoint_type, - tenant_name, object_storage_url, region_name + tenant_name, object_storage_url, region_name, + service_username, service_project_name, service_key :param insecure: Allow to access servers without checking SSL certs. The server's certificate will not be verified. :param ssl_compression: Whether to enable compression at the SSL layer. @@ -1184,6 +1281,7 @@ raise an exception to the caller. Setting this parameter to True will cause a retry after a backoff. + :param timeout: The connect timeout for the HTTP connection. """ self.authurl = authurl self.user = user @@ -1195,18 +1293,24 @@ self.starting_backoff = starting_backoff self.max_backoff = max_backoff self.auth_version = auth_version - self.os_options = os_options or {} + self.os_options = dict(os_options or {}) if tenant_name: self.os_options['tenant_name'] = tenant_name if preauthurl: self.os_options['object_storage_url'] = preauthurl self.url = preauthurl or self.os_options.get('object_storage_url') self.token = preauthtoken or self.os_options.get('auth_token') + if self.os_options.get('service_username', None): + self.service_auth = True + else: + self.service_auth = False + self.service_token = None self.cacert = cacert self.insecure = insecure self.ssl_compression = ssl_compression self.auth_end_time = 0 self.retry_on_ratelimit = retry_on_ratelimit + self.timeout = timeout def close(self): if (self.http_conn and isinstance(self.http_conn, tuple) @@ -1219,18 +1323,39 @@ self.http_conn = None def get_auth(self): - return get_auth(self.authurl, self.user, self.key, + self.url, self.token = get_auth(self.authurl, self.user, self.key, + snet=self.snet, + auth_version=self.auth_version, + os_options=self.os_options, + cacert=self.cacert, + insecure=self.insecure, + timeout=self.timeout) + return self.url, self.token + + def get_service_auth(self): + opts = self.os_options + service_options = {} + service_options['tenant_name'] = opts.get('service_project_name', None) + service_options['region_name'] = opts.get('region_name', None) + service_options['object_storage_url'] = opts.get('object_storage_url', + None) + service_user = opts.get('service_username', None) + service_key = opts.get('service_key', None) + return get_auth(self.authurl, service_user, + service_key, snet=self.snet, auth_version=self.auth_version, - os_options=self.os_options, + os_options=service_options, cacert=self.cacert, - insecure=self.insecure) + insecure=self.insecure, + timeout=self.timeout) def http_connection(self, url=None): return http_connection(url if url else self.url, cacert=self.cacert, insecure=self.insecure, - ssl_compression=self.ssl_compression) + ssl_compression=self.ssl_compression, + timeout=self.timeout) def _add_response_dict(self, target_dict, kwargs): if target_dict is not None and 'response_dict' in kwargs: @@ -1252,13 +1377,17 @@ if not self.url or not self.token: self.url, self.token = self.get_auth() self.http_conn = None + if self.service_auth and not self.service_token: + self.url, self.service_token = self.get_service_auth() + self.http_conn = None self.auth_end_time = time() if not self.http_conn: self.http_conn = self.http_connection() kwargs['http_conn'] = self.http_conn if caller_response_dict is not None: kwargs['response_dict'] = {} - rv = func(self.url, self.token, *args, **kwargs) + rv = func(self.url, self.token, *args, + service_token=self.service_token, **kwargs) self._add_response_dict(caller_response_dict, kwargs) return rv except SSLError: @@ -1275,7 +1404,7 @@ logger.exception(err) raise if err.http_status == 401: - self.url = self.token = None + self.url = self.token = self.service_token = None if retried_auth or not all((self.authurl, self.user, self.key)): diff -Nru python-swiftclient-2.4.0/swiftclient/multithreading.py python-swiftclient-2.6.0/swiftclient/multithreading.py --- python-swiftclient-2.4.0/swiftclient/multithreading.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/swiftclient/multithreading.py 2015-09-07 15:20:14.000000000 +0000 @@ -45,7 +45,7 @@ """ DEFAULT_OFFSET = 14 - def __init__(self, print_stream=sys.stdout, error_stream=sys.stderr): + def __init__(self, print_stream=None, error_stream=None): """ :param print_stream: The stream to which :meth:`print_msg` sends formatted messages. @@ -54,9 +54,10 @@ On Python 2, Unicode messages are encoded to utf8. """ - self.print_stream = print_stream + self.print_stream = print_stream or sys.stdout self.print_pool = ThreadPoolExecutor(max_workers=1) - self.error_stream = error_stream + + self.error_stream = error_stream or sys.stderr self.error_print_pool = ThreadPoolExecutor(max_workers=1) self.error_count = 0 @@ -101,7 +102,7 @@ def _print(self, item, stream=None): if stream is None: stream = self.print_stream - if six.PY2 and isinstance(item, unicode): + if six.PY2 and isinstance(item, six.text_type): item = item.encode('utf8') print(item, file=stream) diff -Nru python-swiftclient-2.4.0/swiftclient/service.py python-swiftclient-2.6.0/swiftclient/service.py --- python-swiftclient-2.4.0/swiftclient/service.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/swiftclient/service.py 2015-09-07 15:20:14.000000000 +0000 @@ -12,6 +12,9 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging +import os + from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT @@ -29,10 +32,7 @@ from six.moves.urllib.parse import quote, unquote from six import Iterator, string_types -try: - import simplejson as json -except ImportError: - import json +import json from swiftclient import Connection @@ -40,12 +40,16 @@ stat_account, stat_container, stat_object ) from swiftclient.utils import ( - config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG + config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG, + parse_api_response, report_traceback ) from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager +logger = logging.getLogger("swiftclient.service") + + class ResultsIterator(Iterator): def __init__(self, futures): self.futures = interruptable_as_completed(futures) @@ -164,6 +168,8 @@ 'read_acl': None, 'write_acl': None, 'out_file': None, + 'out_directory': None, + 'remove_prefix': False, 'no_download': False, 'long': False, 'totals': False, @@ -175,7 +181,8 @@ 'fail_fast': False, 'human': False, 'dir_marker': False, - 'checksum': True + 'checksum': True, + 'shuffle': False } POLICY = 'X-Storage-Policy' @@ -433,16 +440,24 @@ return res except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res raise SwiftError('Account not found', exc=err) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res else: @@ -465,17 +480,25 @@ return res except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res raise SwiftError('Container %r not found' % container, container=container, exc=err) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res else: @@ -504,9 +527,13 @@ }) return res except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res @@ -579,18 +606,26 @@ get_future_result(post) except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': response_dict }) return res - raise SwiftError('Account not found') + raise SwiftError('Account not found', exc=err) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, 'error': err, - 'response_dict': response_dict + 'response_dict': response_dict, + 'traceback': traceback, + 'error_timestamp': err_time }) return res if not objects: @@ -617,23 +652,31 @@ get_future_result(post) except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'action': 'post_container', 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': response_dict }) return res raise SwiftError( "Container '%s' not found" % container, - container=container + container=container, exc=err ) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'action': 'post_container', 'success': False, 'error': err, - 'response_dict': response_dict + 'response_dict': response_dict, + 'traceback': traceback, + 'error_timestamp': err_time }) return res else: @@ -718,9 +761,13 @@ conn.post_object( container, obj, headers=headers, response_dict=result) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res @@ -751,7 +798,7 @@ else: options = self._options - rq = Queue() + rq = Queue(maxsize=10) # Just stop list running away consuming memory if container is None: listing_future = self.thread_manager.container_pool.submit( @@ -773,7 +820,6 @@ @staticmethod def _list_account_job(conn, options, result_queue): marker = '' - success = True error = None try: while True: @@ -802,23 +848,30 @@ marker = items[-1].get('name', items[-1].get('subdir')) except ClientException as err: - success = False + traceback, err_time = report_traceback() + logger.exception(err) if err.http_status != 404: - error = err + error = (err, traceback, err_time) else: - error = SwiftError('Account not found') + error = ( + SwiftError('Account not found', exc=err), + traceback, err_time + ) except Exception as err: - success = False - error = err + traceback, err_time = report_traceback() + logger.exception(err) + error = (err, traceback, err_time) res = { 'action': 'list_account_part', 'container': None, 'prefix': options['prefix'], - 'success': success, + 'success': False, 'marker': marker, - 'error': error, + 'error': error[0], + 'traceback': error[1], + 'error_timestamp': error[2] } result_queue.put(res) result_queue.put(None) @@ -826,7 +879,6 @@ @staticmethod def _list_container_job(conn, container, options, result_queue): marker = '' - success = True error = None try: while True: @@ -851,23 +903,33 @@ marker = items[-1].get('name', items[-1].get('subdir')) except ClientException as err: - success = False + traceback, err_time = report_traceback() + logger.exception(err) if err.http_status != 404: - error = err + error = (err, traceback, err_time) else: - error = SwiftError('Container %r not found' % container, - container=container) + error = ( + SwiftError( + 'Container %r not found' % container, + container=container, exc=err + ), + traceback, + err_time + ) except Exception as err: - success = False - error = err + traceback, err_time = report_traceback() + logger.exception(err) + error = (err, traceback, err_time) res = { 'action': 'list_container_part', 'container': container, 'prefix': options['prefix'], - 'success': success, + 'success': False, 'marker': marker, - 'error': error, + 'error': error[0], + 'traceback': error[1], + 'error_timestamp': error[2] } result_queue.put(res) result_queue.put(None) @@ -891,7 +953,10 @@ 'no_download': False, 'header': [], 'skip_identical': False, - 'out_file': None + 'out_directory': None, + 'out_file': None, + 'remove_prefix': False, + 'shuffle' : False } :returns: A generator for returning the results of the download @@ -913,45 +978,26 @@ try: options_copy = deepcopy(options) options_copy["long"] = False - containers = [] + for part in self.list(options=options_copy): if part["success"]: - containers.extend([ - i['name'] for i in part["listing"] - ]) - else: - raise part["error"] - - shuffle(containers) + containers = [i['name'] for i in part["listing"]] - o_downs = [] - for con in containers: - objs = [] - for part in self.list( - container=con, options=options_copy): - if part["success"]: - objs.extend([ - i['name'] for i in part["listing"] - ]) - else: - raise part["error"] - shuffle(objs) - - o_downs.extend( - self.thread_manager.object_dd_pool.submit( - self._download_object_job, con, obj, - options_copy - ) for obj in objs - ) + if options['shuffle']: + shuffle(containers) - for o_down in interruptable_as_completed(o_downs): - yield o_down.result() + for con in containers: + for res in self._download_container( + con, options_copy): + yield res + else: + raise part["error"] # If we see a 404 here, the listing of the account failed except ClientException as err: if err.http_status != 404: raise - raise SwiftError('Account not found') + raise SwiftError('Account not found', exc=err) elif not objects: if '/' in container: @@ -976,8 +1022,7 @@ for o_down in interruptable_as_completed(o_downs): yield o_down.result() - @staticmethod - def _download_object_job(conn, container, obj, options): + def _download_object_job(self, conn, container, obj, options): out_file = options['out_file'] results_dict = {} @@ -986,7 +1031,16 @@ pseudodir = False path = join(container, obj) if options['yes_all'] else obj path = path.lstrip(os_path_sep) - if options['skip_identical'] and out_file != '-': + options['skip_identical'] = (options['skip_identical'] and + out_file != '-') + + if options['prefix'] and options['remove_prefix']: + path = path[len(options['prefix']):].lstrip('/') + + if options['out_directory']: + path = os.path.join(options['out_directory'], path) + + if options['skip_identical']: filename = out_file if out_file else path try: fp = open(filename, 'rb') @@ -1004,10 +1058,55 @@ try: start_time = time() - headers, body = \ - conn.get_object(container, obj, resp_chunk_size=65536, - headers=req_headers, - response_dict=results_dict) + get_args = {'resp_chunk_size': 65536, + 'headers': req_headers, + 'response_dict': results_dict} + if options['skip_identical']: + # Assume the file is a large object; if we're wrong, the query + # string is ignored and the If-None-Match header will trigger + # the behavior we want + get_args['query_string'] = 'multipart-manifest=get' + + try: + headers, body = conn.get_object(container, obj, **get_args) + except ClientException as e: + if not options['skip_identical']: + raise + if e.http_status != 304: # Only handling Not Modified + raise + + headers = results_dict['headers'] + if 'x-object-manifest' in headers: + # DLO: most likely it has more than one page worth of + # segments and we have an empty file locally + body = [] + elif config_true_value(headers.get('x-static-large-object')): + # SLO: apparently we have a copy of the manifest locally? + # provide no chunking data to force a fresh download + body = [b'[]'] + else: + # Normal object: let it bubble up + raise + + if options['skip_identical']: + if config_true_value(headers.get('x-static-large-object')) or \ + 'x-object-manifest' in headers: + # The request was chunked, so stitch it back together + chunk_data = self._get_chunk_data(conn, container, obj, + headers, b''.join(body)) + else: + chunk_data = None + + if chunk_data is not None: + if self._is_identical(chunk_data, filename): + raise ClientException('Large object is identical', + http_status=304) + + # Large objects are different; start the real download + del get_args['query_string'] + get_args['response_dict'].clear() + headers, body = conn.get_object(container, obj, **get_args) + headers_receipt = time() obj_body = _SwiftReader(path, body, headers) @@ -1084,12 +1183,16 @@ return res except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res = { 'action': 'download_object', 'container': container, 'object': obj, 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': results_dict, 'path': path, 'pseudodir': pseudodir, @@ -1097,14 +1200,17 @@ } return res - def _download_container(self, container, options): + def _submit_page_downloads(self, container, page_generator, options): try: - objects = [] - for part in self.list(container=container, options=options): - if part["success"]: - objects.extend([o["name"] for o in part["listing"]]) - else: - raise part["error"] + list_page = next(page_generator) + except StopIteration: + return None + + if list_page["success"]: + objects = [o["name"] for o in list_page["listing"]] + + if options["shuffle"]: + shuffle(objects) o_downs = [ self.thread_manager.object_dd_pool.submit( @@ -1112,14 +1218,62 @@ ) for obj in objects ] - for o_down in interruptable_as_completed(o_downs): - yield o_down.result() + return o_downs + else: + raise list_page["error"] + def _download_container(self, container, options): + _page_generator = self.list(container=container, options=options) + try: + next_page_downs = self._submit_page_downloads( + container, _page_generator, options + ) except ClientException as err: if err.http_status != 404: raise - raise SwiftError('Container %r not found' % container, - container=container) + raise SwiftError( + 'Container %r not found' % container, + container=container, exc=err + ) + + error = None + while next_page_downs: + page_downs = next_page_downs + next_page_downs = None + + # Start downloading the next page of list results when + # we have completed 80% of the previous page + next_page_triggered = False + next_page_trigger_point = 0.8 * len(page_downs) + + page_results_yielded = 0 + for o_down in interruptable_as_completed(page_downs): + yield o_down.result() + + # Do we need to start the next set of downloads yet? + if not next_page_triggered: + page_results_yielded += 1 + if page_results_yielded >= next_page_trigger_point: + try: + next_page_downs = self._submit_page_downloads( + container, _page_generator, options + ) + except ClientException as err: + # Allow the current page to finish downloading + logger.exception(err) + error = err + except Exception: + # Something unexpected went wrong - cancel + # remaining downloads + for _d in page_downs: + _d.cancel() + raise + finally: + # Stop counting and testing + next_page_triggered = True + + if error: + raise error # Upload related methods # @@ -1157,10 +1311,10 @@ { 'meta': [], - 'headers': [], + 'header': [], 'segment_size': None, 'use_slo': False, - 'segment_container: None, + 'segment_container': None, 'leave_segments': False, 'changed': None, 'skip_identical': False, @@ -1280,12 +1434,16 @@ file_jobs[file_future] = details except OSError as err: # Avoid tying up threads with jobs that will fail + traceback, err_time = report_traceback() + logger.exception(err) res = { 'action': 'upload_object', 'container': container, 'object': o, 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'path': s } rq.put(res) @@ -1384,9 +1542,13 @@ 'response_dict': create_response }) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': create_response }) return res @@ -1425,9 +1587,14 @@ return res except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err}) + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + }) return res try: conn.put_object(container, obj, '', content_length=0, @@ -1439,9 +1606,13 @@ 'response_dict': results_dict}) return res except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': results_dict}) return res @@ -1494,9 +1665,13 @@ return res except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': results_dict, 'attempts': conn.attempts }) @@ -1505,8 +1680,59 @@ results_queue.put(res) return res + def _get_chunk_data(self, conn, container, obj, headers, manifest=None): + chunks = [] + if 'x-object-manifest' in headers: + scontainer, sprefix = headers['x-object-manifest'].split('/', 1) + for part in self.list(scontainer, {'prefix': sprefix}): + if part["success"]: + chunks.extend(part["listing"]) + else: + raise part["error"] + elif config_true_value(headers.get('x-static-large-object')): + if manifest is None: + headers, manifest = conn.get_object( + container, obj, query_string='multipart-manifest=get') + manifest = parse_api_response(headers, manifest) + for chunk in manifest: + if chunk.get('sub_slo'): + scont, sobj = chunk['name'].lstrip('/').split('/', 1) + chunks.extend(self._get_chunk_data( + conn, scont, sobj, {'x-static-large-object': True})) + else: + chunks.append(chunk) + else: + chunks.append({'hash': headers.get('etag').strip('"'), + 'bytes': int(headers.get('content-length'))}) + return chunks + + def _is_identical(self, chunk_data, path): + try: + fp = open(path, 'rb') + except IOError: + return False + + with fp: + for chunk in chunk_data: + to_read = chunk['bytes'] + md5sum = md5() + while to_read: + data = fp.read(min(65536, to_read)) + if not data: + return False + md5sum.update(data) + to_read -= len(data) + if md5sum.hexdigest() != chunk['hash']: + return False + # Each chunk is verified; check that we're at the end of the file + return not fp.read(1) + def _upload_object_job(self, conn, container, source, obj, options, results_queue=None): + if obj.startswith('./') or obj.startswith('.\\'): + obj = obj[2:] + if obj.startswith('/'): + obj = obj[1:] res = { 'action': 'upload_object', 'container': container, @@ -1519,10 +1745,6 @@ path = source res['path'] = path try: - if obj.startswith('./') or obj.startswith('.\\'): - obj = obj[2:] - if obj.startswith('/'): - obj = obj[1:] if path is not None: put_headers = {'x-object-meta-mtime': "%f" % getmtime(path)} else: @@ -1536,32 +1758,27 @@ old_manifest = None old_slo_manifest_paths = [] new_slo_manifest_paths = set() + segment_size = int(0 if options['segment_size'] is None + else options['segment_size']) if (options['changed'] or options['skip_identical'] or not options['leave_segments']): - checksum = None - if options['skip_identical']: - try: - fp = open(path, 'rb') - except IOError: - pass - else: - with fp: - md5sum = md5() - while True: - data = fp.read(65536) - if not data: - break - md5sum.update(data) - checksum = md5sum.hexdigest() try: headers = conn.head_object(container, obj) - if options['skip_identical'] and checksum is not None: - if checksum == headers.get('etag'): - res.update({ - 'success': True, - 'status': 'skipped-identical' - }) - return res + is_slo = config_true_value( + headers.get('x-static-large-object')) + + if options['skip_identical'] or ( + is_slo and not options['leave_segments']): + chunk_data = self._get_chunk_data( + conn, container, obj, headers) + + if options['skip_identical'] and self._is_identical( + chunk_data, path): + res.update({ + 'success': True, + 'status': 'skipped-identical' + }) + return res cl = int(headers.get('content-length')) mt = headers.get('x-object-meta-mtime') @@ -1575,22 +1792,21 @@ return res if not options['leave_segments']: old_manifest = headers.get('x-object-manifest') - if config_true_value( - headers.get('x-static-large-object')): - headers, manifest_data = conn.get_object( - container, obj, - query_string='multipart-manifest=get' - ) - for old_seg in json.loads(manifest_data): + if is_slo: + for old_seg in chunk_data: seg_path = old_seg['name'].lstrip('/') if isinstance(seg_path, text_type): seg_path = seg_path.encode('utf-8') old_slo_manifest_paths.append(seg_path) except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res @@ -1601,8 +1817,8 @@ # a segment job if we're reading from a stream - we may fail if we # go over the single object limit, but this gives us a nice way # to create objects from memory - if (path is not None and options['segment_size'] - and (getsize(path) > int(options['segment_size']))): + if (path is not None and segment_size + and (getsize(path) > segment_size)): res['large_object'] = True seg_container = container + '_segments' if options['segment_container']: @@ -1615,7 +1831,6 @@ segment_start = 0 while segment_start < full_size: - segment_size = int(options['segment_size']) if segment_start + segment_size > full_size: segment_size = full_size - segment_start if options['use_slo']: @@ -1646,9 +1861,11 @@ if not r['success']: errors = True segment_results.append(r) - except Exception as e: + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) errors = True - exceptions.append(e) + exceptions.append((err, traceback, err_time)) if errors: err = ClientException( 'Aborting manifest creation ' @@ -1734,19 +1951,19 @@ if old_manifest or old_slo_manifest_paths: drs = [] + delobjsmap = {} if old_manifest: scontainer, sprefix = old_manifest.split('/', 1) scontainer = unquote(scontainer) sprefix = unquote(sprefix).rstrip('/') + '/' - delobjs = [] - for delobj in conn.get_container(scontainer, - prefix=sprefix)[1]: - delobjs.append(delobj['name']) - for dr in self.delete(container=scontainer, - objects=delobjs): - drs.append(dr) + delobjsmap[scontainer] = [] + for part in self.list(scontainer, {'prefix': sprefix}): + if not part["success"]: + raise part["error"] + delobjsmap[scontainer].extend( + seg['name'] for seg in part['listing']) + if old_slo_manifest_paths: - delobjsmap = {} for seg_to_delete in old_slo_manifest_paths: if seg_to_delete in new_slo_manifest_paths: continue @@ -1755,10 +1972,18 @@ delobjs_cont = delobjsmap.get(scont, []) delobjs_cont.append(sobj) delobjsmap[scont] = delobjs_cont - for (dscont, dsobjs) in delobjsmap.items(): - for dr in self.delete(container=dscont, - objects=dsobjs): - drs.append(dr) + + del_segs = [] + for dscont, dsobjs in delobjsmap.items(): + for dsobj in dsobjs: + del_seg = self.thread_manager.segment_pool.submit( + self._delete_segment, dscont, dsobj, + results_queue=results_queue + ) + del_segs.append(del_seg) + + for del_seg in interruptable_as_completed(del_segs): + drs.append(del_seg.result()) res['segment_delete_results'] = drs # return dict for printing @@ -1769,16 +1994,26 @@ return res except OSError as err: + traceback, err_time = report_traceback() + logger.exception(err) if err.errno == ENOENT: - err = SwiftError('Local file %r not found' % path) + error = SwiftError('Local file %r not found' % path, exc=err) + else: + error = err res.update({ 'success': False, - 'error': err + 'error': error, + 'traceback': traceback, + 'error_timestamp': err_time }) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res @@ -1877,8 +2112,15 @@ try: conn.delete_object(container, obj, response_dict=results_dict) res = {'success': True} - except Exception as e: - res = {'success': False, 'error': e} + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) + res = { + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + } res.update({ 'action': 'delete_segment', @@ -1894,12 +2136,12 @@ def _delete_object(self, conn, container, obj, options, results_queue=None): + res = { + 'action': 'delete_object', + 'container': container, + 'object': obj + } try: - res = { - 'action': 'delete_object', - 'container': container, - 'object': obj - } old_manifest = None query_string = None @@ -1954,8 +2196,14 @@ }) except Exception as err: - res['success'] = False - res['error'] = err + traceback, err_time = report_traceback() + logger.exception(err) + res.update({ + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + }) return res return res @@ -1966,8 +2214,15 @@ try: conn.delete_container(container, response_dict=results_dict) res = {'success': True} - except Exception as e: - res = {'success': False, 'error': e} + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) + res = { + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + } res.update({ 'action': 'delete_container', @@ -1980,29 +2235,34 @@ def _delete_container(self, container, options): try: - objs = [] for part in self.list(container=container): if part["success"]: - objs.extend([o['name'] for o in part['listing']]) + objs = [o['name'] for o in part['listing']] + + o_dels = self.delete( + container=container, objects=objs, options=options + ) + for res in o_dels: + yield res else: raise part["error"] - for res in self.delete( - container=container, objects=objs, options=options): - yield res - con_del = self.thread_manager.container_pool.submit( self._delete_empty_container, container ) con_del_res = get_future_result(con_del) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) con_del_res = { 'action': 'delete_container', 'container': container, 'object': None, 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time } yield con_del_res @@ -2040,7 +2300,7 @@ except ClientException as err: if err.http_status != 404: raise err - raise SwiftError('Account not found') + raise SwiftError('Account not found', exc=err) return res @@ -2075,10 +2335,16 @@ res['status'] = 'cancelled' result_queue.put(res) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) details = futures[f] res = details - res['success'] = False - res['error'] = err + res.update({ + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + }) result_queue.put(res) result_queue.put(None) diff -Nru python-swiftclient-2.4.0/swiftclient/shell.py python-swiftclient-2.6.0/swiftclient/shell.py --- python-swiftclient-2.4.0/swiftclient/shell.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/swiftclient/shell.py 2015-09-07 15:20:14.000000000 +0000 @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function +from __future__ import print_function, unicode_literals import logging import signal @@ -23,6 +23,7 @@ from optparse import OptionParser, OptionGroup, SUPPRESS_HELP from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join +from six import text_type from sys import argv as sys_argv, exit, stderr from time import gmtime, strftime @@ -31,14 +32,19 @@ from swiftclient.multithreading import OutputManager from swiftclient.exceptions import ClientException from swiftclient import __version__ as client_version -from swiftclient.service import SwiftService, SwiftError, SwiftUploadObject +from swiftclient.service import SwiftService, SwiftError, \ + SwiftUploadObject, get_conn from swiftclient.command_helpers import print_account_stats, \ print_container_stats, print_object_stats +try: + from shlex import quote as sh_quote +except ImportError: + from pipes import quote as sh_quote BASENAME = 'swift' -commands = ('delete', 'download', 'list', 'post', - 'stat', 'upload', 'capabilities', 'info', 'tempurl') +commands = ('delete', 'download', 'list', 'post', 'stat', 'upload', + 'capabilities', 'info', 'tempurl', 'auth') def immediate_exit(signum, frame): @@ -60,7 +66,7 @@ for multiple objects. Optional arguments: - --all Delete all containers and objects. + -a, --all Delete all containers and objects. --leave-segments Do not delete segments of manifest objects. --object-threads Number of threads to use for deleting objects. @@ -83,10 +89,10 @@ '', '--object-threads', type=int, default=10, help='Number of threads to use for deleting objects. ' 'Default is 10.') - parser.add_option('', '--container-threads', type=int, - default=10, help='Number of threads to use for ' - 'deleting containers. ' - 'Default is 10.') + parser.add_option( + '', '--container-threads', type=int, + default=10, help='Number of threads to use for deleting containers. ' + 'Default is 10.') (options, args) = parse_args(parser, args) args = args[1:] if (not args and not options.yes_all) or (args and options.yes_all): @@ -106,8 +112,8 @@ if '/' in container: output_manager.error( 'WARNING: / in container name; you ' - 'might have meant %r instead of %r.' % ( - container.replace('/', ' ', 1), container) + "might have meant '%s' instead of '%s'." % + (container.replace('/', ' ', 1), container) ) return objects = args[1:] @@ -146,9 +152,12 @@ st_download_options = '''[--all] [--marker] [--prefix ] - [--output ] [--object-threads ] + [--output ] [--output-dir ] + [--object-threads ] [--container-threads ] [--no-download] - [--skip-identical] + [--skip-identical] [--remove-prefix] + [--header ] + ''' st_download_help = ''' @@ -162,14 +171,21 @@ objects from the container. Optional arguments: - --all Indicates that you really want to download + -a, --all Indicates that you really want to download everything in the account. - --marker Marker to use when starting a container or account + -m, --marker Marker to use when starting a container or account download. - --prefix Only download items beginning with - --output For a single file download, stream the output to + -p, --prefix Only download items beginning with + -r, --remove-prefix An optional flag for --prefix , use this + option to download items without + -o, --output + For a single file download, stream the output to . Specifying "-" as will redirect to stdout. + -D, --output-dir + An optional directory to which to store objects. + By default, all objects are recreated in the current + directory. --object-threads Number of threads to use for downloading objects. Default is 10. @@ -178,12 +194,20 @@ Default is 10. --no-download Perform download(s), but don't actually write anything to disk. - --header + -H, --header Adds a customized request header to the query, like - "Range" or "If-Match". This argument is repeatable. + "Range" or "If-Match". This option may be repeated. Example --header "content-type:text/plain" --skip-identical Skip downloading files that are identical on both sides. + --no-shuffle By default, when downloading a complete account or + container, download order is randomised in order to + to reduce the load on individual drives when multiple + clients are executed simultaneously to download the + same set of objects (e.g. a nightly automated download + script to multiple servers). Enable this option to + submit download jobs to the thread pool in the order + they are listed in the object store. '''.strip("\n") @@ -204,6 +228,14 @@ 'download, stream the output to . ' 'Specifying "-" as will redirect to stdout.') parser.add_option( + '-D', '--output-dir', dest='out_directory', + help='An optional directory to which to store objects. ' + 'By default, all objects are recreated in the current directory.') + parser.add_option( + '-r', '--remove-prefix', action='store_true', dest='remove_prefix', + default=False, help='An optional flag for --prefix , ' + 'use this option to download items without .') + parser.add_option( '', '--object-threads', type=int, default=10, help='Number of threads to use for downloading objects. ' 'Default is 10.') @@ -219,12 +251,20 @@ '-H', '--header', action='append', dest='header', default=[], help='Adds a customized request header to the query, like "Range" or ' - '"If-Match". This argument is repeatable. ' + '"If-Match". This option may be repeated. ' 'Example: --header "content-type:text/plain"') parser.add_option( '--skip-identical', action='store_true', dest='skip_identical', default=False, help='Skip downloading files that are identical on ' 'both sides.') + parser.add_option( + '--no-shuffle', action='store_false', dest='shuffle', + default=True, help='By default, download order is randomised in order ' + 'to reduce the load on individual drives when multiple clients are ' + 'executed simultaneously to download the same set of objects (e.g. a ' + 'nightly automated download script to multiple servers). Enable this ' + 'option to submit download jobs to the thread pool in the order they ' + 'are listed in the object store.') (options, args) = parse_args(parser, args) args = args[1:] if options.out_file == '-': @@ -233,6 +273,12 @@ if options.out_file and len(args) != 2: exit('-o option only allowed for single file downloads') + if not options.prefix: + options.remove_prefix = False + + if options.out_directory and len(args) == 2: + exit('Please use -o option for single file downloads and renames') + if (not args and not options.yes_all) or (args and options.yes_all): output_manager.error('Usage: %s download %s\n%s', BASENAME, st_download_options, st_download_help) @@ -249,8 +295,8 @@ if '/' in container: output_manager.error( 'WARNING: / in container name; you ' - 'might have meant %r instead of %r.' % ( - container.replace('/', ' ', 1), container) + "might have meant '%s' instead of '%s'." % + (container.replace('/', ' ', 1), container) ) return objects = args[1:] @@ -325,6 +371,8 @@ except SwiftError as e: output_manager.error(e.value) + except Exception as e: + output_manager.error(e) st_list_options = '''[--long] [--lh] [--totals] [--prefix ] @@ -338,12 +386,12 @@ [container] Name of container to list object in. Optional arguments: - --long Long listing format, similar to ls -l. + -l, --long Long listing format, similar to ls -l. --lh Report sizes in human readable format similar to ls -lh. - --totals Used with -l or --lh, only report totals. - --prefix Only list items beginning with the prefix. - --delimiter Roll up items with the given delimiter. For containers + -t, --totals Used with -l or --lh, only report totals. + -p, --prefix Only list items beginning with the prefix. + -d, --delimiter Roll up items with the given delimiter. For containers only. See OpenStack Swift API documentation for what this means. '''.strip('\n') @@ -500,7 +548,7 @@ if '/' in container: output_manager.error( 'WARNING: / in container name; you might have ' - 'meant %r instead of %r.' % + "meant '%s' instead of '%s'." % (container.replace('/', ' ', 1), container)) return args = args[1:] @@ -549,17 +597,22 @@ [object] Name of object to post. Optional arguments: - --read-acl Read ACL for containers. Quick summary of ACL syntax: + -r, --read-acl Read ACL for containers. Quick summary of ACL syntax: .r:*, .r:-.example.com, .r:www.example.com, account1, account2:user2 - --write-acl Write ACL for containers. Quick summary of ACL syntax: + -w, --write-acl Write ACL for containers. Quick summary of ACL syntax: account1 account2:user2 - --sync-to Sync To for containers, for multi-cluster replication. - --sync-key Sync Key for containers, for multi-cluster replication. - --meta Sets a meta data item. This option may be repeated. + -t, --sync-to + Sync To for containers, for multi-cluster replication. + -k, --sync-key + Sync Key for containers, for multi-cluster replication. + -m, --meta + Sets a meta data item. This option may be repeated. Example: -m Color:Blue -m Size:Large - --header
Set request headers. This option may be repeated. - Example -H "content-type:text/plain" + -H, --header + Adds a customized request header. + This option may be repeated. Example + -H "content-type:text/plain" -H "Content-Length: 4000" '''.strip('\n') @@ -584,7 +637,8 @@ 'Example: -m Color:Blue -m Size:Large') parser.add_option( '-H', '--header', action='append', dest='header', - default=[], help='Set request headers. This option may be repeated. ' + default=[], help='Adds a customized request header. ' + 'This option may be repeated. ' 'Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') (options, args) = parse_args(parser, args) @@ -604,7 +658,7 @@ if '/' in container: output_manager.error( 'WARNING: / in container name; you might have ' - 'meant %r instead of %r.' % + "meant '%s' instead of '%s'." % (args[0].replace('/', ' ', 1), args[0])) return args = args[1:] @@ -637,8 +691,7 @@ ''' -st_upload_help = ''' -Uploads specified files and directories to the given container. +st_upload_help = ''' Uploads specified files and directories to the given container. Positional arguments: Name of container to upload to. @@ -646,10 +699,11 @@ times for multiple uploads. Optional arguments: - --changed Only upload files that have changed since the last + -c, --changed Only upload files that have changed since the last upload. --skip-identical Skip uploading files that are identical on both sides. - --segment-size Upload files in segments no larger than (in + -S, --segment-size + Upload files in segments no larger than (in Bytes) and then create a "manifest" file that will download all the segments as if it were the original file. @@ -666,9 +720,10 @@ --segment-threads Number of threads to use for uploading object segments. Default is 10. - --header
Set request headers with the syntax header:value. - This option may be repeated. - Example -H "content-type:text/plain". + -H, --header + Adds a customized request header. This option may be + repeated. Example -H "content-type:text/plain" + -H "Content-Length: 4000". --use-slo When used in conjunction with --segment-size it will create a Static Large Object instead of the default Dynamic Large Object. @@ -763,6 +818,9 @@ return options.segment_size = str((1024 ** size_mod) * multiplier) + if int(options.segment_size) <= 0: + output_manager.error("segment-size should be positive") + return _opts = vars(options) _opts['object_uu_threads'] = options.object_threads @@ -841,7 +899,7 @@ msg = ': %s' % error output_manager.warning( 'Warning: failed to create container ' - '%r%s', container, msg + "'%s'%s", container, msg ) else: output_manager.error("%s" % error) @@ -905,6 +963,46 @@ st_info = st_capabilities +st_auth_help = ''' +Display auth related authentication variables in shell friendly format. + + Commands to run to export storage url and auth token into + OS_STORAGE_URL and OS_AUTH_TOKEN: + + swift auth + + Commands to append to a runcom file (e.g. ~/.bashrc, /etc/profile) for + automatic authentication: + + swift auth -v -U test:tester -K testing \ + -A http://localhost:8080/auth/v1.0 + +'''.strip('\n') + + +def st_auth(parser, args, thread_manager): + (options, args) = parse_args(parser, args) + _opts = vars(options) + if options.verbose > 1: + if options.auth_version in ('1', '1.0'): + print('export ST_AUTH=%s' % sh_quote(options.auth)) + print('export ST_USER=%s' % sh_quote(options.user)) + print('export ST_KEY=%s' % sh_quote(options.key)) + else: + print('export OS_IDENTITY_API_VERSION=%s' % sh_quote( + options.auth_version)) + print('export OS_AUTH_VERSION=%s' % sh_quote(options.auth_version)) + print('export OS_AUTH_URL=%s' % sh_quote(options.auth)) + for k, v in sorted(_opts.items()): + if v and k.startswith('os_') and \ + k not in ('os_auth_url', 'os_options'): + print('export %s=%s' % (k.upper(), sh_quote(v))) + else: + conn = get_conn(_opts) + url, token = conn.get_auth() + print('export OS_STORAGE_URL=%s' % sh_quote(url)) + print('export OS_AUTH_TOKEN=%s' % sh_quote(token)) + st_tempurl_options = ' ' @@ -912,20 +1010,33 @@ st_tempurl_help = ''' Generates a temporary URL for a Swift object. -Positions arguments: - [method] An HTTP method to allow for this temporary URL. +Positional arguments: + An HTTP method to allow for this temporary URL. Usually 'GET' or 'PUT'. - [seconds] The amount of time in seconds the temporary URL will - be valid for. - [path] The full path to the Swift object. Example: + The amount of time in seconds the temporary URL will be + valid for; or, if --absolute is passed, the Unix + timestamp when the temporary URL will expire. + The full path to the Swift object. Example: /v1/AUTH_account/c/o. - [key] The secret temporary URL key set on the Swift cluster. + The secret temporary URL key set on the Swift cluster. To set a key, run \'swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"\' + +Optional arguments: + --absolute Interpet the positional argument as a Unix + timestamp rather than a number of seconds in the + future. '''.strip('\n') def st_tempurl(parser, args, thread_manager): + parser.add_option( + '--absolute', action='store_true', + dest='absolute_expiry', default=False, + help=("If present, seconds argument will be interpreted as a Unix " + "timestamp representing when the tempURL should expire, rather " + "than an offset from the current time") + ) (options, args) = parse_args(parser, args) args = args[1:] if len(args) < 4: @@ -942,7 +1053,8 @@ thread_manager.print_msg('WARNING: Non default HTTP method %s for ' 'tempurl specified, possibly an error' % method.upper()) - url = generate_temp_url(path, seconds, key, method) + url = generate_temp_url(path, seconds, key, method, + absolute=options.absolute_expiry) thread_manager.print_msg(url) @@ -1034,8 +1146,10 @@ else: argv = sys_argv + argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] + version = client_version - parser = OptionParser(version='%%prog %s' % version, + parser = OptionParser(version='python-swiftclient %s' % version, usage=''' usage: %%prog [--version] [--help] [--os-help] [--snet] [--verbose] [--debug] [--info] [--quiet] [--auth ] @@ -1057,7 +1171,7 @@ [--os-endpoint-type ] [--os-cacert ] [--insecure] [--no-ssl-compression] - [--help] + [--help] [] Command-line interface to the OpenStack Swift API. @@ -1073,7 +1187,8 @@ or object. upload Uploads files or directories to the given container. capabilities List cluster capabilities. - tempurl Create a temporary URL + tempurl Create a temporary URL. + auth Display auth related environment variables. Examples: %%prog download --help diff -Nru python-swiftclient-2.4.0/swiftclient/utils.py python-swiftclient-2.6.0/swiftclient/utils.py --- python-swiftclient-2.4.0/swiftclient/utils.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/swiftclient/utils.py 2015-09-07 15:20:14.000000000 +0000 @@ -15,10 +15,11 @@ """Miscellaneous utility functions for use with Swift.""" import hashlib import hmac +import json import logging -import time - import six +import time +import traceback TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y')) EMPTY_ETAG = 'd41d8cd98f00b204e9800998ecf8427e' @@ -28,7 +29,7 @@ """ Returns True if the value is either True or a string in TRUE_VALUES. Returns False otherwise. - This function come from swift.common.utils.config_true_value() + This function comes from swift.common.utils.config_true_value() """ return value is True or \ (isinstance(value, six.string_types) and value.lower() in TRUE_VALUES) @@ -64,8 +65,8 @@ return bytes -def generate_temp_url(path, seconds, key, method): - """ Generates a temporary URL that gives unauthenticated access to the +def generate_temp_url(path, seconds, key, method, absolute=False): + """Generates a temporary URL that gives unauthenticated access to the Swift object. :param path: The full path to the Swift object. Example: @@ -84,7 +85,10 @@ if seconds < 0: raise ValueError('seconds must be a positive integer') try: - expiration = int(time.time() + seconds) + if not absolute: + expiration = int(time.time() + seconds) + else: + expiration = int(seconds) except TypeError: raise TypeError('seconds must be an integer') @@ -108,6 +112,30 @@ exp=expiration)) +def parse_api_response(headers, body): + charset = 'utf-8' + # Swift *should* be speaking UTF-8, but check content-type just in case + content_type = headers.get('content-type', '') + if '; charset=' in content_type: + charset = content_type.split('; charset=', 1)[1].split(';', 1)[0] + + return json.loads(body.decode(charset)) + + +def report_traceback(): + """ + Reports a timestamp and full traceback for a given exception. + + :return: Full traceback and timestamp. + """ + try: + formatted_lines = traceback.format_exc() + now = time.time() + return formatted_lines, now + except AttributeError: + return None, None + + class NoopMD5(object): def __init__(self, *a, **kw): pass diff -Nru python-swiftclient-2.4.0/test-requirements.txt python-swiftclient-2.6.0/test-requirements.txt --- python-swiftclient-2.4.0/test-requirements.txt 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/test-requirements.txt 2015-09-07 15:20:14.000000000 +0000 @@ -1,8 +1,8 @@ -hacking>=0.8.0,<0.9 +hacking>=0.10.0,<0.11 coverage>=3.6 discover -mock>=1.0 +mock>=1.2 oslosphinx python-keystoneclient>=0.7.0 sphinx>=1.1.2,<1.2 diff -Nru python-swiftclient-2.4.0/tests/functional/test_swiftclient.py python-swiftclient-2.6.0/tests/functional/test_swiftclient.py --- python-swiftclient-2.4.0/tests/functional/test_swiftclient.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/tests/functional/test_swiftclient.py 2015-09-07 15:20:14.000000000 +0000 @@ -16,7 +16,6 @@ import os import testtools import time -import types from io import BytesIO from six.moves import configparser @@ -45,14 +44,20 @@ '/etc/swift/test.conf') config = configparser.SafeConfigParser({'auth_version': '1'}) config.read(config_file) + self.config = config if config.has_section('func_test'): auth_host = config.get('func_test', 'auth_host') auth_port = config.getint('func_test', 'auth_port') auth_ssl = config.getboolean('func_test', 'auth_ssl') auth_prefix = config.get('func_test', 'auth_prefix') self.auth_version = config.get('func_test', 'auth_version') - self.account = config.get('func_test', 'account') - self.username = config.get('func_test', 'username') + try: + self.account_username = config.get('func_test', + 'account_username') + except configparser.NoOptionError: + account = config.get('func_test', 'account') + username = config.get('func_test', 'username') + self.account_username = "%s:%s" % (account, username) self.password = config.get('func_test', 'password') self.auth_url = "" if auth_ssl: @@ -62,20 +67,24 @@ self.auth_url += "%s:%s%s" % (auth_host, auth_port, auth_prefix) if self.auth_version == "1": self.auth_url += 'v1.0' - self.account_username = "%s:%s" % (self.account, self.username) else: self.skip_tests = True + def _get_connection(self): + """ + Subclasses may override to use different connection setup + """ + return swiftclient.Connection( + self.auth_url, self.account_username, self.password, + auth_version=self.auth_version) + def setUp(self): super(TestFunctional, self).setUp() if self.skip_tests: self.skipTest('SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG') - self.conn = swiftclient.Connection( - self.auth_url, self.account_username, self.password, - auth_version=self.auth_version) - + self.conn = self._get_connection() self.conn.put_container(self.containername) self.conn.put_container(self.containername_2) self.conn.put_object( @@ -256,8 +265,24 @@ hdrs, body = self.conn.get_object( self.containername, self.objectname, resp_chunk_size=10) - self.assertTrue(isinstance(body, types.GeneratorType)) - self.assertEqual(self.test_data, b''.join(body)) + downloaded_contents = b'' + while True: + try: + chunk = next(body) + except StopIteration: + break + downloaded_contents += chunk + self.assertEqual(self.test_data, downloaded_contents) + + # Download in chunks, should also work with read + hdrs, body = self.conn.get_object( + self.containername, self.objectname, + resp_chunk_size=10) + num_bytes = 5 + downloaded_contents = body.read(num_bytes) + self.assertEqual(num_bytes, len(downloaded_contents)) + downloaded_contents += body.read() + self.assertEqual(self.test_data, downloaded_contents) def test_post_account(self): self.conn.post_account({'x-account-meta-data': 'Something'}) @@ -282,3 +307,58 @@ def test_get_capabilities(self): resp = self.conn.get_capabilities() self.assertTrue(resp.get('swift')) + + +class TestUsingKeystone(TestFunctional): + """ + Repeat tests using os_options parameter to Connection. + """ + + def _get_connection(self): + account = username = password = None + if self.auth_version not in ('2', '3'): + self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS') + try: + account = self.config.get('func_test', 'account') + username = self.config.get('func_test', 'username') + password = self.config.get('func_test', 'password') + except Exception: + self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS' + + ' - NO CONFIG') + os_options = {'tenant_name': account} + return swiftclient.Connection( + self.auth_url, username, password, auth_version=self.auth_version, + os_options=os_options) + + def setUp(self): + super(TestUsingKeystone, self).setUp() + + +class TestUsingKeystoneV3(TestFunctional): + """ + Repeat tests using a keystone user with domain specified. + """ + + def _get_connection(self): + account = username = password = project_domain = user_domain = None + if self.auth_version != '3': + self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS') + try: + account = self.config.get('func_test', 'account4') + username = self.config.get('func_test', 'username4') + user_domain = self.config.get('func_test', 'domain4') + project_domain = self.config.get('func_test', 'domain4') + password = self.config.get('func_test', 'password4') + except Exception: + self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS' + + ' - NO CONFIG') + + os_options = {'project_name': account, + 'project_domain_name': project_domain, + 'user_domain_name': user_domain} + return swiftclient.Connection(self.auth_url, username, password, + auth_version=self.auth_version, + os_options=os_options) + + def setUp(self): + super(TestUsingKeystoneV3, self).setUp() diff -Nru python-swiftclient-2.4.0/tests/sample.conf python-swiftclient-2.6.0/tests/sample.conf --- python-swiftclient-2.4.0/tests/sample.conf 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/tests/sample.conf 2015-09-07 15:20:14.000000000 +0000 @@ -12,7 +12,21 @@ #auth_ssl = no #auth_prefix = /v2.0/ -# Primary functional test account (needs admin access to the account) +# Primary functional test account (needs admin access to the account). +# By default the tests use a swiftclient.client.Connection instance with user +# attribute set to 'account:username' based on the options 'account' and +# 'username' specified below. This can be overridden for auth systems that +# expect a different form of user attribute by setting the option +# 'account_username'. +# account_username = test_tester account = test username = tester password = testing + +# Another user is required for keystone v3 specific tests. +# Account must be in a non-default domain. +# (Suffix '4' is used to be consistent with swift functional test config). +#account4 = test4 +#username4 = tester4 +#password4 = testing4 +#domain4 = test-domain diff -Nru python-swiftclient-2.4.0/tests/unit/test_multithreading.py python-swiftclient-2.6.0/tests/unit/test_multithreading.py --- python-swiftclient-2.4.0/tests/unit/test_multithreading.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/tests/unit/test_multithreading.py 2015-09-07 15:20:14.000000000 +0000 @@ -12,7 +12,6 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - import sys import testtools import threading @@ -89,7 +88,7 @@ f.result() except Exception as e: went_boom = True - self.assertEquals('I went boom!', str(e)) + self.assertEqual('I went boom!', str(e)) self.assertTrue(went_boom) # Has the connection been returned to the pool? @@ -113,7 +112,7 @@ f.result() except Exception as e: connection_failed = True - self.assertEquals('This is a failed connection', str(e)) + self.assertEqual('This is a failed connection', str(e)) self.assertTrue(connection_failed) # Make sure we don't lock up on failed connections @@ -123,7 +122,7 @@ f.result() except Exception as e: connection_failed = True - self.assertEquals('This is a failed connection', str(e)) + self.assertEqual('This is a failed connection', str(e)) self.assertTrue(connection_failed) def test_lazy_connections(self): @@ -206,10 +205,12 @@ u'some raw bytes: \u062A\u062A'.encode('utf-8')) thread_manager.print_items([ - ('key', u'value'), - ('object', 'O\xcc\x88bject') + ('key', 'value'), + ('object', u'O\u0308bject'), ]) + thread_manager.print_raw(b'\xffugly\xffraw') + # Now we have a thread for error printing and a thread for # normal print messages self.assertEqual(starting_thread_count + 2, @@ -220,31 +221,23 @@ if six.PY3: over_the = "over the '\u062a\u062a'\n" - # The CaptureStreamBuffer just encodes all bytes written to it by - # mapping chr over the byte string to produce a str. - raw_bytes = ''.join( - map(chr, u'some raw bytes: \u062A\u062A'.encode('utf-8')) - ) else: over_the = "over the u'\\u062a\\u062a'\n" # We write to the CaptureStream so no decoding is performed - raw_bytes = 'some raw bytes: \xd8\xaa\xd8\xaa' self.assertEqual(''.join([ 'one-argument\n', 'one fish, 88 fish\n', 'some\n', 'where\n', - over_the, raw_bytes, + over_the, + u'some raw bytes: \u062a\u062a', ' key: value\n', - ' object: O\xcc\x88bject\n' - ]), out_stream.getvalue()) + u' object: O\u0308bject\n' + ]).encode('utf8') + b'\xffugly\xffraw', out_stream.getvalue()) - first_item = u'I have 99 problems, but a \u062A\u062A is not one\n' - if six.PY2: - first_item = first_item.encode('utf8') self.assertEqual(''.join([ - first_item, + u'I have 99 problems, but a \u062A\u062A is not one\n', 'one-error-argument\n', 'Sometimes\n', '3.1% just\n', 'does not\n', 'work!\n' - ]), err_stream.getvalue()) + ]), err_stream.getvalue().decode('utf8')) self.assertEqual(3, thread_manager.error_count) diff -Nru python-swiftclient-2.4.0/tests/unit/test_service.py python-swiftclient-2.6.0/tests/unit/test_service.py --- python-swiftclient-2.4.0/tests/unit/test_service.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/tests/unit/test_service.py 2015-09-07 15:20:14.000000000 +0000 @@ -14,18 +14,24 @@ # limitations under the License. import mock import os +import six import tempfile import testtools import time + +from concurrent.futures import Future from hashlib import md5 from mock import Mock, PropertyMock from six.moves.queue import Queue, Empty as QueueEmptyError from six import BytesIO +from time import sleep import swiftclient import swiftclient.utils as utils -from swiftclient.client import Connection -from swiftclient.service import SwiftService, SwiftError +from swiftclient.client import Connection, ClientException +from swiftclient.service import ( + SwiftService, SwiftError, SwiftUploadObject +) clean_os_environ = {} @@ -35,6 +41,12 @@ clean_os_environ[key] = '' +if six.PY2: + import __builtin__ as builtins +else: + import builtins + + class TestSwiftPostObject(testtools.TestCase): def setUp(self): @@ -138,25 +150,29 @@ '97ac82a5b825239e782d0339e2d7b910') -class TestServiceDelete(testtools.TestCase): - def setUp(self): - super(TestServiceDelete, self).setUp() - self.opts = {'leave_segments': False, 'yes_all': False} - self.exc = Exception('test_exc') - # Base response to be copied and updated to matched the expected - # response for each test - self.expected = { - 'action': None, # Should be string in the form delete_XX - 'container': 'test_c', - 'object': 'test_o', - 'attempts': 2, - 'response_dict': {}, - 'success': None # Should be a bool - } +class _TestServiceBase(testtools.TestCase): + def _assertDictEqual(self, a, b, m=None): + # assertDictEqual is not available in py2.6 so use a shallow check + # instead + if not m: + m = '{0} != {1}'.format(a, b) + + if hasattr(self, 'assertDictEqual'): + self.assertDictEqual(a, b, m) + else: + self.assertTrue(isinstance(a, dict), + 'First argument is not a dictionary') + self.assertTrue(isinstance(b, dict), + 'Second argument is not a dictionary') + self.assertEqual(len(a), len(b), m) + for k, v in a.items(): + self.assertIn(k, b, m) + self.assertEqual(b[k], v, m) def _get_mock_connection(self, attempts=2): m = Mock(spec=Connection) type(m).attempts = PropertyMock(return_value=attempts) + type(m).auth_end_time = PropertyMock(return_value=4) return m def _get_queue(self, q): @@ -174,18 +190,22 @@ return expected - def _assertDictEqual(self, a, b, m=None): - # assertDictEqual is not available in py2.6 so use a shallow check - # instead - if hasattr(self, 'assertDictEqual'): - self.assertDictEqual(a, b, m) - else: - self.assertTrue(isinstance(a, dict)) - self.assertTrue(isinstance(b, dict)) - self.assertEqual(len(a), len(b), m) - for k, v in a.items(): - self.assertTrue(k in b, m) - self.assertEqual(b[k], v, m) + +class TestServiceDelete(_TestServiceBase): + def setUp(self): + super(TestServiceDelete, self).setUp() + self.opts = {'leave_segments': False, 'yes_all': False} + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': None, # Should be string in the form delete_XX + 'container': 'test_c', + 'object': 'test_o', + 'attempts': 2, + 'response_dict': {}, + 'success': None # Should be a bool + } def test_delete_segment(self): mock_q = Queue() @@ -212,16 +232,23 @@ 'action': 'delete_segment', 'object': 'test_s', 'success': False, - 'error': self.exc + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY }) + before = time.time() r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q) + after = time.time() mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_s', response_dict={} ) self._assertDictEqual(expected_r, r) self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertTrue('Traceback' in r['traceback']) def test_delete_object(self): mock_q = Queue() @@ -248,20 +275,27 @@ expected_r = self._get_expected({ 'action': 'delete_object', 'success': False, - 'error': self.exc + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY }) # _delete_object doesnt populate attempts or response dict if it hits # an error. This may not be the correct behaviour. del expected_r['response_dict'], expected_r['attempts'] + before = time.time() s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) + after = time.time() mock_conn.head_object.assert_called_once_with('test_c', 'test_o') mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={} ) self._assertDictEqual(expected_r, r) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertTrue('Traceback' in r['traceback']) def test_delete_object_slo_support(self): # If SLO headers are present the delete call should include an @@ -338,23 +372,30 @@ ) self._assertDictEqual(expected_r, r) - def test_delete_empty_container_excpetion(self): + def test_delete_empty_container_exception(self): mock_conn = self._get_mock_connection() mock_conn.delete_container = Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'delete_container', 'success': False, 'object': None, - 'error': self.exc + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY }) + before = time.time() s = SwiftService() r = s._delete_empty_container(mock_conn, 'test_c') + after = time.time() mock_conn.delete_container.assert_called_once_with( 'test_c', response_dict={} ) self._assertDictEqual(expected_r, r) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertTrue('Traceback' in r['traceback']) class TestSwiftError(testtools.TestCase): @@ -538,6 +579,230 @@ self.assertRaises(SwiftError, self.suo, []) +class TestServiceList(_TestServiceBase): + def setUp(self): + super(TestServiceList, self).setUp() + self.opts = {'prefix': None, 'long': False, 'delimiter': ''} + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': None, # Should be list_X_part (account or container) + 'container': None, # Should be a string when listing a container + 'prefix': None, + 'success': None # Should be a bool + } + + def test_list_account(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_account_returns = [ + (None, [{'name': 'test_c'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + + expected_r = self._get_expected({ + 'action': 'list_account_part', + 'success': True, + 'listing': [{'name': 'test_c'}], + 'marker': '' + }) + + SwiftService._list_account_job( + mock_conn, self.opts, mock_q + ) + self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + long_opts = dict(self.opts, **{'long': True}) + mock_conn.head_container = Mock(return_value={'test_m': '1'}) + get_account_returns = [ + (None, [{'name': 'test_c'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + + expected_r_long = self._get_expected({ + 'action': 'list_account_part', + 'success': True, + 'listing': [{'name': 'test_c', 'meta': {'test_m': '1'}}], + 'marker': '', + }) + + SwiftService._list_account_job( + mock_conn, long_opts, mock_q + ) + self._assertDictEqual(expected_r_long, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_account_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.get_account = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'list_account_part', + 'success': False, + 'error': self.exc, + 'marker': '', + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + }) + + SwiftService._list_account_job( + mock_conn, self.opts, mock_q) + + mock_conn.get_account.assert_called_once_with( + marker='', prefix=None + ) + self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_container(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_container_returns = [ + (None, [{'name': 'test_o'}]), + (None, []) + ] + mock_conn.get_container = Mock(side_effect=get_container_returns) + + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'test_o'}], + 'marker': '' + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', self.opts, mock_q + ) + self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + long_opts = dict(self.opts, **{'long': True}) + mock_conn.head_container = Mock(return_value={'test_m': '1'}) + get_container_returns = [ + (None, [{'name': 'test_o'}]), + (None, []) + ] + mock_conn.get_container = Mock(side_effect=get_container_returns) + + expected_r_long = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'test_o'}], + 'marker': '' + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', long_opts, mock_q + ) + self._assertDictEqual(expected_r_long, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_container_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.get_container = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': False, + 'error': self.exc, + 'marker': '', + 'error_timestamp': mock.ANY, + 'traceback': mock.ANY + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', self.opts, mock_q + ) + + mock_conn.get_container.assert_called_once_with( + 'test_c', marker='', delimiter='', prefix=None + ) + self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + @mock.patch('swiftclient.service.get_conn') + def test_list_queue_size(self, mock_get_conn): + mock_conn = self._get_mock_connection() + # Return more results than should fit in the results queue + get_account_returns = [ + (None, [{'name': 'container1'}]), + (None, [{'name': 'container2'}]), + (None, [{'name': 'container3'}]), + (None, [{'name': 'container4'}]), + (None, [{'name': 'container5'}]), + (None, [{'name': 'container6'}]), + (None, [{'name': 'container7'}]), + (None, [{'name': 'container8'}]), + (None, [{'name': 'container9'}]), + (None, [{'name': 'container10'}]), + (None, [{'name': 'container11'}]), + (None, [{'name': 'container12'}]), + (None, [{'name': 'container13'}]), + (None, [{'name': 'container14'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_get_conn.return_value = mock_conn + + s = SwiftService(options=self.opts) + lg = s.list() + + # Start the generator + first_list_part = next(lg) + + # Wait for the number of calls to get_account to reach our expected + # value, then let it run some more to make sure the value remains + # stable + count = mock_conn.get_account.call_count + stable = 0 + while mock_conn.get_account.call_count != count or stable < 5: + if mock_conn.get_account.call_count == count: + stable += 1 + else: + count = mock_conn.get_account.call_count + stable = 0 + # The test requires a small sleep to allow other threads to + # execute - in this mocked environment we assume that if the call + # count to get_account has not changed in 0.25s then no more calls + # will be made. + sleep(0.05) + + stable_get_account_call_count = mock_conn.get_account.call_count + + # Collect all remaining results from the generator + list_results = [first_list_part] + list(lg) + + # Make sure the stable call count is correct - this should be 12 calls + # to get_account; + # 1 for first_list_part + # 10 for the values on the queue + # 1 for the value blocking whilst trying to place onto the queue + self.assertEqual(12, stable_get_account_call_count) + + # Make sure all the containers were listed and placed onto the queue + self.assertEqual(15, mock_conn.get_account.call_count) + + # Check the results were all returned + observed_listing = [] + for lir in list_results: + observed_listing.append( + [li['name'] for li in lir['listing']] + ) + expected_listing = [] + for gar in get_account_returns[:-1]: # The empty list is not returned + expected_listing.append( + [li['name'] for li in gar[1]] + ) + self.assertEqual(observed_listing, expected_listing) + + class TestService(testtools.TestCase): def test_upload_with_bad_segment_size(self): @@ -551,24 +816,41 @@ self.assertEqual('Segment size should be an integer value', exc.value) + @mock.patch('swiftclient.service.stat') + @mock.patch('swiftclient.service.getmtime', return_value=1.0) + @mock.patch('swiftclient.service.getsize', return_value=4) + @mock.patch.object(builtins, 'open', return_value=six.StringIO('asdf')) + def test_upload_with_relative_path(self, *args, **kwargs): + service = SwiftService({}) + objects = [{'path': "./test", + 'strt_indx': 2}, + {'path': os.path.join(os.getcwd(), "test"), + 'strt_indx': 1}, + {'path': ".\\test", + 'strt_indx': 2}] + for obj in objects: + with mock.patch('swiftclient.service.Connection') as mock_conn: + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value =\ + 'd41d8cd98f00b204e9800998ecf8427e' + resp_iter = service.upload( + 'c', [SwiftUploadObject(obj['path'])]) + responses = [x for x in resp_iter] + for resp in responses: + self.assertTrue(resp['success']) + self.assertEqual(2, len(responses)) + create_container_resp, upload_obj_resp = responses + self.assertEqual(create_container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + obj['path'][obj['strt_indx']:]) + self.assertEqual(upload_obj_resp['path'], obj['path']) -class TestServiceUpload(testtools.TestCase): - - def _assertDictEqual(self, a, b, m=None): - # assertDictEqual is not available in py2.6 so use a shallow check - # instead - if not m: - m = '{0} != {1}'.format(a, b) - if hasattr(self, 'assertDictEqual'): - self.assertDictEqual(a, b, m) - else: - self.assertIsInstance(a, dict, m) - self.assertIsInstance(b, dict, m) - self.assertEqual(len(a), len(b), m) - for k, v in a.items(): - self.assertIn(k, b, m) - self.assertEqual(b[k], v, m) +class TestServiceUpload(_TestServiceBase): def test_upload_segment_job(self): with tempfile.NamedTemporaryFile() as f: @@ -860,3 +1142,802 @@ contents = mock_conn.put_object.call_args[0][2] self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest()) + + def test_upload_object_job_identical_etag(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-length': 30, + 'etag': md5(b'a' * 30).hexdigest()} + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options={'changed': False, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 0}) + + self.assertTrue(r['success']) + self.assertIn('status', r) + self.assertEqual(r['status'], 'skipped-identical') + self.assertEqual(mock_conn.put_object.call_count, 0) + self.assertEqual(mock_conn.head_object.call_count, 1) + mock_conn.head_object.assert_called_with('test_c', 'test_o') + + def test_upload_object_job_identical_slo_with_nesting(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + seg_etag = md5(b'a' * 10).hexdigest() + submanifest = "[%s]" % ",".join( + ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) + submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() + manifest = "[%s]" % ",".join([ + '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' + '"bytes":20,"hash":"%s"}' % submanifest_etag, + '{"bytes":10,"hash":"%s"}' % seg_etag]) + + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()} + mock_conn.get_object.side_effect = [ + ({}, manifest.encode('ascii')), + ({}, submanifest.encode('ascii'))] + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options={'changed': False, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + + self.assertIsNone(r.get('error')) + self.assertTrue(r['success']) + self.assertEqual('skipped-identical', r.get('status')) + self.assertEqual(0, mock_conn.put_object.call_count) + self.assertEqual([mock.call('test_c', 'test_o')], + mock_conn.head_object.mock_calls) + self.assertEqual([ + mock.call('test_c', 'test_o', + query_string='multipart-manifest=get'), + mock.call('test_c_segments', 'test_sub_slo', + query_string='multipart-manifest=get'), + ], mock_conn.get_object.mock_calls) + + def test_upload_object_job_identical_dlo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + segment_etag = md5(b'a' * 10).hexdigest() + + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'x-object-manifest': 'test_c_segments/test_o/prefix', + 'content-length': 30, + 'etag': md5(segment_etag.encode('ascii') * 3).hexdigest()} + mock_conn.get_container.side_effect = [ + (None, [{"bytes": 10, "hash": segment_etag, + "name": "test_o/prefix/00"}, + {"bytes": 10, "hash": segment_etag, + "name": "test_o/prefix/01"}]), + (None, [{"bytes": 10, "hash": segment_etag, + "name": "test_o/prefix/02"}]), + (None, {})] + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options={'changed': False, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + + self.assertIsNone(r.get('error')) + self.assertTrue(r['success']) + self.assertEqual('skipped-identical', r.get('status')) + self.assertEqual(0, mock_conn.put_object.call_count) + self.assertEqual(1, mock_conn.head_object.call_count) + self.assertEqual(3, mock_conn.get_container.call_count) + mock_conn.head_object.assert_called_with('test_c', 'test_o') + expected = [ + mock.call('test_c_segments', prefix='test_o/prefix', + marker='', delimiter=None), + mock.call('test_c_segments', prefix='test_o/prefix', + marker="test_o/prefix/01", delimiter=None), + mock.call('test_c_segments', prefix='test_o/prefix', + marker="test_o/prefix/02", delimiter=None), + ] + mock_conn.get_container.assert_has_calls(expected) + + +class TestServiceDownload(_TestServiceBase): + + def setUp(self): + super(TestServiceDownload, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + self.opts['no_download'] = True + self.obj_content = b'c' * 10 + self.obj_etag = md5(self.obj_content).hexdigest() + self.obj_len = len(self.obj_content) + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': 'download_object', # Should always be download_object + 'container': 'test_c', + 'object': 'test_o', + 'attempts': 2, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'success': None # Should be a bool + } + + def _readbody(self): + yield self.obj_content + + @mock.patch('swiftclient.service.SwiftService.list') + @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') + @mock.patch('swiftclient.service.interruptable_as_completed') + def test_download_container_job(self, as_comp, sub_page, service_list): + """ + Check that paged downloads work correctly + """ + obj_count = [0] + + def make_counting_generator(object_to_yield, total_count): + # maintain a counter of objects yielded + count = [0] + + def counting_generator(): + while count[0] < 10: + yield object_to_yield + count[0] += 1 + total_count[0] += 1 + return counting_generator() + + obj_count_on_sub_page_call = [] + sub_page_call_count = [0] + + def fake_sub_page(*args): + # keep a record of obj_count when this function is called + obj_count_on_sub_page_call.append(obj_count[0]) + sub_page_call_count[0] += 1 + if sub_page_call_count[0] < 3: + return range(0, 10) + return None + + sub_page.side_effect = fake_sub_page + + r = Mock(spec=Future) + r.result.return_value = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + as_comp.side_effect = [ + make_counting_generator(r, obj_count), + make_counting_generator(r, obj_count) + ] + + s = SwiftService() + down_gen = s._download_container('test_c', self.opts) + results = list(down_gen) + self.assertEqual(20, len(results)) + self.assertEqual(2, as_comp.call_count) + self.assertEqual(3, sub_page_call_count[0]) + self.assertEqual([0, 7, 17], obj_count_on_sub_page_call) + + @mock.patch('swiftclient.service.SwiftService.list') + @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') + @mock.patch('swiftclient.service.interruptable_as_completed') + def test_download_container_job_error( + self, as_comp, sub_page, service_list): + """ + Check that paged downloads work correctly + """ + class BoomError(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + def _make_result(): + r = Mock(spec=Future) + r.result.return_value = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + return r + + as_comp.side_effect = [ + + ] + # We need Futures here because the error will cause a call to .cancel() + sub_page_effects = [ + [_make_result() for _ in range(0, 10)], + BoomError('Go Boom') + ] + sub_page.side_effect = sub_page_effects + # ...but we must also mock the returns to as_completed + as_comp.side_effect = [ + [_make_result() for _ in range(0, 10)] + ] + + s = SwiftService() + self.assertRaises( + BoomError, + lambda: list(s._download_container('test_c', self.opts)) + ) + # This was an unknown error, so make sure we attempt to cancel futures + for spe in sub_page_effects[0]: + spe.cancel.assert_called_once_with() + self.assertEqual(1, as_comp.call_count) + + # Now test ClientException + sub_page_effects = [ + [_make_result() for _ in range(0, 10)], + ClientException('Go Boom') + ] + sub_page.side_effect = sub_page_effects + as_comp.reset_mock() + as_comp.side_effect = [ + [_make_result() for _ in range(0, 10)], + ] + self.assertRaises( + ClientException, + lambda: list(s._download_container('test_c', self.opts)) + ) + # This was a ClientException, so make sure we don't cancel futures + for spe in sub_page_effects[0]: + self.assertFalse(spe.cancel.called) + self.assertEqual(1, as_comp.call_count) + + def test_download_object_job(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb') + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self._assertDictEqual(expected_r, actual_r) + + def test_download_object_job_exception(self): + mock_conn = self._get_mock_connection() + mock_conn.get_object = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'success': False, + 'error': self.exc, + 'error_timestamp': mock.ANY, + 'traceback': mock.ANY + }) + + s = SwiftService() + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', self.opts) + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self._assertDictEqual(expected_r, actual_r) + + def test_download(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + resp = service._download_object_job(mock_conn, + 'c', + 'test', + self.opts) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_output_dir(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['out_directory'] = 'temp_dir' + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'temp_dir/example/test') + + def test_download_with_remove_prefix(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example/' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_remove_prefix_and_remove_slashes(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_output_dir_and_remove_prefix(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['out_directory'] = 'new/dir' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'new/dir/test') + + def test_download_object_job_skip_identical(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + + err = swiftclient.ClientException('Object GET failed', + http_status=304) + + def fake_get(*args, **kwargs): + kwargs['response_dict']['headers'] = {} + raise err + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = fake_get + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': False, + 'error': err, + 'response_dict': {'headers': {}}, + 'path': 'test_o', + 'pseudodir': False, + 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + } + + s = SwiftService() + r = s._download_object_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, + 'header': {}, + 'yes_all': False, + 'skip_identical': True}) + self._assertDictEqual(r, expected_r) + + self.assertEqual(mock_conn.get_object.call_count, 1) + mock_conn.get_object.assert_called_with( + 'test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': md5(b'a' * 30).hexdigest()}, + query_string='multipart-manifest=get', + response_dict=expected_r['response_dict']) + + def test_download_object_job_skip_identical_dlo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + on_disk_md5 = md5(b'a' * 30).hexdigest() + segment_md5 = md5(b'a' * 10).hexdigest() + + mock_conn = mock.Mock() + mock_conn.get_object.return_value = ( + {'x-object-manifest': 'test_c_segments/test_o/prefix'}, [b'']) + mock_conn.get_container.side_effect = [ + (None, [{'name': 'test_o/prefix/1', + 'bytes': 10, 'hash': segment_md5}, + {'name': 'test_o/prefix/2', + 'bytes': 10, 'hash': segment_md5}]), + (None, [{'name': 'test_o/prefix/3', + 'bytes': 10, 'hash': segment_md5}]), + (None, [])] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': False, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + } + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, + 'header': {}, + 'yes_all': False, + 'skip_identical': True}) + + err = r.pop('error') + self.assertEqual("Large object is identical", err.msg) + self.assertEqual(304, err.http_status) + + self._assertDictEqual(r, expected_r) + + self.assertEqual(mock_conn.get_object.call_count, 1) + mock_conn.get_object.assert_called_with( + 'test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict=expected_r['response_dict']) + self.assertEqual(mock_conn.get_container.mock_calls, [ + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker=''), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/2'), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/3')]) + + def test_download_object_job_skip_identical_nested_slo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + on_disk_md5 = md5(b'a' * 30).hexdigest() + + seg_etag = md5(b'a' * 10).hexdigest() + submanifest = "[%s]" % ",".join( + ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) + submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() + manifest = "[%s]" % ",".join([ + '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' + '"bytes":20,"hash":"%s"}' % submanifest_etag, + '{"bytes":10,"hash":"%s"}' % seg_etag]) + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = [ + ({'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()}, + [manifest.encode('ascii')]), + ({'x-static-large-object': True, + 'content-length': 20, + 'etag': submanifest_etag}, + submanifest.encode('ascii'))] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': False, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + } + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, + 'header': {}, + 'yes_all': False, + 'skip_identical': True}) + + err = r.pop('error') + self.assertEqual("Large object is identical", err.msg) + self.assertEqual(304, err.http_status) + + self._assertDictEqual(r, expected_r) + self.assertEqual(mock_conn.get_object.mock_calls, [ + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict={}), + mock.call('test_c_segments', + 'test_sub_slo', + query_string='multipart-manifest=get')]) + + def test_download_object_job_skip_identical_diff_dlo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.write(b'b') + f.flush() + on_disk_md5 = md5(b'a' * 30 + b'b').hexdigest() + segment_md5 = md5(b'a' * 10).hexdigest() + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = [ + ({'x-object-manifest': 'test_c_segments/test_o/prefix'}, + [b'']), + ({'x-object-manifest': 'test_c_segments/test_o/prefix'}, + [b'a' * 30])] + mock_conn.get_container.side_effect = [ + (None, [{'name': 'test_o/prefix/1', + 'bytes': 10, 'hash': segment_md5}, + {'name': 'test_o/prefix/2', + 'bytes': 10, 'hash': segment_md5}]), + (None, [{'name': 'test_o/prefix/3', + 'bytes': 10, 'hash': segment_md5}]), + (None, [])] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + type(mock_conn).auth_end_time = mock.PropertyMock(return_value=14) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': True, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'read_length': 30, + 'attempts': 2, + 'start_time': 0, + 'headers_receipt': 1, + 'finish_time': 2, + 'auth_end_time': mock_conn.auth_end_time, + } + + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True + s = SwiftService() + with mock.patch('swiftclient.service.time', side_effect=range(3)): + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job( + conn=mock_conn, + container='test_c', + obj='test_o', + options=options) + + self._assertDictEqual(r, expected_r) + + self.assertEqual(mock_conn.get_container.mock_calls, [ + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker=''), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/2'), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/3')]) + self.assertEqual(mock_conn.get_object.mock_calls, [ + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict={}), + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + response_dict={})]) + + def test_download_object_job_skip_identical_diff_nested_slo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 29) + f.flush() + on_disk_md5 = md5(b'a' * 29).hexdigest() + + seg_etag = md5(b'a' * 10).hexdigest() + submanifest = "[%s]" % ",".join( + ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) + submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() + manifest = "[%s]" % ",".join([ + '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' + '"bytes":20,"hash":"%s"}' % submanifest_etag, + '{"bytes":10,"hash":"%s"}' % seg_etag]) + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = [ + ({'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()}, + [manifest.encode('ascii')]), + ({'x-static-large-object': True, + 'content-length': 20, + 'etag': submanifest_etag}, + submanifest.encode('ascii')), + ({'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()}, + [b'a' * 30])] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + type(mock_conn).auth_end_time = mock.PropertyMock(return_value=14) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': True, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'read_length': 30, + 'attempts': 2, + 'start_time': 0, + 'headers_receipt': 1, + 'finish_time': 2, + 'auth_end_time': mock_conn.auth_end_time, + } + + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True + s = SwiftService() + with mock.patch('swiftclient.service.time', side_effect=range(3)): + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job( + conn=mock_conn, + container='test_c', + obj='test_o', + options=options) + + self._assertDictEqual(r, expected_r) + self.assertEqual(mock_conn.get_object.mock_calls, [ + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict={}), + mock.call('test_c_segments', + 'test_sub_slo', + query_string='multipart-manifest=get'), + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + response_dict={})]) diff -Nru python-swiftclient-2.4.0/tests/unit/test_shell.py python-swiftclient-2.6.0/tests/unit/test_shell.py --- python-swiftclient-2.4.0/tests/unit/test_shell.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/tests/unit/test_shell.py 2015-09-07 15:20:14.000000000 +0000 @@ -12,16 +12,18 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import unicode_literals from genericpath import getmtime import hashlib -import json import mock import os import tempfile import unittest +import textwrap from testtools import ExpectedException + import six import swiftclient @@ -31,7 +33,9 @@ from os.path import basename, dirname from tests.unit.test_swiftclient import MockHttpTest -from tests.unit.utils import CaptureOutput, fake_get_auth_keystone +from tests.unit.utils import ( + CaptureOutput, fake_get_auth_keystone, _make_fake_import_keystone_client, + FakeKeystone, StubResponse) from swiftclient.utils import EMPTY_ETAG @@ -45,6 +49,11 @@ 'ST_USER': 'test:tester', 'ST_KEY': 'testing' } +clean_os_environ = {} +environ_prefixes = ('ST_', 'OS_') +for key in os.environ: + if any(key.startswith(m) for m in environ_prefixes): + clean_os_environ[key] = '' clean_os_environ = {} environ_prefixes = ('ST_', 'OS_') @@ -129,11 +138,11 @@ with CaptureOutput() as output: swiftclient.shell.main(argv) - self.assertEquals(output.out, - ' Account: AUTH_account\n' - 'Containers: 1\n' - ' Objects: 2\n' - ' Bytes: 3\n') + self.assertEqual(output.out, + ' Account: AUTH_account\n' + 'Containers: 1\n' + ' Objects: 2\n' + ' Bytes: 3\n') @mock.patch('swiftclient.service.Connection') def test_stat_container(self, connection): @@ -151,15 +160,15 @@ with CaptureOutput() as output: swiftclient.shell.main(argv) - self.assertEquals(output.out, - ' Account: AUTH_account\n' - 'Container: container\n' - ' Objects: 1\n' - ' Bytes: 2\n' - ' Read ACL: test2:tester2\n' - 'Write ACL: test3:tester3\n' - ' Sync To: other\n' - ' Sync Key: secret\n') + self.assertEqual(output.out, + ' Account: AUTH_account\n' + 'Container: container\n' + ' Objects: 1\n' + ' Bytes: 2\n' + ' Read ACL: test2:tester2\n' + 'Write ACL: test3:tester3\n' + ' Sync To: other\n' + ' Sync Key: secret\n') @mock.patch('swiftclient.service.Connection') def test_stat_object(self, connection): @@ -177,15 +186,15 @@ with CaptureOutput() as output: swiftclient.shell.main(argv) - self.assertEquals(output.out, - ' Account: AUTH_account\n' - ' Container: container\n' - ' Object: object\n' - ' Content Type: text/plain\n' - 'Content Length: 42\n' - ' Last Modified: yesterday\n' - ' ETag: md5\n' - ' Manifest: manifest\n') + self.assertEqual(output.out, + ' Account: AUTH_account\n' + ' Container: container\n' + ' Object: object\n' + ' Content Type: text/plain\n' + 'Content Length: 42\n' + ' Last Modified: yesterday\n' + ' ETag: md5\n' + ' Manifest: manifest\n') @mock.patch('swiftclient.service.Connection') def test_list_account(self, connection): @@ -203,7 +212,7 @@ mock.call(marker='container', prefix=None)] connection.return_value.get_account.assert_has_calls(calls) - self.assertEquals(output.out, 'container\n') + self.assertEqual(output.out, 'container\n') @mock.patch('swiftclient.service.Connection') def test_list_account_long(self, connection): @@ -220,9 +229,9 @@ mock.call(marker='container', prefix=None)] connection.return_value.get_account.assert_has_calls(calls) - self.assertEquals(output.out, - ' 0 0 1970-01-01 00:00:01 container\n' - ' 0 0\n') + self.assertEqual(output.out, + ' 0 0 1970-01-01 00:00:01 container\n' + ' 0 0\n') # Now test again, this time without returning metadata connection.return_value.head_container.return_value = {} @@ -240,9 +249,9 @@ mock.call(marker='container', prefix=None)] connection.return_value.get_account.assert_has_calls(calls) - self.assertEquals(output.out, - ' 0 0 ????-??-?? ??:??:?? container\n' - ' 0 0\n') + self.assertEqual(output.out, + ' 0 0 ????-??-?? ??:??:?? container\n' + ' 0 0\n') def test_list_account_totals_error(self): # No --lh provided: expect info message about incorrect --totals use @@ -285,7 +294,7 @@ delimiter=None, prefix=None)] connection.return_value.get_container.assert_has_calls(calls) - self.assertEquals(output.out, 'object_a\n') + self.assertEqual(output.out, 'object_a\n') # Test container listing with --long connection.return_value.get_container.side_effect = [ @@ -302,9 +311,9 @@ delimiter=None, prefix=None)] connection.return_value.get_container.assert_has_calls(calls) - self.assertEquals(output.out, - ' 0 123 456 object_a\n' - ' 0\n') + self.assertEqual(output.out, + ' 0 123 456 object_a\n' + ' 0\n') @mock.patch('swiftclient.service.makedirs') @mock.patch('swiftclient.service.Connection') @@ -368,6 +377,56 @@ swiftclient.shell.main(argv) self.assertEqual('objcontent', output.out) + @mock.patch('swiftclient.service.shuffle') + @mock.patch('swiftclient.service.Connection') + def test_download_shuffle(self, connection, mock_shuffle): + # Test that the container and object lists are shuffled + mock_shuffle.side_effect = lambda l: l + connection.return_value.get_object.return_value = [ + {'content-type': 'text/plain', + 'etag': EMPTY_ETAG}, + ''] + + connection.return_value.get_container.side_effect = [ + (None, [{'name': 'object'}]), + (None, [{'name': 'pseudo/'}]), + (None, []), + ] + connection.return_value.auth_end_time = 0 + connection.return_value.attempts = 0 + connection.return_value.get_account.side_effect = [ + (None, [{'name': 'container'}]), + (None, []) + ] + + with mock.patch(BUILTIN_OPEN) as mock_open: + argv = ["", "download", "--all"] + swiftclient.shell.main(argv) + self.assertEqual(3, mock_shuffle.call_count) + mock_shuffle.assert_any_call(['container']) + mock_shuffle.assert_any_call(['object']) + mock_shuffle.assert_any_call(['pseudo/']) + mock_open.assert_called_once_with('container/object', 'wb') + + # Test that the container and object lists are not shuffled + mock_shuffle.reset_mock() + + connection.return_value.get_container.side_effect = [ + (None, [{'name': 'object'}]), + (None, [{'name': 'pseudo/'}]), + (None, []), + ] + connection.return_value.get_account.side_effect = [ + (None, [{'name': 'container'}]), + (None, []) + ] + + with mock.patch(BUILTIN_OPEN) as mock_open: + argv = ["", "download", "--all", "--no-shuffle"] + swiftclient.shell.main(argv) + self.assertEqual(0, mock_shuffle.call_count) + mock_open.assert_called_once_with('container/object', 'wb') + @mock.patch('swiftclient.service.Connection') def test_download_no_content_type(self, connection): connection.return_value.get_object.return_value = [ @@ -458,9 +517,8 @@ 'x-object-meta-mtime': mock.ANY}, response_dict={}) - @mock.patch('swiftclient.shell.walk') @mock.patch('swiftclient.service.Connection') - def test_upload_delete(self, connection, walk): + def test_upload_delete_slo_segments(self, connection): # Upload delete existing segments connection.return_value.head_container.return_value = { 'x-storage-policy': 'one'} @@ -474,9 +532,11 @@ {'x-static-large-object': 'false', # For the 2nd delete call 'content-length': '2'} ] - connection.return_value.get_object.return_value = ({}, json.dumps( - [{'name': 'container1/old_seg1'}, {'name': 'container2/old_seg2'}] - )) + connection.return_value.get_object.return_value = ( + {}, + b'[{"name": "container1/old_seg1"},' + b' {"name": "container2/old_seg2"}]' + ) connection.return_value.put_object.return_value = EMPTY_ETAG swiftclient.shell.main(argv) connection.return_value.put_object.assert_called_with( @@ -489,11 +549,79 @@ expected_delete_calls = [ mock.call( b'container1', b'old_seg1', - query_string=None, response_dict={} + response_dict={} ), mock.call( b'container2', b'old_seg2', - query_string=None, response_dict={} + response_dict={} + ) + ] + self.assertEqual( + sorted(expected_delete_calls), + sorted(connection.return_value.delete_object.mock_calls) + ) + + @mock.patch('swiftclient.service.Connection') + def test_upload_leave_slo_segments(self, connection): + # Test upload overwriting a manifest respects --leave-segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, "--leave-segments"] + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', # For the upload call + 'content-length': '2'}] + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') + def test_upload_delete_dlo_segments(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile] + connection.return_value.head_object.side_effect = [ + {'x-object-manifest': 'container1/prefix', + 'content-length': '0'}, + {}, + {} + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'prefix_a', 'bytes': 0, + 'last_modified': '123T456'}]], + # Have multiple pages worth of DLO segments + [None, [{'name': 'prefix_b', 'bytes': 0, + 'last_modified': '123T456'}]], + [None, []] + ] + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + expected_delete_calls = [ + mock.call( + 'container1', 'prefix_a', + response_dict={} + ), + mock.call( + 'container1', 'prefix_b', + response_dict={} ) ] self.assertEqual( @@ -502,6 +630,28 @@ ) @mock.patch('swiftclient.service.Connection') + def test_upload_leave_dlo_segments(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, "--leave-segments"] + connection.return_value.head_object.side_effect = [ + {'x-object-manifest': 'container1/prefix', + 'content-length': '0'}] + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') def test_upload_segments_to_same_container(self, connection): # Upload in segments to same container connection.return_value.head_object.return_value = { @@ -558,6 +708,38 @@ connection.return_value.delete_object.assert_called_with( 'container', 'object', query_string=None, response_dict={}) + def test_delete_verbose_output_utf8(self): + container = 't\u00e9st_c' + base_argv = ['', '--verbose', 'delete'] + + # simulate container having an object with utf-8 code points in name, + # just returning the object delete result + res = {'success': True, 'response_dict': {}, 'attempts': 2, + 'container': container, 'action': 'delete_object', + 'object': 'obj_t\u00east_o'} + + with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: + with CaptureOutput() as out: + mock_func.return_value = [res] + swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + + mock_func.assert_called_once_with(container=container) + self.assertTrue(out.out.find( + 'obj_t\u00east_o [after 2 attempts]') >= 0, out) + + # simulate empty container + res = {'success': True, 'response_dict': {}, 'attempts': 2, + 'container': container, 'action': 'delete_container'} + + with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: + with CaptureOutput() as out: + mock_func.return_value = [res] + swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + + mock_func.assert_called_once_with(container=container) + self.assertTrue(out.out.find( + 't\u00e9st_c [after 2 attempts]') >= 0, out) + @mock.patch('swiftclient.service.Connection') def test_delete_object(self, connection): argv = ["", "delete", "container", "object"] @@ -569,8 +751,8 @@ def test_delete_verbose_output(self): del_obj_res = {'success': True, 'response_dict': {}, 'attempts': 2, - 'container': 'test_c', 'action': 'delete_object', - 'object': 'test_o'} + 'container': 't\xe9st_c', 'action': 'delete_object', + 'object': 't\xe9st_o'} del_seg_res = del_obj_res.copy() del_seg_res.update({'action': 'delete_segment'}) @@ -578,7 +760,7 @@ del_con_res = del_obj_res.copy() del_con_res.update({'action': 'delete_container', 'object': None}) - test_exc = Exception('test_exc') + test_exc = Exception('t\xe9st_exc') error_res = del_obj_res.copy() error_res.update({'success': False, 'error': test_exc, 'object': None}) @@ -588,39 +770,39 @@ with mock.patch('swiftclient.shell.SwiftService.delete', mock_delete): with CaptureOutput() as out: mock_delete.return_value = [del_obj_res] - swiftclient.shell.main(base_argv + ['test_c', 'test_o']) + swiftclient.shell.main(base_argv + ['t\xe9st_c', 't\xe9st_o']) - mock_delete.assert_called_once_with(container='test_c', - objects=['test_o']) + mock_delete.assert_called_once_with(container='t\xe9st_c', + objects=['t\xe9st_o']) self.assertTrue(out.out.find( - 'test_o [after 2 attempts]') >= 0) + 't\xe9st_o [after 2 attempts]') >= 0) with CaptureOutput() as out: mock_delete.return_value = [del_seg_res] - swiftclient.shell.main(base_argv + ['test_c', 'test_o']) + swiftclient.shell.main(base_argv + ['t\xe9st_c', 't\xe9st_o']) - mock_delete.assert_called_with(container='test_c', - objects=['test_o']) + mock_delete.assert_called_with(container='t\xe9st_c', + objects=['t\xe9st_o']) self.assertTrue(out.out.find( - 'test_c/test_o [after 2 attempts]') >= 0) + 't\xe9st_c/t\xe9st_o [after 2 attempts]') >= 0) with CaptureOutput() as out: mock_delete.return_value = [del_con_res] - swiftclient.shell.main(base_argv + ['test_c']) + swiftclient.shell.main(base_argv + ['t\xe9st_c']) - mock_delete.assert_called_with(container='test_c') + mock_delete.assert_called_with(container='t\xe9st_c') self.assertTrue(out.out.find( - 'test_c [after 2 attempts]') >= 0) + 't\xe9st_c [after 2 attempts]') >= 0) with CaptureOutput() as out: mock_delete.return_value = [error_res] self.assertRaises(SystemExit, swiftclient.shell.main, - base_argv + ['test_c']) + base_argv + ['t\xe9st_c']) - mock_delete.assert_called_with(container='test_c') + mock_delete.assert_called_with(container='t\xe9st_c') self.assertTrue(out.err.find( - 'Error Deleting: test_c: test_exc') >= 0) + 'Error Deleting: t\xe9st_c: t\xe9st_exc') >= 0) @mock.patch('swiftclient.service.Connection') def test_post_account(self, connection): @@ -639,7 +821,7 @@ with ExpectedException(SystemExit): swiftclient.shell.main(argv) - self.assertEquals(output.err, 'bad auth\n') + self.assertEqual(output.err, 'bad auth\n') @mock.patch('swiftclient.service.Connection') def test_post_account_not_found(self, connection): @@ -651,7 +833,7 @@ with ExpectedException(SystemExit): swiftclient.shell.main(argv) - self.assertEquals(output.err, 'Account not found\n') + self.assertEqual(output.err, 'Account not found\n') @mock.patch('swiftclient.service.Connection') def test_post_container(self, connection): @@ -670,7 +852,7 @@ with ExpectedException(SystemExit): swiftclient.shell.main(argv) - self.assertEquals(output.err, 'bad auth\n') + self.assertEqual(output.err, 'bad auth\n') @mock.patch('swiftclient.service.Connection') def test_post_container_not_found_causes_put(self, connection): @@ -728,7 +910,7 @@ with ExpectedException(SystemExit): swiftclient.shell.main(argv) - self.assertEquals(output.err, 'bad auth\n') + self.assertEqual(output.err, 'bad auth\n') def test_post_object_too_many_args(self): argv = ["", "post", "container", "object", "bad_arg"] @@ -740,15 +922,21 @@ self.assertTrue(output.err != '') self.assertTrue(output.err.startswith('Usage')) - @mock.patch('swiftclient.shell.generate_temp_url') + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url(self, temp_url): argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", - "secret_key" - ] - temp_url.return_value = "" + "secret_key"] swiftclient.shell.main(argv) temp_url.assert_called_with( - '/v1/AUTH_account/c/o', 60, 'secret_key', 'GET') + '/v1/AUTH_account/c/o', 60, 'secret_key', 'GET', absolute=False) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_absolute_expiry_temp_url(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key", "--absolute"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', 60, 'secret_key', 'GET', absolute=True) @mock.patch('swiftclient.service.Connection') def test_capabilities(self, connection): @@ -793,19 +981,42 @@ # Test invalid states argv = ["", "upload", "-S", "1234X", "container", "object"] swiftclient.shell.main(argv) - self.assertEquals(output.err, "Invalid segment size\n") + self.assertEqual(output.err, "Invalid segment size\n") output.clear() with ExpectedException(SystemExit): argv = ["", "upload", "-S", "K1234", "container", "object"] swiftclient.shell.main(argv) - self.assertEquals(output.err, "Invalid segment size\n") + self.assertEqual(output.err, "Invalid segment size\n") output.clear() with ExpectedException(SystemExit): argv = ["", "upload", "-S", "K", "container", "object"] swiftclient.shell.main(argv) - self.assertEquals(output.err, "Invalid segment size\n") + self.assertEqual(output.err, "Invalid segment size\n") + + def test_negative_upload_segment_size(self): + with CaptureOutput() as output: + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40K", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40M", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40G", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() class TestSubcommandHelp(unittest.TestCase): @@ -1116,16 +1327,26 @@ # --help returns condensed help message, overrides --os-help opts = {"help": ""} os_opts = {"help": ""} - # "password": "secret", - # "username": "user", - # "auth_url": "http://example.com:5000/v3"} args = _make_args("", opts, os_opts) with CaptureOutput() as out: self.assertRaises(SystemExit, swiftclient.shell.main, args) self.assertTrue(out.find('[--key ]') > 0) self.assertEqual(-1, out.find('--os-username=')) - ## --os-help return os options help + # --os-password, --os-username and --os-auth_url should be ignored + # because --help overrides it + opts = {"help": ""} + os_opts = {"help": "", + "password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + args = _make_args("", opts, os_opts) + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertTrue(out.find('[--key ]') > 0) + self.assertEqual(-1, out.find('--os-username=')) + + # --os-help return os options help opts = {} args = _make_args("", opts, os_opts) with CaptureOutput() as out: @@ -1134,55 +1355,6 @@ self.assertTrue(out.find('--os-username=') > 0) -class FakeKeystone(object): - ''' - Fake keystone client module. Returns given endpoint url and auth token. - ''' - def __init__(self, endpoint, token): - self.calls = [] - self.auth_version = None - self.endpoint = endpoint - self.token = token - - class _Client(): - def __init__(self, endpoint, token, **kwargs): - self.auth_token = token - self.endpoint = endpoint - self.service_catalog = self.ServiceCatalog(endpoint) - - class ServiceCatalog(object): - def __init__(self, endpoint): - self.calls = [] - self.endpoint_url = endpoint - - def url_for(self, **kwargs): - self.calls.append(kwargs) - return self.endpoint_url - - def Client(self, **kwargs): - self.calls.append(kwargs) - self.client = self._Client(endpoint=self.endpoint, token=self.token, - **kwargs) - return self.client - - class Unauthorized(Exception): - pass - - class AuthorizationFailure(Exception): - pass - - class EndpointNotFound(Exception): - pass - - -def _make_fake_import_keystone_client(fake_import): - def _fake_import_keystone_client(auth_version): - fake_import.auth_version = auth_version - return fake_import, fake_import - - return _fake_import_keystone_client - - class TestKeystoneOptions(MockHttpTest): """ Tests to check that options are passed from the command line or @@ -1488,6 +1660,101 @@ }), ]) + def test_auth(self): + headers = { + 'x-auth-token': 'AUTH_tk5b6b12', + 'x-storage-url': 'https://swift.storage.example.com/v1/AUTH_test', + } + mock_resp = self.fake_http_connection(200, headers=headers) + with mock.patch('swiftclient.client.http_connection', new=mock_resp): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'testing', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=https://swift.storage.example.com/v1/AUTH_test + export OS_AUTH_TOKEN=AUTH_tk5b6b12 + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose(self): + with mock.patch('swiftclient.client.http_connection') as mock_conn: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'te$tin&', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export ST_AUTH=https://swift.storage.example.com/auth/v1.0 + export ST_USER=test:tester + export ST_KEY='te$tin&' + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_conn.mock_calls) + + def test_auth_v2(self): + os_options = {'tenant_name': 'demo'} + with mock.patch('swiftclient.client.get_auth_keystone', + new=fake_get_auth_keystone(os_options)): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', 'admin', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=http://url/ + export OS_AUTH_TOKEN=token + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose_v2(self): + with mock.patch('swiftclient.client.get_auth_keystone') \ + as mock_keystone: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', '$eKr3t', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_IDENTITY_API_VERSION=2.0 + export OS_AUTH_VERSION=2.0 + export OS_AUTH_URL=https://keystone.example.com/v2.0/ + export OS_PASSWORD='$eKr3t' + export OS_TENANT_NAME=demo + export OS_USERNAME=demo + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_keystone.mock_calls) + class TestCrossAccountObjectAccess(TestBase, MockHttpTest): """ @@ -1569,8 +1836,8 @@ self.assertRequests([('PUT', self.cont_path), ('PUT', self.obj_path)]) - self.assertEqual(self.obj, out.strip()) - expected_err = 'Warning: failed to create container %r: 403 Fake' \ + self.assertEqual(self.obj[1:], out.strip()) + expected_err = "Warning: failed to create container '%s': 403 Fake" \ % self.cont self.assertEqual(expected_err, out.err.strip()) @@ -1578,7 +1845,6 @@ req_handler = self._fake_cross_account_auth(False, True) fake_conn = self.fake_http_connection(403, 403, on_request=req_handler) - args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) with mock.patch('swiftclient.client._import_keystone_client', @@ -1590,11 +1856,10 @@ swiftclient.shell.main(args) except SystemExit as e: self.fail('Unexpected SystemExit: %s' % e) - self.assertRequests([('PUT', self.cont_path), ('PUT', self.obj_path)]) - self.assertEqual(self.obj, out.strip()) - expected_err = 'Warning: failed to create container %r: 403 Fake' \ + self.assertEqual(self.obj[1:], out.strip()) + expected_err = "Warning: failed to create container '%s': 403 Fake" \ % self.cont self.assertEqual(expected_err, out.err.strip()) @@ -1628,8 +1893,8 @@ self.assert_request(('PUT', segment_path_0)) self.assert_request(('PUT', segment_path_1)) self.assert_request(('PUT', self.obj_path)) - self.assertTrue(self.obj in out.out) - expected_err = 'Warning: failed to create container %r: 403 Fake' \ + self.assertTrue(self.obj[1:] in out.out) + expected_err = "Warning: failed to create container '%s': 403 Fake" \ % self.cont self.assertEqual(expected_err, out.err.strip()) @@ -1718,19 +1983,16 @@ self.assertRequests([('GET', self.obj_path)]) path = '%s%s' % (self.cont, self.obj) - expected_err = 'Error downloading object %r' % path + expected_err = "Error downloading object '%s'" % path self.assertTrue(out.err.startswith(expected_err)) self.assertEqual('', out) def test_list_with_read_access(self): req_handler = self._fake_cross_account_auth(True, False) - resp_body = '{}' - m = hashlib.md5() - m.update(resp_body.encode()) - etag = m.hexdigest() - fake_conn = self.fake_http_connection(403, on_request=req_handler, - etags=[etag], - body=resp_body) + resp_body = b'{}' + resp = StubResponse(403, resp_body, { + 'etag': hashlib.md5(resp_body).hexdigest()}) + fake_conn = self.fake_http_connection(resp, on_request=req_handler) args, env = self._make_cmd('download', cmd_args=[self.cont]) with mock.patch('swiftclient.client._import_keystone_client', diff -Nru python-swiftclient-2.4.0/tests/unit/test_swiftclient.py python-swiftclient-2.6.0/tests/unit/test_swiftclient.py --- python-swiftclient-2.4.0/tests/unit/test_swiftclient.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/tests/unit/test_swiftclient.py 2015-09-07 15:20:14.000000000 +0000 @@ -14,7 +14,6 @@ # limitations under the License. import logging -import json try: from unittest import mock @@ -28,9 +27,9 @@ import tempfile from hashlib import md5 from six.moves.urllib.parse import urlparse -from six.moves import reload_module -from .utils import MockHttpTest, fake_get_auth_keystone, StubResponse +from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, + FakeKeystone, _make_fake_import_keystone_client) from swiftclient.utils import EMPTY_ETAG from swiftclient import client as c @@ -64,36 +63,6 @@ self.assertTrue(value in str(exc)) -class TestJsonImport(testtools.TestCase): - - def tearDown(self): - reload_module(json) - - try: - import simplejson - except ImportError: - pass - else: - reload_module(simplejson) - super(TestJsonImport, self).tearDown() - - def test_any(self): - self.assertTrue(hasattr(c, 'json_loads')) - - def test_no_simplejson_falls_back_to_stdlib_when_reloaded(self): - # break simplejson - try: - import simplejson - except ImportError: - # not installed, so we don't have to break it for these tests - pass - else: - delattr(simplejson, 'loads') # break simple json - reload_module(c) # reload to repopulate json_loads - - self.assertEqual(c.json_loads, json.loads) - - class MockHttpResponse(object): def __init__(self, status=0, headers=None, verify=False): self.status = status @@ -106,20 +75,31 @@ self.headers = {'etag': '"%s"' % EMPTY_ETAG} if headers: self.headers.update(headers) + self.closed = False class Raw(object): - def read(self): - pass - self.raw = Raw() + def __init__(self, headers): + self.headers = headers + + def read(self, **kw): + return "" + + def getheader(self, name, default): + return self.headers.get(name, default) + + self.raw = Raw(headers) def read(self): return "" + def close(self): + self.closed = True + def getheader(self, name, default): return self.headers.get(name, default) def getheaders(self): - return {"key1": "value1", "key2": "value2"} + return dict(self.headers) def fake_response(self): return self @@ -221,8 +201,6 @@ ua = req_headers.get('user-agent', 'XXX-MISSING-XXX') self.assertEqual(ua, 'a-new-default') -# TODO: following tests are placeholders, need more tests, better coverage - class TestGetAuth(MockHttpTest): @@ -260,6 +238,59 @@ # the full plumbing into the requests's 'verify' option self.assertIn('invalid_certificate', str(e)) + def test_auth_v1_timeout(self): + # this test has some overlap with + # TestConnection.test_timeout_passed_down but is required to check that + # get_auth does the right thing when it is not passed a timeout arg + orig_http_connection = c.http_connection + timeouts = [] + + def fake_request_handler(*a, **kw): + if 'timeout' in kw: + timeouts.append(kw['timeout']) + else: + timeouts.append(None) + return MockHttpResponse( + status=200, + headers={ + 'x-auth-token': 'a_token', + 'x-storage-url': 'http://files.example.com/v1/AUTH_user'}) + + def fake_connection(*a, **kw): + url, conn = orig_http_connection(*a, **kw) + conn._request = fake_request_handler + return url, conn + + with mock.patch('swiftclient.client.http_connection', fake_connection): + c.get_auth('http://www.test.com', 'asdf', 'asdf', + auth_version="1.0", timeout=42.0) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + auth_version="1.0", timeout=None) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + auth_version="1.0") + + self.assertEqual(timeouts, [42.0, None, None]) + + def test_auth_v2_timeout(self): + # this test has some overlap with + # TestConnection.test_timeout_passed_down but is required to check that + # get_auth does the right thing when it is not passed a timeout arg + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=dict(tenant_name='tenant'), + auth_version="2.0", timeout=42.0) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=dict(tenant_name='tenant'), + auth_version="2.0", timeout=None) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=dict(tenant_name='tenant'), + auth_version="2.0") + self.assertEqual(3, len(fake_ks.calls)) + timeouts = [call['timeout'] for call in fake_ks.calls] + self.assertEqual([42.0, None, None], timeouts) + def test_auth_v2_with_tenant_name(self): os_options = {'tenant_name': 'asdf'} req_args = {'auth_version': '2.0'} @@ -655,6 +686,41 @@ }), ]) + def test_chunk_size_read_method(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection(200, body='abcde') + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=3) + self.assertTrue(hasattr(resp, 'read')) + self.assertEqual(resp.read(3), 'abc') + self.assertEqual(resp.read(None), 'de') + self.assertEqual(resp.read(), '') + + def test_chunk_size_iter(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection(200, body='abcde') + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=3) + self.assertTrue(hasattr(resp, 'next')) + self.assertEqual(next(resp), 'abc') + self.assertEqual(next(resp), 'de') + self.assertRaises(StopIteration, next, resp) + + def test_chunk_size_read_and_iter(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection(200, body='abcdef') + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertTrue(hasattr(resp, 'read')) + self.assertEqual(resp.read(3), 'abc') + self.assertEqual(next(resp), 'de') + self.assertEqual(resp.read(), 'f') + self.assertRaises(StopIteration, next, resp) + self.assertEqual(resp.read(), '') + class TestHeadObject(MockHttpTest): @@ -805,9 +871,9 @@ contents=contents, chunk_size=chunk_size) - self.assertNotEquals(etag, contents.get_md5sum()) - self.assertEquals(etag, 'badresponseetag') - self.assertEquals(raw_data_md5, contents.get_md5sum()) + self.assertNotEqual(etag, contents.get_md5sum()) + self.assertEqual(etag, 'badresponseetag') + self.assertEqual(raw_data_md5, contents.get_md5sum()) def test_md5_match(self): conn = c.http_connection('http://www.test.com') @@ -830,8 +896,8 @@ contents=contents, chunk_size=chunk_size) - self.assertEquals(raw_data_md5, contents.get_md5sum()) - self.assertEquals(etag, contents.get_md5sum()) + self.assertEqual(raw_data_md5, contents.get_md5sum()) + self.assertEqual(etag, contents.get_md5sum()) def test_params(self): conn = c.http_connection(u'http://www.test.com/') @@ -925,7 +991,7 @@ class TestGetCapabilities(MockHttpTest): def test_ok(self): - conn = self.fake_http_connection(200, body='{}') + conn = self.fake_http_connection(200, body=b'{}') http_conn = conn('http://www.test.com/info') info = c.get_capabilities(http_conn) self.assertRequests([ @@ -946,7 +1012,7 @@ } auth_v1_response = StubResponse(headers=auth_headers) stub_info = {'swift': {'fake': True}} - info_response = StubResponse(body=json.dumps(stub_info)) + info_response = StubResponse(body=b'{"swift":{"fake":true}}') fake_conn = self.fake_http_connection(auth_v1_response, info_response) conn = c.Connection('http://auth.example.com/auth/v1.0', @@ -964,7 +1030,7 @@ fake_keystone = fake_get_auth_keystone( storage_url='http://storage.example.com/v1/AUTH_test') stub_info = {'swift': {'fake': True}} - info_response = StubResponse(body=json.dumps(stub_info)) + info_response = StubResponse(body=b'{"swift":{"fake":true}}') fake_conn = self.fake_http_connection(info_response) os_options = {'project_id': 'test'} @@ -982,7 +1048,7 @@ def test_conn_get_capabilities_with_url_param(self): stub_info = {'swift': {'fake': True}} - info_response = StubResponse(body=json.dumps(stub_info)) + info_response = StubResponse(body=b'{"swift":{"fake":true}}') fake_conn = self.fake_http_connection(info_response) conn = c.Connection('http://auth.example.com/auth/v1.0', @@ -998,7 +1064,7 @@ def test_conn_get_capabilities_with_preauthurl_param(self): stub_info = {'swift': {'fake': True}} - info_response = StubResponse(body=json.dumps(stub_info)) + info_response = StubResponse(body=b'{"swift":{"fake":true}}') fake_conn = self.fake_http_connection(info_response) storage_url = 'http://storage.example.com/v1/AUTH_test' @@ -1014,7 +1080,7 @@ def test_conn_get_capabilities_with_os_options(self): stub_info = {'swift': {'fake': True}} - info_response = StubResponse(body=json.dumps(stub_info)) + info_response = StubResponse(body=b'{"swift":{"fake":true}}') fake_conn = self.fake_http_connection(info_response) storage_url = 'http://storage.example.com/v1/AUTH_test' @@ -1036,6 +1102,29 @@ class TestHTTPConnection(MockHttpTest): + def test_bad_url_scheme(self): + url = u'www.test.com' + exc = self.assertRaises(c.ClientException, c.http_connection, url) + expected = u'Unsupported scheme "" in url "www.test.com"' + self.assertEqual(expected, str(exc)) + + url = u'://www.test.com' + exc = self.assertRaises(c.ClientException, c.http_connection, url) + expected = u'Unsupported scheme "" in url "://www.test.com"' + self.assertEqual(expected, str(exc)) + + url = u'blah://www.test.com' + exc = self.assertRaises(c.ClientException, c.http_connection, url) + expected = u'Unsupported scheme "blah" in url "blah://www.test.com"' + self.assertEqual(expected, str(exc)) + + def test_ok_url_scheme(self): + for scheme in ('http', 'https', 'HTTP', 'HTTPS'): + url = u'%s://www.test.com' % scheme + parsed_url, conn = c.http_connection(url) + self.assertEqual(scheme.lower(), parsed_url.scheme) + self.assertEqual(u'%s://www.test.com' % scheme, conn.url) + def test_ok_proxy(self): conn = c.http_connection(u'http://www.test.com/', proxy='http://localhost:8080') @@ -1057,6 +1146,17 @@ conn = c.http_connection(u'http://www.test.com/', insecure=True) self.assertEqual(conn[1].requests_args['verify'], False) + def test_response_connection_released(self): + _parsed_url, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse() + conn.resp.raw = mock.Mock() + conn.resp.raw.read.side_effect = ["Chunk", ""] + resp = conn.getresponse() + self.assertFalse(resp.closed) + self.assertEqual("Chunk", resp.read()) + self.assertFalse(resp.read()) + self.assertTrue(resp.closed) + class TestConnection(MockHttpTest): @@ -1131,7 +1231,7 @@ for method, args in method_signatures: c.http_connection = self.fake_http_connection( - 200, body='[]', storage_url=static_url) + 200, body=b'[]', storage_url=static_url) method(*args) self.assertEqual(len(self.request_log), 1) for request in self.iter_request_log(): @@ -1233,7 +1333,7 @@ # represenative of the unit under test. The real get_auth # method will always return the os_option dict's # object_storage_url which will be overridden by the - # preauthurl paramater to Connection if it is provided. + # preauthurl parameter to Connection if it is provided. return 'http://www.new.com', 'new' def swap_sleep(*args): @@ -1381,6 +1481,104 @@ ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), ]) + def test_preauth_url_trumps_os_preauth_url(self): + storage_url = 'http://storage.example.com/v1/AUTH_pre_url' + os_storage_url = 'http://storage.example.com/v1/AUTH_os_pre_url' + os_preauth_options = { + 'tenant_name': 'demo', + 'object_storage_url': os_storage_url, + } + orig_os_preauth_options = dict(os_preauth_options) + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2, + preauthurl=storage_url, tenant_name='not_demo') + fake_keystone = fake_get_auth_keystone( + storage_url='http://storage.example.com/v1/AUTH_post_url', + token='post_token') + fake_conn = self.fake_http_connection(200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), + ]) + + # check that Connection has not modified our os_options + self.assertEqual(orig_os_preauth_options, os_preauth_options) + + def test_get_auth_sets_url_and_token(self): + with mock.patch('swiftclient.client.get_auth') as mock_get_auth: + mock_get_auth.return_value = ( + "https://storage.url/v1/AUTH_storage_acct", "AUTH_token" + ) + conn = c.Connection("https://auth.url/auth/v2.0", + "user", "passkey", tenant_name="tenant") + conn.get_auth() + self.assertEqual("https://storage.url/v1/AUTH_storage_acct", conn.url) + self.assertEqual("AUTH_token", conn.token) + + def test_timeout_passed_down(self): + # We want to avoid mocking http_connection(), and most especially + # avoid passing it down in argument. However, we cannot simply + # instantiate C=Connection(), then shim C.http_conn. Doing so would + # avoid some of the code under test (where _retry() invokes + # http_connection()), and would miss get_auth() completely. + # So, with regret, we do mock http_connection(), but with a very + # light shim that swaps out _request() as originally intended. + + orig_http_connection = c.http_connection + + timeouts = [] + + def my_request_handler(*a, **kw): + if 'timeout' in kw: + timeouts.append(kw['timeout']) + else: + timeouts.append(None) + return MockHttpResponse( + status=200, + headers={ + 'x-auth-token': 'a_token', + 'x-storage-url': 'http://files.example.com/v1/AUTH_user'}) + + def shim_connection(*a, **kw): + url, conn = orig_http_connection(*a, **kw) + conn._request = my_request_handler + return url, conn + + # v1 auth + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', timeout=33.0) + with mock.patch.multiple('swiftclient.client', + http_connection=shim_connection, + sleep=mock.DEFAULT): + conn.head_account() + + # 1 call is through get_auth, 1 call is HEAD for account + self.assertEqual(timeouts, [33.0, 33.0]) + + # v2 auth + timeouts = [] + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', timeout=33.0, + os_options=dict(tenant_name='tenant'), auth_version=2.0) + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + with mock.patch.multiple('swiftclient.client', + http_connection=shim_connection, + sleep=mock.DEFAULT): + conn.head_account() + + # check timeout is passed to keystone client + self.assertEqual(1, len(fake_ks.calls)) + self.assertTrue('timeout' in fake_ks.calls[0]) + self.assertEqual(33.0, fake_ks.calls[0].get('timeout')) + # check timeout passed to HEAD for account + self.assertEqual(timeouts, [33.0]) + def test_reset_stream(self): class LocalContents(object): @@ -1441,7 +1639,8 @@ return '' def local_http_connection(url, proxy=None, cacert=None, - insecure=False, ssl_compression=True): + insecure=False, ssl_compression=True, + timeout=None): parsed = urlparse(url) return parsed, LocalConnection() @@ -1634,3 +1833,308 @@ self.assertIsInstance(http_conn_obj, c.HTTPConnection) self.assertFalse(hasattr(http_conn_obj, 'close')) conn.close() + + +class TestServiceToken(MockHttpTest): + + def setUp(self): + super(TestServiceToken, self).setUp() + self.os_options = { + 'object_storage_url': 'http://storage_url.com', + 'service_username': 'service_username', + 'service_project_name': 'service_project_name', + 'service_key': 'service_key'} + + def get_connection(self): + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + os_options=self.os_options) + + self.assertTrue(isinstance(conn, c.Connection)) + conn.get_auth = self.get_auth + conn.get_service_auth = self.get_service_auth + + self.assertEqual(conn.attempts, 0) + self.assertEqual(conn.service_token, None) + + self.assertTrue(isinstance(conn, c.Connection)) + return conn + + def get_auth(self): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl paramater to Connection if it is provided. + return self.os_options.get('object_storage_url'), 'token' + + def get_service_auth(self): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + return self.os_options.get('object_storage_url'), 'stoken' + + def test_service_token_reauth(self): + get_auth_call_list = [] + + def get_auth(url, user, key, **kwargs): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + args = {'url': url, 'user': user, 'key': key, 'kwargs': kwargs} + get_auth_call_list.append(args) + return_dict = {'asdf': 'new', 'service_username': 'newserv'} + storage_url = kwargs['os_options'].get('object_storage_url') + return storage_url, return_dict[user] + + def swap_sleep(*args): + self.swap_sleep_called = True + c.get_auth = get_auth + + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(401, 200)): + with mock.patch('swiftclient.client.sleep', swap_sleep): + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + os_options=self.os_options) + + self.assertEqual(conn.attempts, 0) + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'old') + + conn.head_account() + + self.assertTrue(self.swap_sleep_called) + self.assertEqual(conn.attempts, 2) + # The original 'preauth' storage URL *must* be preserved + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'new') + self.assertEqual(conn.service_token, 'newserv') + + # Check get_auth was called with expected args + auth_args = get_auth_call_list[0] + auth_kwargs = get_auth_call_list[0]['kwargs'] + self.assertEqual('asdf', auth_args['user']) + self.assertEqual('asdf', auth_args['key']) + self.assertEqual('service_key', + auth_kwargs['os_options']['service_key']) + self.assertEqual('service_username', + auth_kwargs['os_options']['service_username']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['service_project_name']) + + auth_args = get_auth_call_list[1] + auth_kwargs = get_auth_call_list[1]['kwargs'] + self.assertEqual('service_username', auth_args['user']) + self.assertEqual('service_key', auth_args['key']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['tenant_name']) + + def test_service_token_get_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + with mock.patch('swiftclient.client.parse_api_response'): + conn = self.get_connection() + conn.get_account() + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/?format=json', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_account() + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com', actual['full_path']) + + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(201)): + conn = self.get_connection() + conn.post_account(headers={}) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com', actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_delete_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(204)): + conn = self.get_connection() + conn.delete_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('DELETE', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_get_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + with mock.patch('swiftclient.client.parse_api_response'): + conn = self.get_connection() + conn.get_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1?format=json', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(201)): + conn = self.get_connection() + conn.post_container('container1', {}) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_put_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.put_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('PUT', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_get_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.get_object('container1', 'obj1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_object('container1', 'obj1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_put_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.put_object('container1', 'obj1', 'a_string') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('PUT', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(202)): + conn = self.get_connection() + conn.post_object('container1', 'obj1', {}) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_delete_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(202)): + conn = self.get_connection() + conn.delete_object('container1', 'obj1', 'a_string') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('DELETE', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1?a_string', + actual['full_path']) + self.assertEqual(conn.attempts, 1) diff -Nru python-swiftclient-2.4.0/tests/unit/test_utils.py python-swiftclient-2.6.0/tests/unit/test_utils.py --- python-swiftclient-2.4.0/tests/unit/test_utils.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/tests/unit/test_utils.py 2015-09-07 15:20:14.000000000 +0000 @@ -132,11 +132,9 @@ self.key = 'correcthorsebatterystaple' self.method = 'GET' - @mock.patch('hmac.HMAC.hexdigest') - @mock.patch('time.time') + @mock.patch('hmac.HMAC.hexdigest', return_value='temp_url_signature') + @mock.patch('time.time', return_value=1400000000) def test_generate_temp_url(self, time_mock, hmac_mock): - time_mock.return_value = 1400000000 - hmac_mock.return_value = 'temp_url_signature' expected_url = ( '/v1/AUTH_account/c/o?' 'temp_url_sig=temp_url_signature&' @@ -145,6 +143,15 @@ self.method) self.assertEqual(url, expected_url) + @mock.patch('hmac.HMAC.hexdigest', return_value="temp_url_signature") + def test_generate_absolute_expiry_temp_url(self, hmac_mock): + expected_url = ('/v1/AUTH_account/c/o?' + 'temp_url_sig=temp_url_signature&' + 'temp_url_expires=2146636800') + url = u.generate_temp_url(self.url, 2146636800, self.key, self.method, + absolute=True) + self.assertEqual(url, expected_url) + def test_generate_temp_url_bad_seconds(self): self.assertRaises(TypeError, u.generate_temp_url, @@ -176,19 +183,19 @@ data = u.ReadableToIterable(f, chunk_size, True) for i, data_chunk in enumerate(data): - self.assertEquals(chunk_size, len(data_chunk)) - self.assertEquals(data_chunk, write_data[i] * chunk_size) + self.assertEqual(chunk_size, len(data_chunk)) + self.assertEqual(data_chunk, write_data[i] * chunk_size) - self.assertEquals(actual_md5sum.hexdigest(), data.get_md5sum()) + self.assertEqual(actual_md5sum.hexdigest(), data.get_md5sum()) def test_md5_creation(self): # Check creation with a real and noop md5 class data = u.ReadableToIterable(None, None, md5=True) - self.assertEquals(md5().hexdigest(), data.get_md5sum()) + self.assertEqual(md5().hexdigest(), data.get_md5sum()) self.assertTrue(isinstance(data.md5sum, type(md5()))) data = u.ReadableToIterable(None, None, md5=False) - self.assertEquals('', data.get_md5sum()) + self.assertEqual('', data.get_md5sum()) self.assertTrue(isinstance(data.md5sum, type(u.NoopMD5()))) def test_unicode(self): @@ -203,14 +210,14 @@ data = u.ReadableToIterable(f, chunk_size, True) x = next(data) - self.assertEquals(2, len(x)) - self.assertEquals(unicode_data[:2], x) + self.assertEqual(2, len(x)) + self.assertEqual(unicode_data[:2], x) x = next(data) - self.assertEquals(1, len(x)) - self.assertEquals(unicode_data[2:], x) + self.assertEqual(1, len(x)) + self.assertEqual(unicode_data[2:], x) - self.assertEquals(actual_md5sum, data.get_md5sum()) + self.assertEqual(actual_md5sum, data.get_md5sum()) class TestLengthWrapper(testtools.TestCase): diff -Nru python-swiftclient-2.4.0/tests/unit/utils.py python-swiftclient-2.6.0/tests/unit/utils.py --- python-swiftclient-2.4.0/tests/unit/utils.py 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/tests/unit/utils.py 2015-09-07 15:20:14.000000000 +0000 @@ -38,8 +38,10 @@ if exc: raise exc('test') # TODO: some way to require auth_url, user and key? - if expected_os_options and actual_os_options != expected_os_options: - return "", None + if expected_os_options: + for key, value in actual_os_options.items(): + if value and value != expected_os_options.get(key): + return "", None if 'required_kwargs' in kwargs: for k, v in kwargs['required_kwargs'].items(): if v != actual_kwargs.get(k): @@ -155,7 +157,10 @@ sleep(0.1) return ' ' rv = self.body[:amt] - self.body = self.body[amt:] + if amt is not None: + self.body = self.body[amt:] + else: + self.body = '' return rv def send(self, amt=None): @@ -208,6 +213,12 @@ self.fake_connect = None self.request_log = [] + # Capture output, since the test-runner stdout/stderr moneky-patching + # won't cover the references to sys.stdout/sys.stderr in + # swiftclient.multithreading + self.capture_output = CaptureOutput() + self.capture_output.__enter__() + def fake_http_connection(*args, **kwargs): self.validateMockedRequestsConsumed() self.request_log = [] @@ -220,7 +231,7 @@ on_request = kwargs.get('on_request') def wrapper(url, proxy=None, cacert=None, insecure=False, - ssl_compression=True): + ssl_compression=True, timeout=None): if storage_url: self.assertEqual(storage_url, url) @@ -368,32 +379,32 @@ # un-hygienic mocking on the swiftclient.client module; which may lead # to some unfortunate test order dependency bugs by way of the broken # window theory if any other modules are similarly patched + self.capture_output.__exit__() reload_module(c) -class CaptureStreamBuffer(object): +class CaptureStreamPrinter(object): """ - CaptureStreamBuffer is used for testing raw byte writing for PY3. Anything - written here is decoded as utf-8 and written to the parent CaptureStream + CaptureStreamPrinter is used for testing unicode writing for PY3. Anything + written here is encoded as utf-8 and written to the parent CaptureStream """ def __init__(self, captured_stream): self._captured_stream = captured_stream - def write(self, bytes_data): + def write(self, data): # No encoding, just convert the raw bytes into a str for testing # The below call also validates that we have a byte string. self._captured_stream.write( - ''.join(map(chr, bytes_data)) - ) + data if isinstance(data, six.binary_type) else data.encode('utf8')) class CaptureStream(object): def __init__(self, stream): self.stream = stream - self._capture = six.StringIO() - self._buffer = CaptureStreamBuffer(self) - self.streams = [self.stream, self._capture] + self._buffer = six.BytesIO() + self._capture = CaptureStreamPrinter(self._buffer) + self.streams = [self._capture] @property def buffer(self): @@ -415,11 +426,11 @@ stream.writelines(*args, **kwargs) def getvalue(self): - return self._capture.getvalue() + return self._buffer.getvalue() def clear(self): - self._capture.truncate(0) - self._capture.seek(0) + self._buffer.truncate(0) + self._buffer.seek(0) class CaptureOutput(object): @@ -457,11 +468,11 @@ @property def out(self): - return self._out.getvalue() + return self._out.getvalue().decode('utf8') @property def err(self): - return self._err.getvalue() + return self._err.getvalue().decode('utf8') def clear(self): self._out.clear() @@ -480,3 +491,52 @@ def __getattr__(self, name): return getattr(self.out, name) + + +class FakeKeystone(object): + ''' + Fake keystone client module. Returns given endpoint url and auth token. + ''' + def __init__(self, endpoint, token): + self.calls = [] + self.auth_version = None + self.endpoint = endpoint + self.token = token + + class _Client(object): + def __init__(self, endpoint, token, **kwargs): + self.auth_token = token + self.endpoint = endpoint + self.service_catalog = self.ServiceCatalog(endpoint) + + class ServiceCatalog(object): + def __init__(self, endpoint): + self.calls = [] + self.endpoint_url = endpoint + + def url_for(self, **kwargs): + self.calls.append(kwargs) + return self.endpoint_url + + def Client(self, **kwargs): + self.calls.append(kwargs) + self.client = self._Client(endpoint=self.endpoint, token=self.token, + **kwargs) + return self.client + + class Unauthorized(Exception): + pass + + class AuthorizationFailure(Exception): + pass + + class EndpointNotFound(Exception): + pass + + +def _make_fake_import_keystone_client(fake_import): + def _fake_import_keystone_client(auth_version): + fake_import.auth_version = auth_version + return fake_import, fake_import + + return _fake_import_keystone_client diff -Nru python-swiftclient-2.4.0/tox.ini python-swiftclient-2.6.0/tox.ini --- python-swiftclient-2.4.0/tox.ini 2015-03-05 23:37:18.000000000 +0000 +++ python-swiftclient-2.6.0/tox.ini 2015-09-07 15:20:14.000000000 +0000 @@ -11,6 +11,7 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = python setup.py testr --testr-args="{posargs}" +passenv = SWIFT_* *_proxy [testenv:pep8] commands = @@ -40,13 +41,14 @@ python setup.py build_sphinx [flake8] -# it's not a bug that we aren't using all of hacking -# H102 -> apache2 license exists -# H103 -> license is apache -# H201 -> no bare excepts -# H501 -> don't use locals() for str formatting -# H903 -> \n not \r\n -ignore = H -select = H102, H103, H201, H501, H903 +# it's not a bug that we aren't using all of hacking, ignore: +# H101: Use TODO(NAME) +# H301: one import per line +# H306: imports not in alphabetical order (time, os) +# H401: docstring should not start with a space +# H403: multi line docstrings should end on a new line +# H404: multi line docstring should start without a leading new line +# H405: multi line docstring summary not separated with an empty line +ignore = H101,H301,H306,H401,H403,H404,H405 show-source = True -exclude = .venv,.tox,dist,doc,test,*egg +exclude = .venv,.tox,dist,doc,*egg