diff -Nru python-swiftclient-3.8.1/AUTHORS python-swiftclient-3.9.0/AUTHORS --- python-swiftclient-3.8.1/AUTHORS 2019-09-13 21:39:58.000000000 +0000 +++ python-swiftclient-3.9.0/AUTHORS 2020-02-13 17:31:20.000000000 +0000 @@ -1,5 +1,6 @@ Alessandro Pilotti (ap@pilotti.it) Alex Gaynor (alex.gaynor@gmail.com) +Alex Schultz (aschultz@redhat.com) Alexandra Settle (alexandra.settle@rackspace.com) Alexis Lee (lxsli@hpe.com) Alistair Coles (alistairncoles@gmail.com) @@ -70,6 +71,7 @@ Juan J. Martinez (juan@memset.com) Jude Job (judeopenstack@gmail.com) Julien Danjou (julien@danjou.info) +kangyufei (kangyf@inspur.com) Kazufumi Noto (noto.kazufumi@gmail.com) Kota Tsuyuzaki (tsuyuzaki.kota@lab.ntt.co.jp) Kun Huang (gareth@unitedstack.com) @@ -117,6 +119,7 @@ Sam Morrison (sorrison@gmail.com) Samuel Merritt (sam@swiftstack.com) Sean Dague (sean@dague.net) +Sébastien Blaisot (sebastien@blaisot.org) Sergey Gotliv (sgotliv@redhat.com) Sergio Cazzolato (sergio.j.cazzolato@intel.com) Shane Wang (shane.wang@intel.com) diff -Nru python-swiftclient-3.8.1/ChangeLog python-swiftclient-3.9.0/ChangeLog --- python-swiftclient-3.8.1/ChangeLog 2019-09-13 21:39:58.000000000 +0000 +++ python-swiftclient-3.9.0/ChangeLog 2020-02-13 17:31:20.000000000 +0000 @@ -1,3 +1,14 @@ +3.9.0 +----- + +* Now tested under Python 3.8. + +* Better clean up connections when using the low-level client.py API. + +* Fixed a display issue when `swift delete` made multiple attempts to bulk + delete objects. + + 3.8.1 ----- diff -Nru python-swiftclient-3.8.1/debian/changelog python-swiftclient-3.9.0/debian/changelog --- python-swiftclient-3.8.1/debian/changelog 2019-09-26 09:38:22.000000000 +0000 +++ python-swiftclient-3.9.0/debian/changelog 2020-03-11 20:36:02.000000000 +0000 @@ -1,3 +1,11 @@ +python-swiftclient (1:3.9.0-0ubuntu1) focal; urgency=medium + + * New upstream release for OpenStack Ussuri. + * d/control: Align (Build-)Depends with upstream. + * d/rules: Switch to pybuild. + + -- Corey Bryant Wed, 11 Mar 2020 16:36:02 -0400 + python-swiftclient (1:3.8.1-0ubuntu1) eoan; urgency=medium * New upstream release for OpenStack Train. diff -Nru python-swiftclient-3.8.1/debian/control python-swiftclient-3.9.0/debian/control --- python-swiftclient-3.8.1/debian/control 2019-09-26 09:38:22.000000000 +0000 +++ python-swiftclient-3.9.0/debian/control 2020-03-11 20:36:02.000000000 +0000 @@ -11,9 +11,6 @@ debhelper (>= 11.1.4~), dh-python, openstack-pkg-tools, - python-all, - python-pbr, - python-setuptools, python3-all, python3-pbr, python3-setuptools, @@ -25,6 +22,7 @@ python3-keystoneclient, python3-mock (>= 1.2.0), python3-openstackdocstheme (>= 1.20.0), + python3-openstacksdk (>= 0.11.0), python3-oslosphinx (>= 4.7.0), python3-reno (>= 2.5.0), python3-requests (>= 1.1.0), diff -Nru python-swiftclient-3.8.1/debian/rules python-swiftclient-3.9.0/debian/rules --- python-swiftclient-3.8.1/debian/rules 2019-09-26 09:38:22.000000000 +0000 +++ python-swiftclient-3.9.0/debian/rules 2020-03-11 20:36:02.000000000 +0000 @@ -1,11 +1,13 @@ #!/usr/bin/make -f +export PYBUILD_NAME=python-swiftclient + PYTHON3S:=$(shell py3versions -vr) include /usr/share/openstack-pkg-tools/pkgos.make %: - dh $@ --with python3,sphinxdoc + dh $@ --with python3,sphinxdoc --buildsystem=pybuild override_dh_install: set -e ; for pyvers in $(PYTHON3S); do \ @@ -32,5 +34,5 @@ override_dh_sphinxdoc: ifeq (,$(findstring nodocs, $(DEB_BUILD_OPTIONS))) sphinx-build -b html doc/source $(CURDIR)/debian/python-swiftclient-doc/usr/share/doc/python-swiftclient/html - dh_sphinxdoc -O--buildsystem=python_distutils + dh_sphinxdoc endif diff -Nru python-swiftclient-3.8.1/.functests python-swiftclient-3.9.0/.functests --- python-swiftclient-3.8.1/.functests 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/.functests 2020-02-13 17:31:20.000000000 +0000 @@ -1,7 +1,7 @@ #!/bin/bash set -e -export OS_TEST_PATH='tests.functional' +export OS_TEST_PATH='test.functional' export PYTHON='coverage run --source swiftclient --parallel-mode' stestr run --concurrency=1 diff -Nru python-swiftclient-3.8.1/lower-constraints.txt python-swiftclient-3.9.0/lower-constraints.txt --- python-swiftclient-3.8.1/lower-constraints.txt 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/lower-constraints.txt 2020-02-13 17:31:20.000000000 +0000 @@ -21,6 +21,7 @@ mock==1.2.0 netaddr==0.7.10 openstackdocstheme==1.20.0 +openstacksdk==0.11.0 oslo.config==1.2.0 pbr==2.0.0 pep8==1.5.7 diff -Nru python-swiftclient-3.8.1/PKG-INFO python-swiftclient-3.9.0/PKG-INFO --- python-swiftclient-3.8.1/PKG-INFO 2019-09-13 21:40:49.000000000 +0000 +++ python-swiftclient-3.9.0/PKG-INFO 2020-02-13 17:32:24.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: python-swiftclient -Version: 3.8.1 +Version: 3.9.0 Summary: OpenStack Object Storage API Client Library Home-page: https://docs.openstack.org/python-swiftclient/latest/ Author: OpenStack @@ -74,5 +74,5 @@ Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 -Provides-Extra: test Provides-Extra: keystone +Provides-Extra: test diff -Nru python-swiftclient-3.8.1/python_swiftclient.egg-info/pbr.json python-swiftclient-3.9.0/python_swiftclient.egg-info/pbr.json --- python-swiftclient-3.8.1/python_swiftclient.egg-info/pbr.json 2019-09-13 21:40:49.000000000 +0000 +++ python-swiftclient-3.9.0/python_swiftclient.egg-info/pbr.json 2020-02-13 17:32:24.000000000 +0000 @@ -1 +1 @@ -{"git_version": "72b90fe", "is_release": true} \ No newline at end of file +{"git_version": "259b98f", "is_release": true} \ No newline at end of file diff -Nru python-swiftclient-3.8.1/python_swiftclient.egg-info/PKG-INFO python-swiftclient-3.9.0/python_swiftclient.egg-info/PKG-INFO --- python-swiftclient-3.8.1/python_swiftclient.egg-info/PKG-INFO 2019-09-13 21:40:49.000000000 +0000 +++ python-swiftclient-3.9.0/python_swiftclient.egg-info/PKG-INFO 2020-02-13 17:32:24.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: python-swiftclient -Version: 3.8.1 +Version: 3.9.0 Summary: OpenStack Object Storage API Client Library Home-page: https://docs.openstack.org/python-swiftclient/latest/ Author: OpenStack @@ -74,5 +74,5 @@ Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 -Provides-Extra: test Provides-Extra: keystone +Provides-Extra: test diff -Nru python-swiftclient-3.8.1/python_swiftclient.egg-info/requires.txt python-swiftclient-3.9.0/python_swiftclient.egg-info/requires.txt --- python-swiftclient-3.8.1/python_swiftclient.egg-info/requires.txt 2019-09-13 21:40:49.000000000 +0000 +++ python-swiftclient-3.9.0/python_swiftclient.egg-info/requires.txt 2020-02-13 17:32:24.000000000 +0000 @@ -13,3 +13,4 @@ keystoneauth1>=3.4.0 mock>=1.2.0 stestr>=2.0.0 +openstacksdk>=0.11.0 diff -Nru python-swiftclient-3.8.1/python_swiftclient.egg-info/SOURCES.txt python-swiftclient-3.9.0/python_swiftclient.egg-info/SOURCES.txt --- python-swiftclient-3.8.1/python_swiftclient.egg-info/SOURCES.txt 2019-09-13 21:40:49.000000000 +0000 +++ python-swiftclient-3.9.0/python_swiftclient.egg-info/SOURCES.txt 2020-02-13 17:32:24.000000000 +0000 @@ -56,6 +56,7 @@ releasenotes/notes/361_notes-59e020e68bcdd709.yaml releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml releasenotes/notes/3_8_1_release-cb5648c3ae69bde1.yaml +releasenotes/notes/3_9_0_release-3c293d277f14ec22.yaml releasenotes/source/conf.py releasenotes/source/current.rst releasenotes/source/index.rst @@ -65,6 +66,7 @@ releasenotes/source/queens.rst releasenotes/source/rocky.rst releasenotes/source/stein.rst +releasenotes/source/train.rst swiftclient/__init__.py swiftclient/authv1.py swiftclient/client.py @@ -75,17 +77,18 @@ swiftclient/shell.py swiftclient/utils.py swiftclient/version.py -tests/__init__.py -tests/sample.conf -tests/functional/__init__.py -tests/functional/test_swiftclient.py -tests/unit/__init__.py -tests/unit/test_authv1.py -tests/unit/test_command_helpers.py -tests/unit/test_multithreading.py -tests/unit/test_service.py -tests/unit/test_shell.py -tests/unit/test_swiftclient.py -tests/unit/test_utils.py -tests/unit/utils.py +test/__init__.py +test/sample.conf +test/functional/__init__.py +test/functional/test_openstacksdk.py +test/functional/test_swiftclient.py +test/unit/__init__.py +test/unit/test_authv1.py +test/unit/test_command_helpers.py +test/unit/test_multithreading.py +test/unit/test_service.py +test/unit/test_shell.py +test/unit/test_swiftclient.py +test/unit/test_utils.py +test/unit/utils.py tools/swift.bash_completion \ No newline at end of file diff -Nru python-swiftclient-3.8.1/releasenotes/notes/3_9_0_release-3c293d277f14ec22.yaml python-swiftclient-3.9.0/releasenotes/notes/3_9_0_release-3c293d277f14ec22.yaml --- python-swiftclient-3.8.1/releasenotes/notes/3_9_0_release-3c293d277f14ec22.yaml 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/releasenotes/notes/3_9_0_release-3c293d277f14ec22.yaml 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,12 @@ +--- +features: + - | + Now tested under Python 3.8. + +fixes: + - | + Better clean up connections when using the low-level ``client.py`` API. + + - | + Fixed a display issue when ``swift delete`` made multiple attempts to + bulk delete objects. diff -Nru python-swiftclient-3.8.1/releasenotes/source/index.rst python-swiftclient-3.9.0/releasenotes/source/index.rst --- python-swiftclient-3.8.1/releasenotes/source/index.rst 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/releasenotes/source/index.rst 2020-02-13 17:31:20.000000000 +0000 @@ -6,6 +6,7 @@ :maxdepth: 1 current + train stein rocky queens diff -Nru python-swiftclient-3.8.1/releasenotes/source/train.rst python-swiftclient-3.9.0/releasenotes/source/train.rst --- python-swiftclient-3.8.1/releasenotes/source/train.rst 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/releasenotes/source/train.rst 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,6 @@ +========================== +Train Series Release Notes +========================== + +.. release-notes:: + :branch: stable/train diff -Nru python-swiftclient-3.8.1/setup.cfg python-swiftclient-3.9.0/setup.cfg --- python-swiftclient-3.8.1/setup.cfg 2019-09-13 21:40:49.000000000 +0000 +++ python-swiftclient-3.9.0/setup.cfg 2020-02-13 17:32:24.000000000 +0000 @@ -50,7 +50,7 @@ [upload_sphinx] upload-dir = doc/build/html -[wheel] +[bdist_wheel] universal = 1 [pbr] diff -Nru python-swiftclient-3.8.1/.stestr.conf python-swiftclient-3.9.0/.stestr.conf --- python-swiftclient-3.8.1/.stestr.conf 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/.stestr.conf 2020-02-13 17:31:20.000000000 +0000 @@ -1,4 +1,4 @@ [DEFAULT] -test_path=${OS_TEST_PATH:-./tests/unit} +test_path=${OS_TEST_PATH:-./test/unit} top_dir=./ diff -Nru python-swiftclient-3.8.1/swiftclient/authv1.py python-swiftclient-3.9.0/swiftclient/authv1.py --- python-swiftclient-3.8.1/swiftclient/authv1.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/swiftclient/authv1.py 2020-02-13 17:31:20.000000000 +0000 @@ -45,6 +45,7 @@ # Note that while we import keystoneauth1 here, we *don't* need to add it to # requirements.txt -- this entire module only makes sense (and should only be # loaded) if keystoneauth is already installed. +from keystoneauth1 import discover from keystoneauth1 import plugin from keystoneauth1 import exceptions from keystoneauth1 import loading @@ -110,11 +111,20 @@ ] def url_for(self, **kwargs): + return self.endpoint_data_for(**kwargs).url + + def endpoint_data_for(self, **kwargs): kwargs.setdefault('interface', 'public') kwargs.setdefault('service_type', None) if kwargs['service_type'] == 'object-store': - return self.storage_url + return discover.EndpointData( + service_type='object-store', + service_name='swift', + interface=kwargs['interface'], + region_name='default', + catalog_url=self.storage_url, + ) # Although our "catalog" includes an identity entry, nothing that uses # url_for() (including `openstack endpoint list`) will know what to do diff -Nru python-swiftclient-3.8.1/swiftclient/client.py python-swiftclient-3.9.0/swiftclient/client.py --- python-swiftclient-3.8.1/swiftclient/client.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/swiftclient/client.py 2020-02-13 17:31:20.000000000 +0000 @@ -438,6 +438,15 @@ if timeout: self.requests_args['timeout'] = timeout + def __del__(self): + """Cleanup resources other than memory""" + if self.request_session: + # The session we create must be closed to free up file descriptors + try: + self.request_session.close() + finally: + self.request_session = None + def _request(self, *arg, **kwarg): """Final wrapper before requests call, to be patched in tests""" return self.request_session.request(*arg, **kwarg) diff -Nru python-swiftclient-3.8.1/swiftclient/shell.py python-swiftclient-3.9.0/swiftclient/shell.py --- python-swiftclient-3.8.1/swiftclient/shell.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/swiftclient/shell.py 2020-02-13 17:31:20.000000000 +0000 @@ -169,7 +169,8 @@ for r in del_iter: c = r.get('container', '') o = r.get('object', '') - a = r.get('attempts') + a = (' [after {0} attempts]'.format(r.get('attempts')) + if r.get('attempts', 1) > 1 else '') if r['action'] == 'bulk_delete': if r['success']: @@ -202,9 +203,6 @@ else: if r['success']: if options['verbose']: - a = (' [after {0} attempts]'.format(a) - if a > 1 else '') - if r['action'] == 'delete_object': if options['yes_all']: p = '{0}/{1}'.format(c, o) diff -Nru python-swiftclient-3.8.1/test/functional/__init__.py python-swiftclient-3.9.0/test/functional/__init__.py --- python-swiftclient-3.8.1/test/functional/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/functional/__init__.py 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,93 @@ +# Copyright (c) 2014 Christian Schwede +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from six.moves import configparser + +TEST_CONFIG = None + + +def _load_config(force_reload=False): + global TEST_CONFIG + if not force_reload and TEST_CONFIG is not None: + return TEST_CONFIG + + config_file = os.environ.get('SWIFT_TEST_CONFIG_FILE', + '/etc/swift/test.conf') + parser = configparser.ConfigParser({'auth_version': '1'}) + parser.read(config_file) + conf = {} + if parser.has_section('func_test'): + if parser.has_option('func_test', 'auth_uri'): + conf['auth_url'] = parser.get('func_test', 'auth_uri') + try: + conf['auth_version'] = parser.get('func_test', 'auth_version') + except configparser.NoOptionError: + last_piece = conf['auth_url'].rstrip('/').rsplit('/', 1)[1] + if last_piece.endswith('.0'): + last_piece = last_piece[:-2] + if last_piece in ('1', '2', '3'): + conf['auth_version'] = last_piece + else: + raise + else: + auth_host = parser.get('func_test', 'auth_host') + auth_port = parser.getint('func_test', 'auth_port') + auth_ssl = parser.getboolean('func_test', 'auth_ssl') + auth_prefix = parser.get('func_test', 'auth_prefix') + conf['auth_version'] = parser.get('func_test', 'auth_version') + if auth_ssl: + auth_url = "https://" + else: + auth_url = "http://" + auth_url += "%s:%s%s" % (auth_host, auth_port, auth_prefix) + if conf['auth_version'] == "1": + auth_url += 'v1.0' + conf['auth_url'] = auth_url + + try: + conf['account_username'] = parser.get('func_test', + 'account_username') + except configparser.NoOptionError: + conf['account'] = parser.get('func_test', 'account') + conf['username'] = parser.get('func_test', 'username') + conf['account_username'] = "%s:%s" % (conf['account'], + conf['username']) + else: + # Still try to get separate account/usernames for keystone tests + try: + conf['account'] = parser.get('func_test', 'account') + conf['username'] = parser.get('func_test', 'username') + except configparser.NoOptionError: + pass + + conf['password'] = parser.get('func_test', 'password') + + # For keystone v3 + try: + conf['account4'] = parser.get('func_test', 'account4') + conf['username4'] = parser.get('func_test', 'username4') + conf['domain4'] = parser.get('func_test', 'domain4') + conf['password4'] = parser.get('func_test', 'password4') + except configparser.NoOptionError: + pass + + TEST_CONFIG = conf + + +try: + _load_config() +except configparser.NoOptionError: + TEST_CONFIG = None # sentinel used in test setup diff -Nru python-swiftclient-3.8.1/test/functional/test_openstacksdk.py python-swiftclient-3.9.0/test/functional/test_openstacksdk.py --- python-swiftclient-3.8.1/test/functional/test_openstacksdk.py 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/functional/test_openstacksdk.py 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,92 @@ +# Copyright (c) 2019 Tim Burke +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import uuid + +import openstack + +from . import TEST_CONFIG + +PREFIX = 'test-swiftclient-' + + +class TestOpenStackSDK(unittest.TestCase): + @classmethod + def setUpClass(cls): + # NB: Only runs for v1 auth, to exercise our keystoneauth plugin + cls.skip_tests = (TEST_CONFIG is None or + TEST_CONFIG['auth_version'] != '1') + if not cls.skip_tests: + cls.conn = openstack.connect( + auth_type='v1password', + auth_url=TEST_CONFIG['auth_url'], + username=TEST_CONFIG['account_username'], + password=TEST_CONFIG['password'], + ) + cls.object_store = cls.conn.object_store + + def setUp(self): + if self.skip_tests: + raise unittest.SkipTest('SKIPPING V1-AUTH TESTS') + + def tearDown(self): + if self.skip_tests: + return + for c in self.object_store.containers(): + if c.name.startswith(PREFIX): + for o in self.object_store.objects(c.name): + self.object_store.delete_object( + o.name, container=c.name) + self.object_store.delete_container(c.name) + + def test_containers(self): + meta = self.object_store.get_account_metadata() + count_before = meta.account_container_count + containers = sorted(PREFIX + str(uuid.uuid4()) + for _ in range(10)) + for c in containers: + self.object_store.create_container(c) + self.assertEqual([ + c.name for c in self.object_store.containers() + if c.name.startswith(PREFIX) + ], containers) + meta = self.object_store.get_account_metadata() + self.assertEqual(count_before + len(containers), + meta.account_container_count) + + def test_objects(self): + container = PREFIX + str(uuid.uuid4()) + self.object_store.create_container(container) + objects = sorted(str(uuid.uuid4()) for _ in range(10)) + for o in objects: + self.object_store.create_object(container, o, data=b'x') + self.assertEqual([ + o.name for o in self.object_store.objects(container) + ], objects) + meta = self.object_store.get_container_metadata(container) + self.assertEqual(len(objects), meta.object_count) + + def test_object_metadata(self): + container = PREFIX + str(uuid.uuid4()) + self.object_store.create_container(container) + obj = str(uuid.uuid4()) + obj_meta = {str(uuid.uuid4()): str(uuid.uuid4()) for _ in range(10)} + # NB: as of 0.36.0, create_object() doesn't play well with passing + # both data and metadata, so we do a PUT then POST + self.object_store.create_object(container, obj, data=b'x') + self.object_store.set_object_metadata(obj, container, **obj_meta) + meta = self.object_store.get_object_metadata(obj, container) + self.assertEqual(obj_meta, meta.metadata) diff -Nru python-swiftclient-3.8.1/test/functional/test_swiftclient.py python-swiftclient-3.9.0/test/functional/test_swiftclient.py --- python-swiftclient-3.8.1/test/functional/test_swiftclient.py 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/functional/test_swiftclient.py 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,518 @@ +# Copyright (c) 2014 Christian Schwede +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import time +from io import BytesIO + +import six + +import swiftclient +from . import TEST_CONFIG + + +class TestFunctional(unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(TestFunctional, self).__init__(*args, **kwargs) + self.skip_tests = (TEST_CONFIG is None) + if not self.skip_tests: + self._get_config() + + self.test_data = b'42' * 10 + self.etag = '2704306ec982238d85d4b235c925d58e' + + self.containername = "functional-tests-container-%s" % int(time.time()) + self.containername_2 = self.containername + '_second' + self.containername_3 = self.containername + '_third' + self.objectname = "functional-tests-object-%s" % int(time.time()) + self.objectname_2 = self.objectname + '_second' + + def _get_config(self): + self.auth_url = TEST_CONFIG['auth_url'] + self.auth_version = TEST_CONFIG['auth_version'] + self.account_username = TEST_CONFIG['account_username'] + self.password = TEST_CONFIG['password'] + + def _get_connection(self): + """ + Subclasses may override to use different connection setup + """ + return swiftclient.Connection( + self.auth_url, self.account_username, self.password, + auth_version=self.auth_version) + + def setUp(self): + super(TestFunctional, self).setUp() + if self.skip_tests: + self.skipTest('SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG') + + self.conn = self._get_connection() + self.conn.put_container(self.containername) + self.conn.put_container(self.containername_2) + self.conn.put_object( + self.containername, self.objectname, self.test_data) + self.conn.put_object( + self.containername, self.objectname_2, self.test_data) + + def tearDown(self): + super(TestFunctional, self).tearDown() + for obj in [self.objectname, self.objectname_2]: + try: + self.conn.delete_object(self.containername, obj) + except swiftclient.ClientException: + pass + + for container in [self.containername, + self.containername_2, + self.containername_3, + self.containername + '_segments']: + try: + self.conn.delete_container(container) + except swiftclient.ClientException: + pass + self.conn.close() + + def _check_account_headers(self, headers): + headers_to_check = [ + 'content-length', + 'x-account-object-count', + 'x-timestamp', + 'x-trans-id', + 'date', + 'x-account-bytes-used', + 'x-account-container-count', + 'content-type', + 'accept-ranges', + ] + for h in headers_to_check: + self.assertIn(h, headers) + self.assertTrue(headers[h]) + + def test_stat_account(self): + headers = self.conn.head_account() + self._check_account_headers(headers) + + def test_list_account(self): + headers, containers = self.conn.get_account() + self._check_account_headers(headers) + + self.assertTrue(len(containers)) + test_container = [c + for c in containers + if c.get('name') == self.containername][0] + self.assertTrue(test_container.get('bytes') >= 0) + self.assertTrue(test_container.get('count') >= 0) + + # Check if list limit is working + headers, containers = self.conn.get_account(limit=1) + self.assertEqual(1, len(containers)) + + # Check full listing + headers, containers = self.conn.get_account(limit=1, full_listing=True) + self.assertTrue(len(containers) >= 2) # there might be more containers + + # Test marker + headers, containers = self.conn.get_account(marker=self.containername) + self.assertTrue(len(containers) >= 1) + self.assertEqual(self.containername_2, containers[0].get('name')) + + # Test prefix + _, containers = self.conn.get_account(prefix='dne') + self.assertEqual(0, len(containers)) + + # Test delimiter + _, containers = self.conn.get_account( + prefix=self.containername, delimiter='_') + self.assertEqual(2, len(containers)) + self.assertEqual(self.containername, containers[0].get('name')) + self.assertTrue( + self.containername_2.startswith(containers[1].get('subdir'))) + + def _check_container_headers(self, headers): + self.assertTrue(headers.get('content-length')) + self.assertTrue(headers.get('x-container-object-count')) + self.assertTrue(headers.get('x-timestamp')) + self.assertTrue(headers.get('x-trans-id')) + self.assertTrue(headers.get('date')) + self.assertTrue(headers.get('x-container-bytes-used')) + self.assertTrue(headers.get('x-container-object-count')) + self.assertTrue(headers.get('content-type')) + self.assertTrue(headers.get('accept-ranges')) + + def test_stat_container(self): + headers = self.conn.head_container(self.containername) + self._check_container_headers(headers) + + def test_list_container(self): + headers, objects = self.conn.get_container(self.containername) + self._check_container_headers(headers) + self.assertTrue(len(objects)) + test_object = [o + for o in objects + if o.get('name') == self.objectname][0] + self.assertEqual(len(self.test_data), test_object.get('bytes')) + self.assertEqual(self.etag, test_object.get('hash')) + self.assertEqual('application/octet-stream', + test_object.get('content_type')) + + # Check if list limit is working + headers, objects = self.conn.get_container(self.containername, limit=1) + self.assertEqual(1, len(objects)) + + # Check full listing + headers, objects = self.conn.get_container( + self.containername, limit=1, full_listing=True) + self.assertEqual(2, len(objects)) + + # Test marker + headers, objects = self.conn.get_container( + self.containername, marker=self.objectname) + self.assertEqual(1, len(objects)) + self.assertEqual(self.objectname_2, objects[0].get('name')) + + def test_create_container(self): + self.conn.put_container(self.containername_3) + self.assertTrue(self.conn.head_container(self.containername_3)) + + def test_delete(self): + self.conn.delete_object(self.containername, self.objectname) + self.conn.delete_object(self.containername, self.objectname_2) + self.conn.delete_container(self.containername) + + # Container HEAD will raise an exception if container doesn't exist + # which is only possible if previous requests succeeded + self.assertRaises( + swiftclient.ClientException, + self.conn.head_container, + self.containername) + + def test_upload_object(self): + # Object with content from string + self.conn.put_object( + self.containername, self.objectname, contents=self.test_data) + hdrs = self.conn.head_object(self.containername, self.objectname) + self.assertEqual(str(len(self.test_data)), + hdrs.get('content-length')) + self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual('application/octet-stream', + hdrs.get('content-type')) + + # Same but with content_type + self.conn.put_object( + self.containername, self.objectname, + content_type='text/plain', contents=self.test_data) + hdrs = self.conn.head_object(self.containername, self.objectname) + self.assertEqual(str(len(self.test_data)), + hdrs.get('content-length')) + self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual('text/plain', + hdrs.get('content-type')) + + # Same but with content-type in headers + self.conn.put_object( + self.containername, self.objectname, + headers={'Content-Type': 'text/plain'}, contents=self.test_data) + hdrs = self.conn.head_object(self.containername, self.objectname) + self.assertEqual(str(len(self.test_data)), + hdrs.get('content-length')) + self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual('text/plain', + hdrs.get('content-type')) + + # content_type rewrites content-type in headers + self.conn.put_object( + self.containername, self.objectname, + content_type='image/jpeg', + headers={'Content-Type': 'text/plain'}, contents=self.test_data) + hdrs = self.conn.head_object(self.containername, self.objectname) + self.assertEqual(str(len(self.test_data)), + hdrs.get('content-length')) + self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual('image/jpeg', + hdrs.get('content-type')) + + # Same but with content-length + self.conn.put_object( + self.containername, self.objectname, + contents=self.test_data, content_length=len(self.test_data)) + hdrs = self.conn.head_object(self.containername, self.objectname) + self.assertEqual(str(len(self.test_data)), + hdrs.get('content-length')) + self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual('application/octet-stream', hdrs.get('content-type')) + + # Content from File-like object + fileobj = BytesIO(self.test_data) + self.conn.put_object( + self.containername, self.objectname, contents=fileobj) + hdrs = self.conn.head_object(self.containername, self.objectname) + self.assertEqual(str(len(self.test_data)), + hdrs.get('content-length')) + self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual('application/octet-stream', hdrs.get('content-type')) + + # Content from File-like object, but read in chunks + fileobj = BytesIO(self.test_data) + self.conn.put_object( + self.containername, self.objectname, + contents=fileobj, content_length=len(self.test_data), + chunk_size=10) + hdrs = self.conn.head_object(self.containername, self.objectname) + self.assertEqual(str(len(self.test_data)), + hdrs.get('content-length')) + self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual('application/octet-stream', hdrs.get('content-type')) + + # Wrong etag arg, should raise an exception + self.assertRaises( + swiftclient.ClientException, + self.conn.put_object, + self.containername, self.objectname, + contents=self.test_data, etag='invalid') + + def test_download_object(self): + # Download whole object + hdrs, body = self.conn.get_object(self.containername, self.objectname) + self.assertEqual(self.test_data, body) + + # Download in chunks, should return a generator + hdrs, body = self.conn.get_object( + self.containername, self.objectname, + resp_chunk_size=10) + downloaded_contents = b'' + while True: + try: + chunk = next(body) + except StopIteration: + break + downloaded_contents += chunk + self.assertEqual(self.test_data, downloaded_contents) + + # Download in chunks, should also work with read + hdrs, body = self.conn.get_object( + self.containername, self.objectname, + resp_chunk_size=10) + num_bytes = 5 + downloaded_contents = body.read(num_bytes) + self.assertEqual(num_bytes, len(downloaded_contents)) + downloaded_contents += body.read() + self.assertEqual(self.test_data, downloaded_contents) + + def test_put_object_using_generator(self): + # verify that put using a generator yielding empty strings does not + # cause connection to be closed + def data(): + yield b"should" + yield b"" + yield b" tolerate" + yield b"" + yield b" empty chunks" + + self.conn.put_object( + self.containername, self.objectname, data()) + hdrs, body = self.conn.get_object(self.containername, self.objectname) + self.assertEqual(b"should tolerate empty chunks", body) + + def test_download_object_retry_chunked(self): + resp_chunk_size = 2 + hdrs, body = self.conn.get_object(self.containername, + self.objectname, + resp_chunk_size=resp_chunk_size) + data = next(body) + self.assertEqual(self.test_data[:resp_chunk_size], data) + self.assertTrue(1, self.conn.attempts) + for chunk in body.resp: + # Flush remaining data from underlying response + # (simulate a dropped connection) + pass + # Trigger the retry + for chunk in body: + data += chunk + self.assertEqual(self.test_data, data) + self.assertEqual(2, self.conn.attempts) + + def test_download_object_retry_chunked_auth_failure(self): + resp_chunk_size = 2 + self.conn.token = 'invalid' + hdrs, body = self.conn.get_object(self.containername, + self.objectname, + resp_chunk_size=resp_chunk_size) + self.assertEqual(2, self.conn.attempts) + for chunk in body.resp: + # Flush remaining data from underlying response + # (simulate a dropped connection) + pass + + self.conn.token = 'invalid' + data = next(body) + self.assertEqual(4, self.conn.attempts) + + for chunk in body: + data += chunk + + self.assertEqual(self.test_data, data) + self.assertEqual(4, self.conn.attempts) + + def test_download_object_non_chunked(self): + hdrs, body = self.conn.get_object(self.containername, self.objectname) + data = body + self.assertEqual(self.test_data, data) + self.assertTrue(1, self.conn.attempts) + + hdrs, body = self.conn.get_object(self.containername, self.objectname, + resp_chunk_size=0) + data = body + self.assertEqual(self.test_data, data) + self.assertTrue(1, self.conn.attempts) + + def test_post_account(self): + self.conn.post_account({'x-account-meta-data': 'Something'}) + headers = self.conn.head_account() + self.assertEqual('Something', headers.get('x-account-meta-data')) + + def test_post_container(self): + self.conn.post_container( + self.containername, {'x-container-meta-color': 'Something'}) + + headers = self.conn.head_container(self.containername) + self.assertEqual('Something', headers.get('x-container-meta-color')) + + def test_post_object(self): + self.conn.post_object(self.containername, + self.objectname, + {'x-object-meta-color': 'Something', + 'x-object-meta-uni': b'\xd8\xaa'.decode('utf8'), + 'x-object-meta-int': 123, + 'x-object-meta-float': 45.67, + 'x-object-meta-bool': False}) + + headers = self.conn.head_object(self.containername, self.objectname) + self.assertEqual('Something', headers.get('x-object-meta-color')) + self.assertEqual(b'\xd8\xaa'.decode('utf-8'), + headers.get('x-object-meta-uni')) + self.assertEqual('123', headers.get('x-object-meta-int')) + self.assertEqual('45.67', headers.get('x-object-meta-float')) + self.assertEqual('False', headers.get('x-object-meta-bool')) + + def test_post_object_unicode_header_name(self): + self.conn.post_object(self.containername, + self.objectname, + {u'x-object-meta-\U0001f44d': u'\U0001f44d'}) + + # Note that we can't actually read this header back on py3; see + # https://bugs.python.org/issue37093 + # We'll have to settle for just testing that the POST doesn't blow up + # with a UnicodeDecodeError + if six.PY2: + headers = self.conn.head_object( + self.containername, self.objectname) + self.assertIn(u'x-object-meta-\U0001f44d', headers) + self.assertEqual(u'\U0001f44d', + headers.get(u'x-object-meta-\U0001f44d')) + + def test_copy_object(self): + self.conn.put_object( + self.containername, self.objectname, self.test_data) + self.conn.copy_object(self.containername, + self.objectname, + headers={'x-object-meta-color': 'Something'}) + + headers = self.conn.head_object(self.containername, self.objectname) + self.assertEqual('Something', headers.get('x-object-meta-color')) + + self.conn.copy_object(self.containername, + self.objectname, + headers={'x-object-meta-taste': 'Second'}) + + headers = self.conn.head_object(self.containername, self.objectname) + self.assertEqual('Something', headers.get('x-object-meta-color')) + self.assertEqual('Second', headers.get('x-object-meta-taste')) + + destination = "/%s/%s" % (self.containername, self.objectname_2) + self.conn.copy_object(self.containername, + self.objectname, + destination, + headers={'x-object-meta-taste': 'Second'}) + headers, data = self.conn.get_object(self.containername, + self.objectname_2) + self.assertEqual(self.test_data, data) + self.assertEqual('Something', headers.get('x-object-meta-color')) + self.assertEqual('Second', headers.get('x-object-meta-taste')) + + destination = "/%s/%s" % (self.containername, self.objectname_2) + self.conn.copy_object(self.containername, + self.objectname, + destination, + headers={'x-object-meta-color': 'Else'}, + fresh_metadata=True) + + headers = self.conn.head_object(self.containername, self.objectname_2) + self.assertEqual('Else', headers.get('x-object-meta-color')) + self.assertIsNone(headers.get('x-object-meta-taste')) + + def test_get_capabilities(self): + resp = self.conn.get_capabilities() + self.assertTrue(resp.get('swift')) + + +class TestUsingKeystone(TestFunctional): + """ + Repeat tests using os_options parameter to Connection. + """ + + def _get_connection(self): + account = username = None + if self.auth_version not in ('2', '3'): + self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS') + try: + account = TEST_CONFIG['account'] + username = TEST_CONFIG['username'] + except KeyError: + self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS' + + ' - NO CONFIG') + + return swiftclient.Connection( + self.auth_url, username, self.password, + auth_version=self.auth_version, + os_options={'tenant_name': account}) + + +class TestUsingKeystoneV3(TestFunctional): + """ + Repeat tests using a keystone user with domain specified. + """ + + def _get_connection(self): + account = username = password = project_domain = user_domain = None + if self.auth_version != '3': + self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS') + + try: + account = TEST_CONFIG['account4'] + username = TEST_CONFIG['username4'] + user_domain = TEST_CONFIG['domain4'] + project_domain = TEST_CONFIG['domain4'] + password = TEST_CONFIG['password4'] + except KeyError: + self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS' + + ' - NO CONFIG') + + os_options = {'project_name': account, + 'project_domain_name': project_domain, + 'user_domain_name': user_domain} + return swiftclient.Connection(self.auth_url, username, password, + auth_version=self.auth_version, + os_options=os_options) diff -Nru python-swiftclient-3.8.1/test/sample.conf python-swiftclient-3.9.0/test/sample.conf --- python-swiftclient-3.8.1/test/sample.conf 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/sample.conf 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,32 @@ +[func_test] +# sample config +auth_host = 127.0.0.1 +auth_port = 8080 +auth_ssl = no +auth_prefix = /auth/ +## sample config for Swift with Keystone v2 API +# For keystone v3 change auth_version to 3 and auth_prefix to /v3/ +#auth_version = 2 +#auth_host = localhost +#auth_port = 5000 +#auth_ssl = no +#auth_prefix = /v2.0/ + +# Primary functional test account (needs admin access to the account). +# By default the tests use a swiftclient.client.Connection instance with user +# attribute set to 'account:username' based on the options 'account' and +# 'username' specified below. This can be overridden for auth systems that +# expect a different form of user attribute by setting the option +# 'account_username'. +# account_username = test_tester +account = test +username = tester +password = testing + +# Another user is required for keystone v3 specific tests. +# Account must be in a non-default domain. +# (Suffix '4' is used to be consistent with swift functional test config). +#account4 = test4 +#username4 = tester4 +#password4 = testing4 +#domain4 = test-domain diff -Nru python-swiftclient-3.8.1/test/unit/test_authv1.py python-swiftclient-3.9.0/test/unit/test_authv1.py --- python-swiftclient-3.8.1/test/unit/test_authv1.py 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/unit/test_authv1.py 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,246 @@ +# Copyright 2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +import datetime +import json +import mock +import unittest +from keystoneauth1 import plugin +from keystoneauth1 import loading +from keystoneauth1 import exceptions +from swiftclient import authv1 + + +class TestDataNoAccount(object): + options = dict( + auth_url='http://saio:8080/auth/v1.0', + username='test:tester', + password='testing') + storage_url = 'http://saio:8080/v1/AUTH_test' + expected_endpoint = storage_url + token = 'token' + + +class TestDataWithAccount(object): + options = dict( + auth_url='http://saio:8080/auth/v1.0', + username='test2:tester2', + project_name='SOME_other_account', + password='testing2') + storage_url = 'http://saio:8080/v1/AUTH_test2' + expected_endpoint = 'http://saio:8080/v1/SOME_other_account' + token = 'other_token' + + +class TestPluginLoading(TestDataNoAccount, unittest.TestCase): + def test_can_load(self): + loader = loading.get_plugin_loader('v1password') + self.assertIsInstance(loader, authv1.PasswordLoader) + + auth_plugin = loader.load_from_options(**self.options) + self.assertIsInstance(auth_plugin, authv1.PasswordPlugin) + + self.assertEqual(self.options['auth_url'], auth_plugin.auth_url) + self.assertEqual(self.options['username'], auth_plugin.user) + self.assertEqual(self.options.get('project_name'), auth_plugin.account) + self.assertEqual(self.options['password'], auth_plugin.key) + + def test_get_state(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.assertIsNone(auth_plugin.get_auth_state()) + + with mock.patch('swiftclient.authv1.time.time', return_value=1234.56): + auth_plugin.auth_ref = authv1.AccessInfoV1( + self.options['auth_url'], + self.storage_url, + self.options.get('project_name'), + self.options['username'], + self.token, + 60) + + expected = json.dumps({ + 'auth_url': self.options['auth_url'], + 'username': self.options['username'], + 'account': self.options.get('project_name'), + 'issued': 1234.56, + 'storage_url': self.storage_url, + 'auth_token': self.token, + 'expires': 1234.56 + 60, + }, sort_keys=True) + self.assertEqual(expected, auth_plugin.auth_ref.get_state()) + self.assertEqual(expected, auth_plugin.get_auth_state()) + + def test_set_state(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.assertIsNone(auth_plugin.auth_ref) + + auth_plugin.auth_ref = object() + auth_plugin.set_auth_state(None) + self.assertIsNone(auth_plugin.get_auth_state()) + + state = json.dumps({ + 'auth_url': self.options['auth_url'], + 'username': self.options['username'], + 'account': self.options.get('project_name'), + 'issued': 1234.56, + 'storage_url': self.storage_url, + 'auth_token': self.token, + 'expires': None, + }, sort_keys=True) + auth_plugin.set_auth_state(state) + self.assertIsInstance(auth_plugin.auth_ref, authv1.AccessInfoV1) + + self.assertEqual(self.options['username'], + auth_plugin.auth_ref.username) + self.assertEqual(self.options['auth_url'], + auth_plugin.auth_ref.auth_url) + self.assertEqual(self.storage_url, auth_plugin.auth_ref.storage_url) + self.assertEqual(self.options.get('project_name'), auth_plugin.account) + self.assertEqual(self.token, auth_plugin.auth_ref.auth_token) + self.assertEqual(1234.56, auth_plugin.auth_ref._issued) + self.assertIs(datetime.datetime, type(auth_plugin.auth_ref.issued)) + self.assertIsNone(auth_plugin.auth_ref._expires) + self.assertIsNone(auth_plugin.auth_ref.expires) + + +class TestPluginLoadingWithAccount(TestDataWithAccount, TestPluginLoading): + pass + + +class TestPlugin(TestDataNoAccount, unittest.TestCase): + def setUp(self): + self.mock_session = mock.MagicMock() + self.mock_response = self.mock_session.get.return_value + self.mock_response.status_code = 200 + self.mock_response.headers = { + 'X-Auth-Token': self.token, + 'X-Storage-Url': self.storage_url, + } + + def test_get_access(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + with mock.patch('swiftclient.authv1.time.time', return_value=1234.56): + access = auth_plugin.get_access(self.mock_session) + + self.assertEqual(self.mock_session.get.mock_calls, [mock.call( + self.options['auth_url'], authenticated=False, log=False, headers={ + 'X-Auth-User': self.options['username'], + 'X-Auth-Key': self.options['password'], + })]) + + self.assertEqual(self.options['username'], access.username) + # `openstack token issue` requires a user_id property + self.assertEqual(self.options['username'], access.user_id) + self.assertEqual(self.storage_url, access.storage_url) + self.assertEqual(self.token, access.auth_token) + self.assertEqual(1234.56, access._issued) + self.assertIs(datetime.datetime, type(auth_plugin.auth_ref.issued)) + self.assertIsNone(access.expires) + + # `openstack catalog list/show` require a catalog property + catalog = access.service_catalog.catalog + self.assertEqual('swift', catalog[0].get('name')) + self.assertEqual('object-store', catalog[0].get('type')) + self.assertIn('endpoints', catalog[0]) + self.assertIn(self.storage_url, [ + e.get('publicURL') for e in catalog[0]['endpoints']]) + + def test_get_access_with_expiry(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.headers['X-Auth-Token-Expires'] = '78.9' + with mock.patch('swiftclient.authv1.time.time', + return_value=1234.56) as mock_time: + access = auth_plugin.get_access(self.mock_session) + self.assertEqual(1234.56 + 78.9, access._expires) + self.assertIs(datetime.datetime, + type(auth_plugin.auth_ref.expires)) + + self.assertIs(True, access.will_expire_soon(90)) + self.assertIs(False, access.will_expire_soon(60)) + self.assertEqual(3, len(mock_time.mock_calls)) + + def test_get_access_bad_expiry(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.headers['X-Auth-Token-Expires'] = 'foo' + access = auth_plugin.get_access(self.mock_session) + self.assertIsNone(access.expires) + + self.assertIs(False, access.will_expire_soon(60)) + self.assertIs(False, access.will_expire_soon(1e20)) + + def test_get_access_bad_status(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.status_code = 401 + self.assertRaises(exceptions.InvalidResponse, + auth_plugin.get_access, self.mock_session) + + def test_get_access_missing_token(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.headers.pop('X-Auth-Token') + self.assertRaises(exceptions.InvalidResponse, + auth_plugin.get_access, self.mock_session) + + def test_get_access_accepts_storage_token(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.headers.pop('X-Auth-Token') + self.mock_response.headers['X-Storage-Token'] = 'yet another token' + access = auth_plugin.get_access(self.mock_session) + self.assertEqual('yet another token', access.auth_token) + + def test_get_access_missing_url(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + self.mock_response.headers.pop('X-Storage-Url') + self.assertRaises(exceptions.InvalidResponse, + auth_plugin.get_access, self.mock_session) + + def test_get_endpoint(self): + auth_plugin = authv1.PasswordPlugin(**self.options) + + object_store_endpoint = auth_plugin.get_endpoint( + self.mock_session, service_type='object-store') + self.assertEqual(object_store_endpoint, self.expected_endpoint) + + auth_endpoint = auth_plugin.get_endpoint( + self.mock_session, interface=plugin.AUTH_INTERFACE) + self.assertEqual(auth_endpoint, self.options['auth_url']) + + with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: + auth_plugin.get_endpoint(self.mock_session) + self.assertEqual('public endpoint for None service not found', + str(exc_mgr.exception)) + + with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: + auth_plugin.get_endpoint( + self.mock_session, service_type='identity', region_name='DFW') + self.assertEqual( + 'public endpoint for identity service in DFW region not found', + str(exc_mgr.exception)) + + with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: + auth_plugin.get_endpoint( + self.mock_session, service_type='image', service_name='glance') + self.assertEqual( + 'public endpoint for image service named glance not found', + str(exc_mgr.exception)) + + with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: + auth_plugin.get_endpoint( + self.mock_session, service_type='compute', service_name='nova', + region_name='IAD') + self.assertEqual('public endpoint for compute service named nova in ' + 'IAD region not found', str(exc_mgr.exception)) + + +class TestPluginWithAccount(TestDataWithAccount, TestPlugin): + pass diff -Nru python-swiftclient-3.8.1/test/unit/test_command_helpers.py python-swiftclient-3.9.0/test/unit/test_command_helpers.py --- python-swiftclient-3.8.1/test/unit/test_command_helpers.py 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/unit/test_command_helpers.py 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,249 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from six import StringIO +import unittest + +from swiftclient import command_helpers as h +from swiftclient.multithreading import OutputManager + + +class TestStatHelpers(unittest.TestCase): + + def setUp(self): + super(TestStatHelpers, self).setUp() + conn_attrs = { + 'url': 'http://storage/v1/a', + 'token': 'tk12345', + } + self.conn = mock.MagicMock(**conn_attrs) + self.options = {'human': False, 'verbose': 1} + self.stdout = StringIO() + self.stderr = StringIO() + self.output_manager = OutputManager(self.stdout, self.stderr) + + def assertOut(self, expected): + real = self.stdout.getvalue() + # commonly if we strip of blank lines we have a match + try: + self.assertEqual(expected.strip('\n'), + real.strip('\n')) + except AssertionError: + # could be anything, try to find typos line by line + expected_lines = [line.lstrip() for line in + expected.splitlines() if line.strip()] + real_lines = [line.lstrip() for line in + real.splitlines() if line.strip()] + for expected, real in zip(expected_lines, real_lines): + self.assertEqual(expected, real) + # not a typo, might be an indent thing, hopefully you can spot it + raise + + def test_stat_account_human(self): + self.options['human'] = True + # stub head_account + stub_headers = { + 'x-account-container-count': 42, + 'x-account-object-count': 1000000, + 'x-account-bytes-used': 2 ** 30, + } + self.conn.head_account.return_value = stub_headers + + with self.output_manager as output_manager: + items, headers = h.stat_account(self.conn, self.options) + h.print_account_stats(items, headers, output_manager) + expected = """ + Account: a +Containers: 42 + Objects: 976K + Bytes: 1.0G +""" + self.assertOut(expected) + + def test_stat_account_verbose(self): + self.options['verbose'] += 1 + # stub head_account + stub_headers = { + 'x-account-container-count': 42, + 'x-account-object-count': 1000000, + 'x-account-bytes-used': 2 ** 30, + } + self.conn.head_account.return_value = stub_headers + + with self.output_manager as output_manager: + items, headers = h.stat_account(self.conn, self.options) + h.print_account_stats(items, headers, output_manager) + expected = """ +StorageURL: http://storage/v1/a +Auth Token: tk12345 + Account: a +Containers: 42 + Objects: 1000000 + Bytes: 1073741824 +""" + self.assertOut(expected) + + def test_stat_account_policy_stat(self): + # stub head_account + stub_headers = { + 'x-account-container-count': 42, + 'x-account-object-count': 1000000, + 'x-account-bytes-used': 2 ** 30, + 'x-account-storage-policy-nada-object-count': 1000000, + 'x-account-storage-policy-nada-bytes-used': 2 ** 30, + } + self.conn.head_account.return_value = stub_headers + + with self.output_manager as output_manager: + items, headers = h.stat_account(self.conn, self.options) + h.print_account_stats(items, headers, output_manager) + expected = """ + Account: a + Containers: 42 + Objects: 1000000 + Bytes: 1073741824 +Objects in policy "nada": 1000000 + Bytes in policy "nada": 1073741824 +""" + self.assertOut(expected) + + def test_stat_account_policy_stat_with_container_counts(self): + # stub head_account + stub_headers = { + 'x-account-container-count': 42, + 'x-account-object-count': 1000000, + 'x-account-bytes-used': 2 ** 30, + 'x-account-storage-policy-nada-container-count': 10, + 'x-account-storage-policy-nada-object-count': 1000000, + 'x-account-storage-policy-nada-bytes-used': 2 ** 30, + } + self.conn.head_account.return_value = stub_headers + + with self.output_manager as output_manager: + items, headers = h.stat_account(self.conn, self.options) + h.print_account_stats(items, headers, output_manager) + expected = """ + Account: a + Containers: 42 + Objects: 1000000 + Bytes: 1073741824 +Containers in policy "nada": 10 + Objects in policy "nada": 1000000 + Bytes in policy "nada": 1073741824 +""" + self.assertOut(expected) + + def test_stat_container_human(self): + self.options['human'] = True + # stub head container request + stub_headers = { + 'x-container-object-count': 10 ** 6, + 'x-container-bytes-used': 2 ** 30, + } + self.conn.head_container.return_value = stub_headers + args = ('c',) + with self.output_manager as output_manager: + items, headers = h.stat_container(self.conn, self.options, *args) + h.print_container_stats(items, headers, output_manager) + expected = """ + Account: a +Container: c + Objects: 976K + Bytes: 1.0G + Read ACL: +Write ACL: + Sync To: + Sync Key: +""" + self.assertOut(expected) + + def test_stat_container_verbose(self): + self.options['verbose'] += 1 + # stub head container request + stub_headers = { + 'x-container-object-count': 10 ** 6, + 'x-container-bytes-used': 2 ** 30, + } + self.conn.head_container.return_value = stub_headers + args = ('c',) + with self.output_manager as output_manager: + items, headers = h.stat_container(self.conn, self.options, *args) + h.print_container_stats(items, headers, output_manager) + expected = """ + URL: http://storage/v1/a/c +Auth Token: tk12345 + Account: a + Container: c + Objects: 1000000 + Bytes: 1073741824 + Read ACL: + Write ACL: + Sync To: + Sync Key: +""" + self.assertOut(expected) + + def test_stat_object_human(self): + self.options['human'] = True + # stub head object request + stub_headers = { + 'content-length': 2 ** 20, + 'x-object-meta-color': 'blue', + 'etag': '68b329da9893e34099c7d8ad5cb9c940', + 'content-encoding': 'gzip', + } + self.conn.head_object.return_value = stub_headers + args = ('c', 'o') + with self.output_manager as output_manager: + items, headers = h.stat_object(self.conn, self.options, *args) + h.print_object_stats(items, headers, output_manager) + expected = """ + Account: a + Container: c + Object: o + Content Length: 1.0M + ETag: 68b329da9893e34099c7d8ad5cb9c940 + Meta Color: blue +Content-Encoding: gzip +""" + self.assertOut(expected) + + def test_stat_object_verbose(self): + self.options['verbose'] += 1 + # stub head object request + stub_headers = { + 'content-length': 2 ** 20, + 'x-object-meta-color': 'blue', + 'etag': '68b329da9893e34099c7d8ad5cb9c940', + 'content-encoding': 'gzip', + } + self.conn.head_object.return_value = stub_headers + args = ('c', 'o') + with self.output_manager as output_manager: + items, headers = h.stat_object(self.conn, self.options, *args) + h.print_object_stats(items, headers, output_manager) + expected = """ + URL: http://storage/v1/a/c/o + Auth Token: tk12345 + Account: a + Container: c + Object: o + Content Length: 1048576 + ETag: 68b329da9893e34099c7d8ad5cb9c940 + Meta Color: blue +Content-Encoding: gzip +""" + self.assertOut(expected) diff -Nru python-swiftclient-3.8.1/test/unit/test_multithreading.py python-swiftclient-3.9.0/test/unit/test_multithreading.py --- python-swiftclient-3.8.1/test/unit/test_multithreading.py 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/unit/test_multithreading.py 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,240 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +import unittest +import threading +import six + +from concurrent.futures import as_completed +from six.moves.queue import Queue, Empty +from time import sleep + +from swiftclient import multithreading as mt +from .utils import CaptureStream + + +class ThreadTestCase(unittest.TestCase): + def setUp(self): + super(ThreadTestCase, self).setUp() + self.got_items = Queue() + self.got_args_kwargs = Queue() + self.starting_thread_count = threading.active_count() + + def _func(self, conn, item, *args, **kwargs): + self.got_items.put((conn, item)) + self.got_args_kwargs.put((args, kwargs)) + + if item == 'sleep': + sleep(1) + if item == 'go boom': + raise Exception('I went boom!') + + return 'success' + + def _create_conn(self): + return "This is a connection" + + def _create_conn_fail(self): + raise Exception("This is a failed connection") + + def assertQueueContains(self, queue, expected_contents): + got_contents = [] + try: + while True: + got_contents.append(queue.get(timeout=0.1)) + except Empty: + pass + if isinstance(expected_contents, set): + got_contents = set(got_contents) + self.assertEqual(expected_contents, got_contents) + + +class TestConnectionThreadPoolExecutor(ThreadTestCase): + def setUp(self): + super(TestConnectionThreadPoolExecutor, self).setUp() + self.input_queue = Queue() + self.stored_results = [] + + def tearDown(self): + super(TestConnectionThreadPoolExecutor, self).tearDown() + + def test_submit_good_connection(self): + ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 1) + with ctpe as pool: + # Try submitting a job that should succeed + f = pool.submit(self._func, "succeed") + f.result() + self.assertQueueContains( + self.got_items, + [("This is a connection", "succeed")] + ) + + # Now a job that fails + try: + f = pool.submit(self._func, "go boom") + f.result() + except Exception as e: + self.assertEqual('I went boom!', str(e)) + else: + self.fail('I never went boom!') + + # Has the connection been returned to the pool? + f = pool.submit(self._func, "succeed") + f.result() + self.assertQueueContains( + self.got_items, + [ + ("This is a connection", "go boom"), + ("This is a connection", "succeed") + ] + ) + + def test_submit_bad_connection(self): + ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn_fail, 1) + with ctpe as pool: + # Now a connection that fails + try: + f = pool.submit(self._func, "succeed") + f.result() + except Exception as e: + self.assertEqual('This is a failed connection', str(e)) + else: + self.fail('The connection did not fail') + + # Make sure we don't lock up on failed connections + try: + f = pool.submit(self._func, "go boom") + f.result() + except Exception as e: + self.assertEqual('This is a failed connection', str(e)) + else: + self.fail('The connection did not fail') + + def test_lazy_connections(self): + ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 10) + with ctpe as pool: + # Submit multiple jobs sequentially - should only use 1 conn + f = pool.submit(self._func, "succeed") + f.result() + f = pool.submit(self._func, "succeed") + f.result() + f = pool.submit(self._func, "succeed") + f.result() + + expected_connections = [(0, "This is a connection")] + expected_connections.extend([(x, None) for x in range(1, 10)]) + + self.assertQueueContains( + pool._connections, expected_connections + ) + + ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 10) + with ctpe as pool: + fs = [] + f1 = pool.submit(self._func, "sleep") + f2 = pool.submit(self._func, "sleep") + f3 = pool.submit(self._func, "sleep") + fs.extend([f1, f2, f3]) + + expected_connections = [ + (0, "This is a connection"), + (1, "This is a connection"), + (2, "This is a connection") + ] + expected_connections.extend([(x, None) for x in range(3, 10)]) + + for f in as_completed(fs): + f.result() + + self.assertQueueContains( + pool._connections, expected_connections + ) + + +class TestOutputManager(unittest.TestCase): + + def test_instantiation(self): + output_manager = mt.OutputManager() + + self.assertEqual(sys.stdout, output_manager.print_stream) + self.assertEqual(sys.stderr, output_manager.error_stream) + + def test_printers(self): + out_stream = CaptureStream(sys.stdout) + err_stream = CaptureStream(sys.stderr) + starting_thread_count = threading.active_count() + + with mt.OutputManager( + print_stream=out_stream, + error_stream=err_stream) as thread_manager: + + # Sanity-checking these gives power to the previous test which + # looked at the default values of thread_manager.print/error_stream + self.assertEqual(out_stream, thread_manager.print_stream) + self.assertEqual(err_stream, thread_manager.error_stream) + + # No printing has happened yet, so no new threads + self.assertEqual(starting_thread_count, + threading.active_count()) + + thread_manager.print_msg('one-argument') + thread_manager.print_msg('one %s, %d fish', 'fish', 88) + thread_manager.error('I have %d problems, but a %s is not one', + 99, u'\u062A\u062A') + thread_manager.print_msg('some\n%s\nover the %r', 'where', + u'\u062A\u062A') + thread_manager.error('one-error-argument') + thread_manager.error('Sometimes\n%.1f%% just\ndoes not\nwork!', + 3.14159) + thread_manager.print_raw( + u'some raw bytes: \u062A\u062A'.encode('utf-8')) + + thread_manager.print_items([ + ('key', 'value'), + ('object', u'O\u0308bject'), + ]) + + thread_manager.print_raw(b'\xffugly\xffraw') + + # Now we have a thread for error printing and a thread for + # normal print messages + self.assertEqual(starting_thread_count + 2, + threading.active_count()) + + # The threads should have been cleaned up + self.assertEqual(starting_thread_count, threading.active_count()) + + if six.PY3: + over_the = "over the '\u062a\u062a'\n" + else: + over_the = "over the u'\\u062a\\u062a'\n" + # We write to the CaptureStream so no decoding is performed + self.assertEqual(''.join([ + 'one-argument\n', + 'one fish, 88 fish\n', + 'some\n', 'where\n', + over_the, + u'some raw bytes: \u062a\u062a', + ' key: value\n', + u' object: O\u0308bject\n' + ]).encode('utf8') + b'\xffugly\xffraw', out_stream.getvalue()) + + self.assertEqual(''.join([ + u'I have 99 problems, but a \u062A\u062A is not one\n', + 'one-error-argument\n', + 'Sometimes\n', '3.1% just\n', 'does not\n', 'work!\n' + ]), err_stream.getvalue().decode('utf8')) + + self.assertEqual(3, thread_manager.error_count) diff -Nru python-swiftclient-3.8.1/test/unit/test_service.py python-swiftclient-3.9.0/test/unit/test_service.py --- python-swiftclient-3.8.1/test/unit/test_service.py 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/unit/test_service.py 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,2909 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2014 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import unicode_literals +import contextlib +import mock +import os +import six +import tempfile +import unittest +import time + +from concurrent.futures import Future +from hashlib import md5 +from mock import Mock, PropertyMock +from six.moves.queue import Queue, Empty as QueueEmptyError +from six import BytesIO +from time import sleep + +import swiftclient +import swiftclient.utils as utils +from swiftclient.client import Connection, ClientException +from swiftclient.service import ( + SwiftService, SwiftError, SwiftUploadObject +) + +from test.unit import utils as test_utils + + +clean_os_environ = {} +environ_prefixes = ('ST_', 'OS_') +for key in os.environ: + if any(key.startswith(m) for m in environ_prefixes): + clean_os_environ[key] = '' + + +if six.PY2: + import __builtin__ as builtins +else: + import builtins + + +class TestSwiftPostObject(unittest.TestCase): + + def setUp(self): + super(TestSwiftPostObject, self).setUp() + self.spo = swiftclient.service.SwiftPostObject + + def test_create(self): + spo = self.spo('obj_name') + + self.assertEqual(spo.object_name, 'obj_name') + self.assertIsNone(spo.options) + + def test_create_with_invalid_name(self): + # empty strings are not allowed as names + self.assertRaises(SwiftError, self.spo, '') + + # names cannot be anything but strings + self.assertRaises(SwiftError, self.spo, 1) + + +class TestSwiftCopyObject(unittest.TestCase): + + def setUp(self): + super(TestSwiftCopyObject, self).setUp() + self.sco = swiftclient.service.SwiftCopyObject + + def test_create(self): + sco = self.sco('obj_name') + + self.assertEqual(sco.object_name, 'obj_name') + self.assertIsNone(sco.destination) + self.assertFalse(sco.fresh_metadata) + + sco = self.sco('obj_name', + {'destination': '/dest', 'fresh_metadata': True}) + + self.assertEqual(sco.object_name, 'obj_name') + self.assertEqual(sco.destination, '/dest/obj_name') + self.assertTrue(sco.fresh_metadata) + + sco = self.sco('obj_name', + {'destination': '/dest/new_obj/a', + 'fresh_metadata': False}) + + self.assertEqual(sco.object_name, 'obj_name') + self.assertEqual(sco.destination, '/dest/new_obj/a') + self.assertFalse(sco.fresh_metadata) + + def test_create_with_invalid_name(self): + # empty strings are not allowed as names + self.assertRaises(SwiftError, self.sco, '') + + # names cannot be anything but strings + self.assertRaises(SwiftError, self.sco, 1) + + +class TestSwiftReader(unittest.TestCase): + + def setUp(self): + super(TestSwiftReader, self).setUp() + self.sr = swiftclient.service._SwiftReader + self.md5_type = type(md5()) + + def test_create(self): + sr = self.sr('path', 'body', {}) + + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + + self.assertIsNone(sr._actual_md5) + + def test_create_with_large_object_headers(self): + # md5 should not be initialized if large object headers are present + sr = self.sr('path', 'body', {'x-object-manifest': 'test', + 'etag': '"%s"' % ('0' * 32)}) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + self.assertIsNone(sr._actual_md5) + + sr = self.sr('path', 'body', {'x-static-large-object': 'test', + 'etag': '"%s"' % ('0' * 32)}) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + self.assertIsNone(sr._actual_md5) + + def test_create_with_content_range_header(self): + # md5 should not be initialized if large object headers are present + sr = self.sr('path', 'body', {'content-range': 'bytes 0-3/10', + 'etag': '"%s"' % ('0' * 32)}) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + self.assertIsNone(sr._actual_md5) + + def test_create_with_ignore_checksum(self): + # md5 should not be initialized if checksum is False + sr = self.sr('path', 'body', {}, False) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + self.assertIsNone(sr._actual_md5) + + def test_create_with_content_length(self): + sr = self.sr('path', 'body', {'content-length': 5}) + + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertEqual(sr._content_length, 5) + self.assertFalse(sr._expected_md5) + + self.assertIsNone(sr._actual_md5) + + # Check Contentlength raises error if it isn't an integer + self.assertRaises(SwiftError, self.sr, 'path', 'body', + {'content-length': 'notanint'}) + + def test_iterator_usage(self): + def _consume(sr): + for _ in sr: + pass + + sr = self.sr('path', BytesIO(b'body'), {}) + _consume(sr) + + # Check error is raised if expected etag doesn't match calculated md5. + # md5 for a SwiftReader that has done nothing is + # d41d8cd98f00b204e9800998ecf8427e i.e md5 of nothing + sr = self.sr('path', BytesIO(b'body'), + {'etag': md5(b'doesntmatch').hexdigest()}) + self.assertRaises(SwiftError, _consume, sr) + + sr = self.sr('path', BytesIO(b'body'), + {'etag': md5(b'body').hexdigest()}) + _consume(sr) + + # Should still work if etag was quoted + sr = self.sr('path', BytesIO(b'body'), + {'etag': '"%s"' % md5(b'body').hexdigest()}) + _consume(sr) + + # Check error is raised if SwiftReader doesn't read the same length + # as the content length it is created with + sr = self.sr('path', BytesIO(b'body'), {'content-length': 5}) + self.assertRaises(SwiftError, _consume, sr) + + sr = self.sr('path', BytesIO(b'body'), {'content-length': 4}) + _consume(sr) + + # Check that the iterator generates expected length and etag values + sr = self.sr('path', ['abc'.encode()] * 3, + {'content-length': 9, + 'etag': md5('abc'.encode() * 3).hexdigest()}) + _consume(sr) + self.assertEqual(sr._actual_read, 9) + self.assertEqual(sr._actual_md5.hexdigest(), + md5('abc'.encode() * 3).hexdigest()) + + +class _TestServiceBase(unittest.TestCase): + def _get_mock_connection(self, attempts=2): + m = Mock(spec=Connection) + type(m).attempts = PropertyMock(return_value=attempts) + type(m).auth_end_time = PropertyMock(return_value=4) + return m + + def _get_queue(self, q): + # Instead of blocking pull items straight from the queue. + # expects at least one item otherwise the test will fail. + try: + return q.get_nowait() + except QueueEmptyError: + self.fail('Expected item in queue but found none') + + def _get_expected(self, update=None): + expected = self.expected.copy() + if update: + expected.update(update) + + return expected + + +class TestServiceDelete(_TestServiceBase): + def setUp(self): + super(TestServiceDelete, self).setUp() + self.opts = {'leave_segments': False, 'yes_all': False} + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': None, # Should be string in the form delete_XX + 'container': 'test_c', + 'object': 'test_o', + 'attempts': 2, + 'response_dict': {}, + 'success': None # Should be a bool + } + + def test_delete_segment(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + expected_r = self._get_expected({ + 'action': 'delete_segment', + 'object': 'test_s', + 'success': True, + }) + + r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q) + + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_s', response_dict={} + ) + self.assertEqual(expected_r, r) + self.assertEqual(expected_r, self._get_queue(mock_q)) + + def test_delete_segment_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.delete_object = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'delete_segment', + 'object': 'test_s', + 'success': False, + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + }) + + before = time.time() + r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q) + after = time.time() + + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_s', response_dict={} + ) + self.assertEqual(expected_r, r) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertIn('Traceback', r['traceback']) + + def test_delete_object(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.head_object = Mock(return_value={}) + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': True + }) + + s = SwiftService() + r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) + + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_o', query_string=None, response_dict={}, + headers={} + ) + self.assertEqual(expected_r, r) + + def test_delete_object_with_headers(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.head_object = Mock(return_value={}) + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': True + }) + opt_c = self.opts.copy() + opt_c['header'] = ['Skip-Middleware: Test'] + + s = SwiftService() + r = s._delete_object(mock_conn, 'test_c', 'test_o', opt_c, mock_q) + + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'}, + query_string='symlink=get') + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_o', query_string=None, response_dict={}, + headers={'Skip-Middleware': 'Test'} + ) + self.assertEqual(expected_r, r) + + def test_delete_object_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.delete_object = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': False, + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + }) + # _delete_object doesn't populate attempts or response dict if it hits + # an error. This may not be the correct behaviour. + del expected_r['response_dict'], expected_r['attempts'] + + before = time.time() + s = SwiftService() + r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) + after = time.time() + + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_o', query_string=None, response_dict={}, + headers={} + ) + self.assertEqual(expected_r, r) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertIn('Traceback', r['traceback']) + + def test_delete_object_slo_support(self): + # If SLO headers are present the delete call should include an + # additional query string to cause the right delete server side + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.head_object = Mock( + return_value={'x-static-large-object': True} + ) + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': True + }) + + s = SwiftService() + r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) + + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_o', + query_string='multipart-manifest=delete', + response_dict={}, + headers={} + ) + self.assertEqual(expected_r, r) + + def test_delete_object_dlo_support(self): + mock_q = Queue() + s = SwiftService() + mock_conn = self._get_mock_connection() + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': True, + 'dlo_segments_deleted': True + }) + # A DLO object is determined in _delete_object by heading the object + # and checking for the existence of a x-object-manifest header. + # Mock that here. + mock_conn.head_object = Mock( + return_value={'x-object-manifest': 'manifest_c/manifest_p'} + ) + mock_conn.get_container = Mock( + side_effect=[(None, [{'name': 'test_seg_1'}, + {'name': 'test_seg_2'}]), + (None, {})] + ) + + def get_mock_list_conn(options): + return mock_conn + + with mock.patch('swiftclient.service.get_conn', get_mock_list_conn): + r = s._delete_object( + mock_conn, 'test_c', 'test_o', self.opts, mock_q + ) + + self.assertEqual(expected_r, r) + expected = [ + mock.call('test_c', 'test_o', query_string=None, response_dict={}, + headers={}), + mock.call('manifest_c', 'test_seg_1', response_dict={}), + mock.call('manifest_c', 'test_seg_2', response_dict={})] + mock_conn.delete_object.assert_has_calls(expected, any_order=True) + + def test_delete_empty_container(self): + mock_conn = self._get_mock_connection() + expected_r = self._get_expected({ + 'action': 'delete_container', + 'success': True, + 'object': None + }) + + r = SwiftService._delete_empty_container(mock_conn, 'test_c', + self.opts) + + mock_conn.delete_container.assert_called_once_with( + 'test_c', response_dict={}, headers={} + ) + self.assertEqual(expected_r, r) + + def test_delete_empty_container_with_headers(self): + mock_conn = self._get_mock_connection() + expected_r = self._get_expected({ + 'action': 'delete_container', + 'success': True, + 'object': None + }) + opt_c = self.opts.copy() + opt_c['header'] = ['Skip-Middleware: Test'] + + r = SwiftService._delete_empty_container(mock_conn, 'test_c', opt_c) + + mock_conn.delete_container.assert_called_once_with( + 'test_c', response_dict={}, headers={'Skip-Middleware': 'Test'} + ) + self.assertEqual(expected_r, r) + + def test_delete_empty_container_exception(self): + mock_conn = self._get_mock_connection() + mock_conn.delete_container = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'delete_container', + 'success': False, + 'object': None, + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + }) + + before = time.time() + s = SwiftService() + r = s._delete_empty_container(mock_conn, 'test_c', {}) + after = time.time() + + mock_conn.delete_container.assert_called_once_with( + 'test_c', response_dict={}, headers={} + ) + self.assertEqual(expected_r, r) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertIn('Traceback', r['traceback']) + + @mock.patch.object(swiftclient.service.SwiftService, 'capabilities', + lambda *a: {'action': 'capabilities', + 'timestamp': time.time(), + 'success': True, + 'capabilities': { + 'bulk_delete': + {'max_deletes_per_request': 10}} + }) + def test_bulk_delete_page_size(self): + # make a list of 100 objects + obj_list = ['x%02d' % i for i in range(100)] + errors = [] + + # _bulk_delete_page_size uses 2x the number of threads to determine + # if if there are "many" object to delete or not + + # format is: [(thread_count, expected result), ...] + obj_threads_exp = [ + (10, 10), # something small + (49, 10), # just under the bounds + (50, 1), # cutover point + (51, 1), # just over bounds + (100, 1), # something big + ] + for thread_count, exp in obj_threads_exp: + s = SwiftService(options={'object_dd_threads': thread_count}) + res = s._bulk_delete_page_size(obj_list) + if res != exp: + msg = 'failed for thread_count %d: got %r expected %r' % \ + (thread_count, res, exp) + errors.append(msg) + if errors: + self.fail('_bulk_delete_page_size() failed\n' + '\n'.join(errors)) + + +class TestSwiftError(unittest.TestCase): + + def test_is_exception(self): + se = SwiftError(5) + self.assertIsInstance(se, Exception) + + def test_empty_swifterror_creation(self): + se = SwiftError(5) + + self.assertEqual(se.value, 5) + self.assertIsNone(se.container) + self.assertIsNone(se.obj) + self.assertIsNone(se.segment) + self.assertIsNone(se.exception) + + self.assertEqual(str(se), '5') + + def test_swifterror_creation(self): + test_exc = Exception('test exc') + se = SwiftError(5, 'con', 'obj', 'seg', test_exc) + + self.assertEqual(se.value, 5) + self.assertEqual(se.container, 'con') + self.assertEqual(se.obj, 'obj') + self.assertEqual(se.segment, 'seg') + self.assertEqual(se.exception, test_exc) + + self.assertEqual(str(se), '5 container:con object:obj segment:seg') + + +class TestServiceUtils(unittest.TestCase): + + def setUp(self): + super(TestServiceUtils, self).setUp() + with mock.patch.dict(swiftclient.service.environ, clean_os_environ): + swiftclient.service._default_global_options = \ + swiftclient.service._build_default_global_options() + self.opts = swiftclient.service._default_global_options.copy() + + def test_process_options_defaults(self): + # The only actions that should be taken on default options set is + # to change the auth version to v2.0 and create the os_options dict + opt_c = self.opts.copy() + + swiftclient.service.process_options(opt_c) + + self.assertIn('os_options', opt_c) + del opt_c['os_options'] + self.assertEqual(opt_c['auth_version'], '2.0') + opt_c['auth_version'] = '1.0' + + self.assertEqual(opt_c, self.opts) + + def test_process_options_auth_version(self): + # auth_version should be set to 2.0 + # if it isn't already set to 3.0 + # and the v1 command line arguments aren't present + opt_c = self.opts.copy() + + # Check v3 isn't changed + opt_c['auth_version'] = '3' + swiftclient.service.process_options(opt_c) + self.assertEqual(opt_c['auth_version'], '3') + + # Check v1 isn't changed if user, key and auth are set + opt_c = self.opts.copy() + opt_c['auth_version'] = '1' + opt_c['auth'] = True + opt_c['user'] = True + opt_c['key'] = True + swiftclient.service.process_options(opt_c) + self.assertEqual(opt_c['auth_version'], '1') + + def test_process_options_new_style_args(self): + # checks new style args are copied to old style + # when old style don't exist + opt_c = self.opts.copy() + + opt_c['auth'] = '' + opt_c['user'] = '' + opt_c['key'] = '' + opt_c['os_auth_url'] = 'os_auth' + opt_c['os_username'] = 'os_user' + opt_c['os_password'] = 'os_pass' + swiftclient.service.process_options(opt_c) + self.assertEqual(opt_c['auth_version'], '2.0') + self.assertEqual(opt_c['auth'], 'os_auth') + self.assertEqual(opt_c['user'], 'os_user') + self.assertEqual(opt_c['key'], 'os_pass') + + # Check old style args are left alone if they exist + opt_c = self.opts.copy() + opt_c['auth'] = 'auth' + opt_c['user'] = 'user' + opt_c['key'] = 'key' + opt_c['os_auth_url'] = 'os_auth' + opt_c['os_username'] = 'os_user' + opt_c['os_password'] = 'os_pass' + swiftclient.service.process_options(opt_c) + self.assertEqual(opt_c['auth_version'], '1.0') + self.assertEqual(opt_c['auth'], 'auth') + self.assertEqual(opt_c['user'], 'user') + self.assertEqual(opt_c['key'], 'key') + + def test_split_headers(self): + mock_headers = ['color:blue', 'SIZE: large'] + expected = {'Color': 'blue', 'Size': 'large'} + + actual = swiftclient.service.split_headers(mock_headers) + self.assertEqual(expected, actual) + + def test_split_headers_prefix(self): + mock_headers = ['color:blue', 'size:large'] + expected = {'Prefix-Color': 'blue', 'Prefix-Size': 'large'} + + actual = swiftclient.service.split_headers(mock_headers, 'prefix-') + self.assertEqual(expected, actual) + + def test_split_headers_list_of_tuples(self): + mock_headers = [('color', 'blue'), ('size', 'large')] + expected = {'Prefix-Color': 'blue', 'Prefix-Size': 'large'} + + actual = swiftclient.service.split_headers(mock_headers, 'prefix-') + self.assertEqual(expected, actual) + + def test_split_headers_dict(self): + expected = {'Color': 'blue', 'Size': 'large'} + + actual = swiftclient.service.split_headers(expected) + self.assertEqual(expected, actual) + + def test_split_headers_error(self): + self.assertRaises(SwiftError, swiftclient.service.split_headers, + ['notvalid']) + self.assertRaises(SwiftError, swiftclient.service.split_headers, + [('also', 'not', 'valid')]) + + +class TestSwiftUploadObject(unittest.TestCase): + + def setUp(self): + self.suo = swiftclient.service.SwiftUploadObject + super(TestSwiftUploadObject, self).setUp() + + def test_create_with_string(self): + suo = self.suo('source') + self.assertEqual(suo.source, 'source') + self.assertEqual(suo.object_name, 'source') + self.assertIsNone(suo.options) + + suo = self.suo('source', 'obj_name') + self.assertEqual(suo.source, 'source') + self.assertEqual(suo.object_name, 'obj_name') + self.assertIsNone(suo.options) + + suo = self.suo('source', 'obj_name', {'opt': '123'}) + self.assertEqual(suo.source, 'source') + self.assertEqual(suo.object_name, 'obj_name') + self.assertEqual(suo.options, {'opt': '123'}) + + def test_create_with_file(self): + with tempfile.TemporaryFile() as mock_file: + # Check error is raised if no object name is provided with a + # filelike object + self.assertRaises(SwiftError, self.suo, mock_file) + + # Check that empty strings are invalid object names + self.assertRaises(SwiftError, self.suo, mock_file, '') + + suo = self.suo(mock_file, 'obj_name') + self.assertEqual(suo.source, mock_file) + self.assertEqual(suo.object_name, 'obj_name') + self.assertIsNone(suo.options) + + suo = self.suo(mock_file, 'obj_name', {'opt': '123'}) + self.assertEqual(suo.source, mock_file) + self.assertEqual(suo.object_name, 'obj_name') + self.assertEqual(suo.options, {'opt': '123'}) + + def test_create_with_no_source(self): + suo = self.suo(None, 'obj_name') + self.assertIsNone(suo.source) + self.assertEqual(suo.object_name, 'obj_name') + self.assertIsNone(suo.options) + + # Check error is raised if source is None without an object name + self.assertRaises(SwiftError, self.suo, None) + + def test_create_with_invalid_source(self): + # Source can only be None, string or filelike object, + # check an error is raised with an invalid type. + self.assertRaises(SwiftError, self.suo, []) + + +class TestServiceList(_TestServiceBase): + def setUp(self): + super(TestServiceList, self).setUp() + self.opts = {'prefix': None, 'long': False, 'delimiter': ''} + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': None, # Should be list_X_part (account or container) + 'container': None, # Should be a string when listing a container + 'prefix': None, + 'success': None # Should be a bool + } + + def test_list_account(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_account_returns = [ + (None, [{'name': 'test_c'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + + expected_r = self._get_expected({ + 'action': 'list_account_part', + 'success': True, + 'listing': [{'name': 'test_c'}], + 'marker': '' + }) + + SwiftService._list_account_job( + mock_conn, self.opts, mock_q + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + long_opts = dict(self.opts, **{'long': True}) + mock_conn.head_container = Mock(return_value={'test_m': '1'}) + get_account_returns = [ + (None, [{'name': 'test_c'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + + expected_r_long = self._get_expected({ + 'action': 'list_account_part', + 'success': True, + 'listing': [{'name': 'test_c', 'meta': {'test_m': '1'}}], + 'marker': '', + }) + + SwiftService._list_account_job( + mock_conn, long_opts, mock_q + ) + self.assertEqual(expected_r_long, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_account_with_headers(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_account_returns = [ + (None, [{'name': 'test_c'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + + expected_r = self._get_expected({ + 'action': 'list_account_part', + 'success': True, + 'listing': [{'name': 'test_c'}], + 'marker': '' + }) + opt_c = self.opts.copy() + opt_c['header'] = ['Skip-Middleware: True'] + SwiftService._list_account_job( + mock_conn, opt_c, mock_q + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + self.assertEqual(mock_conn.get_account.mock_calls, [ + mock.call(headers={'Skip-Middleware': 'True'}, marker='', + prefix=None), + mock.call(headers={'Skip-Middleware': 'True'}, marker='test_c', + prefix=None)]) + + def test_list_account_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.get_account = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'list_account_part', + 'success': False, + 'error': self.exc, + 'marker': '', + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + }) + + SwiftService._list_account_job( + mock_conn, self.opts, mock_q) + + mock_conn.get_account.assert_called_once_with( + marker='', prefix=None, headers={} + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_container(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_container_returns = [ + (None, [{'name': 'test_o'}]), + (None, []) + ] + mock_conn.get_container = Mock(side_effect=get_container_returns) + + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'test_o'}], + 'marker': '' + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', self.opts, mock_q + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + long_opts = dict(self.opts, **{'long': True}) + mock_conn.head_container = Mock(return_value={'test_m': '1'}) + get_container_returns = [ + (None, [{'name': 'test_o'}]), + (None, []) + ] + mock_conn.get_container = Mock(side_effect=get_container_returns) + + expected_r_long = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'test_o'}], + 'marker': '' + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', long_opts, mock_q + ) + self.assertEqual(expected_r_long, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_container_marker(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + + get_container_returns = [ + (None, [{'name': 'b'}, {'name': 'c'}]), + (None, []) + ] + mock_get_cont = Mock(side_effect=get_container_returns) + mock_conn.get_container = mock_get_cont + + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'b'}, {'name': 'c'}], + 'marker': 'b' + }) + + _opts = self.opts.copy() + _opts['marker'] = 'b' + SwiftService._list_container_job(mock_conn, 'test_c', _opts, mock_q) + + # This does not test if the marker is propagated, because we always + # get the final call to the get_container with the final item 'c', + # even if marker wasn't set. This test just makes sure the whole + # stack works in a sane way. + mock_kw = mock_get_cont.call_args[1] + self.assertEqual(mock_kw['marker'], 'c') + + # This tests that the lower levels get the marker delivered. + self.assertEqual(expected_r, self._get_queue(mock_q)) + + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_container_with_headers(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_container_returns = [ + (None, [{'name': 'test_o'}]), + (None, []) + ] + mock_conn.get_container = Mock(side_effect=get_container_returns) + + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'test_o'}], + 'marker': '' + }) + + opt_c = self.opts.copy() + opt_c['header'] = ['Skip-Middleware: Test'] + + SwiftService._list_container_job( + mock_conn, 'test_c', opt_c, mock_q + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + self.assertEqual(mock_conn.get_container.mock_calls, [ + mock.call('test_c', headers={'Skip-Middleware': 'Test'}, + delimiter='', marker='', prefix=None), + mock.call('test_c', headers={'Skip-Middleware': 'Test'}, + delimiter='', marker='test_o', prefix=None)]) + + def test_list_container_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.get_container = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': False, + 'error': self.exc, + 'marker': '', + 'error_timestamp': mock.ANY, + 'traceback': mock.ANY + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', self.opts, mock_q + ) + + mock_conn.get_container.assert_called_once_with( + 'test_c', marker='', delimiter='', prefix=None, headers={} + ) + self.assertEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + @mock.patch('swiftclient.service.get_conn') + def test_list_queue_size(self, mock_get_conn): + mock_conn = self._get_mock_connection() + # Return more results than should fit in the results queue + get_account_returns = [ + (None, [{'name': 'container1'}]), + (None, [{'name': 'container2'}]), + (None, [{'name': 'container3'}]), + (None, [{'name': 'container4'}]), + (None, [{'name': 'container5'}]), + (None, [{'name': 'container6'}]), + (None, [{'name': 'container7'}]), + (None, [{'name': 'container8'}]), + (None, [{'name': 'container9'}]), + (None, [{'name': 'container10'}]), + (None, [{'name': 'container11'}]), + (None, [{'name': 'container12'}]), + (None, [{'name': 'container13'}]), + (None, [{'name': 'container14'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_get_conn.return_value = mock_conn + + s = SwiftService(options=self.opts) + lg = s.list() + + # Start the generator + first_list_part = next(lg) + + # Wait for the number of calls to get_account to reach our expected + # value, then let it run some more to make sure the value remains + # stable + count = mock_conn.get_account.call_count + stable = 0 + while mock_conn.get_account.call_count != count or stable < 5: + if mock_conn.get_account.call_count == count: + stable += 1 + else: + count = mock_conn.get_account.call_count + stable = 0 + # The test requires a small sleep to allow other threads to + # execute - in this mocked environment we assume that if the call + # count to get_account has not changed in 0.25s then no more calls + # will be made. + sleep(0.05) + + stable_get_account_call_count = mock_conn.get_account.call_count + + # Collect all remaining results from the generator + list_results = [first_list_part] + list(lg) + + # Make sure the stable call count is correct - this should be 12 calls + # to get_account; + # 1 for first_list_part + # 10 for the values on the queue + # 1 for the value blocking whilst trying to place onto the queue + self.assertEqual(12, stable_get_account_call_count) + + # Make sure all the containers were listed and placed onto the queue + self.assertEqual(15, mock_conn.get_account.call_count) + + # Check the results were all returned + observed_listing = [] + for lir in list_results: + observed_listing.append( + [li['name'] for li in lir['listing']] + ) + expected_listing = [] + for gar in get_account_returns[:-1]: # The empty list is not returned + expected_listing.append( + [li['name'] for li in gar[1]] + ) + self.assertEqual(observed_listing, expected_listing) + + +class TestService(unittest.TestCase): + + def test_upload_with_bad_segment_size(self): + for bad in ('ten', '1234X', '100.3'): + options = {'segment_size': bad} + try: + service = SwiftService(options) + next(service.upload('c', 'o')) + self.fail('Expected SwiftError when segment_size=%s' % bad) + except SwiftError as exc: + self.assertEqual('Segment size should be an integer value', + exc.value) + + @mock.patch('swiftclient.service.stat') + @mock.patch('swiftclient.service.getmtime', return_value=1.0) + @mock.patch('swiftclient.service.getsize', return_value=4) + def test_upload_with_relative_path(self, *args, **kwargs): + service = SwiftService({}) + objects = [{'path': "./testobj", + 'strt_indx': 2}, + {'path': os.path.join(os.getcwd(), "testobj"), + 'strt_indx': 1}, + {'path': ".\\testobj", + 'strt_indx': 2}] + for obj in objects: + with mock.patch('swiftclient.service.Connection') as mock_conn, \ + mock.patch.object(builtins, 'open') as mock_open: + mock_open.return_value = six.StringIO('asdf') + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value =\ + 'd41d8cd98f00b204e9800998ecf8427e' + resp_iter = service.upload( + 'c', [SwiftUploadObject(obj['path'])]) + responses = [x for x in resp_iter] + for resp in responses: + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(2, len(responses)) + create_container_resp, upload_obj_resp = responses + self.assertEqual(create_container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + obj['path'][obj['strt_indx']:]) + self.assertEqual(upload_obj_resp['path'], obj['path']) + self.assertTrue(mock_open.return_value.closed) + + @mock.patch('swiftclient.service.Connection') + def test_upload_stream(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + segment_etag = md5(b'A' * 1024).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + segment_etag + options = {'use_slo': True, 'segment_size': 1024} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertFalse('error' in resp) + self.assertTrue(resp['success']) + self.assertEqual(5, len(responses)) + container_resp, segment_container_resp = responses[0:2] + segment_response = responses[2:4] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertTrue(upload_obj_resp['large_object']) + self.assertIn('manifest_response_dict', upload_obj_resp) + self.assertEqual(upload_obj_resp['manifest_response_dict'], {}) + for i, resp in enumerate(segment_response): + self.assertEqual(i, resp['segment_index']) + self.assertEqual(1024, resp['segment_size']) + self.assertEqual('d47b127bc2de2d687ddc82dac354c415', + resp['segment_etag']) + self.assertTrue(resp['segment_location'].endswith( + '/0000000%d' % i)) + self.assertTrue(resp['segment_location'].startswith( + '/container_segments/streamed')) + + @mock.patch('swiftclient.service.Connection') + def test_upload_stream_fits_in_one_segment(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + whole_etag = md5(b'A' * 2048).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + whole_etag + options = {'use_slo': True, 'segment_size': 10240} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertNotIn('error', resp) + self.assertTrue(resp['success']) + self.assertEqual(3, len(responses)) + container_resp, segment_container_resp = responses[0:2] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertFalse(upload_obj_resp['large_object']) + self.assertNotIn('manifest_response_dict', upload_obj_resp) + + +class TestServiceUpload(_TestServiceBase): + + @contextlib.contextmanager + def assert_open_results_are_closed(self): + opened_files = [] + builtin_open = builtins.open + + def open_wrapper(*a, **kw): + opened_files.append((builtin_open(*a, **kw), a, kw)) + return opened_files[-1][0] + + with mock.patch.object(builtins, 'open', open_wrapper): + yield + for fp, args, kwargs in opened_files: + formatted_args = [repr(a) for a in args] + formatted_args.extend('%s=%r' % kv for kv in kwargs.items()) + formatted_args = ', '.join(formatted_args) + self.assertTrue(fp.closed, + 'Failed to close open(%s)' % formatted_args) + + def test_upload_object_job_file_with_unicode_path(self): + # Uploading a file results in the file object being wrapped in a + # LengthWrapper. This test sets the options in such a way that much + # of _upload_object_job is skipped bringing the critical path down + # to around 60 lines to ease testing. + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + expected_r = { + 'action': 'upload_object', + 'attempts': 2, + 'container': 'test_c', + 'headers': {}, + 'large_object': True, + 'object': 'テスト/dummy.dat', + 'manifest_response_dict': {}, + 'segment_results': [{'action': 'upload_segment', + 'success': True}] * 3, + 'status': 'uploaded', + 'success': True, + } + expected_mtime = '%f' % os.path.getmtime(f.name) + + mock_conn = mock.Mock() + mock_conn.put_object.return_value = '' + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + with mock.patch.object(s, '_upload_segment_job') as mock_job: + mock_job.return_value = { + 'action': 'upload_segment', + 'success': True} + + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='テスト/dummy.dat', + options=dict(s._options, + segment_size=10, + leave_segments=True)) + + mtime = r['headers']['x-object-meta-mtime'] + self.assertEqual(expected_mtime, mtime) + del r['headers']['x-object-meta-mtime'] + + self.assertEqual( + 'test_c_segments/%E3%83%86%E3%82%B9%E3%83%88/dummy.dat/' + + '%s/30/10/' % mtime, r['headers']['x-object-manifest']) + del r['headers']['x-object-manifest'] + + self.assertEqual(r['path'], f.name) + del r['path'] + + self.assertEqual(r, expected_r) + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with('test_c', 'テスト/dummy.dat', + '', + content_length=0, + headers={}, + response_dict={}) + + def test_upload_segment_job(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 10) + f.write(b'b' * 10) + f.write(b'c' * 10) + f.flush() + + # run read() when put_object is called to calculate md5sum + def _consuming_conn(*a, **kw): + contents = a[2] + contents.read() # Force md5 calculation + return contents.get_md5sum() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _consuming_conn + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + expected_r = { + 'action': 'upload_segment', + 'for_container': 'test_c', + 'for_object': 'test_o', + 'segment_index': 2, + 'segment_size': 10, + 'segment_location': '/test_c_segments/test_s_1', + 'log_line': 'test_o segment 2', + 'success': True, + 'response_dict': {}, + 'segment_etag': md5(b'b' * 10).hexdigest(), + 'attempts': 2, + } + + s = SwiftService() + with self.assert_open_results_are_closed(): + r = s._upload_segment_job(conn=mock_conn, + path=f.name, + container='test_c', + segment_name='test_s_1', + segment_start=10, + segment_size=10, + segment_index=2, + obj_name='test_o', + options={'segment_container': None, + 'checksum': True}) + + self.assertEqual(r, expected_r) + + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) + contents = mock_conn.put_object.call_args[0][2] + self.assertIsInstance(contents, utils.LengthWrapper) + self.assertEqual(len(contents), 10) + + def test_upload_stream_segment(self): + common_params = { + 'segment_container': 'segments', + 'segment_name': 'test_stream_2', + 'container': 'test_stream', + 'object': 'stream_object', + } + tests = [ + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1024}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 512}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 512).hexdigest()}}, + # 0-sized segment should not be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 0}, + 'put_object_args': {}, + 'expected': { + 'complete': True}}, + # 0-sized objects should be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 0, + 'content_size': 0}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'').hexdigest()}}, + # Test boundary conditions + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 1023}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 1023).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 2047}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 2047).hexdigest()}}, + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1025}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + ] + + for test_args in tests: + params = test_args['test_params'] + stream = test_utils.FakeStream(params['content_size']) + segment_size = params['segment_size'] + segment_index = params['segment_index'] + + def _fake_put_object(*args, **kwargs): + contents = args[2] + # Consume and compute md5 + return md5(contents).hexdigest() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _fake_put_object + + s = SwiftService() + resp = s._upload_stream_segment( + conn=mock_conn, + container=common_params['container'], + object_name=common_params['object'], + segment_container=common_params['segment_container'], + segment_name=common_params['segment_name'], + segment_size=segment_size, + segment_index=segment_index, + headers={}, + fd=stream) + expected_args = test_args['expected'] + put_args = test_args['put_object_args'] + expected_response = { + 'segment_size': min(len(stream), segment_size), + 'complete': expected_args['complete'], + 'success': True, + } + if len(stream) or segment_index == 0: + segment_location = '/%s/%s' % (put_args['container'], + put_args['object']) + expected_response.update( + {'segment_index': segment_index, + 'segment_location': segment_location, + 'segment_etag': expected_args['segment_etag'], + 'for_object': common_params['object']}) + mock_conn.put_object.assert_called_once_with( + put_args['container'], + put_args['object'], + mock.ANY, + content_length=min(len(stream), segment_size), + headers={'etag': expected_args['segment_etag']}, + response_dict=mock.ANY) + else: + self.assertEqual([], mock_conn.put_object.mock_calls) + expected_response.update( + {'segment_index': None, + 'segment_location': None, + 'segment_etag': None}) + self.assertEqual(expected_response, resp) + + def test_etag_mismatch_with_ignore_checksum(self): + def _consuming_conn(*a, **kw): + contents = a[2] + contents.read() # Force md5 calculation + return 'badresponseetag' + + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 10) + f.write(b'b' * 10) + f.write(b'c' * 10) + f.flush() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _consuming_conn + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_segment_job(conn=mock_conn, + path=f.name, + container='test_c', + segment_name='test_s_1', + segment_start=10, + segment_size=10, + segment_index=2, + obj_name='test_o', + options={'segment_container': None, + 'checksum': False}) + + self.assertIsNone(r.get('error')) + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) + contents = mock_conn.put_object.call_args[0][2] + # Check that md5sum is not calculated. + self.assertEqual(contents.get_md5sum(), '') + + def test_upload_segment_job_etag_mismatch(self): + def _consuming_conn(*a, **kw): + contents = a[2] + contents.read() # Force md5 calculation + return 'badresponseetag' + + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 10) + f.write(b'b' * 10) + f.write(b'c' * 10) + f.flush() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _consuming_conn + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + with self.assert_open_results_are_closed(): + r = s._upload_segment_job(conn=mock_conn, + path=f.name, + container='test_c', + segment_name='test_s_1', + segment_start=10, + segment_size=10, + segment_index=2, + obj_name='test_o', + options={'segment_container': None, + 'checksum': True}) + + self.assertIn('md5 mismatch', str(r.get('error'))) + + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) + contents = mock_conn.put_object.call_args[0][2] + self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest()) + + def test_upload_object_job_file(self): + # Uploading a file results in the file object being wrapped in a + # LengthWrapper. This test sets the options in such a way that much + # of _upload_object_job is skipped bringing the critical path down + # to around 60 lines to ease testing. + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + expected_r = { + 'action': 'upload_object', + 'attempts': 2, + 'container': 'test_c', + 'headers': {}, + 'large_object': False, + 'object': 'test_o', + 'response_dict': {}, + 'status': 'uploaded', + 'success': True, + } + expected_mtime = '%f' % os.path.getmtime(f.name) + + # run read() when put_object is called to calculate md5sum + # md5sum is verified in _upload_object_job. + def _consuming_conn(*a, **kw): + contents = a[2] + contents.read() # Force md5 calculation + return contents.get_md5sum() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _consuming_conn + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + with self.assert_open_results_are_closed(): + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options=dict(s._options, + leave_segments=True)) + + mtime = r['headers']['x-object-meta-mtime'] + self.assertEqual(expected_mtime, mtime) + del r['headers']['x-object-meta-mtime'] + + self.assertEqual(r['path'], f.name) + del r['path'] + + self.assertEqual(r, expected_r) + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with('test_c', 'test_o', + mock.ANY, + content_length=30, + headers={}, + response_dict={}) + contents = mock_conn.put_object.call_args[0][2] + self.assertIsInstance(contents, utils.LengthWrapper) + self.assertEqual(len(contents), 30) + + @mock.patch('swiftclient.service.time', return_value=1400000000) + def test_upload_object_job_stream(self, time_mock): + # Streams are wrapped as ReadableToIterable + with tempfile.TemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + f.seek(0) + expected_r = { + 'action': 'upload_object', + 'attempts': 2, + 'container': 'test_c', + 'headers': {}, + 'large_object': False, + 'object': 'test_o', + 'response_dict': {}, + 'status': 'uploaded', + 'success': True, + 'path': None, + } + expected_mtime = 1400000000 + + mock_conn = mock.Mock() + mock_conn.put_object.return_value = '' + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f, + obj='test_o', + options=dict(s._options, + leave_segments=True)) + + mtime = float(r['headers']['x-object-meta-mtime']) + self.assertEqual(mtime, expected_mtime) + del r['headers']['x-object-meta-mtime'] + + self.assertEqual(r, expected_r) + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with('test_c', 'test_o', + mock.ANY, + content_length=None, + headers={}, + response_dict={}) + contents = mock_conn.put_object.call_args[0][2] + self.assertIsInstance(contents, utils.ReadableToIterable) + self.assertEqual(contents.chunk_size, 65536) + # next retrieves the first chunk of the stream or len(chunk_size) + # or less, it also forces the md5 to be calculated. + self.assertEqual(next(contents), b'a' * 30) + self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest()) + + def test_upload_object_job_etag_mismatch(self): + # The etag test for both streams and files use the same code + # so only one test should be needed. + def _consuming_conn(*a, **kw): + contents = a[2] + contents.read() # Force md5 calculation + return 'badresponseetag' + + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _consuming_conn + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options=dict(s._options, + leave_segments=True)) + + self.assertIs(r['success'], False) + self.assertIn('md5 mismatch', str(r.get('error'))) + + self.assertEqual(mock_conn.put_object.call_count, 1) + expected_headers = {'x-object-meta-mtime': mock.ANY} + mock_conn.put_object.assert_called_with('test_c', 'test_o', + mock.ANY, + content_length=30, + headers=expected_headers, + response_dict={}) + + contents = mock_conn.put_object.call_args[0][2] + self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest()) + + def test_upload_object_job_identical_etag(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-length': 30, + 'etag': md5(b'a' * 30).hexdigest()} + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options={'changed': False, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 0}) + + self.assertIsNone(r.get('error')) + self.assertIs(True, r['success']) + self.assertEqual(r.get('status'), 'skipped-identical') + self.assertEqual(mock_conn.put_object.call_count, 0) + self.assertEqual(mock_conn.head_object.call_count, 1) + mock_conn.head_object.assert_called_with('test_c', 'test_o') + + def test_upload_object_job_identical_slo_with_nesting(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + seg_etag = md5(b'a' * 10).hexdigest() + submanifest = "[%s]" % ",".join( + ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) + submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() + manifest = "[%s]" % ",".join([ + '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' + '"bytes":20,"hash":"%s"}' % submanifest_etag, + '{"bytes":10,"hash":"%s"}' % seg_etag]) + + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()} + mock_conn.get_object.side_effect = [ + ({}, manifest.encode('ascii')), + ({}, submanifest.encode('ascii'))] + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options={'changed': False, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + + self.assertIsNone(r.get('error')) + self.assertIs(True, r['success']) + self.assertEqual('skipped-identical', r.get('status')) + self.assertEqual(0, mock_conn.put_object.call_count) + self.assertEqual([mock.call('test_c', 'test_o')], + mock_conn.head_object.mock_calls) + self.assertEqual([ + mock.call('test_c', 'test_o', + query_string='multipart-manifest=get'), + mock.call('test_c_segments', 'test_sub_slo', + query_string='multipart-manifest=get'), + ], mock_conn.get_object.mock_calls) + + def test_upload_object_job_identical_dlo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + segment_etag = md5(b'a' * 10).hexdigest() + + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'x-object-manifest': 'test_c_segments/test_o/prefix', + 'content-length': 30, + 'etag': md5(segment_etag.encode('ascii') * 3).hexdigest()} + mock_conn.get_container.side_effect = [ + (None, [{"bytes": 10, "hash": segment_etag, + "name": "test_o/prefix/00"}, + {"bytes": 10, "hash": segment_etag, + "name": "test_o/prefix/01"}]), + (None, [{"bytes": 10, "hash": segment_etag, + "name": "test_o/prefix/02"}]), + (None, {})] + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='test_o', + options={'changed': False, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + + self.assertIsNone(r.get('error')) + self.assertIs(True, r['success']) + self.assertEqual('skipped-identical', r.get('status')) + self.assertEqual(0, mock_conn.put_object.call_count) + self.assertEqual(1, mock_conn.head_object.call_count) + self.assertEqual(3, mock_conn.get_container.call_count) + mock_conn.head_object.assert_called_with('test_c', 'test_o') + expected = [ + mock.call('test_c_segments', prefix='test_o/prefix', + marker='', delimiter=None, headers={}), + mock.call('test_c_segments', prefix='test_o/prefix', + marker="test_o/prefix/01", delimiter=None, + headers={}), + mock.call('test_c_segments', prefix='test_o/prefix', + marker="test_o/prefix/02", delimiter=None, + headers={}), + ] + mock_conn.get_container.assert_has_calls(expected) + + def test_make_upload_objects(self): + check_names_pseudo_to_expected = { + (('/absolute/file/path',), ''): ['absolute/file/path'], + (('relative/file/path',), ''): ['relative/file/path'], + (('/absolute/file/path',), '/absolute/pseudo/dir'): [ + 'absolute/pseudo/dir/absolute/file/path'], + (('/absolute/file/path',), 'relative/pseudo/dir'): [ + 'relative/pseudo/dir/absolute/file/path'], + (('relative/file/path',), '/absolute/pseudo/dir'): [ + 'absolute/pseudo/dir/relative/file/path'], + (('relative/file/path',), 'relative/pseudo/dir'): [ + 'relative/pseudo/dir/relative/file/path'], + } + errors = [] + for (filenames, pseudo_folder), expected in \ + check_names_pseudo_to_expected.items(): + actual = SwiftService._make_upload_objects( + filenames, pseudo_folder=pseudo_folder) + try: + self.assertEqual(expected, [o.object_name for o in actual]) + except AssertionError as e: + msg = 'given (%r, %r) expected %r, got %s' % ( + filenames, pseudo_folder, expected, e) + errors.append(msg) + self.assertFalse(errors, "\nERRORS:\n%s" % '\n'.join(errors)) + + def test_create_dir_marker_job_unchanged(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'application/directory', + 'content-length': '0', + 'x-object-meta-mtime': '1.234000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.getmtime', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + path='test', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': 'test', + 'headers': {'x-object-meta-mtime': '1.234000'}, + # NO response dict! + 'success': True, + }, r) + self.assertEqual([], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_unchanged_old_type(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'text/directory', + 'content-length': '0', + 'x-object-meta-mtime': '1.000000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + # NO response dict! + 'success': True, + }, r) + self.assertEqual([], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_overwrites_bad_type(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'text/plain', + 'content-length': '0', + 'x-object-meta-mtime': '1.000000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + 'response_dict': {}, + 'success': True, + }, r) + self.assertEqual([mock.call( + 'test_c', 'test_o', '', + content_length=0, + content_type='application/directory', + headers={'x-object-meta-mtime': '1.000000'}, + response_dict={})], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_missing(self): + mock_conn = mock.Mock() + mock_conn.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + 'response_dict': {}, + 'success': True, + }, r) + self.assertEqual([mock.call( + 'test_c', 'test_o', '', + content_length=0, + content_type='application/directory', + headers={'x-object-meta-mtime': '1.000000'}, + response_dict={})], mock_conn.put_object.mock_calls) + + +class TestServiceDownload(_TestServiceBase): + + def setUp(self): + super(TestServiceDownload, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + self.opts['no_download'] = True + self.obj_content = b'c' * 10 + self.obj_etag = md5(self.obj_content).hexdigest() + self.obj_len = len(self.obj_content) + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': 'download_object', # Should always be download_object + 'container': 'test_c', + 'object': 'test_o', + 'attempts': 2, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'success': None # Should be a bool + } + + def _readbody(self): + yield self.obj_content + + @mock.patch('swiftclient.service.SwiftService.list') + @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') + @mock.patch('swiftclient.service.interruptable_as_completed') + def test_download_container_job(self, as_comp, sub_page, service_list): + """ + Check that paged downloads work correctly + """ + obj_count = [0] + + def make_counting_generator(object_to_yield, total_count): + # maintain a counter of objects yielded + count = [0] + + def counting_generator(): + while count[0] < 10: + yield object_to_yield + count[0] += 1 + total_count[0] += 1 + return counting_generator() + + obj_count_on_sub_page_call = [] + sub_page_call_count = [0] + + def fake_sub_page(*args): + # keep a record of obj_count when this function is called + obj_count_on_sub_page_call.append(obj_count[0]) + sub_page_call_count[0] += 1 + if sub_page_call_count[0] < 3: + return range(0, 10) + return None + + sub_page.side_effect = fake_sub_page + + r = Mock(spec=Future) + r.result.return_value = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + as_comp.side_effect = [ + make_counting_generator(r, obj_count), + make_counting_generator(r, obj_count) + ] + + s = SwiftService() + down_gen = s._download_container('test_c', self.opts) + results = list(down_gen) + self.assertEqual(20, len(results)) + self.assertEqual(2, as_comp.call_count) + self.assertEqual(3, sub_page_call_count[0]) + self.assertEqual([0, 7, 17], obj_count_on_sub_page_call) + + @mock.patch('swiftclient.service.SwiftService.list') + @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') + @mock.patch('swiftclient.service.interruptable_as_completed') + def test_download_container_job_error( + self, as_comp, sub_page, service_list): + """ + Check that paged downloads work correctly + """ + class BoomError(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + def _make_result(): + r = Mock(spec=Future) + r.result.return_value = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + return r + + as_comp.side_effect = [ + + ] + # We need Futures here because the error will cause a call to .cancel() + sub_page_effects = [ + [_make_result() for _ in range(0, 10)], + BoomError('Go Boom') + ] + sub_page.side_effect = sub_page_effects + # ...but we must also mock the returns to as_completed + as_comp.side_effect = [ + [_make_result() for _ in range(0, 10)] + ] + + s = SwiftService() + self.assertRaises( + BoomError, + lambda: list(s._download_container('test_c', self.opts)) + ) + # This was an unknown error, so make sure we attempt to cancel futures + for spe in sub_page_effects[0]: + spe.cancel.assert_called_once_with() + self.assertEqual(1, as_comp.call_count) + + # Now test ClientException + sub_page_effects = [ + [_make_result() for _ in range(0, 10)], + ClientException('Go Boom') + ] + sub_page.side_effect = sub_page_effects + as_comp.reset_mock() + as_comp.side_effect = [ + [_make_result() for _ in range(0, 10)], + ] + self.assertRaises( + ClientException, + lambda: list(s._download_container('test_c', self.opts)) + ) + # This was a ClientException, so make sure we don't cancel futures + for spe in sub_page_effects[0]: + self.assertFalse(spe.cancel.called) + self.assertEqual(1, as_comp.call_count) + + def test_download_object_job(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + + def test_download_object_job_with_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': '1454113727.682512'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + mock_utime.assert_called_once_with( + 'test_o', (1454113727.682512, 1454113727.682512)) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + + def test_download_object_job_bad_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': 'foo'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + self.assertEqual(0, len(mock_utime.mock_calls)) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + + def test_download_object_job_ignore_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': '1454113727.682512'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + _opts['ignore_mtime'] = True + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + self.assertEqual([], mock_utime.mock_calls) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + + def test_download_object_job_exception(self): + mock_conn = self._get_mock_connection() + mock_conn.get_object = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'success': False, + 'error': self.exc, + 'error_timestamp': mock.ANY, + 'traceback': mock.ANY + }) + + s = SwiftService() + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', self.opts) + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + + def test_download(self): + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'test', + self.opts) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'test') + self.assertEqual(resp['path'], 'test') + + @mock.patch('swiftclient.service.interruptable_as_completed') + @mock.patch('swiftclient.service.SwiftService._download_container') + @mock.patch('swiftclient.service.SwiftService._download_object_job') + def test_download_with_objects_empty(self, mock_down_obj, + mock_down_cont, mock_as_comp): + fake_future = Future() + fake_future.set_result(1) + mock_as_comp.return_value = [fake_future] + service = SwiftService() + next(service.download('c', [], self.opts), None) + mock_down_obj.assert_not_called() + mock_down_cont.assert_not_called() + + next(service.download('c', options=self.opts), None) + self.assertTrue(mock_down_cont.called) + + def test_download_with_output_dir(self): + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['out_directory'] = 'temp_dir' + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'temp_dir/example/test') + + def test_download_with_remove_prefix(self): + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example/' + options['remove_prefix'] = True + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_remove_prefix_and_remove_slashes(self): + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['remove_prefix'] = True + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_output_dir_and_remove_prefix(self): + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['out_directory'] = 'new/dir' + options['remove_prefix'] = True + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'new/dir/test') + + def test_download_object_job_skip_identical(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + + err = swiftclient.ClientException('Object GET failed', + http_status=304) + + def fake_get(*args, **kwargs): + kwargs['response_dict']['headers'] = {} + raise err + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = fake_get + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': False, + 'error': err, + 'response_dict': {'headers': {}}, + 'path': 'test_o', + 'pseudodir': False, + 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + } + + s = SwiftService() + r = s._download_object_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, + 'header': {}, + 'yes_all': False, + 'skip_identical': True}) + self.assertEqual(r, expected_r) + + self.assertEqual(mock_conn.get_object.call_count, 1) + mock_conn.get_object.assert_called_with( + 'test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': md5(b'a' * 30).hexdigest()}, + query_string='multipart-manifest=get', + response_dict=expected_r['response_dict']) + + def test_download_object_job_skip_identical_dlo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + on_disk_md5 = md5(b'a' * 30).hexdigest() + segment_md5 = md5(b'a' * 10).hexdigest() + + mock_conn = mock.Mock() + mock_conn.get_object.return_value = ( + {'x-object-manifest': 'test_c_segments/test_o/prefix'}, [b'']) + mock_conn.get_container.side_effect = [ + (None, [{'name': 'test_o/prefix/1', + 'bytes': 10, 'hash': segment_md5}, + {'name': 'test_o/prefix/2', + 'bytes': 10, 'hash': segment_md5}]), + (None, [{'name': 'test_o/prefix/3', + 'bytes': 10, 'hash': segment_md5}]), + (None, [])] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': False, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + } + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, + 'header': {}, + 'yes_all': False, + 'skip_identical': True}) + + err = r.pop('error') + self.assertEqual("Large object is identical", err.msg) + self.assertEqual(304, err.http_status) + + self.assertEqual(r, expected_r) + + self.assertEqual(mock_conn.get_object.call_count, 1) + mock_conn.get_object.assert_called_with( + 'test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict=expected_r['response_dict']) + self.assertEqual(mock_conn.get_container.mock_calls, [ + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='', + headers={}), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/2', + headers={}), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/3', + headers={})]) + + def test_download_object_job_skip_identical_nested_slo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + on_disk_md5 = md5(b'a' * 30).hexdigest() + + seg_etag = md5(b'a' * 10).hexdigest() + submanifest = "[%s]" % ",".join( + ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) + submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() + manifest = "[%s]" % ",".join([ + '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' + '"bytes":20,"hash":"%s"}' % submanifest_etag, + '{"bytes":10,"hash":"%s"}' % seg_etag]) + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = [ + ({'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()}, + [manifest.encode('ascii')]), + ({'x-static-large-object': True, + 'content-length': 20, + 'etag': submanifest_etag}, + submanifest.encode('ascii'))] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': False, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY + } + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, + 'header': {}, + 'yes_all': False, + 'skip_identical': True}) + + err = r.pop('error') + self.assertEqual("Large object is identical", err.msg) + self.assertEqual(304, err.http_status) + + self.assertEqual(r, expected_r) + self.assertEqual(mock_conn.get_object.mock_calls, [ + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict={}), + mock.call('test_c_segments', + 'test_sub_slo', + query_string='multipart-manifest=get')]) + + def test_download_object_job_skip_identical_diff_dlo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.write(b'b') + f.flush() + on_disk_md5 = md5(b'a' * 30 + b'b').hexdigest() + segment_md5 = md5(b'a' * 10).hexdigest() + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = [ + ({'x-object-manifest': 'test_c_segments/test_o/prefix'}, + [b'']), + ({'x-object-manifest': 'test_c_segments/test_o/prefix'}, + [b'a' * 30])] + mock_conn.get_container.side_effect = [ + (None, [{'name': 'test_o/prefix/1', + 'bytes': 10, 'hash': segment_md5}, + {'name': 'test_o/prefix/2', + 'bytes': 10, 'hash': segment_md5}]), + (None, [{'name': 'test_o/prefix/3', + 'bytes': 10, 'hash': segment_md5}]), + (None, [])] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + type(mock_conn).auth_end_time = mock.PropertyMock(return_value=14) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': True, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'read_length': 30, + 'attempts': 2, + 'start_time': 0, + 'headers_receipt': 1, + 'finish_time': 2, + 'auth_end_time': mock_conn.auth_end_time, + } + + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True + s = SwiftService() + with mock.patch('swiftclient.service.time', side_effect=range(3)): + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job( + conn=mock_conn, + container='test_c', + obj='test_o', + options=options) + + self.assertEqual(r, expected_r) + + self.assertEqual(mock_conn.get_container.mock_calls, [ + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='', + headers={}), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/2', + headers={}), + mock.call('test_c_segments', + delimiter=None, + prefix='test_o/prefix', + marker='test_o/prefix/3', + headers={})]) + self.assertEqual(mock_conn.get_object.mock_calls, [ + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict={}), + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + response_dict={})]) + + def test_download_object_job_skip_identical_diff_nested_slo(self): + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 29) + f.flush() + on_disk_md5 = md5(b'a' * 29).hexdigest() + + seg_etag = md5(b'a' * 10).hexdigest() + submanifest = "[%s]" % ",".join( + ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) + submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() + manifest = "[%s]" % ",".join([ + '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' + '"bytes":20,"hash":"%s"}' % submanifest_etag, + '{"bytes":10,"hash":"%s"}' % seg_etag]) + + mock_conn = mock.Mock() + mock_conn.get_object.side_effect = [ + ({'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()}, + [manifest.encode('ascii')]), + ({'x-static-large-object': True, + 'content-length': 20, + 'etag': submanifest_etag}, + submanifest.encode('ascii')), + ({'x-static-large-object': True, + 'content-length': 30, + 'etag': md5(submanifest_etag.encode('ascii') + + seg_etag.encode('ascii')).hexdigest()}, + [b'a' * 30])] + + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + type(mock_conn).auth_end_time = mock.PropertyMock(return_value=14) + expected_r = { + 'action': 'download_object', + 'container': 'test_c', + 'object': 'test_o', + 'success': True, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'read_length': 30, + 'attempts': 2, + 'start_time': 0, + 'headers_receipt': 1, + 'finish_time': 2, + 'auth_end_time': mock_conn.auth_end_time, + } + + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True + s = SwiftService() + with mock.patch('swiftclient.service.time', side_effect=range(3)): + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + r = s._download_object_job( + conn=mock_conn, + container='test_c', + obj='test_o', + options=options) + + self.assertEqual(r, expected_r) + self.assertEqual(mock_conn.get_object.mock_calls, [ + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + query_string='multipart-manifest=get', + response_dict={}), + mock.call('test_c_segments', + 'test_sub_slo', + query_string='multipart-manifest=get'), + mock.call('test_c', + 'test_o', + resp_chunk_size=65536, + headers={'If-None-Match': on_disk_md5}, + response_dict={})]) + + +class TestServicePost(_TestServiceBase): + + def setUp(self): + super(TestServicePost, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + + @mock.patch('swiftclient.service.MultiThreadingManager') + @mock.patch('swiftclient.service.ResultsIterator') + def test_object_post(self, res_iter, thread_manager): + """ + Check post method translates strings and objects to _post_object_job + calls correctly + """ + tm_instance = Mock() + thread_manager.return_value = tm_instance + + self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) + spo = swiftclient.service.SwiftPostObject( + "test_spo", + {'meta': ["meta1:test2"], "header": ["hdr1:test2"]}) + + SwiftService().post('test_c', ['test_o', spo], self.opts) + + calls = [ + mock.call( + SwiftService._post_object_job, 'test_c', 'test_o', + { + "X-Object-Meta-Meta1": "test1", + "Hdr1": "test1"}, + {}), + mock.call( + SwiftService._post_object_job, 'test_c', 'test_spo', + { + "X-Object-Meta-Meta1": "test2", + "Hdr1": "test2"}, + {}), + ] + tm_instance.object_uu_pool.submit.assert_has_calls(calls) + self.assertEqual( + tm_instance.object_uu_pool.submit.call_count, len(calls)) + + res_iter.assert_called_with( + [tm_instance.object_uu_pool.submit()] * len(calls)) + + +class TestServiceCopy(_TestServiceBase): + + def setUp(self): + super(TestServiceCopy, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + + @mock.patch('swiftclient.service.MultiThreadingManager') + @mock.patch('swiftclient.service.interruptable_as_completed') + def test_object_copy(self, inter_compl, thread_manager): + """ + Check copy method translates strings and objects to _copy_object_job + calls correctly + """ + tm_instance = Mock() + thread_manager.return_value = tm_instance + + self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) + sco = swiftclient.service.SwiftCopyObject( + "test_sco", + options={'meta': ["meta1:test2"], "header": ["hdr1:test2"], + 'destination': "/cont_new/test_sco"}) + + res = SwiftService().copy('test_c', ['test_o', sco], self.opts) + res = list(res) + + calls = [ + mock.call( + SwiftService._create_container_job, 'cont_new', headers={}), + ] + tm_instance.container_pool.submit.assert_has_calls(calls, + any_order=True) + self.assertEqual( + tm_instance.container_pool.submit.call_count, len(calls)) + + calls = [ + mock.call( + SwiftService._copy_object_job, 'test_c', 'test_o', + None, + { + "X-Object-Meta-Meta1": "test1", + "Hdr1": "test1"}, + False), + mock.call( + SwiftService._copy_object_job, 'test_c', 'test_sco', + '/cont_new/test_sco', + { + "X-Object-Meta-Meta1": "test2", + "Hdr1": "test2"}, + False), + ] + tm_instance.object_uu_pool.submit.assert_has_calls(calls) + self.assertEqual( + tm_instance.object_uu_pool.submit.call_count, len(calls)) + + inter_compl.assert_called_with( + [tm_instance.object_uu_pool.submit()] * len(calls)) + + def test_object_copy_fail_dest(self): + """ + Destination in incorrect format and destination with object + used when multiple objects are copied raises SwiftError + """ + with self.assertRaises(SwiftError): + list(SwiftService().copy('test_c', ['test_o'], + {'destination': 'cont'})) + with self.assertRaises(SwiftError): + list(SwiftService().copy('test_c', ['test_o', 'test_o2'], + {'destination': '/cont/obj'})) diff -Nru python-swiftclient-3.8.1/test/unit/test_shell.py python-swiftclient-3.9.0/test/unit/test_shell.py --- python-swiftclient-3.8.1/test/unit/test_shell.py 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/unit/test_shell.py 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,3422 @@ +# Copyright (c) 2014 Christian Schwede +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import unicode_literals + +import contextlib +from genericpath import getmtime +import getpass +import hashlib +import json +import logging +import mock +import os +import tempfile +import unittest +import textwrap +from time import localtime, mktime, strftime, strptime + +import six +import sys + +import swiftclient +from swiftclient.service import SwiftError +import swiftclient.shell +import swiftclient.utils + +from os.path import basename, dirname +from .utils import ( + CaptureOutput, fake_get_auth_keystone, + FakeKeystone, StubResponse, MockHttpTest) +from swiftclient.utils import ( + EMPTY_ETAG, EXPIRES_ISO8601_FORMAT, + SHORT_EXPIRES_ISO8601_FORMAT, TIME_ERRMSG) + +try: + from requests.packages.urllib3.exceptions import InsecureRequestWarning +except ImportError: + InsecureRequestWarning = None + +if six.PY2: + BUILTIN_OPEN = '__builtin__.open' +else: + BUILTIN_OPEN = 'builtins.open' + +mocked_os_environ = { + 'ST_AUTH': 'http://localhost:8080/auth/v1.0', + 'ST_USER': 'test:tester', + 'ST_KEY': 'testing' +} +clean_os_environ = {} +environ_prefixes = ('ST_', 'OS_') +for key in os.environ: + if any(key.startswith(m) for m in environ_prefixes): + clean_os_environ[key] = '' + + +def _make_args(cmd, opts, os_opts, separator='-', flags=None, cmd_args=None): + """ + Construct command line arguments for given options. + """ + args = [""] + flags = flags or [] + for k, v in opts.items(): + args.append("--" + k.replace("_", "-")) + if v is not None: + args.append(v) + for k, v in os_opts.items(): + args.append("--os" + separator + k.replace("_", separator)) + if v is not None: + args.append(v) + for flag in flags: + args.append('--%s' % flag) + if cmd: + args.append(cmd) + if cmd_args: + args.extend(cmd_args) + return args + + +def _make_env(opts, os_opts): + """ + Construct a dict of environment variables for given options. + """ + env = {} + for k, v in opts.items(): + key = 'ST_' + k.upper().replace('-', '_') + env[key] = v + for k, v in os_opts.items(): + key = 'OS_' + k.upper().replace('-', '_') + env[key] = v + return env + + +def _make_cmd(cmd, opts, os_opts, use_env=False, flags=None, cmd_args=None): + flags = flags or [] + if use_env: + # set up fake environment variables and make a minimal command line + env = _make_env(opts, os_opts) + args = _make_args(cmd, {}, {}, separator='-', flags=flags, + cmd_args=cmd_args) + else: + # set up empty environment and make full command line + env = {} + args = _make_args(cmd, opts, os_opts, separator='-', flags=flags, + cmd_args=cmd_args) + return args, env + + +@contextlib.contextmanager +def patch_disable_warnings(): + if InsecureRequestWarning is None: + # If InsecureRequestWarning isn't available, disbale_warnings won't + # be either; they both came in with + # https://github.com/requests/requests/commit/811ee4e and left again + # in https://github.com/requests/requests/commit/8e17600 + yield None + else: + with mock.patch('requests.packages.urllib3.disable_warnings') \ + as patched: + yield patched + + +@mock.patch.dict(os.environ, mocked_os_environ) +class TestShell(unittest.TestCase): + def setUp(self): + super(TestShell, self).setUp() + tmpfile = tempfile.NamedTemporaryFile(delete=False) + self.tmpfile = tmpfile.name + + def tearDown(self): + try: + os.remove(self.tmpfile) + except OSError: + pass + super(TestShell, self).tearDown() + + @mock.patch('swiftclient.service.Connection') + def test_stat_account(self, connection): + argv = ["", "stat"] + return_headers = { + 'x-account-container-count': '1', + 'x-account-object-count': '2', + 'x-account-bytes-used': '3', + 'content-length': 0, + 'date': ''} + connection.return_value.head_account.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + 'Containers: 1\n' + ' Objects: 2\n' + ' Bytes: 3\n') + + @mock.patch('swiftclient.service.Connection') + def test_stat_account_with_headers(self, connection): + argv = ["", "stat", "-H", "Skip-Middleware: Test"] + return_headers = { + 'x-account-container-count': '1', + 'x-account-object-count': '2', + 'x-account-bytes-used': '3', + 'content-length': 0, + 'date': ''} + connection.return_value.head_account.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + 'Containers: 1\n' + ' Objects: 2\n' + ' Bytes: 3\n') + self.assertEqual(connection.return_value.head_account.mock_calls, [ + mock.call(headers={'Skip-Middleware': 'Test'})]) + + @mock.patch('swiftclient.service.Connection') + def test_stat_container(self, connection): + return_headers = { + 'x-container-object-count': '1', + 'x-container-bytes-used': '2', + 'x-container-read': 'test2:tester2', + 'x-container-write': 'test3:tester3', + 'x-container-sync-to': 'other', + 'x-container-sync-key': 'secret', + } + argv = ["", "stat", "container"] + connection.return_value.head_container.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + 'Container: container\n' + ' Objects: 1\n' + ' Bytes: 2\n' + ' Read ACL: test2:tester2\n' + 'Write ACL: test3:tester3\n' + ' Sync To: other\n' + ' Sync Key: secret\n') + + @mock.patch('swiftclient.service.Connection') + def test_stat_container_with_headers(self, connection): + return_headers = { + 'x-container-object-count': '1', + 'x-container-bytes-used': '2', + 'x-container-read': 'test2:tester2', + 'x-container-write': 'test3:tester3', + 'x-container-sync-to': 'other', + 'x-container-sync-key': 'secret', + } + argv = ["", "stat", "container", "-H", "Skip-Middleware: Test"] + connection.return_value.head_container.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + 'Container: container\n' + ' Objects: 1\n' + ' Bytes: 2\n' + ' Read ACL: test2:tester2\n' + 'Write ACL: test3:tester3\n' + ' Sync To: other\n' + ' Sync Key: secret\n') + self.assertEqual(connection.return_value.head_container.mock_calls, [ + mock.call('container', headers={'Skip-Middleware': 'Test'})]) + + @mock.patch('swiftclient.service.Connection') + def test_stat_object(self, connection): + return_headers = { + 'x-object-manifest': 'manifest', + 'etag': 'md5', + 'last-modified': 'yesterday', + 'content-type': 'text/plain', + 'content-length': 42, + } + argv = ["", "stat", "container", "object"] + connection.return_value.head_object.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + ' Container: container\n' + ' Object: object\n' + ' Content Type: text/plain\n' + 'Content Length: 42\n' + ' Last Modified: yesterday\n' + ' ETag: md5\n' + ' Manifest: manifest\n') + + @mock.patch('swiftclient.service.Connection') + def test_stat_object_with_headers(self, connection): + return_headers = { + 'x-object-manifest': 'manifest', + 'etag': 'md5', + 'last-modified': 'yesterday', + 'content-type': 'text/plain', + 'content-length': 42, + } + argv = ["", "stat", "container", "object", + "-H", "Skip-Middleware: Test"] + connection.return_value.head_object.return_value = return_headers + connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEqual(output.out, + ' Account: AUTH_account\n' + ' Container: container\n' + ' Object: object\n' + ' Content Type: text/plain\n' + 'Content Length: 42\n' + ' Last Modified: yesterday\n' + ' ETag: md5\n' + ' Manifest: manifest\n') + self.assertEqual(connection.return_value.head_object.mock_calls, [ + mock.call('container', 'object', + headers={'Skip-Middleware': 'Test'})]) + + @mock.patch('swiftclient.service.Connection') + def test_list_json(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, [{'name': u'\u263A', 'some-custom-key': 'and value'}]], + [None, []], + ] + + argv = ["", "list", "--json"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={}), + mock.call(marker='container', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + + listing = [{'name': 'container'}, + {'name': u'\u263A', 'some-custom-key': 'and value'}] + expected = json.dumps(listing, sort_keys=True, indent=2) + '\n' + self.assertEqual(output.out, expected) + + @mock.patch('swiftclient.service.Connection') + def test_list_account(self, connection): + # Test account listing + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, []], + ] + + argv = ["", "list"] + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={}), + mock.call(marker='container', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + + self.assertEqual(output.out, 'container\n') + + @mock.patch('swiftclient.service.Connection') + def test_list_account_with_headers(self, connection): + # Test account listing + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, []], + ] + + argv = ["", "list", '-H', 'Skip-Custom-Middleware: True'] + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, + headers={'Skip-Custom-Middleware': 'True'}), + mock.call(marker='container', prefix=None, + headers={'Skip-Custom-Middleware': 'True'})] + connection.return_value.get_account.assert_has_calls(calls) + + self.assertEqual(output.out, 'container\n') + + @mock.patch('swiftclient.service.Connection') + def test_list_account_long(self, connection): + # Test account listing + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container', 'bytes': 0, 'count': 0}]], + [None, []], + ] + + argv = ["", "list", "--lh"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={}), + mock.call(marker='container', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + + self.assertEqual(output.out, + ' 0 0 1970-01-01 00:00:01 container\n' + ' 0 0\n') + + # Now test again, this time without returning metadata + connection.return_value.head_container.return_value = {} + + # Test account listing + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container', 'bytes': 0, 'count': 0}]], + [None, []], + ] + + argv = ["", "list", "--lh"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={}), + mock.call(marker='container', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + + self.assertEqual(output.out, + ' 0 0 ????-??-?? ??:??:?? container\n' + ' 0 0\n') + + def test_list_account_totals_error(self): + # No --lh provided: expect info message about incorrect --totals use + argv = ["", "list", "--totals"] + + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(output.err, + "Listing totals only works with -l or --lh.\n") + + @mock.patch('swiftclient.service.Connection') + def test_list_account_totals(self, connection): + + # Test account listing, only total count and size + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container1', 'bytes': 1, 'count': 2}, + {'name': 'container2', 'bytes': 2, 'count': 4}]], + [None, []], + ] + + argv = ["", "list", "--lh", "--totals"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + self.assertEqual(output.out, ' 6 3\n') + + @mock.patch('swiftclient.service.Connection') + def test_list_container(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object_a'}]], + [None, []], + ] + argv = ["", "list", "container"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [ + mock.call('container', marker='', + delimiter=None, prefix=None, headers={}), + mock.call('container', marker='object_a', + delimiter=None, prefix=None, headers={})] + connection.return_value.get_container.assert_has_calls(calls) + + self.assertEqual(output.out, 'object_a\n') + + # Test container listing with --long + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object_a', 'bytes': 0, + 'content_type': 'type/content', + 'last_modified': '123T456'}]], + [None, []], + ] + argv = ["", "list", "container", "--long"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [ + mock.call('container', marker='', + delimiter=None, prefix=None, headers={}), + mock.call('container', marker='object_a', + delimiter=None, prefix=None, headers={})] + connection.return_value.get_container.assert_has_calls(calls) + + self.assertEqual(output.out, + ' 0 123 456' + ' type/content object_a\n' + ' 0\n') + + @mock.patch('swiftclient.service.Connection') + def test_list_container_with_headers(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object_a'}]], + [None, []], + ] + argv = ["", "list", "container", "-H", "Skip-Middleware: Test"] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [ + mock.call('container', marker='', + delimiter=None, prefix=None, + headers={'Skip-Middleware': 'Test'}), + mock.call('container', marker='object_a', + delimiter=None, prefix=None, + headers={'Skip-Middleware': 'Test'})] + connection.return_value.get_container.assert_has_calls(calls) + + self.assertEqual(output.out, 'object_a\n') + + @mock.patch('swiftclient.service.makedirs') + @mock.patch('swiftclient.service.Connection') + def test_download(self, connection, makedirs): + objcontent = six.BytesIO(b'objcontent') + connection.return_value.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent), + ({'content-type': 'text/plain', + 'etag': EMPTY_ETAG}, + '') + ] + + # Test downloading whole container + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}]], + [None, [{'name': 'pseudo/'}]], + [None, []], + ] + connection.return_value.auth_end_time = 0 + connection.return_value.attempts = 0 + + with mock.patch(BUILTIN_OPEN) as mock_open: + argv = ["", "download", "container"] + swiftclient.shell.main(argv) + calls = [mock.call('container', 'object', + headers={}, resp_chunk_size=65536, + response_dict={}), + mock.call('container', 'pseudo/', + headers={}, resp_chunk_size=65536, + response_dict={})] + connection.return_value.get_object.assert_has_calls( + calls, any_order=True) + mock_open.assert_called_once_with('object', 'wb', 65536) + self.assertEqual([mock.call('pseudo')], makedirs.mock_calls) + makedirs.reset_mock() + + # Test downloading single object + objcontent = six.BytesIO(b'objcontent') + connection.return_value.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent) + ] + with mock.patch(BUILTIN_OPEN) as mock_open: + argv = ["", "download", "container", "object"] + swiftclient.shell.main(argv) + connection.return_value.get_object.assert_called_with( + 'container', 'object', headers={}, resp_chunk_size=65536, + response_dict={}) + mock_open.assert_called_with('object', 'wb', 65536) + self.assertEqual([], makedirs.mock_calls) + + # Test downloading without md5 checks + objcontent = six.BytesIO(b'objcontent') + connection.return_value.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent) + ] + with mock.patch(BUILTIN_OPEN) as mock_open, mock.patch( + 'swiftclient.service._SwiftReader') as sr: + argv = ["", "download", "container", "object", "--ignore-check"] + swiftclient.shell.main(argv) + connection.return_value.get_object.assert_called_with( + 'container', 'object', headers={}, resp_chunk_size=65536, + response_dict={}) + mock_open.assert_called_with('object', 'wb', 65536) + sr.assert_called_once_with('object', mock.ANY, mock.ANY, False) + self.assertEqual([], makedirs.mock_calls) + + # Test downloading single object to stdout + objcontent = six.BytesIO(b'objcontent') + connection.return_value.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent) + ] + with CaptureOutput() as output: + argv = ["", "download", "--output", "-", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual('objcontent', output.out) + + @mock.patch('swiftclient.service.shuffle') + @mock.patch('swiftclient.service.Connection') + def test_download_shuffle(self, connection, mock_shuffle): + # Test that the container and object lists are shuffled + mock_shuffle.side_effect = lambda l: l + connection.return_value.get_object.return_value = [ + {'content-type': 'text/plain', + 'etag': EMPTY_ETAG}, + ''] + + connection.return_value.get_container.side_effect = [ + (None, [{'name': 'object'}]), + (None, [{'name': 'pseudo/'}]), + (None, []), + ] + connection.return_value.auth_end_time = 0 + connection.return_value.attempts = 0 + connection.return_value.get_account.side_effect = [ + (None, [{'name': 'container'}]), + (None, []) + ] + + with mock.patch(BUILTIN_OPEN) as mock_open: + with mock.patch('swiftclient.service.makedirs') as mock_mkdir: + argv = ["", "download", "--all"] + swiftclient.shell.main(argv) + self.assertEqual(3, mock_shuffle.call_count) + mock_shuffle.assert_any_call(['container']) + mock_shuffle.assert_any_call(['object']) + mock_shuffle.assert_any_call(['pseudo/']) + mock_open.assert_called_once_with('container/object', 'wb', 65536) + self.assertEqual([ + mock.call('container'), + mock.call('container/pseudo'), + ], mock_mkdir.mock_calls) + + # Test that the container and object lists are not shuffled + mock_shuffle.reset_mock() + + connection.return_value.get_container.side_effect = [ + (None, [{'name': 'object'}]), + (None, [{'name': 'pseudo/'}]), + (None, []), + ] + connection.return_value.get_account.side_effect = [ + (None, [{'name': 'container'}]), + (None, []) + ] + + with mock.patch(BUILTIN_OPEN) as mock_open: + with mock.patch('swiftclient.service.makedirs') as mock_mkdir: + argv = ["", "download", "--all", "--no-shuffle"] + swiftclient.shell.main(argv) + self.assertEqual(0, mock_shuffle.call_count) + mock_open.assert_called_once_with('container/object', 'wb', 65536) + self.assertEqual([ + mock.call('container'), + mock.call('container/pseudo'), + ], mock_mkdir.mock_calls) + + @mock.patch('swiftclient.service.Connection') + def test_download_no_content_type(self, connection): + connection.return_value.get_object.return_value = [ + {'etag': EMPTY_ETAG}, + ''] + + # Test downloading whole container + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}]], + [None, [{'name': 'pseudo/'}]], + [None, []], + ] + connection.return_value.auth_end_time = 0 + connection.return_value.attempts = 0 + + with mock.patch(BUILTIN_OPEN) as mock_open: + with mock.patch('swiftclient.service.makedirs') as mock_mkdir: + argv = ["", "download", "container"] + swiftclient.shell.main(argv) + calls = [mock.call('container', 'object', + headers={}, resp_chunk_size=65536, + response_dict={}), + mock.call('container', 'pseudo/', + headers={}, resp_chunk_size=65536, + response_dict={})] + connection.return_value.get_object.assert_has_calls( + calls, any_order=True) + mock_open.assert_called_once_with('object', 'wb', 65536) + self.assertEqual([ + mock.call('pseudo'), + ], mock_mkdir.mock_calls) + + @mock.patch('swiftclient.shell.walk') + @mock.patch('swiftclient.service.Connection') + def test_upload(self, connection, walk): + connection.return_value.head_object.return_value = { + 'content-length': '0'} + connection.return_value.put_object.return_value = EMPTY_ETAG + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, + "-H", "X-Storage-Policy:one", + "--meta", "Color:Blue"] + swiftclient.shell.main(argv) + connection.return_value.put_container.assert_called_once_with( + 'container', + {'X-Storage-Policy': 'one'}, + response_dict={}) + + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY, + 'X-Storage-Policy': 'one', + 'X-Object-Meta-Color': 'Blue'}, + response_dict={}) + + # upload to pseudo-folder (via param) + argv = ["", "upload", "container/pseudo-folder/nested", self.tmpfile, + "-H", "X-Storage-Policy:one"] + swiftclient.shell.main(argv) + connection.return_value.put_container.assert_called_with( + 'container', + {'X-Storage-Policy': 'one'}, + response_dict={}) + + connection.return_value.put_object.assert_called_with( + 'container', + 'pseudo-folder/nested' + self.tmpfile, + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY, + 'X-Storage-Policy': 'one'}, + response_dict={}) + + # Upload whole directory + argv = ["", "upload", "container", "/tmp"] + _tmpfile = self.tmpfile + _tmpfile_dir = dirname(_tmpfile) + _tmpfile_base = basename(_tmpfile) + walk.return_value = [(_tmpfile_dir, [], [_tmpfile_base])] + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + + # Upload in segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + argv = ["", "upload", "container", self.tmpfile, "-S", "10"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + swiftclient.shell.main(argv) + expected_calls = [mock.call('container', + {'X-Storage-Policy': mock.ANY}, + response_dict={}), + mock.call('container_segments', + {'X-Storage-Policy': mock.ANY}, + response_dict={})] + connection.return_value.put_container.has_calls(expected_calls) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + '', + content_length=0, + headers={'x-object-manifest': mock.ANY, + 'x-object-meta-mtime': mock.ANY}, + response_dict={}) + + # upload in segments to pseudo-folder (via param) + connection.reset_mock() + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + argv = ["", "upload", "container/pseudo-folder/nested", + self.tmpfile, "-S", "10", "--use-slo"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + swiftclient.shell.main(argv) + expected_calls = [mock.call('container', + {}, + response_dict={}), + mock.call('container_segments', + {'X-Storage-Policy': 'one'}, + response_dict={})] + connection.return_value.put_container.assert_has_calls(expected_calls) + connection.return_value.put_object.assert_called_with( + 'container', + 'pseudo-folder/nested' + self.tmpfile, + mock.ANY, + headers={ + 'x-object-meta-mtime': mock.ANY, + }, + query_string='multipart-manifest=put', + response_dict=mock.ANY) + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_object_with_account_readonly(self, upload): + argv = ["", "upload", "container", self.tmpfile] + upload.return_value = [ + {"success": False, + "headers": {}, + "container": 'container', + "action": 'create_container', + "error": swiftclient.ClientException( + 'Container PUT failed', + http_status=403, + http_reason='Forbidden', + http_response_content=b'

Forbidden

') + }] + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + self.assertTrue(output.err != '') + warning_msg = "Warning: failed to create container 'container': " \ + "403 Forbidden" + self.assertTrue(output.err.startswith(warning_msg)) + + @mock.patch('swiftclient.service.Connection') + def test_upload_delete_slo_segments(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile] + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', # For the upload call + 'content-length': '2'}, + {'x-static-large-object': 'false', # For the 1st delete call + 'content-length': '2'}, + {'x-static-large-object': 'false', # For the 2nd delete call + 'content-length': '2'} + ] + connection.return_value.get_object.return_value = ( + {}, + b'[{"name": "container1/old_seg1"},' + b' {"name": "container2/old_seg2"}]' + ) + connection.return_value.put_object.return_value = EMPTY_ETAG + # create the delete_object child mock here in attempt to fix + # https://bugs.launchpad.net/python-swiftclient/+bug/1480223 + connection.return_value.delete_object.return_value = None + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + expected_delete_calls = [ + mock.call( + 'container1', 'old_seg1', + response_dict={} + ), + mock.call( + 'container2', 'old_seg2', + response_dict={} + ) + ] + self.assertEqual( + sorted(expected_delete_calls), + sorted(connection.return_value.delete_object.mock_calls) + ) + + @mock.patch('swiftclient.service.Connection') + def test_upload_over_symlink_to_slo(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', + 'content-location': '/v1/a/c/manifest', + 'content-length': '2'}, + ] + connection.return_value.get_object.return_value = ( + {'content-location': '/v1/a/c/manifest'}, + b'[{"name": "container1/old_seg1"},' + b' {"name": "container2/old_seg2"}]' + ) + connection.return_value.put_object.return_value = EMPTY_ETAG + connection.return_value.delete_object.return_value = None + argv = ["", "upload", "container", self.tmpfile] + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + self.assertEqual([], connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') + def test_upload_leave_slo_segments(self, connection): + # Test upload overwriting a manifest respects --leave-segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, "--leave-segments"] + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', # For the upload call + 'content-length': '2'}] + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') + def test_reupload_leaves_slo_segments(self, connection): + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + mtime = '{:.6f}'.format(os.path.getmtime(self.tmpfile)) + expected_segments = [ + 'container_segments/{}/slo/{}/20/10/{:08d}'.format( + self.tmpfile[1:], mtime, i) + for i in range(2) + ] + + # Test re-upload overwriting a manifest doesn't remove + # segments it just wrote + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, + "--use-slo", "-S", "10"] + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', # For the upload call + 'content-length': '20'}] + connection.return_value.get_object.return_value = ( + {}, + # we've already *got* the expected manifest! + json.dumps([ + {'name': seg} for seg in expected_segments + ]).encode('ascii') + ) + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile[1:], # drop leading / + mock.ANY, + headers={'x-object-meta-mtime': mtime}, + query_string='multipart-manifest=put', + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') + def test_upload_delete_dlo_segments(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile] + connection.return_value.head_object.side_effect = [ + {'x-object-manifest': 'container1/prefix', + 'content-length': '0'}, + {}, + {} + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'prefix_a', 'bytes': 0, + 'last_modified': '123T456'}]], + # Have multiple pages worth of DLO segments + [None, [{'name': 'prefix_b', 'bytes': 0, + 'last_modified': '123T456'}]], + [None, []] + ] + connection.return_value.put_object.return_value = EMPTY_ETAG + # create the delete_object child mock here in attempt to fix + # https://bugs.launchpad.net/python-swiftclient/+bug/1480223 + connection.return_value.delete_object.return_value = None + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + expected_delete_calls = [ + mock.call( + 'container1', 'prefix_a', + response_dict={} + ), + mock.call( + 'container1', 'prefix_b', + response_dict={} + ) + ] + self.assertEqual( + sorted(expected_delete_calls), + sorted(connection.return_value.delete_object.mock_calls) + ) + + @mock.patch('swiftclient.service.Connection') + def test_upload_leave_dlo_segments(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, "--leave-segments"] + connection.return_value.head_object.side_effect = [ + {'x-object-manifest': 'container1/prefix', + 'content-length': '0'}] + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') + def test_upload_segments_to_same_container(self, connection): + # Upload in segments to same container + connection.return_value.head_object.return_value = { + 'content-length': '0'} + connection.return_value.attempts = 0 + connection.return_value.put_object.return_value = EMPTY_ETAG + argv = ["", "upload", "container", self.tmpfile, "-S", "10", + "-C", "container"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + swiftclient.shell.main(argv) + connection.return_value.put_container.assert_called_once_with( + 'container', {}, response_dict={}) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + '', + content_length=0, + headers={'x-object-manifest': mock.ANY, + 'x-object-meta-mtime': mock.ANY}, + response_dict={}) + + @mock.patch('swiftclient.shell.io.open') + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin(self, upload_mock, io_open_mock): + def fake_open(fd, mode): + mock_io = mock.Mock() + mock_io.fileno.return_value = fd + return mock_io + + io_open_mock.side_effect = fake_open + + argv = ["", "upload", "container", "-", "--object-name", "foo"] + swiftclient.shell.main(argv) + upload_mock.assert_called_once_with("container", mock.ANY) + # This is a little convoluted: we want to examine the first call ([0]), + # the argv list([1]), the second parameter ([1]), and the first + # element. This is because the upload method takes a container and a + # list of SwiftUploadObjects. + swift_upload_obj = upload_mock.mock_calls[0][1][1][0] + self.assertEqual(sys.stdin.fileno(), swift_upload_obj.source.fileno()) + io_open_mock.assert_called_once_with(sys.stdin.fileno(), mode='rb') + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_no_name(self, upload_mock): + argv = ["", "upload", "container", "-"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find('object-name must be specified') >= 0) + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_and_others(self, upload_mock): + argv = ["", "upload", "container", "-", "foo", "--object-name", "bar"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find( + 'upload from stdin cannot be used') >= 0) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 0) + @mock.patch('swiftclient.service.Connection') + def test_delete_bad_threads(self, mock_connection): + mock_connection.return_value.get_container.return_value = (None, []) + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = _make_cmd( + 'delete', {}, {}, cmd_args=['cont'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = _make_cmd( + 'delete', {}, {}, cmd_args=['cont'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--container-threads", "-1"]) + check_bad(["--container-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--container-threads", "1"]) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_delete_account(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}, {'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]], + [None, []], + [None, [{'name': 'object'}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all"] + connection.return_value.head_object.return_value = {} + connection.return_value.delete_object.return_value = None + swiftclient.shell.main(argv) + connection.return_value.delete_object.assert_has_calls([ + mock.call('container', 'object', query_string=None, + response_dict={}, headers={}), + mock.call('container', 'obj\xe9ct2', query_string=None, + response_dict={}, headers={}), + mock.call('container2', 'object', query_string=None, + response_dict={}, headers={})], any_order=True) + self.assertEqual(3, connection.return_value.delete_object.call_count, + 'Expected 3 calls but found\n%r' + % connection.return_value.delete_object.mock_calls) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}, headers={}), + mock.call('container2', response_dict={}, headers={}), + mock.call('empty_container', response_dict={}, headers={})]) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 10) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_account(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}, {'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'object3'}]], + [None, []], + [None, [{'name': 'object'}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all", "--object-threads", "2"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + self.assertEqual( + 3, len(connection.return_value.post_account.mock_calls), + 'Expected 3 calls but found\n%r' + % connection.return_value.post_account.mock_calls) + # POSTs for same container are made in parallel so expect any order + for expected in [ + mock.call(query_string='bulk-delete', + data=b'/container/object\n/container/obj%C3%A9ct2\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b'/container/object3\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={})]: + self.assertIn(expected, + connection.return_value.post_account.mock_calls[:2]) + # POSTs for different containers are made sequentially so expect order + self.assertEqual( + mock.call(query_string='bulk-delete', + data=b'/container2/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + connection.return_value.post_account.mock_calls[2]) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}, headers={}), + mock.call('container2', response_dict={}, headers={}), + mock.call('empty_container', response_dict={}, headers={})]) + + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_account_with_capabilities(self, connection): + connection.return_value.get_capabilities.return_value = { + 'bulk_delete': { + 'max_deletes_per_request': 10000, + 'max_failed_deletes': 1000, + }, + } + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, [{'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all", "--object-threads", "1"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + self.assertEqual( + connection.return_value.post_account.mock_calls, [ + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container/object\n', + b'/container/obj%C3%A9ct2\n', + b'/container/z_object\n', + b'/container/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container2/object\n', + b'/container2/obj%C3%A9ct2\n', + b'/container2/z_object\n', + b'/container2/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={})]) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}, headers={}), + mock.call('container2', response_dict={}, headers={}), + mock.call('empty_container', response_dict={}, headers={})]) + self.assertEqual(connection.return_value.get_capabilities.mock_calls, + [mock.call(None)]) # only one /info request + + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_account_with_capabilities_and_pages(self, connection): + connection.return_value.get_capabilities.return_value = { + 'bulk_delete': { + 'max_deletes_per_request': 2, + 'max_failed_deletes': 1000, + }, + } + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, [{'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all", "--object-threads", "1"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + # check that each bulk call was only called with 2 objects + self.assertEqual( + connection.return_value.post_account.mock_calls, [ + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container/object\n', + b'/container/obj%C3%A9ct2\n', + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container/z_object\n', + b'/container/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container2/object\n', + b'/container2/obj%C3%A9ct2\n', + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container2/z_object\n', + b'/container2/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={})]) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}, headers={}), + mock.call('container2', response_dict={}, headers={}), + mock.call('empty_container', response_dict={}, headers={})]) + self.assertEqual(connection.return_value.get_capabilities.mock_calls, + [mock.call(None)]) # only one /info request + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_delete_container(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}]], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "container"] + connection.return_value.head_object.return_value = {} + swiftclient.shell.main(argv) + connection.return_value.delete_container.assert_called_with( + 'container', response_dict={}, headers={}) + connection.return_value.delete_object.assert_called_with( + 'container', 'object', query_string=None, response_dict={}, + headers={}) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_delete_container_headers(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}]], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "container", "-H", "Skip-Middleware: Test"] + connection.return_value.head_object.return_value = {} + swiftclient.shell.main(argv) + connection.return_value.delete_container.assert_called_with( + 'container', response_dict={}, + headers={'Skip-Middleware': 'Test'}) + connection.return_value.delete_object.assert_called_with( + 'container', 'object', query_string=None, response_dict={}, + headers={'Skip-Middleware': 'Test'}) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 10) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_container(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}]], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "container"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + connection.return_value.post_account.assert_called_with( + query_string='bulk-delete', data=b'/container/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}) + connection.return_value.delete_container.assert_called_with( + 'container', response_dict={}, headers={}) + + def test_delete_verbose_output_utf8(self): + container = 't\u00e9st_c' + base_argv = ['', '--verbose', 'delete'] + + # simulate container having an object with utf-8 code points in name, + # just returning the object delete result + res = {'success': True, 'response_dict': {}, 'attempts': 2, + 'container': container, 'action': 'delete_object', + 'object': 'obj_t\u00east_o'} + + with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: + with CaptureOutput() as out: + mock_func.return_value = [res] + swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + + mock_func.assert_called_once_with(container=container) + self.assertTrue(out.out.find( + 'obj_t\u00east_o [after 2 attempts]') >= 0, out) + + # simulate empty container + res = {'success': True, 'response_dict': {}, 'attempts': 2, + 'container': container, 'action': 'delete_container'} + + with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: + with CaptureOutput() as out: + mock_func.return_value = [res] + swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + + mock_func.assert_called_once_with(container=container) + self.assertTrue(out.out.find( + 't\u00e9st_c [after 2 attempts]') >= 0, out) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_delete_per_object(self, connection): + argv = ["", "delete", "container", "object"] + connection.return_value.head_object.return_value = {} + connection.return_value.attempts = 0 + swiftclient.shell.main(argv) + connection.return_value.delete_object.assert_called_with( + 'container', 'object', query_string=None, response_dict={}, + headers={}) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 10) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_object(self, connection): + argv = ["", "delete", "container", "object"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + connection.return_value.attempts = 0 + with CaptureOutput() as out: + swiftclient.shell.main(argv) + connection.return_value.post_account.assert_called_with( + query_string='bulk-delete', data=b'/container/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}) + self.assertEqual('object\n', out.out) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 10) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_object_with_retry(self, connection): + argv = ["", "delete", "container", "object"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + connection.return_value.attempts = 3 + with CaptureOutput() as out: + swiftclient.shell.main(argv) + connection.return_value.post_account.assert_called_with( + query_string='bulk-delete', data=b'/container/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}) + self.assertEqual('object [after 3 attempts]\n', out.out) + + def test_delete_verbose_output(self): + del_obj_res = {'success': True, 'response_dict': {}, 'attempts': 2, + 'container': 't\xe9st_c', 'action': 'delete_object', + 'object': 't\xe9st_o'} + + del_seg_res = del_obj_res.copy() + del_seg_res.update({'action': 'delete_segment'}) + + del_con_res = del_obj_res.copy() + del_con_res.update({'action': 'delete_container', 'object': None}) + + test_exc = Exception('t\xe9st_exc') + error_res = del_obj_res.copy() + error_res.update({'success': False, 'error': test_exc, 'object': None}) + + mock_delete = mock.Mock() + base_argv = ['', '--verbose', 'delete'] + + with mock.patch('swiftclient.shell.SwiftService.delete', mock_delete): + with CaptureOutput() as out: + mock_delete.return_value = [del_obj_res] + swiftclient.shell.main(base_argv + ['t\xe9st_c', 't\xe9st_o']) + + mock_delete.assert_called_once_with(container='t\xe9st_c', + objects=['t\xe9st_o']) + self.assertTrue(out.out.find( + 't\xe9st_o [after 2 attempts]') >= 0) + + with CaptureOutput() as out: + mock_delete.return_value = [del_seg_res] + swiftclient.shell.main(base_argv + ['t\xe9st_c', 't\xe9st_o']) + + mock_delete.assert_called_with(container='t\xe9st_c', + objects=['t\xe9st_o']) + self.assertTrue(out.out.find( + 't\xe9st_c/t\xe9st_o [after 2 attempts]') >= 0) + + with CaptureOutput() as out: + mock_delete.return_value = [del_con_res] + swiftclient.shell.main(base_argv + ['t\xe9st_c']) + + mock_delete.assert_called_with(container='t\xe9st_c') + self.assertTrue(out.out.find( + 't\xe9st_c [after 2 attempts]') >= 0) + + with CaptureOutput() as out: + mock_delete.return_value = [error_res] + self.assertRaises(SystemExit, + swiftclient.shell.main, + base_argv + ['t\xe9st_c']) + + mock_delete.assert_called_with(container='t\xe9st_c') + self.assertTrue(out.err.find( + 'Error Deleting: t\xe9st_c: t\xe9st_exc') >= 0) + + @mock.patch('swiftclient.service.Connection') + def test_post_account(self, connection): + argv = ["", "post"] + swiftclient.shell.main(argv) + connection.return_value.post_account.assert_called_with( + headers={}, response_dict={}) + + @mock.patch('swiftclient.service.Connection') + def test_post_account_bad_auth(self, connection): + argv = ["", "post"] + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException( + 'bad auth', http_response_headers={'X-Trans-Id': 'trans_id'}) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, + 'bad auth\nFailed Transaction ID: trans_id\n') + + # do it again with a unicode token + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException( + 'bad auth', http_response_headers={ + 'X-Trans-Id': 'non\u2011utf8'}) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, + 'bad auth\n' + 'Failed Transaction ID: non\u2011utf8\n') + + # do it again with a wonky token + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException( + 'bad auth', http_response_headers={ + 'X-Trans-Id': b'non\xffutf8'}) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, + 'bad auth\nFailed Transaction ID: non%FFutf8\n') + + @mock.patch('swiftclient.service.Connection') + def test_post_account_not_found(self, connection): + argv = ["", "post"] + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException('test', http_status=404) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, 'Account not found\n') + + @mock.patch('swiftclient.service.Connection') + def test_post_container(self, connection): + argv = ["", "post", "container"] + swiftclient.shell.main(argv) + connection.return_value.post_container.assert_called_with( + 'container', headers={}, response_dict={}) + + @mock.patch('swiftclient.service.Connection') + def test_post_container_bad_auth(self, connection): + argv = ["", "post", "container"] + connection.return_value.post_container.side_effect = \ + swiftclient.ClientException('bad auth') + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, 'bad auth\n') + + @mock.patch('swiftclient.service.Connection') + def test_post_container_not_found_causes_put(self, connection): + argv = ["", "post", "container"] + connection.return_value.post_container.side_effect = \ + swiftclient.ClientException('test', http_status=404) + swiftclient.shell.main(argv) + self.assertEqual('container', + connection.return_value.put_container.call_args[0][0]) + + def test_post_container_with_bad_name(self): + argv = ["", "post", "conta/iner"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('WARNING: / in')) + + @mock.patch('swiftclient.service.Connection') + def test_post_container_with_options(self, connection): + argv = ["", "post", "container", + "--read-acl", "test2:tester2", + "--write-acl", "test3:tester3 test4", + "--sync-to", "othersite", + "--sync-key", "secret", + ] + swiftclient.shell.main(argv) + connection.return_value.post_container.assert_called_with( + 'container', headers={ + 'X-Container-Write': 'test3:tester3 test4', + 'X-Container-Read': 'test2:tester2', + 'X-Container-Sync-Key': 'secret', + 'X-Container-Sync-To': 'othersite'}, response_dict={}) + + @mock.patch('swiftclient.service.Connection') + def test_post_object(self, connection): + argv = ["", "post", "container", "object", + "--meta", "Color:Blue", + "--header", "content-type:text/plain" + ] + swiftclient.shell.main(argv) + connection.return_value.post_object.assert_called_with( + 'container', 'object', headers={ + 'Content-Type': 'text/plain', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) + + @mock.patch('swiftclient.service.Connection') + def test_post_object_bad_auth(self, connection): + argv = ["", "post", "container", "object"] + connection.return_value.post_object.side_effect = \ + swiftclient.ClientException("bad auth") + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, 'bad auth\n') + + def test_post_object_too_many_args(self): + argv = ["", "post", "container", "object", "bad_arg"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('Usage')) + + @mock.patch('swiftclient.service.Connection') + def test_copy_object_no_destination(self, connection): + argv = ["", "copy", "container", "object", + "--meta", "Color:Blue", + "--header", "content-type:text/plain" + ] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + connection.return_value.copy_object.assert_called_with( + 'container', 'object', destination=None, fresh_metadata=False, + headers={ + 'Content-Type': 'text/plain', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) + self.assertEqual(output.out, 'container/object copied to \n') + + @mock.patch('swiftclient.service.Connection') + def test_copy_object(self, connection): + argv = ["", "copy", "container", "object", + "--meta", "Color:Blue", + "--header", "content-type:text/plain", + "--destination", "/c/o" + ] + with CaptureOutput() as output: + swiftclient.shell.main(argv) + connection.return_value.copy_object.assert_called_with( + 'container', 'object', destination="/c/o", + fresh_metadata=False, + headers={ + 'Content-Type': 'text/plain', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) + self.assertEqual( + output.out, + 'created container c\ncontainer/object copied to /c/o\n' + ) + + @mock.patch('swiftclient.service.Connection') + def test_copy_object_fresh_metadata(self, connection): + argv = ["", "copy", "container", "object", + "--meta", "Color:Blue", "--fresh-metadata", + "--header", "content-type:text/plain", + "--destination", "/c/o" + ] + swiftclient.shell.main(argv) + connection.return_value.copy_object.assert_called_with( + 'container', 'object', destination="/c/o", fresh_metadata=True, + headers={ + 'Content-Type': 'text/plain', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) + + @mock.patch('swiftclient.service.Connection') + def test_copy_two_objects(self, connection): + argv = ["", "copy", "container", "object", "object2", + "--meta", "Color:Blue"] + connection.return_value.copy_object.return_value = None + swiftclient.shell.main(argv) + calls = [ + mock.call( + 'container', 'object', destination=None, + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}), + mock.call( + 'container', 'object2', destination=None, + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}) + ] + connection.return_value.copy_object.assert_has_calls( + calls, any_order=True) + self.assertEqual(len(connection.return_value.copy_object.mock_calls), + len(calls)) + + @mock.patch('swiftclient.service.Connection') + def test_copy_two_objects_destination(self, connection): + argv = ["", "copy", "container", "object", "object2", + "--meta", "Color:Blue", "--destination", "/c"] + connection.return_value.copy_object.return_value = None + swiftclient.shell.main(argv) + calls = [ + mock.call( + 'container', 'object', destination="/c/object", + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}), + mock.call( + 'container', 'object2', destination="/c/object2", + fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, + response_dict={}) + ] + connection.return_value.copy_object.assert_has_calls( + calls, any_order=True) + self.assertEqual(len(connection.return_value.copy_object.mock_calls), + len(calls)) + + @mock.patch('swiftclient.service.Connection') + def test_copy_two_objects_bad_destination(self, connection): + argv = ["", "copy", "container", "object", "object2", + "--meta", "Color:Blue", "--destination", "/c/o"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual( + output.err, + 'Combination of multiple objects and destination ' + 'including object is invalid\n') + + @mock.patch('swiftclient.service.Connection') + def test_copy_object_bad_auth(self, connection): + argv = ["", "copy", "container", "object"] + connection.return_value.copy_object.side_effect = \ + swiftclient.ClientException("bad auth") + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, 'bad auth\n') + + def test_copy_object_not_enough_args(self): + argv = ["", "copy", "container"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('Usage')) + + def test_copy_bad_container(self): + argv = ["", "copy", "cont/ainer", "object"] + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('WARN')) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_prefix_based(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/", + "secret_key", "--prefix-based"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=True, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_iso8601_in(self, temp_url): + dates = ('1970-01-01T00:01:00Z', '1970-01-01T00:01:00', + '1970-01-01') + for d in dates: + argv = ["", "tempurl", "GET", d, "/v1/AUTH_account/c/", + "secret_key"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/', d, 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_iso8601_out(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/", + "secret_key", "--iso8601"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, + iso8601=True, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_absolute_expiry_temp_url(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key", "--absolute"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True, + iso8601=False, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_with_ip_range(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key", "--ip-range", "1.2.3.4"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=False, ip_range='1.2.3.4') + + def test_temp_url_output(self): + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" + expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "http://saio:8080/v1/a/c/o", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + expected = "http://saio:8080%s" % expected + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/", + "secret_key", "--absolute", "--prefix"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' + expected = ("/v1/a/c/?temp_url_sig=%s&temp_url_expires=60" + "&temp_url_prefix=\n" % sig) + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/", + "secret_key", "--absolute", "--prefix", '--iso8601'] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' + expires = '1970-01-01T00:01:00Z' + expected = ("/v1/a/c/?temp_url_sig=%s&temp_url_expires=%s" + "&temp_url_prefix=\n" % (sig, expires)) + self.assertEqual(expected, output.out) + + dates = ("1970-01-01T00:01:00Z", + strftime(EXPIRES_ISO8601_FORMAT[:-1], localtime(60))) + for d in dates: + argv = ["", "tempurl", "GET", d, "/v1/a/c/o", + "secret_key"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" + expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig + self.assertEqual(expected, output.out) + + ts = str(int( + mktime(strptime('2005-05-01', SHORT_EXPIRES_ISO8601_FORMAT)))) + + argv = ["", "tempurl", "GET", ts, "/v1/a/c/", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + expected = output.out + + argv = ["", "tempurl", "GET", '2005-05-01', "/v1/a/c/", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--ip-range", "1.2.3.4"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "6a6ec8efa4be53904ecba8d055d841e24a937c98" + expected = ( + "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60" + "&temp_url_ip_range=1.2.3.4\n" % sig + ) + self.assertEqual(expected, output.out) + + def test_temp_url_error_output(self): + expected = 'path must be full path to an object e.g. /v1/a/c/o\n' + for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o', + 'http://saio/v1/a/c', 'http://v1/a/c/o'): + argv = ["", "tempurl", "GET", "60", bad_path, + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + self.assertEqual(expected, output.err, + 'Expected %r but got %r for path %r' % + (expected, output.err, bad_path)) + + expected = 'path must at least contain /v1/a/c/\n' + argv = ["", "tempurl", "GET", "60", '/v1/a/c', + "secret_key", "--absolute", '--prefix-based'] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + self.assertEqual(expected, output.err, + 'Expected %r but got %r for path %r' % + (expected, output.err, '/v1/a/c')) + + expected = TIME_ERRMSG + '\n' + for bad_time in ('not_an_int', '-1', '2015-05', '2015-05-01T01:00'): + argv = ["", "tempurl", "GET", bad_time, '/v1/a/c/o', + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + self.assertEqual(expected, output.err, + 'Expected %r but got %r for time %r' % + (expected, output.err, bad_time)) + + @mock.patch('swiftclient.service.Connection') + def test_capabilities(self, connection): + argv = ["", "capabilities"] + connection.return_value.get_capabilities.return_value = {'swift': None} + swiftclient.shell.main(argv) + connection.return_value.get_capabilities.assert_called_with(None) + + @mock.patch('swiftclient.service.Connection') + def test_capabilities_json(self, connection): + capabilities = { + 'slo': {'min_segment_size': 1000000}, + 'some': [{'arbitrary': 'nested'}, {'crazy': 'structure'}], + 'swift': {'version': '2.5.0'}} + + connection.return_value.get_capabilities.return_value = capabilities + argv = ["", "capabilities", "--json"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + expected = json.dumps(capabilities, sort_keys=True, indent=2) + '\n' + self.assertEqual(expected, output.out) + connection.return_value.get_capabilities.assert_called_with(None) + + def test_human_readable_upload_segment_size(self): + def _check_expected(x, expected): + actual = x.call_args_list[-1][1]["options"]["segment_size"] + self.assertEqual(int(actual), expected) + + mock_swift = mock.MagicMock(spec=swiftclient.shell.SwiftService) + with mock.patch("swiftclient.shell.SwiftService", mock_swift): + with CaptureOutput(suppress_systemexit=True) as output: + # Test new behaviour with both upper and lower case + # trailing characters + argv = ["", "upload", "-S", "1B", "container", "object"] + swiftclient.shell.main(argv) + _check_expected(mock_swift, 1) + + argv = ["", "upload", "-S", "1K", "container", "object"] + swiftclient.shell.main(argv) + _check_expected(mock_swift, 1024) + + argv = ["", "upload", "-S", "1m", "container", "object"] + swiftclient.shell.main(argv) + _check_expected(mock_swift, 1048576) + + argv = ["", "upload", "-S", "1G", "container", "object"] + swiftclient.shell.main(argv) + _check_expected(mock_swift, 1073741824) + + # Test old behaviour is not affected + argv = ["", "upload", "-S", "12345", "container", "object"] + swiftclient.shell.main(argv) + _check_expected(mock_swift, 12345) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + # Test invalid states + argv = ["", "upload", "-S", "1234X", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "Invalid segment size\n") + output.clear() + + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S", "K1234", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "Invalid segment size\n") + output.clear() + + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S", "K", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "Invalid segment size\n") + + def test_negative_upload_segment_size(self): + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S", "-40", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S=-40K", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S=-40M", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() + with self.assertRaises(SystemExit): + argv = ["", "upload", "-S=-40G", "container", "object"] + swiftclient.shell.main(argv) + self.assertEqual(output.err, "segment-size should be positive\n") + output.clear() + + +class TestSubcommandHelp(unittest.TestCase): + + def test_subcommand_help(self): + for command in swiftclient.shell.commands: + help_var = 'st_%s_help' % command + options_var = 'st_%s_options' % command + self.assertTrue(hasattr(swiftclient.shell, help_var)) + with CaptureOutput() as out: + argv = ['', command, '--help'] + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + expected = 'Usage: swift %s %s\n%s' % ( + command, vars(swiftclient.shell).get(options_var, "\n"), + vars(swiftclient.shell)[help_var]) + self.assertEqual(out.strip('\n'), expected) + + def test_no_help(self): + with CaptureOutput() as out: + argv = ['', 'bad_command', '--help'] + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + expected = 'no such command: bad_command' + self.assertEqual(out.strip('\n'), expected) + + +@mock.patch.dict(os.environ, mocked_os_environ) +class TestDebugAndInfoOptions(unittest.TestCase): + @mock.patch('logging.basicConfig') + @mock.patch('swiftclient.service.Connection') + def test_option_after_posarg(self, connection, mock_logging): + argv = ["", "stat", "--info"] + swiftclient.shell.main(argv) + mock_logging.assert_called_with(level=logging.INFO) + + argv = ["", "stat", "--debug"] + swiftclient.shell.main(argv) + mock_logging.assert_called_with(level=logging.DEBUG) + + @mock.patch('logging.basicConfig') + @mock.patch('swiftclient.service.Connection') + def test_debug_trumps_info(self, connection, mock_logging): + argv_scenarios = (["", "stat", "--info", "--debug"], + ["", "stat", "--debug", "--info"], + ["", "--info", "stat", "--debug"], + ["", "--debug", "stat", "--info"], + ["", "--info", "--debug", "stat"], + ["", "--debug", "--info", "stat"]) + for argv in argv_scenarios: + mock_logging.reset_mock() + swiftclient.shell.main(argv) + try: + mock_logging.assert_called_once_with(level=logging.DEBUG) + except AssertionError: + self.fail('Unexpected call(s) %r for args %r' + % (mock_logging.call_args_list, argv)) + + +class TestBase(unittest.TestCase): + """ + Provide some common methods to subclasses + """ + def _remove_swift_env_vars(self): + self._environ_vars = {} + keys = list(os.environ.keys()) + for k in keys: + if (k in ('ST_KEY', 'ST_USER', 'ST_AUTH') or + k.startswith('OS_')): + self._environ_vars[k] = os.environ.pop(k) + + def _replace_swift_env_vars(self): + os.environ.update(self._environ_vars) + + +class TestParsing(TestBase): + + def setUp(self): + super(TestParsing, self).setUp() + self._remove_swift_env_vars() + + def tearDown(self): + self._replace_swift_env_vars() + super(TestParsing, self).tearDown() + + def _make_fake_command(self, result): + def fake_command(parser, args, thread_manager): + result[0], result[1] = swiftclient.shell.parse_args(parser, args) + return fake_command + + def _verify_opts(self, actual_opts, expected_opts, expected_os_opts=None, + expected_os_opts_dict=None): + """ + Check parsed options are correct. + + :param expected_opts: v1 style options. + :param expected_os_opts: openstack style options. + :param expected_os_opts_dict: openstack options that should be found in + the os_options dict. + """ + expected_os_opts = expected_os_opts or {} + expected_os_opts_dict = expected_os_opts_dict or {} + # check the expected opts are set + for key, v in expected_opts.items(): + actual = actual_opts.get(key) + self.assertEqual(v, actual, 'Expected %s for key %s, found %s' % + (v, key, actual)) + + for key, v in expected_os_opts.items(): + actual = actual_opts.get("os_" + key) + self.assertEqual(v, actual, 'Expected %s for key %s, found %s' % + (v, key, actual)) + + # check the os_options dict values are set + self.assertIn('os_options', actual_opts) + actual_os_opts_dict = actual_opts['os_options'] + expected_os_opts_keys = ['project_name', 'region_name', + 'tenant_name', + 'user_domain_name', 'endpoint_type', + 'object_storage_url', 'project_domain_id', + 'user_id', 'user_domain_id', 'tenant_id', + 'service_type', 'project_id', 'auth_token', + 'project_domain_name'] + for key in expected_os_opts_keys: + self.assertIn(key, actual_os_opts_dict) + cli_key = key + if key == 'object_storage_url': + # exceptions to the pattern... + cli_key = 'storage_url' + if cli_key in expected_os_opts_dict: + expect = expected_os_opts_dict[cli_key] + else: + expect = None + actual = actual_os_opts_dict[key] + self.assertEqual(expect, actual, 'Expected %s for %s, got %s' + % (expect, key, actual)) + for key in actual_os_opts_dict: + self.assertIn(key, expected_os_opts_keys) + + # check that equivalent keys have equal values + equivalents = [('os_username', 'user'), + ('os_auth_url', 'auth'), + ('os_password', 'key')] + for pair in equivalents: + self.assertEqual(actual_opts.get(pair[0]), + actual_opts.get(pair[1])) + + def test_minimum_required_args_v3(self): + opts = {"auth_version": "3"} + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + + # username with domain is sufficient in args because keystone will + # assume user is in default domain + args = _make_args("stat", opts, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, {}) + + # check its ok to have user_id instead of username + os_opts = {"password": "secret", + "auth_url": "http://example.com:5000/v3"} + os_opts_dict = {"user_id": "user_ID"} + all_os_opts = os_opts.copy() + all_os_opts.update(os_opts_dict) + + args = _make_args("stat", opts, all_os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + # check no user credentials required if token and url supplied + os_opts = {} + os_opts_dict = {"storage_url": "http://example.com:8080/v1", + "auth_token": "0123abcd"} + + args = _make_args("stat", opts, os_opts_dict, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + def test_sloppy_versions(self): + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3", + "identity-api-version": "3.0"} + + # check os_identity_api_version=3.0 is mapped to auth_version=3 + args = _make_args("stat", {}, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '3'} # NB: not '3.0' + expected_os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check os_identity_api_version=2 is mapped to auth_version=2.0 + # A somewhat contrived scenario - we need to pass in the v1 style opts + # to prevent auth version defaulting to 2.0 due to lack of v1 style + # options. That way we can actually verify that the sloppy 2 was + # interpreted and mapped to 2.0 + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v2.0", + "identity-api-version": "2"} + opts = {"key": "secret", + "user": "user", + "auth": "http://example.com:5000/v2.0"} + args = _make_args("stat", opts, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '2.0'} # NB: not '2' + expected_os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v2.0"} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + def test_os_identity_api_version(self): + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3", + "identity-api-version": "3"} + + # check os_identity_api_version is sufficient in place of auth_version + args = _make_args("stat", {}, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '3'} + expected_os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check again using environment variables + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that last of auth-version, os-identity-api-version is preferred + args = _make_args("stat", {}, os_opts, '-') + ['--auth-version', '2.0'] + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '2.0'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # now put auth_version ahead of os-identity-api-version + args = _make_args("stat", {"auth_version": "2.0"}, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '3'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that OS_AUTH_VERSION overrides OS_IDENTITY_API_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'OS_AUTH_VERSION': '2.0'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '2.0'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that ST_AUTH_VERSION overrides OS_IDENTITY_API_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'ST_AUTH_VERSION': '2.0'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that ST_AUTH_VERSION overrides OS_AUTH_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'ST_AUTH_VERSION': '2.0', 'OS_AUTH_VERSION': '3'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + def test_args_v3(self): + opts = {"auth_version": "3"} + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + os_opts_dict = {"user_id": "user_ID", + "project_id": "project_ID", + "tenant_id": "tenant_ID", + "project_domain_id": "project_domain_ID", + "user_domain_id": "user_domain_ID", + "tenant_name": "tenant", + "project_name": "project", + "project_domain_name": "project_domain", + "user_domain_name": "user_domain", + "auth_token": "token", + "storage_url": "http://example.com:8080/v1", + "region_name": "region", + "service_type": "service", + "endpoint_type": "endpoint"} + all_os_opts = os_opts.copy() + all_os_opts.update(os_opts_dict) + + # check using hyphen separator + args = _make_args("stat", opts, all_os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + # check using underscore separator + args = _make_args("stat", opts, all_os_opts, '_') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + # check using environment variables + args = _make_args("stat", {}, {}) + env = _make_env(opts, all_os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + # check again using OS_AUTH_VERSION instead of ST_AUTH_VERSION + env = _make_env({}, all_os_opts) + env.update({'OS_AUTH_VERSION': '3'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], opts, os_opts, os_opts_dict) + + def test_command_args_v3(self): + result = [None, None] + fake_command = self._make_fake_command(result) + opts = {"auth_version": "3"} + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + args = _make_args("stat", opts, os_opts) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self.assertEqual(['stat'], result[1]) + with mock.patch('swiftclient.shell.st_stat', fake_command): + args = args + ["container_name"] + swiftclient.shell.main(args) + self.assertEqual(["stat", "container_name"], result[1]) + + def test_insufficient_args_v3(self): + opts = {"auth_version": "3"} + os_opts = {"password": "secret", + "auth_url": "http://example.com:5000/v3"} + args = _make_args("stat", opts, os_opts) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn( + 'Auth version 3 requires either OS_USERNAME or OS_USER_ID', + str(cm.exception)) + + os_opts = {"username": "user", + "auth_url": "http://example.com:5000/v3"} + args = _make_args("stat", opts, os_opts) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_PASSWORD', str(cm.exception)) + + os_opts = {"username": "user", + "password": "secret"} + args = _make_args("stat", opts, os_opts) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_AUTH_URL', str(cm.exception)) + + def test_password_prompt(self): + def do_test(opts, os_opts, auth_version): + args = _make_args("stat", opts, os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + return_value='input_pwd') as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertEqual('input_pwd', result[0]['key']) + self.assertEqual('input_pwd', result[0]['os_password']) + + # ctrl-D + with self.assertRaises(SystemExit) as cm: + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + side_effect=EOFError) as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertIn( + 'Auth version %s requires' % auth_version, str(cm.exception)) + + # force getpass to think it needs to use raw input + with self.assertRaises(SystemExit) as cm: + with mock.patch('getpass.getpass', getpass.fallback_getpass): + swiftclient.shell.main(args) + self.assertIn( + 'Input stream incompatible', str(cm.exception)) + + opts = {"prompt": None, "user": "bob", "key": "secret", + "auth": "http://example.com:8080/auth/v1.0"} + do_test(opts, {}, '1.0') + os_opts = {"username": "user", + "password": "secret", + "auth_url": "http://example.com:5000/v3"} + opts = {"auth_version": "2.0", "prompt": None} + do_test(opts, os_opts, '2.0') + opts = {"auth_version": "3", "prompt": None} + do_test(opts, os_opts, '3') + + def test_no_tenant_name_or_id_v2(self): + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3", + "tenant_name": "", + "tenant_id": ""} + + with CaptureOutput() as output: + args = _make_args("stat", {}, os_opts) + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertEqual(output.err.strip(), 'No tenant specified') + + with CaptureOutput() as output: + args = _make_args("stat", {}, os_opts, cmd_args=["testcontainer"]) + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertEqual(output.err.strip(), 'No tenant specified') + + def test_no_tenant_name_or_id_v3(self): + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3", + "tenant_name": "", + "tenant_id": ""} + + with CaptureOutput() as output: + args = _make_args("stat", {"auth_version": "3"}, os_opts) + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertEqual(output.err.strip(), + 'No project name or project id specified.') + + with CaptureOutput() as output: + args = _make_args("stat", {"auth_version": "3"}, + os_opts, cmd_args=["testcontainer"]) + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertEqual(output.err.strip(), + 'No project name or project id specified.') + + def test_insufficient_env_vars_v3(self): + args = _make_args("stat", {}, {}) + opts = {"auth_version": "3"} + os_opts = {"password": "secret", + "auth_url": "http://example.com:5000/v3"} + env = _make_env(opts, os_opts) + with mock.patch.dict(os.environ, env): + self.assertRaises(SystemExit, swiftclient.shell.main, args) + + os_opts = {"username": "user", + "auth_url": "http://example.com:5000/v3"} + env = _make_env(opts, os_opts) + with mock.patch.dict(os.environ, env): + self.assertRaises(SystemExit, swiftclient.shell.main, args) + + os_opts = {"username": "user", + "password": "secret"} + env = _make_env(opts, os_opts) + with mock.patch.dict(os.environ, env): + self.assertRaises(SystemExit, swiftclient.shell.main, args) + + def test_help(self): + # --help returns condensed help message + opts = {"help": None} + os_opts = {} + args = _make_args(None, opts, os_opts) + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertTrue(out.find('[--key ]') > 0) + self.assertEqual(-1, out.find('--os-username=')) + + # --help returns condensed help message, overrides --os-help + opts = {"help": None} + os_opts = {"help": None} + args = _make_args("", opts, os_opts) + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertTrue(out.find('[--key ]') > 0) + self.assertEqual(-1, out.find('--os-username=')) + + # --os-password, --os-username and --os-auth_url should be ignored + # because --help overrides it + opts = {"help": None} + os_opts = {"help": None, + "password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + args = _make_args("", opts, os_opts) + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertTrue(out.find('[--key ]') > 0) + self.assertEqual(-1, out.find('--os-username=')) + + # --os-help return os options help + opts = {} + args = _make_args("", opts, os_opts) + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertTrue(out.find('[--key ]') > 0) + self.assertTrue(out.find('--os-username=') > 0) + + +class TestKeystoneOptions(MockHttpTest): + """ + Tests to check that options are passed from the command line or + environment variables through to the keystone client interface. + """ + all_os_opts = {'password': 'secret', + 'username': 'user', + 'auth-url': 'http://example.com:5000/v3', + 'user-domain-name': 'userdomain', + 'user-id': 'userid', + 'user-domain-id': 'userdomainid', + 'tenant-name': 'tenantname', + 'tenant-id': 'tenantid', + 'project-name': 'projectname', + 'project-id': 'projectid', + 'project-domain-id': 'projectdomainid', + 'project-domain-name': 'projectdomain', + 'cacert': 'foo', + 'cert': 'minnie', + 'key': 'mickey'} + catalog_opts = {'service-type': 'my-object-store', + 'endpoint-type': 'public', + 'region-name': 'my-region'} + flags = ['insecure', 'debug'] + + # options that are given default values in code if missing from CLI + defaults = {'auth-version': '2.0', + 'service-type': 'object-store', + 'endpoint-type': 'publicURL'} + + def _build_os_opts(self, keys): + os_opts = {} + for k in keys: + os_opts[k] = self.all_os_opts.get(k, self.catalog_opts.get(k)) + return os_opts + + def _test_options_passed_to_keystone(self, cmd, opts, os_opts, + flags=None, use_env=False, + cmd_args=None, no_auth=False): + flags = flags or [] + if use_env: + # set up fake environment variables and make a minimal command line + env = _make_env(opts, os_opts) + args = _make_args(cmd, {}, {}, separator='-', flags=flags, + cmd_args=cmd_args) + else: + # set up empty environment and make full command line + env = {} + args = _make_args(cmd, opts, os_opts, separator='-', flags=flags, + cmd_args=cmd_args) + ks_endpoint = 'http://example.com:8080/v1/AUTH_acc' + ks_token = 'fake_auth_token' + # check correct auth version gets used + key = 'auth-version' + fake_ks = FakeKeystone(endpoint=ks_endpoint, token=ks_token) + if no_auth: + fake_ks2 = fake_ks3 = None + elif opts.get(key, self.defaults.get(key)) == '2.0': + fake_ks2 = fake_ks + fake_ks3 = None + else: + fake_ks2 = None + fake_ks3 = fake_ks + # fake_conn will check that storage_url and auth_token are as expected + endpoint = os_opts.get('storage-url', ks_endpoint) + token = os_opts.get('auth-token', ks_token) + fake_conn = self.fake_http_connection(204, headers={}, + storage_url=endpoint, + auth_token=token) + + with mock.patch('swiftclient.client.ksclient_v2', fake_ks2), \ + mock.patch('swiftclient.client.ksclient_v3', fake_ks3), \ + mock.patch('swiftclient.client.http_connection', fake_conn), \ + mock.patch.dict(os.environ, env, clear=True), \ + patch_disable_warnings() as mock_disable_warnings: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + except SwiftError as err: + self.fail('Unexpected SwiftError: %s' % err) + + if InsecureRequestWarning is not None: + if 'insecure' in flags: + self.assertEqual([mock.call(InsecureRequestWarning)], + mock_disable_warnings.mock_calls) + else: + self.assertEqual([], mock_disable_warnings.mock_calls) + + if no_auth: + # We patched out both keystoneclient versions to be None; + # they *can't* have been used and if we tried to, we would + # have raised ClientExceptions + return + + # check args passed to keystone Client __init__ + self.assertEqual(len(fake_ks.calls), 1) + actual_args = fake_ks.calls[0] + for key in self.all_os_opts.keys(): + expected = os_opts.get(key, self.defaults.get(key)) + key = key.replace('-', '_') + self.assertIn(key, actual_args) + self.assertEqual(expected, actual_args[key], + 'Expected %s for key %s, found %s' + % (expected, key, actual_args[key])) + for flag in flags: + self.assertIn(flag, actual_args) + self.assertTrue(actual_args[flag]) + + check_attr = True + # check args passed to ServiceCatalog.url_for() method + self.assertEqual(len(fake_ks.client.service_catalog.calls), 1) + actual_args = fake_ks.client.service_catalog.calls[0] + for key in self.catalog_opts.keys(): + expected = os_opts.get(key, self.defaults.get(key)) + key = key.replace('-', '_') + if key == 'region_name': + key = 'filter_value' + if expected is None: + check_attr = False + self.assertNotIn(key, actual_args) + self.assertNotIn('attr', actual_args) + continue + self.assertIn(key, actual_args) + self.assertEqual(expected, actual_args[key], + 'Expected %s for key %s, found %s' + % (expected, key, actual_args[key])) + if check_attr: + key, v = 'attr', 'region' + self.assertIn(key, actual_args) + self.assertEqual(v, actual_args[key], + 'Expected %s for key %s, found %s' + % (v, key, actual_args[key])) + + def _test_options(self, opts, os_opts, flags=None, no_auth=False): + # repeat test for different commands using env and command line options + for cmd in ('stat', 'post'): + self._test_options_passed_to_keystone(cmd, opts, os_opts, + flags=flags, no_auth=no_auth) + self._test_options_passed_to_keystone(cmd, opts, os_opts, + flags=flags, use_env=True, + no_auth=no_auth) + + def test_all_args_passed_to_keystone(self): + # check that all possible command line args are passed to keystone + opts = {'auth-version': '3'} + os_opts = dict(self.all_os_opts) + os_opts.update(self.catalog_opts) + self._test_options(opts, os_opts, flags=self.flags) + + opts = {'auth-version': '2.0'} + self._test_options(opts, os_opts, flags=self.flags) + + opts = {} + self.defaults['auth-version'] = '3' + self._test_options(opts, os_opts, flags=self.flags) + + for o in ('user-domain-name', 'user-domain-id', + 'project-domain-name', 'project-domain-id'): + os_opts.pop(o) + self.defaults['auth-version'] = '2.0' + self._test_options(opts, os_opts, flags=self.flags) + + def test_catalog_options_and_flags_not_required_v3(self): + # check that all possible command line args are passed to keystone + opts = {'auth-version': '3'} + os_opts = dict(self.all_os_opts) + self._test_options(opts, os_opts, flags=None) + + def test_ok_option_combinations_v3(self): + opts = {'auth-version': '3'} + keys = ('username', 'password', 'tenant-name', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('user-id', 'password', 'tenant-name', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('user-id', 'password', 'tenant-id', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('user-id', 'password', 'project-name', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('user-id', 'password', 'project-id', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + def test_ok_option_combinations_v2(self): + opts = {'auth-version': '2.0'} + keys = ('username', 'password', 'tenant-name', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'password', 'tenant-id', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + # allow auth_version to default to 2.0 + opts = {} + keys = ('username', 'password', 'tenant-name', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'password', 'tenant-id', 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + # ...except when it should be 3 + self.defaults['auth-version'] = '3' + keys = ('username', 'user-domain-name', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'user-domain-id', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'project-domain-name', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + keys = ('username', 'project-domain-id', 'password', 'project-name', + 'auth-url') + os_opts = self._build_os_opts(keys) + self._test_options(opts, os_opts) + + def test_url_and_token_provided_on_command_line(self): + endpoint = 'http://alternate.com:8080/v1/AUTH_another' + token = 'alternate_auth_token' + os_opts = {'auth-token': token, + 'storage-url': endpoint} + opts = {'auth-version': '3'} + self._test_options(opts, os_opts, no_auth=True) + + opts = {'auth-version': '2.0'} + self._test_options(opts, os_opts, no_auth=True) + + def test_url_provided_on_command_line(self): + endpoint = 'http://alternate.com:8080/v1/AUTH_another' + os_opts = {'username': 'username', + 'password': 'password', + 'project-name': 'projectname', + 'auth-url': 'http://example.com:5000/v3', + 'storage-url': endpoint} + opts = {'auth-version': '3'} + self._test_options(opts, os_opts) + + opts = {'auth-version': '2.0'} + self._test_options(opts, os_opts) + + +@mock.patch.dict(os.environ, clean_os_environ) +class TestAuth(MockHttpTest): + + def test_pre_authed_request(self): + url = 'https://swift.storage.example.com/v1/AUTH_test' + token = 'AUTH_tk5b6b12' + + pre_auth_env = { + 'OS_STORAGE_URL': url, + 'OS_AUTH_TOKEN': token, + } + fake_conn = self.fake_http_connection(200) + with mock.patch('swiftclient.client.http_connection', new=fake_conn): + with mock.patch.dict(os.environ, pre_auth_env): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', {'x-auth-token': token}), + ]) + + # and again with re-auth + pre_auth_env.update(mocked_os_environ) + pre_auth_env['OS_AUTH_TOKEN'] = 'expired' + fake_conn = self.fake_http_connection(401, 200, 200, headers={ + 'x-auth-token': token + '_new', + 'x-storage-url': url + '_not_used', + }) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + with mock.patch.dict(os.environ, pre_auth_env): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', { + 'x-auth-token': 'expired', + }), + ('GET', mocked_os_environ['ST_AUTH'], '', { + 'x-auth-user': mocked_os_environ['ST_USER'], + 'x-auth-key': mocked_os_environ['ST_KEY'], + }), + ('HEAD', url, '', { + 'x-auth-token': token + '_new', + }), + ]) + + def test_os_pre_authed_request(self): + url = 'https://swift.storage.example.com/v1/AUTH_test' + token = 'AUTH_tk5b6b12' + + pre_auth_env = { + 'OS_STORAGE_URL': url, + 'OS_AUTH_TOKEN': token, + } + fake_conn = self.fake_http_connection(200) + with mock.patch('swiftclient.client.http_connection', new=fake_conn): + with mock.patch.dict(os.environ, pre_auth_env): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', {'x-auth-token': token}), + ]) + + # and again with re-auth + os_environ = { + 'OS_AUTH_URL': 'https://keystone.example.com/v2.0/', + 'OS_TENANT_NAME': 'demo', + 'OS_USERNAME': 'demo', + 'OS_PASSWORD': 'admin', + } + os_environ.update(pre_auth_env) + os_environ['OS_AUTH_TOKEN'] = 'expired' + + fake_conn = self.fake_http_connection(401, 200) + fake_keystone = fake_get_auth_keystone(storage_url=url + '_not_used', + token=token + '_new') + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + get_auth_keystone=fake_keystone, + sleep=mock.DEFAULT): + with mock.patch.dict(os.environ, os_environ): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', { + 'x-auth-token': 'expired', + }), + ('HEAD', url, '', { + 'x-auth-token': token + '_new', + }), + ]) + + def test_auth(self): + headers = { + 'x-auth-token': 'AUTH_tk5b6b12', + 'x-storage-url': 'https://swift.storage.example.com/v1/AUTH_test', + } + mock_resp = self.fake_http_connection(200, headers=headers) + with mock.patch('swiftclient.client.http_connection', new=mock_resp): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'testing', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=https://swift.storage.example.com/v1/AUTH_test + export OS_AUTH_TOKEN=AUTH_tk5b6b12 + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose(self): + with mock.patch('swiftclient.client.http_connection') as mock_conn: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'te$tin&', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export ST_AUTH=https://swift.storage.example.com/auth/v1.0 + export ST_USER=test:tester + export ST_KEY='te$tin&' + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_conn.mock_calls) + + def test_auth_v2(self): + os_options = {'tenant_name': 'demo'} + with mock.patch('swiftclient.client.get_auth_keystone', + new=fake_get_auth_keystone(os_options)): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', 'admin', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=http://url/ + export OS_AUTH_TOKEN=token + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose_v2(self): + with mock.patch('swiftclient.client.get_auth_keystone') \ + as mock_keystone: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', '$eKr3t', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_IDENTITY_API_VERSION=2.0 + export OS_AUTH_VERSION=2.0 + export OS_AUTH_URL=https://keystone.example.com/v2.0/ + export OS_PASSWORD='$eKr3t' + export OS_TENANT_NAME=demo + export OS_USERNAME=demo + """ + self.assertEqual(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_keystone.mock_calls) + + +class TestCrossAccountObjectAccess(TestBase, MockHttpTest): + """ + Tests to verify use of --os-storage-url will actually + result in the object request being sent despite account + read/write access and container write access being denied. + """ + def setUp(self): + super(TestCrossAccountObjectAccess, self).setUp() + self._remove_swift_env_vars() + temp_file = tempfile.NamedTemporaryFile(delete=False) + temp_file.file.write(b'01234567890123456789') + temp_file.file.flush() + self.obj = temp_file.name + self.url = 'http://alternate.com:8080/v1' + + # account tests will attempt to access + self.account = 'AUTH_alice' + + # keystone returns endpoint for another account + self.fake_ks = FakeKeystone( + endpoint='http://example.com:8080/v1/AUTH_bob', + token='bob_token') + + self.cont = 'c1' + self.cont_path = '/v1/%s/%s' % (self.account, self.cont) + self.obj_path = '%s%s' % (self.cont_path, self.obj) + + self.os_opts = {'username': 'bob', + 'password': 'password', + 'project-name': 'proj_bob', + 'auth-url': 'http://example.com:5000/v3', + 'storage-url': '%s/%s' % (self.url, self.account)} + self.opts = {'auth-version': '3'} + + def tearDown(self): + try: + os.remove(self.obj) + except OSError: + pass + self._replace_swift_env_vars() + super(TestCrossAccountObjectAccess, self).tearDown() + + def _make_cmd(self, cmd, cmd_args=None): + return _make_cmd(cmd, self.opts, self.os_opts, cmd_args=cmd_args) + + def _fake_cross_account_auth(self, read_ok, write_ok): + def on_request(method, path, *args, **kwargs): + """ + Modify response code to 200 if cross account permissions match. + """ + status = 403 + if (path.startswith('/v1/%s/%s' % (self.account, self.cont)) and + read_ok and method in ('GET', 'HEAD')): + status = 200 + elif (path.startswith('/v1/%s/%s%s' + % (self.account, self.cont, self.obj)) and + write_ok and method in ('PUT', 'POST', 'DELETE')): + status = 200 + return status + return on_request + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_upload_bad_threads(self, mock_connection): + mock_connection.return_value.put_object.return_value = EMPTY_ETAG + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = self._make_cmd( + 'upload', cmd_args=[self.cont, self.obj] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = self._make_cmd( + 'upload', + cmd_args=[self.cont, self.obj, '--leave-segments'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--segment-threads", "-1"]) + check_bad(["--segment-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--segment-threads", "1"]) + + def test_upload_with_read_write_access(self): + req_handler = self._fake_cross_account_auth(True, True) + fake_conn = self.fake_http_connection(403, 403, + on_request=req_handler) + + args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, + '--leave-segments']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + self.assertRequests([('PUT', self.cont_path), + ('PUT', self.obj_path)]) + self.assertEqual(self.obj[1:], out.strip()) + expected_err = "Warning: failed to create container '%s': 403 Fake" \ + % self.cont + self.assertEqual(expected_err, out.err.strip()) + + def test_upload_with_write_only_access(self): + req_handler = self._fake_cross_account_auth(False, True) + fake_conn = self.fake_http_connection(403, 403, + on_request=req_handler) + args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, + '--leave-segments']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + self.assertRequests([('PUT', self.cont_path), + ('PUT', self.obj_path)]) + self.assertEqual(self.obj[1:], out.strip()) + expected_err = "Warning: failed to create container '%s': 403 Fake" \ + % self.cont + self.assertEqual(expected_err, out.err.strip()) + + def test_segment_upload_with_write_only_access(self): + req_handler = self._fake_cross_account_auth(False, True) + fake_conn = self.fake_http_connection(403, 403, 403, 403, + on_request=req_handler) + + args, env = self._make_cmd('upload', + cmd_args=[self.cont, self.obj, + '--leave-segments', + '--segment-size=10', + '--segment-container=%s' + % self.cont]) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + segment_time = getmtime(self.obj) + segment_path_0 = '%s/%f/20/10/00000000' % (self.obj_path, segment_time) + segment_path_1 = '%s/%f/20/10/00000001' % (self.obj_path, segment_time) + # Note that the order of segment PUTs cannot be asserted, so test for + # existence in request log individually + self.assert_request(('PUT', self.cont_path)) + self.assert_request(('PUT', segment_path_0)) + self.assert_request(('PUT', segment_path_1)) + self.assert_request(('PUT', self.obj_path)) + self.assertIn(self.obj[1:], out.out) + expected_err = "Warning: failed to create container '%s': 403 Fake" \ + % self.cont + self.assertEqual(expected_err, out.err.strip()) + + def test_segment_upload_with_write_only_access_segments_container(self): + fake_conn = self.fake_http_connection( + 403, # PUT c1 + # HEAD c1 to get storage policy + StubResponse(200, headers={'X-Storage-Policy': 'foo'}), + 403, # PUT c1_segments + 201, # PUT c1_segments/...00 + 201, # PUT c1_segments/...01 + 201, # PUT c1/... + ) + + args, env = self._make_cmd('upload', + cmd_args=[self.cont, self.obj, + '--leave-segments', + '--segment-size=10']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + segment_time = getmtime(self.obj) + segment_path_0 = '%s_segments%s/%f/20/10/00000000' % ( + self.cont_path, self.obj, segment_time) + segment_path_1 = '%s_segments%s/%f/20/10/00000001' % ( + self.cont_path, self.obj, segment_time) + # Note that the order of segment PUTs cannot be asserted, so test for + # existence in request log individually + self.assert_request(('PUT', self.cont_path)) + self.assert_request(('PUT', self.cont_path + '_segments', '', { + 'X-Auth-Token': 'bob_token', + 'X-Storage-Policy': 'foo', + 'Content-Length': '0', + })) + self.assert_request(('PUT', segment_path_0)) + self.assert_request(('PUT', segment_path_1)) + self.assert_request(('PUT', self.obj_path)) + self.assertIn(self.obj[1:], out.out) + expected_err = ("Warning: failed to create container '%s': 403 Fake\n" + "Warning: failed to create container '%s': 403 Fake" + ) % (self.cont, self.cont + '_segments') + self.assertEqual(expected_err, out.err.strip()) + + def test_upload_with_no_access(self): + fake_conn = self.fake_http_connection(403, 403) + + args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, + '--leave-segments']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + self.fail('Expected SystemExit') + except SystemExit: + pass + + self.assertRequests([('PUT', self.cont_path), + ('PUT', self.obj_path)]) + expected_err = 'Object PUT failed: http://1.2.3.4%s 403 Fake' \ + % self.obj_path + self.assertIn(expected_err, out.err) + self.assertEqual('', out) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_download_bad_threads(self, mock_connection): + mock_connection.return_value.get_object.return_value = [{}, ''] + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = self._make_cmd( + 'download', cmd_args=[self.cont, self.obj] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = self._make_cmd( + 'download', + cmd_args=[self.cont, self.obj, '--no-download'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--container-threads", "-1"]) + check_bad(["--container-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--container-threads", "1"]) + + def test_download_with_read_write_access(self): + req_handler = self._fake_cross_account_auth(True, True) + fake_conn = self.fake_http_connection(403, on_request=req_handler, + etags=[EMPTY_ETAG]) + + args, env = self._make_cmd('download', cmd_args=[self.cont, + self.obj.lstrip('/'), + '--no-download']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + self.assertRequests([('GET', self.obj_path)]) + self.assertTrue(out.out.startswith(self.obj.lstrip('/'))) + self.assertEqual('', out.err) + + def test_download_with_read_only_access(self): + req_handler = self._fake_cross_account_auth(True, False) + fake_conn = self.fake_http_connection(403, on_request=req_handler, + etags=[EMPTY_ETAG]) + + args, env = self._make_cmd('download', cmd_args=[self.cont, + self.obj.lstrip('/'), + '--no-download']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + self.assertRequests([('GET', self.obj_path)]) + self.assertTrue(out.out.startswith(self.obj.lstrip('/'))) + self.assertEqual('', out.err) + + def test_download_with_no_access(self): + fake_conn = self.fake_http_connection(403) + args, env = self._make_cmd('download', cmd_args=[self.cont, + self.obj.lstrip('/'), + '--no-download']) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + self.fail('Expected SystemExit') + except SystemExit: + pass + + self.assertRequests([('GET', self.obj_path)]) + path = '%s%s' % (self.cont, self.obj) + expected_err = "Error downloading object '%s'" % path + self.assertTrue(out.err.startswith(expected_err)) + self.assertEqual('', out) + + def test_list_with_read_access(self): + req_handler = self._fake_cross_account_auth(True, False) + resp_body = b'{}' + resp = StubResponse(403, resp_body, { + 'etag': hashlib.md5(resp_body).hexdigest()}) + fake_conn = self.fake_http_connection(resp, on_request=req_handler) + + args, env = self._make_cmd('download', cmd_args=[self.cont]) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + self.assertRequests([('GET', '%s?format=json' % self.cont_path)]) + self.assertEqual('', out) + self.assertEqual('', out.err) + + def test_list_with_no_access(self): + fake_conn = self.fake_http_connection(403) + + args, env = self._make_cmd('download', cmd_args=[self.cont]) + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + self.fail('Expected SystemExit') + except SystemExit: + pass + + self.assertRequests([('GET', '%s?format=json' % self.cont_path)]) + self.assertEqual('', out) + self.assertTrue(out.err.startswith('Container GET failed:')) + + +class TestCrossAccountObjectAccessUsingEnv(TestCrossAccountObjectAccess): + """ + Repeat super-class tests using environment variables rather than command + line to set options. + """ + + def _make_cmd(self, cmd, cmd_args=None): + return _make_cmd(cmd, self.opts, self.os_opts, cmd_args=cmd_args, + use_env=True) diff -Nru python-swiftclient-3.8.1/test/unit/test_swiftclient.py python-swiftclient-3.9.0/test/unit/test_swiftclient.py --- python-swiftclient-3.8.1/test/unit/test_swiftclient.py 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/unit/test_swiftclient.py 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,3328 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import json +import logging +import mock +import six +import socket +import string +import unittest +import warnings +import tempfile +from hashlib import md5 +from six import binary_type +from six.moves.urllib.parse import urlparse +from requests.exceptions import RequestException + +from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, + FakeKeystone) + +from swiftclient.utils import EMPTY_ETAG +from swiftclient.exceptions import ClientException +from swiftclient import client as c +import swiftclient.utils +import swiftclient + + +class TestClientException(unittest.TestCase): + + def test_is_exception(self): + self.assertTrue(issubclass(c.ClientException, Exception)) + + def test_format(self): + exc = c.ClientException('something failed') + self.assertIn('something failed', str(exc)) + test_kwargs = ( + 'scheme', + 'host', + 'port', + 'path', + 'query', + 'status', + 'reason', + 'device', + 'response_content', + ) + for value in test_kwargs: + kwargs = { + 'http_%s' % value: value, + } + exc = c.ClientException('test', **kwargs) + self.assertIn(value, str(exc)) + + def test_attrs(self): + test_kwargs = ( + 'scheme', + 'host', + 'port', + 'path', + 'query', + 'status', + 'reason', + 'device', + 'response_content', + 'response_headers', + ) + for value in test_kwargs: + key = 'http_%s' % value + kwargs = {key: value} + exc = c.ClientException('test', **kwargs) + self.assertIs(True, hasattr(exc, key)) + self.assertEqual(getattr(exc, key), value) + + +class MockHttpResponse(object): + def __init__(self, status=0, headers=None, verify=False): + self.status = status + self.status_code = status + self.reason = "OK" + self.buffer = [] + self.requests_params = None + self.verify = verify + self.md5sum = md5() + self.headers = {'etag': '"%s"' % EMPTY_ETAG} + if headers: + self.headers.update(headers) + self.closed = False + + class Raw(object): + def __init__(self, headers): + self.headers = headers + + def read(self, **kw): + return "" + + def getheader(self, name, default): + return self.headers.get(name, default) + + self.raw = Raw(headers) + + def read(self): + return "" + + def close(self): + self.closed = True + + def getheader(self, name, default): + return self.headers.get(name, default) + + def getheaders(self): + return dict(self.headers).items() + + def fake_response(self): + return self + + def _fake_request(self, *arg, **kwarg): + self.status = 200 + self.requests_params = kwarg + if self.verify: + for chunk in kwarg['data']: + self.md5sum.update(chunk) + + # This simulate previous httplib implementation that would do a + # putrequest() and then use putheader() to send header. + for k, v in kwarg['headers'].items(): + self.buffer.append((k, v)) + return self.fake_response() + + +class TestHttpHelpers(MockHttpTest): + + def test_quote(self): + value = b'bytes\xff' + self.assertEqual('bytes%FF', c.quote(value)) + value = 'native string' + self.assertEqual('native%20string', c.quote(value)) + value = u'unicode string' + self.assertEqual('unicode%20string', c.quote(value)) + value = u'unicode:\xe9\u20ac' + self.assertEqual('unicode%3A%C3%A9%E2%82%AC', c.quote(value)) + + def test_parse_header_string(self): + value = b'bytes' + self.assertEqual(u'bytes', c.parse_header_string(value)) + value = u'unicode:\xe9\u20ac' + self.assertEqual(u'unicode:\xe9\u20ac', c.parse_header_string(value)) + value = 'native%20string' + self.assertEqual(u'native string', c.parse_header_string(value)) + + value = b'encoded%20bytes%E2%82%AC' + self.assertEqual(u'encoded bytes\u20ac', c.parse_header_string(value)) + value = 'encoded%20unicode%E2%82%AC' + self.assertEqual(u'encoded unicode\u20ac', + c.parse_header_string(value)) + + value = b'bad%20bytes%ff%E2%82%AC' + self.assertEqual(u'bad%20bytes%ff%E2%82%AC', + c.parse_header_string(value)) + value = u'bad%20unicode%ff\u20ac' + self.assertEqual(u'bad%20unicode%ff\u20ac', + c.parse_header_string(value)) + + value = b'really%20bad\xffbytes' + self.assertEqual(u'really%2520bad%FFbytes', + c.parse_header_string(value)) + + def test_http_connection(self): + url = 'http://www.test.com' + _junk, conn = c.http_connection(url) + self.assertIs(type(conn), c.HTTPConnection) + url = 'https://www.test.com' + _junk, conn = c.http_connection(url) + self.assertIs(type(conn), c.HTTPConnection) + url = 'ftp://www.test.com' + self.assertRaises(c.ClientException, c.http_connection, url) + + def test_encode_meta_headers(self): + headers = {'abc': '123', + u'x-container-meta-\u0394': 123, + u'x-account-meta-\u0394': 12.3, + u'x-object-meta-\u0394': True} + + r = swiftclient.encode_meta_headers(headers) + + self.assertEqual(len(headers), len(r)) + # ensure non meta headers are not encoded + self.assertIs(type(r.get('abc')), binary_type) + del r['abc'] + + for k, v in r.items(): + self.assertIs(type(k), binary_type) + self.assertIs(type(v), binary_type) + self.assertIn(v, (b'123', b'12.3', b'True')) + + def test_set_user_agent_default(self): + _junk, conn = c.http_connection('http://www.example.com') + req_headers = {} + + def my_request_handler(*a, **kw): + req_headers.update(kw.get('headers', {})) + conn._request = my_request_handler + + # test the default + conn.request('GET', '/') + ua = req_headers.get('user-agent', 'XXX-MISSING-XXX') + self.assertTrue(ua.startswith('python-swiftclient-')) + + def test_set_user_agent_per_request_override(self): + _junk, conn = c.http_connection('http://www.example.com') + req_headers = {} + + def my_request_handler(*a, **kw): + req_headers.update(kw.get('headers', {})) + conn._request = my_request_handler + + # test if it's actually set + conn.request('GET', '/', headers={'User-Agent': 'Me'}) + ua = req_headers.get('user-agent', 'XXX-MISSING-XXX') + self.assertEqual(ua, b'Me', req_headers) + + def test_set_user_agent_default_override(self): + _junk, conn = c.http_connection( + 'http://www.example.com', + default_user_agent='a-new-default') + req_headers = {} + + def my_request_handler(*a, **kw): + req_headers.update(kw.get('headers', {})) + conn._request = my_request_handler + + # test setting a default + conn._request = my_request_handler + conn.request('GET', '/') + ua = req_headers.get('user-agent', 'XXX-MISSING-XXX') + self.assertEqual(ua, 'a-new-default') + + +class TestGetAuth(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf') + self.assertIsNone(url) + self.assertIsNone(token) + + def test_invalid_auth(self): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + auth_version="foo") + + def test_auth_v1(self): + c.http_connection = self.fake_http_connection(200, auth_v1=True) + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + auth_version="1.0") + self.assertEqual(url, 'storageURL') + self.assertEqual(token, 'someauthtoken') + + def test_auth_v1_insecure(self): + c.http_connection = self.fake_http_connection(200, 200, auth_v1=True) + url, token = c.get_auth('http://www.test.com/invalid_cert', + 'asdf', 'asdf', + auth_version='1.0', + insecure=True) + self.assertEqual(url, 'storageURL') + self.assertEqual(token, 'someauthtoken') + + with self.assertRaises(c.ClientException) as exc_context: + c.get_auth('http://www.test.com/invalid_cert', + 'asdf', 'asdf', auth_version='1.0') + # TODO: this test is really on validating the mock and not the + # the full plumbing into the requests's 'verify' option + self.assertIn('invalid_certificate', str(exc_context.exception)) + + def test_auth_v1_timeout(self): + # this test has some overlap with + # TestConnection.test_timeout_passed_down but is required to check that + # get_auth does the right thing when it is not passed a timeout arg + orig_http_connection = c.http_connection + timeouts = [] + + def fake_request_handler(*a, **kw): + if 'timeout' in kw: + timeouts.append(kw['timeout']) + else: + timeouts.append(None) + return MockHttpResponse( + status=200, + headers={ + 'x-auth-token': 'a_token', + 'x-storage-url': 'http://files.example.com/v1/AUTH_user'}) + + def fake_connection(*a, **kw): + url, conn = orig_http_connection(*a, **kw) + conn._request = fake_request_handler + return url, conn + + with mock.patch('swiftclient.client.http_connection', fake_connection): + c.get_auth('http://www.test.com', 'asdf', 'asdf', + auth_version="1.0", timeout=42.0) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + auth_version="1.0", timeout=None) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + auth_version="1.0") + + self.assertEqual(timeouts, [42.0, None, None]) + + def test_auth_v2_timeout(self): + # this test has some overlap with + # TestConnection.test_timeout_passed_down but is required to check that + # get_auth does the right thing when it is not passed a timeout arg + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): + c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=dict(tenant_name='tenant'), + auth_version="2.0", timeout=42.0) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=dict(tenant_name='tenant'), + auth_version="2.0", timeout=None) + c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=dict(tenant_name='tenant'), + auth_version="2.0") + self.assertEqual(3, len(fake_ks.calls)) + timeouts = [call['timeout'] for call in fake_ks.calls] + self.assertEqual([42.0, None, None], timeouts) + + def test_auth_v2_with_tenant_name(self): + os_options = {'tenant_name': 'asdf'} + req_args = {'auth_version': '2.0'} + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_tenant_id(self): + os_options = {'tenant_id': 'asdf'} + req_args = {'auth_version': '2.0'} + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_project_name(self): + os_options = {'project_name': 'asdf'} + req_args = {'auth_version': '2.0'} + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_project_id(self): + os_options = {'project_id': 'asdf'} + req_args = {'auth_version': '2.0'} + + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_no_tenant_name_or_tenant_id(self): + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone({})): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options={}, + auth_version='2.0') + + def test_auth_v2_with_tenant_name_none_and_tenant_id_none(self): + os_options = {'tenant_name': None, + 'tenant_id': None} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options)): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options=os_options, + auth_version='2.0') + + def test_auth_v2_with_tenant_user_in_user(self): + tenant_option = {'tenant_name': 'foo'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(tenant_option)): + url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf', + os_options={}, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_tenant_name_no_os_options(self): + tenant_option = {'tenant_name': 'asdf'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(tenant_option)): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + tenant_name='asdf', + os_options={}, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_os_options(self): + os_options = {'service_type': 'object-store', + 'endpoint_type': 'internalURL', + 'tenant_name': 'asdf'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options)): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_tenant_user_in_user_no_os_options(self): + tenant_option = {'tenant_name': 'foo'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(tenant_option)): + url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf', + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_with_os_region_name(self): + os_options = {'region_name': 'good-region', + 'tenant_name': 'asdf'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options)): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_auth_v2_no_endpoint(self): + os_options = {'region_name': 'unknown_region', + 'tenant_name': 'asdf'} + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, c.ClientException)): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + + def test_auth_v2_ks_exception(self): + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone({}, c.ClientException)): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options={}, + auth_version='2.0') + + def test_auth_v2_cacert(self): + os_options = {'tenant_name': 'foo'} + auth_url_secure = 'https://www.tests.com' + auth_url_insecure = 'https://www.tests.com/self-signed-certificate' + + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, None)): + url, token = c.get_auth(auth_url_secure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=False) + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + url, token = c.get_auth(auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cacert='ca.pem', insecure=False) + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=False) + + def test_auth_v2_insecure(self): + os_options = {'tenant_name': 'foo'} + auth_url_secure = 'https://www.tests.com' + auth_url_insecure = 'https://www.tests.com/invalid-certificate' + + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, None)): + url, token = c.get_auth(auth_url_secure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + url, token = c.get_auth(auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=True) + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=False) + + def test_auth_v2_cert(self): + os_options = {'tenant_name': 'foo'} + auth_url_no_sslauth = 'https://www.tests.com' + auth_url_sslauth = 'https://www.tests.com/client-certificate' + + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, None)): + url, token = c.get_auth(auth_url_no_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + url, token = c.get_auth(auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cert='minnie', cert_key='mickey') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + self.assertRaises(c.ClientException, c.get_auth, + auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertRaises(c.ClientException, c.get_auth, + auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cert='minnie') + + def test_auth_v3_with_tenant_name(self): + # check the correct auth version is passed to get_auth_keystone + os_options = {'tenant_name': 'asdf'} + req_args = {'auth_version': '3'} + + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="3") + + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_get_keystone_client_2_0(self): + # check the correct auth version is passed to get_auth_keystone + os_options = {'tenant_name': 'asdf'} + req_args = {'auth_version': '2.0'} + + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_keystoneclient_2_0('http://www.test.com', + 'asdf', 'asdf', + os_options=os_options) + + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + def test_get_auth_keystone_versionless(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('http://authurl', 'user', 'key', {}) + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://authurl/v3', fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_versionless_auth_version_set(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): + c.get_auth_keystone('http://auth_url', 'user', 'key', + {}, auth_version='2.0') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://auth_url/v2.0', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_versionful(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('http://auth_url/v3', 'user', 'key', + {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://auth_url/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionful(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('https://192.168.8.8/identity/v3', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionless(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('https://192.168.8.8/identity', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_auth_keystone_url_some_junk_nonsense(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', + token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('http://blah.example.com/v2moo', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + # v2 looks sorta version-y, but it's not an exact match, so this is + # probably about just as bad as anything else we might guess at + self.assertEqual('http://blah.example.com/v2moo/v3', + fake_ks.calls[0].get('auth_url')) + + def test_auth_with_session(self): + mock_session = mock.MagicMock() + mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' + mock_session.get_token.return_value = 'token' + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + session=mock_session) + self.assertEqual(url, 'http://storagehost/v1/acct') + self.assertTrue(token) + + +class TestGetAccount(MockHttpTest): + + def test_no_content(self): + c.http_connection = self.fake_http_connection(204) + value = c.get_account('http://www.test.com/v1/acct', 'asdf')[1] + self.assertEqual(value, []) + self.assertRequests([ + ('GET', '/v1/acct?format=json', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + def test_param_marker(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&marker=marker") + c.get_account('http://www.test.com/v1/acct', 'asdf', marker='marker') + self.assertRequests([ + ('GET', '/v1/acct?format=json&marker=marker', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + def test_param_limit(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&limit=10") + c.get_account('http://www.test.com/v1/acct', 'asdf', limit=10) + self.assertRequests([ + ('GET', '/v1/acct?format=json&limit=10', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + def test_param_prefix(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&prefix=asdf/") + c.get_account('http://www.test.com/v1/acct', 'asdf', prefix='asdf/') + self.assertRequests([ + ('GET', '/v1/acct?format=json&prefix=asdf/', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + def test_param_end_marker(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&end_marker=end_marker") + c.get_account('http://www.test.com/v1/acct', 'asdf', + end_marker='end_marker') + self.assertRequests([ + ('GET', '/v1/acct?format=json&end_marker=end_marker', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + def test_param_delimiter(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&delimiter=-") + c.get_account('http://www.test.com/v1/acct', 'asdf', + delimiter='-') + self.assertRequests([ + ('GET', '/v1/acct?format=json&delimiter=-', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + +class TestHeadAccount(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200, headers={ + 'x-account-meta-color': 'blue', + }) + resp_headers = c.head_account('http://www.tests.com', 'asdf') + self.assertEqual(resp_headers['x-account-meta-color'], 'blue') + self.assertRequests([ + ('HEAD', 'http://www.tests.com', '', {'x-auth-token': 'asdf'}) + ]) + + def test_server_error(self): + body = 'c' * 65 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_account('http://www.tests.com', 'asdf') + e = exc_context.exception + self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_status, 500) + self.assertRequests([ + ('HEAD', 'http://www.tests.com', '', {'x-auth-token': 'asdf'}) + ]) + # TODO: this is a fairly brittle test of the __repr__ on the + # ClientException which should probably be in a targeted test + new_body = "[first 60 chars of response] " + body[0:60] + self.assertEqual(e.__str__()[-89:], new_body) + + +class TestPostAccount(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200, headers={ + 'X-Account-Meta-Color': 'blue', + }, body='foo') + headers = {'x-account-meta-shape': 'square'} + resp_headers, body = c.post_account( + 'http://www.tests.com/path/to/account', 'asdf', + headers, query_string='bar=baz', + data='some data') + self.assertEqual('blue', resp_headers.get('x-account-meta-color')) + self.assertEqual('foo', body) + self.assertRequests([ + ('POST', 'http://www.tests.com/path/to/account?bar=baz', + 'some data', {'x-auth-token': 'asdf', + 'x-account-meta-shape': 'square'}) + ]) + # Check that we didn't mutate the request ehader dict + self.assertEqual(headers, {'x-account-meta-shape': 'square'}) + + def test_server_error(self): + body = 'c' * 65 + c.http_connection = self.fake_http_connection(500, body=body) + with self.assertRaises(c.ClientException) as exc_mgr: + c.post_account('http://www.tests.com', 'asdf', {}) + self.assertEqual(exc_mgr.exception.http_response_content, body) + self.assertEqual(exc_mgr.exception.http_status, 500) + self.assertRequests([ + ('POST', 'http://www.tests.com', None, {'x-auth-token': 'asdf'}) + ]) + # TODO: this is a fairly brittle test of the __repr__ on the + # ClientException which should probably be in a targeted test + new_body = "[first 60 chars of response] " + body[0:60] + self.assertEqual(exc_mgr.exception.__str__()[-89:], new_body) + + +class TestGetContainer(MockHttpTest): + + def test_no_content(self): + c.http_connection = self.fake_http_connection(204) + value = c.get_container('http://www.test.com/v1/acct', 'token', + 'container')[1] + self.assertEqual(value, []) + self.assertRequests([ + ('GET', '/v1/acct/container?format=json', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_param_marker(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&marker=marker") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + marker='marker') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&marker=marker', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_param_limit(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&limit=10") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + limit=10) + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&limit=10', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_param_prefix(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&prefix=asdf/") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + prefix='asdf/') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&prefix=asdf/', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_param_delimiter(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&delimiter=/") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + delimiter='/') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&delimiter=/', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_param_end_marker(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&end_marker=end_marker") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + end_marker='end_marker') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&end_marker=end_marker', + '', {'x-auth-token': 'token', 'accept-encoding': 'gzip'}), + ]) + + def test_param_path(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&path=asdf") + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + path='asdf') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&path=asdf', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'token'}), + ]) + + def test_request_headers(self): + c.http_connection = self.fake_http_connection( + 204, query_string="format=json") + conn = c.http_connection('http://www.test.com') + headers = {'x-client-key': 'client key'} + c.get_container('url_is_irrelevant', 'TOKEN', 'container', + http_conn=conn, headers=headers) + self.assertRequests([ + ('GET', '/container?format=json', '', { + 'x-auth-token': 'TOKEN', + 'x-client-key': 'client key', + 'accept-encoding': 'gzip', + }), + ]) + + def test_query_string(self): + c.http_connection = self.fake_http_connection( + 200, query_string="format=json&hello=20", body=b'[]') + c.get_container('http://www.test.com', 'asdf', 'asdf', + query_string="hello=20") + self.assertRequests([ + ('GET', '/asdf?format=json&hello=20', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + + +class TestHeadContainer(MockHttpTest): + + def test_head_ok(self): + fake_conn = self.fake_http_connection( + 200, headers={'x-container-meta-color': 'blue'}) + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + resp = c.head_container('https://example.com/v1/AUTH_test', + 'token', 'container') + self.assertEqual(resp['x-container-meta-color'], 'blue') + self.assertRequests([ + ('HEAD', 'https://example.com/v1/AUTH_test/container', '', + {'x-auth-token': 'token'}), + ]) + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_container('http://www.test.com', 'asdf', 'container') + e = exc_context.exception + self.assertRequests([ + ('HEAD', '/container', '', {'x-auth-token': 'asdf'}), + ]) + self.assertEqual(e.http_status, 500) + self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_response_headers, headers) + + +class TestPutContainer(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + value = c.put_container('http://www.test.com', 'token', 'container') + self.assertIsNone(value) + self.assertRequests([ + ('PUT', '/container', '', { + 'x-auth-token': 'token', + 'content-length': '0'}), + ]) + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.put_container('http://www.test.com', 'token', 'container') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + self.assertRequests([ + ('PUT', '/container', '', { + 'x-auth-token': 'token', + 'content-length': '0'}), + ]) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(200, + query_string="hello=20") + c.put_container('http://www.test.com', 'asdf', 'asdf', + query_string="hello=20") + for req in self.iter_request_log(): + self.assertEqual(req['method'], 'PUT') + self.assertEqual(req['parsed_path'].path, '/asdf') + self.assertEqual(req['parsed_path'].query, 'hello=20') + self.assertEqual(req['headers']['x-auth-token'], 'asdf') + + +class TestDeleteContainer(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + value = c.delete_container('http://www.test.com', 'token', 'container') + self.assertIsNone(value) + self.assertRequests([ + ('DELETE', '/container', '', { + 'x-auth-token': 'token'}), + ]) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(200, + query_string="hello=20") + c.delete_container('http://www.test.com', 'token', 'container', + query_string="hello=20") + self.assertRequests([ + ('DELETE', 'http://www.test.com/container?hello=20', '', { + 'x-auth-token': 'token'}) + ]) + + +class TestGetObject(MockHttpTest): + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(200, + query_string="hello=20") + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf', + query_string="hello=20") + self.assertRequests([ + ('GET', '/asdf/asdf?hello=20', '', { + 'x-auth-token': 'asdf'}), + ]) + + def test_get_object_as_string(self): + c.http_connection = self.fake_http_connection(200, body='abcde') + __, resp = c.get_object('http://storage.example.com', 'TOKEN', + 'container_name', 'object_name') + self.assertEqual(resp, 'abcde') + + def test_request_headers(self): + c.http_connection = self.fake_http_connection(200) + conn = c.http_connection('http://www.test.com') + headers = {'Range': 'bytes=1-2'} + c.get_object('url_is_irrelevant', 'TOKEN', 'container', 'object', + http_conn=conn, headers=headers) + self.assertRequests([ + ('GET', '/container/object', '', { + 'x-auth-token': 'TOKEN', + 'range': 'bytes=1-2', + }), + ]) + + def test_response_headers(self): + c.http_connection = self.fake_http_connection( + 200, headers={'X-Utf-8-Header': b't%c3%a9st', + 'X-Non-Utf-8-Header': b'%ff', + 'X-Binary-Header': b'\xff'}) + conn = c.http_connection('http://www.test.com') + headers, data = c.get_object('url_is_irrelevant', 'TOKEN', + 'container', 'object', http_conn=conn) + self.assertEqual(u't\xe9st', headers.get('x-utf-8-header', '')) + self.assertEqual(u'%ff', headers.get('x-non-utf-8-header', '')) + self.assertEqual(u'%FF', headers.get('x-binary-header', '')) + + def test_chunk_size_read_method(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection(200, body='abcde') + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=3) + self.assertTrue(hasattr(resp, 'read')) + self.assertEqual(resp.read(3), 'abc') + self.assertEqual(resp.read(None), 'de') + self.assertEqual(resp.read(), '') + + def test_chunk_size_iter(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection(200, body='abcde') + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=3) + self.assertTrue(hasattr(resp, 'next')) + self.assertEqual(next(resp), 'abc') + self.assertEqual(next(resp), 'de') + self.assertRaises(StopIteration, next, resp) + + def test_chunk_size_read_and_iter(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection(200, body='abcdef') + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertTrue(hasattr(resp, 'read')) + self.assertEqual(resp.read(3), 'abc') + self.assertEqual(next(resp), 'de') + self.assertEqual(resp.read(), 'f') + self.assertRaises(StopIteration, next, resp) + self.assertEqual(resp.read(), '') + + def test_chunk_size_iter_chunked_no_retry(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection( + 200, body='abcdef', headers={'Transfer-Encoding': 'chunked'}) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + # simulate a dropped connection + resp.resp.read() + self.assertRaises(StopIteration, next, resp) + + def test_chunk_size_iter_retry(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url', 'tToken') + c.http_connection = self.fake_http_connection( + StubResponse(200, 'abcdef', {'etag': 'some etag', + 'content-length': '6'}), + StubResponse(206, 'cdef', {'etag': 'some etag', + 'content-length': '4', + 'content-range': 'bytes 2-5/6'}), + StubResponse(206, 'ef', {'etag': 'some etag', + 'content-length': '2', + 'content-range': 'bytes 4-5/6'}), + ) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + self.assertEqual(1, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'cd') + self.assertEqual(2, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'ef') + self.assertEqual(3, conn.attempts) + self.assertRaises(StopIteration, next, resp) + self.assertRequests([ + ('GET', '/asdf/asdf', '', { + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=2-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=4-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ]) + + def test_chunk_size_iter_retry_no_range_support(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url', 'tToken') + c.http_connection = self.fake_http_connection(*[ + StubResponse(200, 'abcdef', {'etag': 'some etag', + 'content-length': '6'}) + ] * 3) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + self.assertEqual(1, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'cd') + self.assertEqual(2, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'ef') + self.assertEqual(3, conn.attempts) + self.assertRaises(StopIteration, next, resp) + self.assertRequests([ + ('GET', '/asdf/asdf', '', { + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=2-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=4-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ]) + + def test_chunk_size_iter_retry_bad_range_response(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url', 'tToken') + c.http_connection = self.fake_http_connection( + StubResponse(200, 'abcdef', {'etag': 'some etag', + 'content-length': '6'}), + StubResponse(206, 'abcdef', {'etag': 'some etag', + 'content-length': '6', + 'content-range': 'chunk 1-2/3'}) + ) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + self.assertEqual(1, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertRaises(c.ClientException, next, resp) + self.assertRequests([ + ('GET', '/asdf/asdf', '', { + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=2-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ]) + + def test_get_object_with_resp_chunk_size_zero(self): + def get_connection(self): + def get_auth(): + return 'http://auth.test.com', 'token' + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + self.assertIs(type(conn), c.Connection) + conn.get_auth = get_auth + self.assertEqual(conn.attempts, 0) + return conn + + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = get_connection(self) + conn.get_object('container1', 'obj1', resp_chunk_size=0) + self.assertEqual(conn.attempts, 1) + + +class TestHeadObject(MockHttpTest): + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + + def test_request_headers(self): + c.http_connection = self.fake_http_connection(204) + conn = c.http_connection('http://www.test.com') + headers = {'x-client-key': 'client key'} + c.head_object('url_is_irrelevant', 'TOKEN', 'container', + 'asdf', http_conn=conn, headers=headers) + self.assertRequests([ + ('HEAD', '/container/asdf', '', { + 'x-auth-token': 'TOKEN', + 'x-client-key': 'client key', + }), + ]) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(204) + conn = c.http_connection('http://www.test.com') + query_string = 'foo=bar' + c.head_object('url_is_irrelevant', 'token', 'container', 'key', + http_conn=conn, query_string=query_string) + self.assertRequests([ + ('HEAD', '/container/key?foo=bar', '', {'x-auth-token': 'token'}) + ]) + + +class TestPutObject(MockHttpTest): + + @mock.patch('swiftclient.requests.__version__', '2.2.0') + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4) + value = c.put_object(*args) + self.assertIsInstance(value, six.string_types) + self.assertEqual(value, EMPTY_ETAG) + self.assertRequests([ + ('PUT', '/container/obj', 'body', { + 'x-auth-token': 'TOKEN', + 'content-length': '4', + 'content-type': '' + }), + ]) + + def test_unicode_ok(self): + conn = c.http_connection(u'http://www.test.com/') + mock_file = six.StringIO(u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') + args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + mock_file) + text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + headers = {'X-Header1': text, + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:fg:lp'} + + resp = MockHttpResponse() + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + value = c.put_object(*args, headers=headers, http_conn=conn) + self.assertIsInstance(value, six.string_types) + # Test for RFC-2616 encoded symbols + self.assertIn(("a-b", b".x:yz mn:fg:lp"), + resp.buffer) + # Test unicode header + self.assertIn(('x-header1', text.encode('utf8')), + resp.buffer) + + def test_chunk_warning(self): + conn = c.http_connection('http://www.test.com/') + mock_file = six.StringIO('asdf') + args = ('asdf', 'asdf', 'asdf', 'asdf', mock_file) + resp = MockHttpResponse() + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + with warnings.catch_warnings(record=True) as w: + c.put_object(*args, chunk_size=20, headers={}, http_conn=conn) + self.assertEqual(len(w), 0) + + body = 'c' * 60 + c.http_connection = self.fake_http_connection(200, body=body) + args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') + with warnings.catch_warnings(record=True) as w: + c.put_object(*args, chunk_size=20) + self.assertEqual(len(w), 1) + self.assertTrue(issubclass(w[-1].category, UserWarning)) + + @mock.patch('swiftclient.requests.__version__', '2.2.0') + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') + with self.assertRaises(c.ClientException) as exc_context: + c.put_object(*args) + e = exc_context.exception + self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_response_headers, headers) + self.assertEqual(e.http_status, 500) + self.assertRequests([ + ('PUT', '/asdf/asdf', 'asdf', { + 'x-auth-token': 'asdf', + 'content-type': ''}), + ]) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(200, + query_string="hello=20") + c.put_object('http://www.test.com', 'asdf', 'asdf', 'asdf', + query_string="hello=20") + for req in self.iter_request_log(): + self.assertEqual(req['method'], 'PUT') + self.assertEqual(req['parsed_path'].path, '/asdf/asdf') + self.assertEqual(req['parsed_path'].query, 'hello=20') + self.assertEqual(req['headers']['x-auth-token'], 'asdf') + + def test_raw_upload(self): + # Raw upload happens when content_length is passed to put_object + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + raw_data = b'asdf' * 256 + raw_data_len = len(raw_data) + + for kwarg in ({'headers': {'Content-Length': str(raw_data_len)}}, + {'content_length': raw_data_len}): + with tempfile.TemporaryFile() as mock_file: + mock_file.write(raw_data) + mock_file.seek(0) + + c.put_object(url='http://www.test.com', http_conn=conn, + contents=mock_file, **kwarg) + + req_data = resp.requests_params['data'] + self.assertIs(type(req_data), swiftclient.utils.LengthWrapper) + self.assertEqual(raw_data_len, len(req_data.read())) + + def test_chunk_upload(self): + # Chunked upload happens when no content_length is passed to put_object + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + raw_data = b'asdf' * 256 + chunk_size = 16 + + with tempfile.TemporaryFile() as mock_file: + mock_file.write(raw_data) + mock_file.seek(0) + + c.put_object(url='http://www.test.com', http_conn=conn, + contents=mock_file, chunk_size=chunk_size) + req_data = resp.requests_params['data'] + self.assertTrue(hasattr(req_data, '__iter__')) + data = b'' + for chunk in req_data: + self.assertEqual(chunk_size, len(chunk)) + data += chunk + self.assertEqual(data, raw_data) + + def test_iter_upload(self): + def data(): + for chunk in ('foo', '', 'bar'): + yield chunk + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn, + contents=data()) + req_headers = resp.requests_params['headers'] + self.assertNotIn('Content-Length', req_headers) + req_data = resp.requests_params['data'] + self.assertTrue(hasattr(req_data, '__iter__')) + # If we emit an empty chunk, requests will go ahead and send it, + # causing the server to close the connection. So make sure we don't + # do that. + self.assertEqual(['foo', 'bar'], list(req_data)) + + def test_md5_mismatch(self): + conn = c.http_connection('http://www.test.com') + resp = MockHttpResponse(status=200, verify=True, + headers={'etag': '"badresponseetag"'}) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + raw_data = b'asdf' * 256 + raw_data_md5 = md5(raw_data).hexdigest() + chunk_size = 16 + + with tempfile.TemporaryFile() as mock_file: + mock_file.write(raw_data) + mock_file.seek(0) + + contents = swiftclient.utils.ReadableToIterable(mock_file, + md5=True) + + etag = c.put_object(url='http://www.test.com', + http_conn=conn, + contents=contents, + chunk_size=chunk_size) + + self.assertNotEqual(etag, contents.get_md5sum()) + self.assertEqual(etag, 'badresponseetag') + self.assertEqual(raw_data_md5, contents.get_md5sum()) + + def test_md5_match(self): + conn = c.http_connection('http://www.test.com') + raw_data = b'asdf' * 256 + raw_data_md5 = md5(raw_data).hexdigest() + resp = MockHttpResponse(status=200, verify=True, + headers={'etag': '"' + raw_data_md5 + '"'}) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + chunk_size = 16 + + with tempfile.TemporaryFile() as mock_file: + mock_file.write(raw_data) + mock_file.seek(0) + contents = swiftclient.utils.ReadableToIterable(mock_file, + md5=True) + + etag = c.put_object(url='http://www.test.com', + http_conn=conn, + contents=contents, + chunk_size=chunk_size) + + self.assertEqual(raw_data_md5, contents.get_md5sum()) + self.assertEqual(etag, contents.get_md5sum()) + + def test_params(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn, + etag='1234-5678', content_type='text/plain') + request_header = resp.requests_params['headers'] + self.assertEqual(request_header['etag'], b'1234-5678') + self.assertEqual(request_header['content-type'], b'text/plain') + + @mock.patch('swiftclient.requests.__version__', '2.2.0') + def test_no_content_type_old_requests(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn) + request_header = resp.requests_params['headers'] + self.assertEqual(request_header['content-type'], b'') + + @mock.patch('swiftclient.requests.__version__', '2.4.0') + def test_no_content_type_new_requests(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn) + request_header = resp.requests_params['headers'] + self.assertNotIn('content-type', request_header) + + def test_content_type_in_headers(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + # title-case header + hdrs = {'Content-Type': 'text/Plain'} + c.put_object(url='http://www.test.com', http_conn=conn, headers=hdrs) + request_header = resp.requests_params['headers'] + self.assertEqual(request_header['content-type'], b'text/Plain') + + # method param overrides headers + c.put_object(url='http://www.test.com', http_conn=conn, headers=hdrs, + content_type='image/jpeg') + request_header = resp.requests_params['headers'] + self.assertEqual(request_header['content-type'], b'image/jpeg') + + +class TestPostObject(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + delete_at = 2.1 # not str! we don't know what other devs will use! + args = ('http://www.test.com', 'token', 'container', 'obj', + {'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}) + c.post_object(*args) + self.assertRequests([ + ('POST', '/container/obj', '', { + 'x-auth-token': 'token', + 'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}), + ]) + # Check that the request header dict didn't get mutated + self.assertEqual(args[-1], { + 'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at, + }) + + def test_unicode_ok(self): + conn = c.http_connection(u'http://www.test.com/') + args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') + text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + headers = {'X-Header1': text, + b'X-Header2': 'value', + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:kl:qr', + 'X-Object-Meta-Header-not-encoded': text, + b'X-Object-Meta-Header-encoded': 'value'} + + resp = MockHttpResponse() + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + c.post_object(*args, headers=headers, http_conn=conn) + # Test for RFC-2616 encoded symbols + self.assertIn(('a-b', b".x:yz mn:kl:qr"), resp.buffer) + # Test unicode header + self.assertIn(('x-header1', text.encode('utf8')), + resp.buffer) + self.assertIn((b'x-object-meta-header-not-encoded', + text.encode('utf8')), resp.buffer) + self.assertIn((b'x-object-meta-header-encoded', b'value'), + resp.buffer) + self.assertIn((b'x-header2', b'value'), resp.buffer) + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + args = ('http://www.test.com', 'token', 'container', 'obj', {}) + with self.assertRaises(c.ClientException) as exc_context: + c.post_object(*args) + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + self.assertRequests([ + ('POST', 'http://www.test.com/container/obj', '', { + 'x-auth-token': 'token', + }), + ]) + + +class TestCopyObject(MockHttpTest): + + def test_server_error(self): + c.http_connection = self.fake_http_connection(500) + self.assertRaises( + c.ClientException, c.copy_object, + 'http://www.test.com/v1/AUTH', 'asdf', 'asdf', 'asdf') + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj') + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + }), + ]) + + def test_service_token(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object('http://www.test.com/v1/AUTH', None, 'container', + 'obj', destination='/container2/obj', + service_token="TOKEN") + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Service-Token': 'TOKEN', + 'Destination': '/container2/obj', + + }), + ]) + + def test_headers(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj', + headers={'some-hdr': 'a', 'other-hdr': 'b'}) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + 'some-hdr': 'a', + 'other-hdr': 'b', + }), + ]) + + def test_fresh_metadata_default(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + '/container2/obj', {'x-fresh-metadata': 'hdr-value'}) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + 'X-Fresh-Metadata': 'hdr-value', + }), + ]) + + def test_fresh_metadata_true(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj', + headers={'x-fresh-metadata': 'hdr-value'}, + fresh_metadata=True) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'X-Auth-Token': 'token', + 'Destination': '/container2/obj', + 'X-Fresh-Metadata': 'true', + }), + ]) + + def test_fresh_metadata_false(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', + destination='/container2/obj', + headers={'x-fresh-metadata': 'hdr-value'}, + fresh_metadata=False) + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'x-auth-token': 'token', + 'Destination': '/container2/obj', + 'X-Fresh-Metadata': 'false', + }), + ]) + + def test_no_destination(self): + c.http_connection = self.fake_http_connection(200) + c.copy_object( + 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj') + self.assertRequests([ + ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { + 'x-auth-token': 'token', + 'Destination': '/container/obj', + }), + ]) + + +class TestDeleteObject(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200) + c.delete_object('http://www.test.com', 'token', 'container', 'obj') + self.assertRequests([ + ('DELETE', 'http://www.test.com/container/obj', '', { + 'x-auth-token': 'token', + }), + ]) + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.delete_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + + def test_query_string(self): + c.http_connection = self.fake_http_connection(200, + query_string="hello=20") + c.delete_object('http://www.test.com', 'token', 'container', 'obj', + query_string="hello=20") + self.assertRequests([ + ('DELETE', 'http://www.test.com/container/obj?hello=20', '', { + 'x-auth-token': 'token', + }), + ]) + + +class TestGetCapabilities(MockHttpTest): + + def test_ok(self): + conn = self.fake_http_connection(200, body=b'{}') + http_conn = conn('http://www.test.com/info') + info = c.get_capabilities(http_conn) + self.assertRequests([ + ('GET', '/info', '', {'Accept-Encoding': 'gzip'}), + ]) + self.assertEqual(info, {}) + self.assertTrue(http_conn[1].resp.has_been_read) + + def test_server_error(self): + body = 'c' * 60 + headers = {'foo': 'bar'} + conn = self.fake_http_connection( + StubResponse(500, body, headers)) + http_conn = conn('http://www.test.com/info') + with self.assertRaises(c.ClientException) as exc_context: + c.get_capabilities(http_conn) + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) + + def test_conn_get_capabilities_with_auth(self): + auth_headers = { + 'x-auth-token': 'token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_test' + } + auth_v1_response = StubResponse(headers=auth_headers) + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=b'{"swift":{"fake":true}}') + fake_conn = self.fake_http_connection(auth_v1_response, info_response) + + conn = c.Connection('http://auth.example.com/auth/v1.0', + 'user', 'key') + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', '/auth/v1.0', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'key'}), + ('GET', 'http://storage.example.com/info', '', { + 'accept-encoding': 'gzip'}), + ]) + + def test_conn_get_capabilities_with_os_auth(self): + fake_keystone = fake_get_auth_keystone( + storage_url='http://storage.example.com/v1/AUTH_test') + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=b'{"swift":{"fake":true}}') + fake_conn = self.fake_http_connection(info_response) + + os_options = {'project_id': 'test'} + conn = c.Connection('http://keystone.example.com/v3.0', + 'user', 'key', os_options=os_options, + auth_version=3) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://storage.example.com/info'), + ]) + + def test_conn_get_capabilities_with_url_param(self): + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=b'{"swift":{"fake":true}}') + fake_conn = self.fake_http_connection(info_response) + + conn = c.Connection('http://auth.example.com/auth/v1.0', + 'user', 'key') + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities( + 'http://other-storage.example.com/info') + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://other-storage.example.com/info'), + ]) + + def test_conn_get_capabilities_with_preauthurl_param(self): + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=b'{"swift":{"fake":true}}') + fake_conn = self.fake_http_connection(info_response) + + storage_url = 'http://storage.example.com/v1/AUTH_test' + conn = c.Connection('http://auth.example.com/auth/v1.0', + 'user', 'key', preauthurl=storage_url) + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://storage.example.com/info'), + ]) + + def test_conn_get_capabilities_with_os_options(self): + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=b'{"swift":{"fake":true}}') + fake_conn = self.fake_http_connection(info_response) + + storage_url = 'http://storage.example.com/v1/AUTH_test' + os_options = { + 'project_id': 'test', + 'object_storage_url': storage_url, + } + conn = c.Connection('http://keystone.example.com/v3.0', + 'user', 'key', os_options=os_options, + auth_version=3) + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://storage.example.com/info'), + ]) + + +class TestHTTPConnection(MockHttpTest): + + def test_bad_url_scheme(self): + url = u'www.test.com' + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception + expected = u'Unsupported scheme "" in url "www.test.com"' + self.assertEqual(expected, str(exc)) + + url = u'://www.test.com' + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception + expected = u'Unsupported scheme "" in url "://www.test.com"' + self.assertEqual(expected, str(exc)) + + url = u'blah://www.test.com' + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception + expected = u'Unsupported scheme "blah" in url "blah://www.test.com"' + self.assertEqual(expected, str(exc)) + + def test_ok_url_scheme(self): + for scheme in ('http', 'https', 'HTTP', 'HTTPS'): + url = u'%s://www.test.com' % scheme + parsed_url, conn = c.http_connection(url) + self.assertEqual(scheme.lower(), parsed_url.scheme) + self.assertEqual(u'%s://www.test.com' % scheme, conn.url) + + def test_ok_proxy(self): + conn = c.http_connection(u'http://www.test.com/', + proxy='http://localhost:8080') + self.assertEqual(conn[1].requests_args['proxies']['http'], + 'http://localhost:8080') + + def test_bad_proxy(self): + try: + c.http_connection(u'http://www.test.com/', proxy='localhost:8080') + except c.ClientException as e: + self.assertEqual(e.msg, "Proxy's missing scheme") + + def test_cacert(self): + conn = c.http_connection(u'http://www.test.com/', + cacert='/dev/urandom') + self.assertEqual(conn[1].requests_args['verify'], '/dev/urandom') + + def test_insecure(self): + conn = c.http_connection(u'http://www.test.com/', insecure=True) + self.assertEqual(conn[1].requests_args['verify'], False) + + def test_cert(self): + conn = c.http_connection(u'http://www.test.com/', cert='minnie') + self.assertEqual(conn[1].requests_args['cert'], 'minnie') + + def test_cert_key(self): + conn = c.http_connection( + u'http://www.test.com/', cert='minnie', cert_key='mickey') + self.assertEqual(conn[1].requests_args['cert'], ('minnie', 'mickey')) + + def test_response_connection_released(self): + _parsed_url, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse() + conn.resp.raw = mock.Mock() + conn.resp.raw.read.side_effect = ["Chunk", ""] + resp = conn.getresponse() + self.assertFalse(resp.closed) + self.assertEqual("Chunk", resp.read()) + self.assertFalse(resp.read()) + self.assertTrue(resp.closed) + + @unittest.skipIf(six.PY3, 'python2 specific test') + def test_response_python2_headers(self): + '''Test utf-8 headers in Python 2. + ''' + _, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse( + status=200, + headers={ + '\xd8\xaa-unicode': '\xd8\xaa-value', + 'empty-header': '' + } + ) + + resp = conn.getresponse() + self.assertEqual( + '\xd8\xaa-value', resp.getheader('\xd8\xaa-unicode')) + self.assertEqual( + '\xd8\xaa-value', resp.getheader('\xd8\xaa-UNICODE')) + self.assertEqual('', resp.getheader('empty-header')) + self.assertEqual( + dict([('\xd8\xaa-unicode', '\xd8\xaa-value'), + ('empty-header', ''), + ('etag', '"%s"' % EMPTY_ETAG)]), + dict(resp.getheaders())) + + @unittest.skipIf(six.PY2, 'python3 specific test') + def test_response_python3_headers(self): + '''Test latin1-encoded headers in Python 3. + ''' + _, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse( + status=200, + headers={ + b'\xd8\xaa-unicode'.decode('iso-8859-1'): + b'\xd8\xaa-value'.decode('iso-8859-1'), + 'empty-header': '' + } + ) + + resp = conn.getresponse() + self.assertEqual( + '\u062a-value', resp.getheader('\u062a-unicode')) + self.assertEqual( + '\u062a-value', resp.getheader('\u062a-UNICODE')) + self.assertEqual('', resp.getheader('empty-header')) + self.assertEqual( + dict([('\u062a-unicode', '\u062a-value'), + ('empty-header', ''), + ('etag', ('"%s"' % EMPTY_ETAG))]), + dict(resp.getheaders())) + + +class TestConnection(MockHttpTest): + + def test_instance(self): + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + self.assertEqual(conn.retries, 5) + + def test_instance_kwargs(self): + args = {'user': 'ausername', + 'key': 'secretpass', + 'authurl': 'http://www.test.com', + 'tenant_name': 'atenant'} + conn = c.Connection(**args) + self.assertEqual(type(conn), c.Connection) + + def test_instance_kwargs_token(self): + args = {'preauthtoken': 'atoken123', + 'preauthurl': 'http://www.test.com:8080/v1/AUTH_123456'} + conn = c.Connection(**args) + self.assertEqual(conn.url, args['preauthurl']) + self.assertEqual(conn.token, args['preauthtoken']) + + def test_instance_kwargs_os_token(self): + storage_url = 'http://storage.example.com/v1/AUTH_test' + token = 'token' + args = { + 'os_options': { + 'object_storage_url': storage_url, + 'auth_token': token, + } + } + conn = c.Connection(**args) + self.assertEqual(conn.url, storage_url) + self.assertEqual(conn.token, token) + + def test_instance_kwargs_token_precedence(self): + storage_url = 'http://storage.example.com/v1/AUTH_test' + token = 'token' + args = { + 'preauthurl': storage_url, + 'preauthtoken': token, + 'os_options': { + 'auth_token': 'less-specific-token', + 'object_storage_url': 'less-specific-storage-url', + } + } + conn = c.Connection(**args) + self.assertEqual(conn.url, storage_url) + self.assertEqual(conn.token, token) + + def test_storage_url_override(self): + static_url = 'http://overridden.storage.url' + conn = c.Connection('http://auth.url/', 'some_user', 'some_key', + os_options={ + 'object_storage_url': static_url}) + method_signatures = ( + (conn.head_account, []), + (conn.get_account, []), + (conn.head_container, ('asdf',)), + (conn.get_container, ('asdf',)), + (conn.put_container, ('asdf',)), + (conn.delete_container, ('asdf',)), + (conn.head_object, ('asdf', 'asdf')), + (conn.get_object, ('asdf', 'asdf')), + (conn.put_object, ('asdf', 'asdf', 'asdf')), + (conn.post_object, ('asdf', 'asdf', {})), + (conn.delete_object, ('asdf', 'asdf')), + ) + + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.storage.url', 'tToken') + + for method, args in method_signatures: + c.http_connection = self.fake_http_connection( + 200, body=b'[]', storage_url=static_url) + method(*args) + self.assertEqual(len(self.request_log), 1) + for request in self.iter_request_log(): + self.assertEqual(request['parsed_path'].netloc, + 'overridden.storage.url') + self.assertEqual(request['headers']['x-auth-token'], + 'tToken') + + def test_get_capabilities(self): + conn = c.Connection() + with mock.patch('swiftclient.client.get_capabilities') as get_cap: + conn.get_capabilities('http://storage2.test.com') + parsed = get_cap.call_args[0][0][0] + self.assertEqual(parsed.path, '/info') + self.assertEqual(parsed.netloc, 'storage2.test.com') + conn.get_auth = lambda: ('http://storage.test.com/v1/AUTH_test', + 'token') + conn.get_capabilities() + parsed = get_cap.call_args[0][0][0] + self.assertEqual(parsed.path, '/info') + self.assertEqual(parsed.netloc, 'storage.test.com') + + def test_retry(self): + def quick_sleep(*args): + pass + c.sleep = quick_sleep + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + code_iter = [500] * (conn.retries + 1) + c.http_connection = self.fake_http_connection(*code_iter) + + self.assertRaises(c.ClientException, conn.head_account) + self.assertEqual(conn.attempts, conn.retries + 1) + + def test_retry_on_ratelimit(self): + + def quick_sleep(*args): + pass + c.sleep = quick_sleep + + # test retries + conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf', + retry_on_ratelimit=True) + code_iter = [200] + [498] * (conn.retries + 1) + auth_resp_headers = { + 'x-auth-token': 'asdf', + 'x-storage-url': 'http://storage/v1/test', + } + c.http_connection = self.fake_http_connection( + *code_iter, headers=auth_resp_headers) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) + self.assertEqual(conn.attempts, conn.retries + 1) + + # test default no-retry + c.http_connection = self.fake_http_connection( + 200, 498, + headers=auth_resp_headers) + conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf') + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) + self.assertEqual(conn.attempts, 1) + + def test_retry_with_socket_error(self): + def quick_sleep(*args): + pass + c.sleep = quick_sleep + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = socket.error + self.assertRaises(socket.error, conn.head_account) + self.assertEqual(mock_auth.call_count, 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + def test_retry_with_force_auth_retry_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(exception): + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = exception + self.assertRaises(exception, conn.head_account) + self.assertEqual(mock_auth.call_count, conn.retries + 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + do_test(socket.error) + do_test(RequestException) + + def test_retry_with_force_auth_retry_client_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(http_status, count): + + def mock_http_connection(*args, **kwargs): + raise ClientException('fake', http_status=http_status) + + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = mock_http_connection + self.assertRaises(ClientException, conn.head_account) + self.assertEqual(mock_auth.call_count, count) + self.assertEqual(conn.attempts, count) + + # sanity, in case of 401, the auth will be called only twice because of + # retried_auth mechanism + do_test(401, 2) + # others will be tried until retry limits + do_test(408, 6) + do_test(500, 6) + do_test(503, 6) + + def test_resp_read_on_server_error(self): + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', retries=0) + + def get_auth(*args, **kwargs): + return 'http://www.new.com', 'new' + conn.get_auth = get_auth + self.url, self.token = conn.get_auth() + + method_signatures = ( + (conn.head_account, []), + (conn.get_account, []), + (conn.head_container, ('asdf',)), + (conn.get_container, ('asdf',)), + (conn.put_container, ('asdf',)), + (conn.delete_container, ('asdf',)), + (conn.head_object, ('asdf', 'asdf')), + (conn.get_object, ('asdf', 'asdf')), + (conn.put_object, ('asdf', 'asdf', 'asdf')), + (conn.post_object, ('asdf', 'asdf', {})), + (conn.delete_object, ('asdf', 'asdf')), + ) + + for method, args in method_signatures: + c.http_connection = self.fake_http_connection(500) + self.assertRaises(c.ClientException, method, *args) + requests = list(self.iter_request_log()) + self.assertEqual(len(requests), 1) + for req in requests: + msg = '%s did not read resp on server error' % method.__name__ + self.assertTrue(req['resp'].has_been_read, msg) + + def test_reauth(self): + c.http_connection = self.fake_http_connection(401, 200) + + def get_auth(*args, **kwargs): + # this mock, and by extension this test are not + # representative of the unit under test. The real get_auth + # method will always return the os_option dict's + # object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + return 'http://www.new.com', 'new' + + def swap_sleep(*args): + self.swap_sleep_called = True + c.get_auth = get_auth + c.sleep = swap_sleep + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + ) + + self.assertEqual(conn.attempts, 0) + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'old') + + conn.head_account() + + self.assertTrue(self.swap_sleep_called) + self.assertEqual(conn.attempts, 2) + self.assertEqual(conn.url, 'http://www.new.com') + self.assertEqual(conn.token, 'new') + + def test_reauth_preauth(self): + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', + preauthurl='http://storage.example.com/v1/AUTH_test', + preauthtoken='expired') + auth_v1_response = StubResponse(200, headers={ + 'x-auth-token': 'token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_user', + }) + fake_conn = self.fake_http_connection(401, auth_v1_response, 200) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'expired'}), + ('GET', 'http://auth.example.com', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'password'}), + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'token'}), + ]) + + def test_reauth_os_preauth(self): + os_preauth_options = { + 'tenant_name': 'demo', + 'object_storage_url': 'http://storage.example.com/v1/AUTH_test', + 'auth_token': 'expired', + } + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2) + fake_keystone = fake_get_auth_keystone(os_preauth_options) + fake_conn = self.fake_http_connection(401, 200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'expired'}), + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'token'}), + ]) + + def test_session_no_invalidate(self): + mock_session = mock.MagicMock() + mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' + mock_session.get_token.return_value = 'expired' + mock_session.invalidate.return_value = False + conn = c.Connection(session=mock_session) + fake_conn = self.fake_http_connection(401) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + self.assertRaises(c.ClientException, conn.head_account) + self.assertEqual(mock_session.get_token.mock_calls, [mock.call()]) + self.assertEqual(mock_session.invalidate.mock_calls, [mock.call()]) + + def test_session_can_invalidate(self): + mock_session = mock.MagicMock() + mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' + mock_session.get_token.side_effect = ['expired', 'token'] + mock_session.invalidate.return_value = True + conn = c.Connection(session=mock_session) + fake_conn = self.fake_http_connection(401, 200) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/acct', '', {'x-auth-token': 'expired'}), + ('HEAD', '/v1/acct', '', {'x-auth-token': 'token'}), + ]) + self.assertEqual(mock_session.get_token.mock_calls, [ + mock.call(), mock.call()]) + self.assertEqual(mock_session.invalidate.mock_calls, [mock.call()]) + + def test_preauth_token_with_no_storage_url_requires_auth(self): + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', + preauthtoken='expired') + auth_v1_response = StubResponse(200, headers={ + 'x-auth-token': 'token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_user', + }) + fake_conn = self.fake_http_connection(auth_v1_response, 200) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('GET', 'http://auth.example.com', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'password'}), + ('HEAD', '/v1/AUTH_user', '', {'x-auth-token': 'token'}), + ]) + + def test_os_preauth_token_with_no_storage_url_requires_auth(self): + os_preauth_options = { + 'tenant_name': 'demo', + 'auth_token': 'expired', + } + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2) + storage_url = 'http://storage.example.com/v1/AUTH_user' + fake_keystone = fake_get_auth_keystone(storage_url=storage_url) + fake_conn = self.fake_http_connection(200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_user', '', {'x-auth-token': 'token'}), + ]) + + def test_preauth_url_trumps_auth_url(self): + storage_url = 'http://storage.example.com/v1/AUTH_pre_url' + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', + preauthurl=storage_url) + auth_v1_response = StubResponse(200, headers={ + 'x-auth-token': 'post_token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_post_url', + }) + fake_conn = self.fake_http_connection(auth_v1_response, 200) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('GET', 'http://auth.example.com', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'password'}), + ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), + ]) + + def test_os_preauth_url_trumps_auth_url(self): + storage_url = 'http://storage.example.com/v1/AUTH_pre_url' + os_preauth_options = { + 'tenant_name': 'demo', + 'object_storage_url': storage_url, + } + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2) + fake_keystone = fake_get_auth_keystone( + storage_url='http://storage.example.com/v1/AUTH_post_url', + token='post_token') + fake_conn = self.fake_http_connection(200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), + ]) + + def test_preauth_url_trumps_os_preauth_url(self): + storage_url = 'http://storage.example.com/v1/AUTH_pre_url' + os_storage_url = 'http://storage.example.com/v1/AUTH_os_pre_url' + os_preauth_options = { + 'tenant_name': 'demo', + 'object_storage_url': os_storage_url, + } + orig_os_preauth_options = dict(os_preauth_options) + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2, + preauthurl=storage_url, tenant_name='not_demo') + fake_keystone = fake_get_auth_keystone( + storage_url='http://storage.example.com/v1/AUTH_post_url', + token='post_token') + fake_conn = self.fake_http_connection(200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), + ]) + + # check that Connection has not modified our os_options + self.assertEqual(orig_os_preauth_options, os_preauth_options) + + def test_get_auth_sets_url_and_token(self): + with mock.patch('swiftclient.client.get_auth') as mock_get_auth: + mock_get_auth.return_value = ( + "https://storage.url/v1/AUTH_storage_acct", "AUTH_token" + ) + conn = c.Connection("https://auth.url/auth/v2.0", + "user", "passkey", tenant_name="tenant") + conn.get_auth() + self.assertEqual("https://storage.url/v1/AUTH_storage_acct", conn.url) + self.assertEqual("AUTH_token", conn.token) + + def test_timeout_passed_down(self): + # We want to avoid mocking http_connection(), and most especially + # avoid passing it down in argument. However, we cannot simply + # instantiate C=Connection(), then shim C.http_conn. Doing so would + # avoid some of the code under test (where _retry() invokes + # http_connection()), and would miss get_auth() completely. + # So, with regret, we do mock http_connection(), but with a very + # light shim that swaps out _request() as originally intended. + + orig_http_connection = c.http_connection + + timeouts = [] + + def my_request_handler(*a, **kw): + if 'timeout' in kw: + timeouts.append(kw['timeout']) + else: + timeouts.append(None) + return MockHttpResponse( + status=200, + headers={ + 'x-auth-token': 'a_token', + 'x-storage-url': 'http://files.example.com/v1/AUTH_user'}) + + def shim_connection(*a, **kw): + url, conn = orig_http_connection(*a, **kw) + conn._request = my_request_handler + return url, conn + + # v1 auth + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', timeout=33.0) + with mock.patch.multiple('swiftclient.client', + http_connection=shim_connection, + sleep=mock.DEFAULT): + conn.head_account() + + # 1 call is through get_auth, 1 call is HEAD for account + self.assertEqual(timeouts, [33.0, 33.0]) + + # v2 auth + timeouts = [] + os_options = {'tenant_name': 'tenant', 'auth_token': 'meta-token'} + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', timeout=33.0, + os_options=os_options, auth_version=2.0) + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): + with mock.patch.multiple('swiftclient.client', + http_connection=shim_connection, + sleep=mock.DEFAULT): + conn.head_account() + + # check timeout is passed to keystone client + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual(33.0, fake_ks.calls[0].get('timeout')) + # check timeout passed to HEAD for account + self.assertEqual(timeouts, [33.0]) + + # check token passed to keystone client + self.assertIn('token', fake_ks.calls[0]) + self.assertEqual('meta-token', fake_ks.calls[0].get('token')) + + def test_reset_stream(self): + + class LocalContents(object): + + def __init__(self, tell_value=0): + self.data = six.BytesIO(string.ascii_letters.encode() * 10) + self.data.seek(tell_value) + self.reads = [] + self.seeks = [] + self.tells = [] + + def tell(self): + self.tells.append(self.data.tell()) + return self.tells[-1] + + def seek(self, position, mode=0): + self.seeks.append((position, mode)) + self.data.seek(position, mode) + + def read(self, size=-1): + read_data = self.data.read(size) + self.reads.append((size, read_data)) + return read_data + + class LocalConnection(object): + + def __init__(self, parsed_url=None): + self.reason = "" + if parsed_url: + self.host = parsed_url.netloc + self.port = parsed_url.netloc + + def putrequest(self, *args, **kwargs): + self.send('PUT', *args, **kwargs) + + def putheader(self, *args, **kwargs): + return + + def endheaders(self, *args, **kwargs): + return + + def send(self, *args, **kwargs): + data = kwargs.get('data') + if data is not None: + if hasattr(data, 'read'): + data.read() + else: + for datum in data: + pass + raise socket.error('oops') + + def request(self, *args, **kwargs): + return + + def getresponse(self, *args, **kwargs): + self.status = 200 + return self + + def getheader(self, *args, **kwargs): + return 'header' + + def getheaders(self): + return [('key1', 'value1'), ('key2', 'value2')] + + def read(self, *args, **kwargs): + return '' + + def close(self): + pass + + def local_http_connection(url, proxy=None, cacert=None, + insecure=False, cert=None, cert_key=None, + ssl_compression=True, timeout=None): + parsed = urlparse(url) + return parsed, LocalConnection() + + with mock.patch.object(c, 'http_connection', local_http_connection): + conn = c.Connection('http://www.example.com', 'asdf', 'asdf', + retries=1, starting_backoff=.0001) + + contents = LocalContents() + exc = None + try: + conn.put_object('c', 'o', contents) + except socket.error as err: + exc = err + self.assertEqual(contents.tells, [0]) + self.assertEqual(contents.seeks, [(0, 0)]) + # four reads: two in the initial pass, two in the retry + self.assertEqual(4, len(contents.reads)) + self.assertEqual((65536, b''), contents.reads[1]) + self.assertEqual((65536, b''), contents.reads[3]) + self.assertEqual(str(exc), 'oops') + + contents = LocalContents(tell_value=123) + exc = None + try: + conn.put_object('c', 'o', contents) + except socket.error as err: + exc = err + self.assertEqual(contents.tells, [123]) + self.assertEqual(contents.seeks, [(123, 0)]) + # four reads: two in the initial pass, two in the retry + self.assertEqual(4, len(contents.reads)) + self.assertEqual((65536, b''), contents.reads[1]) + self.assertEqual((65536, b''), contents.reads[3]) + self.assertEqual(str(exc), 'oops') + + contents = LocalContents(tell_value=123) + wrapped_contents = swiftclient.utils.LengthWrapper( + contents, 6, md5=True) + exc = None + try: + conn.put_object('c', 'o', wrapped_contents) + except socket.error as err: + exc = err + self.assertEqual(contents.tells, [123]) + self.assertEqual(contents.seeks, [(123, 0)]) + self.assertEqual(contents.reads, [(6, b'tuvwxy')] * 2) + self.assertEqual(str(exc), 'oops') + self.assertEqual(md5(b'tuvwxy').hexdigest(), + wrapped_contents.get_md5sum()) + + contents = LocalContents() + contents.tell = None + exc = None + try: + conn.put_object('c', 'o', contents) + except c.ClientException as err: + exc = err + self.assertEqual(contents.seeks, []) + self.assertEqual(str(exc), "put_object('c', 'o', ...) failure " + "and no ability to reset contents for reupload.") + + def test_get_container(self): + headers = {'X-Favourite-Pet': 'Aardvark'} + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, body=b'{}')): + with mock.patch('swiftclient.client.get_auth', + lambda *a, **k: ('http://url:8080/v1/a', 'token')): + conn = c.Connection() + conn.get_container('c1', prefix='p', limit=5, + headers=headers) + self.assertEqual(1, len(self.request_log), self.request_log) + self.assertRequests([ + ('GET', '/v1/a/c1?format=json&limit=5&prefix=p', '', { + 'x-auth-token': 'token', + 'X-Favourite-Pet': 'Aardvark', + 'accept-encoding': 'gzip', + }), + ]) + self.assertEqual(conn.attempts, 1) + + def test_head_container(self): + headers = {'X-Favourite-Pet': 'Aardvark'} + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, body=b'{}')): + with mock.patch('swiftclient.client.get_auth', + lambda *a, **k: ('http://url:8080/v1/a', 'token')): + conn = c.Connection() + conn.head_container('c1', headers=headers) + self.assertEqual(1, len(self.request_log), self.request_log) + self.assertRequests([ + ('HEAD', '/v1/a/c1', '', { + 'x-auth-token': 'token', + 'X-Favourite-Pet': 'Aardvark', + }), + ]) + self.assertEqual(conn.attempts, 1) + + def test_head_object(self): + headers = {'X-Favourite-Pet': 'Aardvark'} + query_string = 'foo=bar' + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + with mock.patch('swiftclient.client.get_auth', + lambda *a, **k: ('http://url:8080/v1/a', 'token')): + conn = c.Connection() + conn.head_object('c1', 'o1', + headers=headers, query_string=query_string) + self.assertEqual(1, len(self.request_log), self.request_log) + self.assertRequests([ + ('HEAD', '/v1/a/c1/o1?foo=bar', '', { + 'x-auth-token': 'token', + 'X-Favourite-Pet': 'Aardvark', + }), + ]) + self.assertEqual(conn.attempts, 1) + + +class TestResponseDict(MockHttpTest): + """ + Verify handling of optional response_dict argument. + """ + calls = [('post_account', {}), + ('post_container', 'c', {}), + ('put_container', 'c'), + ('delete_container', 'c'), + ('post_object', 'c', 'o', {}), + ('put_object', 'c', 'o', 'body'), + ('copy_object', 'c', 'o'), + ('delete_object', 'c', 'o')] + + def fake_get_auth(*args, **kwargs): + return 'http://url', 'token' + + def test_response_dict_with_auth_error(self): + def bad_get_auth(*args, **kwargs): + raise c.ClientException('test') + + for call in self.calls: + resp_dict = {'test': 'should be untouched'} + with mock.patch('swiftclient.client.get_auth', + bad_get_auth): + conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') + self.assertRaises(c.ClientException, getattr(conn, call[0]), + *call[1:], response_dict=resp_dict) + + self.assertEqual({'test': 'should be untouched'}, resp_dict) + + def test_response_dict_with_request_error(self): + for call in self.calls: + resp_dict = {'test': 'should be untouched'} + with mock.patch('swiftclient.client.get_auth', + self.fake_get_auth): + exc = c.ClientException('test') + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, exc=exc)): + conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') + self.assertRaises(c.ClientException, + getattr(conn, call[0]), + *call[1:], + response_dict=resp_dict) + + self.assertEqual('should be untouched', resp_dict.get('test')) + self.assertEqual([{}], resp_dict.get('response_dicts')) + + def test_response_dict(self): + # test response_dict is populated and + # new list of response_dicts is created + for call in self.calls: + resp_dict = {'test': 'should be untouched'} + with mock.patch('swiftclient.client.get_auth', + self.fake_get_auth): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') + getattr(conn, call[0])(*call[1:], response_dict=resp_dict) + + self.assertEqual('should be untouched', + resp_dict.pop('test', None)) + self.assertEqual('Fake', resp_dict.get('reason')) + self.assertEqual(200, resp_dict.get('status')) + self.assertIn('headers', resp_dict) + self.assertEqual('yes', resp_dict['headers'].get('x-works')) + children = resp_dict.pop('response_dicts', []) + self.assertEqual(1, len(children)) + self.assertEqual(resp_dict, children[0]) + + def test_response_dict_with_existing(self): + # check response_dict is populated and new dict is appended + # to existing response_dicts list + for call in self.calls: + resp_dict = {'test': 'should be untouched', + 'response_dicts': [{'existing': 'response dict'}]} + with mock.patch('swiftclient.client.get_auth', + self.fake_get_auth): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') + getattr(conn, call[0])(*call[1:], response_dict=resp_dict) + + self.assertEqual('should be untouched', + resp_dict.pop('test', None)) + self.assertEqual('Fake', resp_dict.get('reason')) + self.assertEqual(200, resp_dict.get('status')) + self.assertIn('headers', resp_dict) + self.assertEqual('yes', resp_dict['headers'].get('x-works')) + children = resp_dict.pop('response_dicts', []) + self.assertEqual(2, len(children)) + self.assertEqual({'existing': 'response dict'}, children[0]) + self.assertEqual(resp_dict, children[1]) + + +class TestLogging(MockHttpTest): + """ + Make sure all the lines in http_log are covered. + """ + + def setUp(self): + super(TestLogging, self).setUp() + self.swiftclient_logger = logging.getLogger("swiftclient") + self.log_level = self.swiftclient_logger.getEffectiveLevel() + self.swiftclient_logger.setLevel(logging.INFO) + + def tearDown(self): + self.swiftclient_logger.setLevel(self.log_level) + super(TestLogging, self).tearDown() + + def test_put_ok(self): + c.http_connection = self.fake_http_connection(200) + args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') + value = c.put_object(*args) + self.assertIsInstance(value, six.string_types) + + def test_head_error(self): + c.http_connection = self.fake_http_connection(500) + self.assertRaises(c.ClientException, c.head_object, + 'http://www.test.com', 'asdf', 'asdf', 'asdf') + + def test_get_error(self): + c.http_connection = self.fake_http_connection(404) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_status, 404) + + def test_content_encoding_gzip_body_is_logged_decoded(self): + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + data = {"test": u"\u2603"} + decoded_body = json.dumps(data).encode('utf-8') + gz.write(decoded_body) + gz.close() + # stub a gzip encoded body + body = buf.getvalue() + headers = {'content-encoding': 'gzip'} + # ... and make a content-encoding gzip error response + stub_response = StubResponse(500, body, headers) + with mock.patch('swiftclient.client.logger.info') as mock_log: + # ... if the client gets such a response + c.http_connection = self.fake_http_connection(stub_response) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_status, 500) + # it will log the decoded body + self.assertEqual([ + mock.call('REQ: %s', u'curl -i http://www.test.com/asdf/asdf ' + '-X GET -H "X-Auth-Token: ..."'), + mock.call('RESP STATUS: %s %s', 500, 'Fake'), + mock.call('RESP HEADERS: %s', {'content-encoding': 'gzip'}), + mock.call('RESP BODY: %s', decoded_body) + ], mock_log.mock_calls) + + def test_redact_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + unicode_token_encoded = unicode_token_value.encode('utf8') + set_cookie_value = 'X-Auth-Token=%s' % token_value + set_cookie_encoded = set_cookie_value.encode('utf8') + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded + }}, + MockHttpResponse( + status=200, + headers={ + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded, + 'Etag': b'mock_etag', + 'Set-Cookie': set_cookie_encoded + } + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value[:16] + '...', output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value[:8] + '...', output) + self.assertIn('Set-Cookie', output) + self.assertIn(set_cookie_value[:16] + '...', output) + self.assertNotIn(token_value, output) + self.assertNotIn(unicode_token_value, output) + self.assertNotIn(set_cookie_value, output) + + def test_show_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + c.logger_settings['redact_sensitive_headers'] = False + unicode_token_encoded = unicode_token_value.encode('utf8') + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded + }}, + MockHttpResponse( + status=200, + headers=[ + ('X-Auth-Token', token_encoded), + ('X-Storage-Token', unicode_token_encoded), + ('Etag', b'mock_etag') + ] + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value, output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value, output) + + @mock.patch('swiftclient.client.logger.debug') + def test_unicode_path(self, mock_log): + path = u'http://swift/v1/AUTH_account-\u062a'.encode('utf-8') + c.http_log(['GET', path], {}, + MockHttpResponse(status=200, headers=[]), '') + request_log_line = mock_log.mock_calls[0] + self.assertEqual('REQ: %s', request_log_line[1][0]) + self.assertEqual(u'curl -i -X GET %s' % path.decode('utf-8'), + request_log_line[1][1]) + + +class TestCloseConnection(MockHttpTest): + + def test_close_none(self): + c.http_connection = self.fake_http_connection() + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + self.assertIsNone(conn.http_conn) + conn.close() + self.assertIsNone(conn.http_conn) + # Can re-close + conn.close() + self.assertIsNone(conn.http_conn) + + def test_close_ok(self): + url = 'http://www.test.com' + conn = c.Connection(url, 'asdf', 'asdf') + self.assertIsNone(conn.http_conn) + conn.http_conn = c.http_connection(url) + self.assertEqual(type(conn.http_conn), tuple) + self.assertEqual(len(conn.http_conn), 2) + http_conn_obj = conn.http_conn[1] + self.assertIsInstance(http_conn_obj, c.HTTPConnection) + self.assertTrue(hasattr(http_conn_obj, 'close')) + conn.close() + + +class TestServiceToken(MockHttpTest): + + def setUp(self): + super(TestServiceToken, self).setUp() + self.os_options = { + 'object_storage_url': 'http://storage_url.com', + 'service_username': 'service_username', + 'service_project_name': 'service_project_name', + 'service_key': 'service_key'} + + def get_connection(self): + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + os_options=self.os_options) + + self.assertIs(type(conn), c.Connection) + conn.get_auth = self.get_auth + conn.get_service_auth = self.get_service_auth + + self.assertEqual(conn.attempts, 0) + self.assertIsNone(conn.service_token) + + self.assertIs(type(conn), c.Connection) + return conn + + def get_auth(self): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + return self.os_options.get('object_storage_url'), 'token' + + def get_service_auth(self): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + return self.os_options.get('object_storage_url'), 'stoken' + + def test_service_token_reauth(self): + get_auth_call_list = [] + + def get_auth(url, user, key, **kwargs): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + args = {'url': url, 'user': user, 'key': key, 'kwargs': kwargs} + get_auth_call_list.append(args) + return_dict = {'asdf': 'new', 'service_username': 'newserv'} + storage_url = kwargs['os_options'].get('object_storage_url') + return storage_url, return_dict[user] + + def swap_sleep(*args): + self.swap_sleep_called = True + c.get_auth = get_auth + + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(401, 200)): + with mock.patch('swiftclient.client.sleep', swap_sleep): + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + os_options=self.os_options) + + self.assertEqual(conn.attempts, 0) + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'old') + + conn.head_account() + + self.assertTrue(self.swap_sleep_called) + self.assertEqual(conn.attempts, 2) + # The original 'preauth' storage URL *must* be preserved + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'new') + self.assertEqual(conn.service_token, 'newserv') + + # Check get_auth was called with expected args + auth_args = get_auth_call_list[0] + auth_kwargs = get_auth_call_list[0]['kwargs'] + self.assertEqual('asdf', auth_args['user']) + self.assertEqual('asdf', auth_args['key']) + self.assertEqual('service_key', + auth_kwargs['os_options']['service_key']) + self.assertEqual('service_username', + auth_kwargs['os_options']['service_username']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['service_project_name']) + + auth_args = get_auth_call_list[1] + auth_kwargs = get_auth_call_list[1]['kwargs'] + self.assertEqual('service_username', auth_args['user']) + self.assertEqual('service_key', auth_args['key']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['tenant_name']) + + def test_service_token_reauth_retries_0(self): + get_auth_call_list = [] + + def get_auth(url, user, key, **kwargs): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + args = {'url': url, 'user': user, 'key': key, 'kwargs': kwargs} + get_auth_call_list.append(args) + return_dict = {'asdf': 'new', 'service_username': 'newserv'} + storage_url = kwargs['os_options'].get('object_storage_url') + return storage_url, return_dict[user] + + def swap_sleep(*args): + self.swap_sleep_called = True + c.get_auth = get_auth + + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(401, 200)): + with mock.patch('swiftclient.client.sleep', swap_sleep): + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + os_options=self.os_options, + retries=0) + + self.assertEqual(conn.attempts, 0) + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'old') + + conn.head_account() + + self.assertTrue(self.swap_sleep_called) + self.assertEqual(conn.attempts, 2) + # The original 'preauth' storage URL *must* be preserved + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'new') + self.assertEqual(conn.service_token, 'newserv') + + # Check get_auth was called with expected args + auth_args = get_auth_call_list[0] + auth_kwargs = get_auth_call_list[0]['kwargs'] + self.assertEqual('asdf', auth_args['user']) + self.assertEqual('asdf', auth_args['key']) + self.assertEqual('service_key', + auth_kwargs['os_options']['service_key']) + self.assertEqual('service_username', + auth_kwargs['os_options']['service_username']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['service_project_name']) + + auth_args = get_auth_call_list[1] + auth_kwargs = get_auth_call_list[1]['kwargs'] + self.assertEqual('service_username', auth_args['user']) + self.assertEqual('service_key', auth_args['key']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['tenant_name']) + + # Ensure this is not an endless loop - it fails after the second 401 + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(401, 401, 401, 401)): + with mock.patch('swiftclient.client.sleep', swap_sleep): + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + os_options=self.os_options, + retries=0) + + self.assertEqual(conn.attempts, 0) + self.assertRaises(c.ClientException, conn.head_account) + self.assertEqual(conn.attempts, 2) + unused_responses = list(self.fake_connect.code_iter) + self.assertEqual(unused_responses, [401, 401]) + + def test_service_token_get_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + with mock.patch('swiftclient.client.parse_api_response'): + conn = self.get_connection() + conn.get_account() + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/?format=json', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_account() + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com', actual['full_path']) + + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(201)): + conn = self.get_connection() + conn.post_account(headers={}) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com', actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_delete_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(204)): + conn = self.get_connection() + conn.delete_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('DELETE', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_get_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + with mock.patch('swiftclient.client.parse_api_response'): + conn = self.get_connection() + conn.get_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1?format=json', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_get_container_full_listing(self): + # verify service token is sent with each request for a full listing + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, 200)): + with mock.patch('swiftclient.client.parse_api_response') as resp: + resp.side_effect = ([{"name": "obj1"}], []) + conn = self.get_connection() + conn.get_container('container1', full_listing=True) + self.assertEqual(2, len(self.request_log), self.request_log) + expected_urls = iter(( + 'http://storage_url.com/container1?format=json', + 'http://storage_url.com/container1?format=json&marker=obj1' + )) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual(next(expected_urls), + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_container(self): + headers = {'X-Container-Meta-Color': 'blue'} + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(201)): + conn = self.get_connection() + conn.post_container('container1', headers) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + # Check that we didn't mutate the request header dict + self.assertEqual(headers, {'X-Container-Meta-Color': 'blue'}) + + def test_service_token_put_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.put_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('PUT', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_get_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.get_object('container1', 'obj1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_object('container1', 'obj1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_put_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.put_object('container1', 'obj1', 'a_string') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('PUT', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(202)): + conn = self.get_connection() + conn.post_object('container1', 'obj1', {}) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_delete_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(202)): + conn = self.get_connection() + conn.delete_object('container1', 'obj1', query_string='a_string') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('DELETE', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1?a_string', + actual['full_path']) + self.assertEqual(conn.attempts, 1) diff -Nru python-swiftclient-3.8.1/test/unit/test_utils.py python-swiftclient-3.9.0/test/unit/test_utils.py --- python-swiftclient-3.8.1/test/unit/test_utils.py 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/unit/test_utils.py 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,679 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import json +import unittest +import mock +import six +import tempfile +from time import gmtime, localtime, mktime, strftime, strptime +from hashlib import md5, sha1 + +from swiftclient import utils as u + + +class TestConfigTrueValue(unittest.TestCase): + + def test_TRUE_VALUES(self): + for v in u.TRUE_VALUES: + self.assertEqual(v, v.lower()) + + @mock.patch.object(u, 'TRUE_VALUES', 'hello world'.split()) + def test_config_true_value(self): + for val in 'hello world HELLO WORLD'.split(): + self.assertIs(True, u.config_true_value(val)) + self.assertIs(True, u.config_true_value(True)) + self.assertIs(False, u.config_true_value('foo')) + self.assertIs(False, u.config_true_value(False)) + + +class TestPrtBytes(unittest.TestCase): + + def test_zero_bytes(self): + bytes_ = 0 + raw = '0' + human = '0' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_one_byte(self): + bytes_ = 1 + raw = '1' + human = '1' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_less_than_one_k(self): + bytes_ = (2 ** 10) - 1 + raw = '1023' + human = '1023' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_one_k(self): + bytes_ = 2 ** 10 + raw = '1024' + human = '1.0K' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_a_decimal_k(self): + bytes_ = (3 * 2 ** 10) + 512 + raw = '3584' + human = '3.5K' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_a_bit_less_than_one_meg(self): + bytes_ = (2 ** 20) - (2 ** 10) + raw = '1047552' + human = '1023K' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_just_a_hair_less_than_one_meg(self): + bytes_ = (2 ** 20) - (2 ** 10) + 1 + raw = '1047553' + human = '1.0M' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_one_meg(self): + bytes_ = 2 ** 20 + raw = '1048576' + human = '1.0M' + self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_ten_meg(self): + bytes_ = 10 * 2 ** 20 + human = '10M' + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_bit_less_than_ten_meg(self): + bytes_ = (10 * 2 ** 20) - (100 * 2 ** 10) + human = '9.9M' + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_just_a_hair_less_than_ten_meg(self): + bytes_ = (10 * 2 ** 20) - 1 + human = '10.0M' + self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) + + def test_a_yotta(self): + bytes_ = 42 * 2 ** 80 + self.assertEqual('42Y', u.prt_bytes(bytes_, True).lstrip()) + + def test_overflow(self): + bytes_ = 2 ** 90 + self.assertEqual('1024Y', u.prt_bytes(bytes_, True).lstrip()) + + +class TestTempURL(unittest.TestCase): + url = '/v1/AUTH_account/c/o' + seconds = 3600 + key = 'correcthorsebatterystaple' + method = 'GET' + expected_url = url + ('?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600') + expected_body = '\n'.join([ + method, + '1400003600', + url, + ]).encode('utf-8') + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + url = u.generate_temp_url(self.url, self.seconds, + self.key, self.method) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + self.assertEqual(url, self.expected_url) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(), + mock.call(key, self.expected_body, sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(self.url)) + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_ip_range(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + ip_ranges = [ + '1.2.3.4', '1.2.3.4/24', '2001:db8::', + b'1.2.3.4', b'1.2.3.4/24', b'2001:db8::', + ] + path = '/v1/AUTH_account/c/o/' + expected_url = path + ('?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600' + '&temp_url_ip_range=') + for ip_range in ip_ranges: + hmac_mock.reset_mock() + url = u.generate_temp_url(path, self.seconds, + self.key, self.method, + ip_range=ip_range) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + + if isinstance(ip_range, six.binary_type): + ip_range_expected_url = ( + expected_url + ip_range.decode('utf-8') + ) + expected_body = '\n'.join([ + 'ip=' + ip_range.decode('utf-8'), + self.method, + '1400003600', + path, + ]).encode('utf-8') + else: + ip_range_expected_url = expected_url + ip_range + expected_body = '\n'.join([ + 'ip=' + ip_range, + self.method, + '1400003600', + path, + ]).encode('utf-8') + + self.assertEqual(url, ip_range_expected_url) + + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(key, expected_body, sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(path)) + + @mock.patch('hmac.HMAC') + def test_generate_temp_url_iso8601_argument(self, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', + self.key, self.method) + self.assertEqual(url, self.expected_url) + + # Don't care about absolute arg. + url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', + self.key, self.method, absolute=True) + self.assertEqual(url, self.expected_url) + + lt = localtime() + expires = strftime(u.EXPIRES_ISO8601_FORMAT[:-1], lt) + + if not isinstance(self.expected_url, six.string_types): + expected_url = self.expected_url.replace( + b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) + else: + expected_url = self.expected_url.replace( + '1400003600', str(int(mktime(lt)))) + url = u.generate_temp_url(self.url, expires, + self.key, self.method) + self.assertEqual(url, expected_url) + + expires = strftime(u.SHORT_EXPIRES_ISO8601_FORMAT, lt) + lt = strptime(expires, u.SHORT_EXPIRES_ISO8601_FORMAT) + + if not isinstance(self.expected_url, six.string_types): + expected_url = self.expected_url.replace( + b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) + else: + expected_url = self.expected_url.replace( + '1400003600', str(int(mktime(lt)))) + url = u.generate_temp_url(self.url, expires, + self.key, self.method) + self.assertEqual(url, expected_url) + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_iso8601_output(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + url = u.generate_temp_url(self.url, self.seconds, + self.key, self.method, + iso8601=True) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + + expires = strftime(u.EXPIRES_ISO8601_FORMAT, gmtime(1400003600)) + if not isinstance(self.url, six.string_types): + self.assertTrue(url.endswith(bytes(expires, 'utf-8'))) + else: + self.assertTrue(url.endswith(expires)) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(), + mock.call(key, self.expected_body, sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(self.url)) + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_prefix(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + prefixes = ['', 'o', 'p0/p1/'] + for p in prefixes: + hmac_mock.reset_mock() + path = '/v1/AUTH_account/c/' + p + expected_url = path + ('?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600' + '&temp_url_prefix=' + p) + expected_body = '\n'.join([ + self.method, + '1400003600', + 'prefix:' + path, + ]).encode('utf-8') + url = u.generate_temp_url(path, self.seconds, + self.key, self.method, prefix=True) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + self.assertEqual(url, expected_url) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(key, expected_body, sha1), + mock.call().hexdigest(), + ]) + + self.assertIsInstance(url, type(path)) + + def test_generate_temp_url_invalid_path(self): + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(b'/v1/a/c/\xff', self.seconds, self.key, + self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be representable as UTF-8') + + @mock.patch('hmac.HMAC.hexdigest', return_value="temp_url_signature") + def test_generate_absolute_expiry_temp_url(self, hmac_mock): + if isinstance(self.expected_url, six.binary_type): + expected_url = self.expected_url.replace( + b'1400003600', b'2146636800') + else: + expected_url = self.expected_url.replace( + u'1400003600', u'2146636800') + url = u.generate_temp_url(self.url, 2146636800, self.key, self.method, + absolute=True) + self.assertEqual(url, expected_url) + + def test_generate_temp_url_bad_time(self): + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, 'not_an_int', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, -1, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, 1.1, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, '-1', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, '1.1', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, '2015-05', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url( + self.url, '2015-05-01T01:00', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + def test_generate_temp_url_bad_path(self): + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('/v1/a/c', 60, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be full path to an object e.g. /v1/a/c/o') + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('v1/a/c/o', 60, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be full path to an object e.g. /v1/a/c/o') + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('blah/v1/a/c/o', 60, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be full path to an object e.g. /v1/a/c/o') + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('/v1//c/o', 60, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be full path to an object e.g. /v1/a/c/o') + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('/v1/a/c/', 60, self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], + 'path must be full path to an object e.g. /v1/a/c/o') + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('/v1/a/c', 60, self.key, self.method, + prefix=True) + self.assertEqual(exc_manager.exception.args[0], + 'path must at least contain /v1/a/c/') + + +class TestTempURLUnicodePathAndKey(TestTempURL): + url = u'/v1/\u00e4/c/\u00f3' + key = u'k\u00e9y' + expected_url = (u'%s?temp_url_sig=temp_url_signature' + u'&temp_url_expires=1400003600') % url + expected_body = u'\n'.join([ + u'GET', + u'1400003600', + url, + ]).encode('utf-8') + + +class TestTempURLUnicodePathBytesKey(TestTempURL): + url = u'/v1/\u00e4/c/\u00f3' + key = u'k\u00e9y'.encode('utf-8') + expected_url = (u'%s?temp_url_sig=temp_url_signature' + u'&temp_url_expires=1400003600') % url + expected_body = '\n'.join([ + u'GET', + u'1400003600', + url, + ]).encode('utf-8') + + +class TestTempURLBytesPathUnicodeKey(TestTempURL): + url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') + key = u'k\u00e9y' + expected_url = url + (b'?temp_url_sig=temp_url_signature' + b'&temp_url_expires=1400003600') + expected_body = b'\n'.join([ + b'GET', + b'1400003600', + url, + ]) + + +class TestTempURLBytesPathAndKey(TestTempURL): + url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') + key = u'k\u00e9y'.encode('utf-8') + expected_url = url + (b'?temp_url_sig=temp_url_signature' + b'&temp_url_expires=1400003600') + expected_body = b'\n'.join([ + b'GET', + b'1400003600', + url, + ]) + + +class TestTempURLBytesPathAndNonUtf8Key(TestTempURL): + url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') + key = b'k\xffy' + expected_url = url + (b'?temp_url_sig=temp_url_signature' + b'&temp_url_expires=1400003600') + expected_body = b'\n'.join([ + b'GET', + b'1400003600', + url, + ]) + + +class TestReadableToIterable(unittest.TestCase): + + def test_iter(self): + chunk_size = 4 + write_data = tuple(x.encode() for x in ('a', 'b', 'c', 'd')) + actual_md5sum = md5() + + with tempfile.TemporaryFile() as f: + for x in write_data: + f.write(x * chunk_size) + actual_md5sum.update(x * chunk_size) + f.seek(0) + data = u.ReadableToIterable(f, chunk_size, True) + + for i, data_chunk in enumerate(data): + self.assertEqual(chunk_size, len(data_chunk)) + self.assertEqual(data_chunk, write_data[i] * chunk_size) + + self.assertEqual(actual_md5sum.hexdigest(), data.get_md5sum()) + + def test_md5_creation(self): + # Check creation with a real and noop md5 class + data = u.ReadableToIterable(None, None, md5=True) + self.assertEqual(md5().hexdigest(), data.get_md5sum()) + self.assertIs(type(md5()), type(data.md5sum)) + + data = u.ReadableToIterable(None, None, md5=False) + self.assertEqual('', data.get_md5sum()) + self.assertIs(u.NoopMD5, type(data.md5sum)) + + def test_unicode(self): + # Check no errors are raised if unicode data is feed in. + unicode_data = u'abc' + actual_md5sum = md5(unicode_data.encode()).hexdigest() + chunk_size = 2 + + with tempfile.TemporaryFile(mode='w+') as f: + f.write(unicode_data) + f.seek(0) + data = u.ReadableToIterable(f, chunk_size, True) + + x = next(data) + self.assertEqual(2, len(x)) + self.assertEqual(unicode_data[:2], x) + + x = next(data) + self.assertEqual(1, len(x)) + self.assertEqual(unicode_data[2:], x) + + self.assertEqual(actual_md5sum, data.get_md5sum()) + + +class TestLengthWrapper(unittest.TestCase): + + def test_stringio(self): + contents = six.StringIO(u'a' * 50 + u'b' * 50) + contents.seek(22) + data = u.LengthWrapper(contents, 42, True) + s = u'a' * 28 + u'b' * 14 + read_data = u''.join(iter(data.read, '')) + + self.assertEqual(42, len(data)) + self.assertEqual(42, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) + + data.reset() + self.assertEqual(md5().hexdigest(), data.get_md5sum()) + + read_data = u''.join(iter(data.read, '')) + self.assertEqual(42, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) + + def test_bytesio(self): + contents = six.BytesIO(b'a' * 50 + b'b' * 50) + contents.seek(22) + data = u.LengthWrapper(contents, 42, True) + s = b'a' * 28 + b'b' * 14 + read_data = b''.join(iter(data.read, '')) + + self.assertEqual(42, len(data)) + self.assertEqual(42, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + def test_tempfile(self): + with tempfile.NamedTemporaryFile(mode='wb') as f: + f.write(b'a' * 100) + f.flush() + contents = open(f.name, 'rb') + data = u.LengthWrapper(contents, 42, True) + s = b'a' * 42 + read_data = b''.join(iter(data.read, '')) + + self.assertEqual(42, len(data)) + self.assertEqual(42, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + def test_segmented_file(self): + with tempfile.NamedTemporaryFile(mode='wb') as f: + segment_length = 1024 + segments = ('a', 'b', 'c', 'd') + for c in segments: + f.write((c * segment_length).encode()) + f.flush() + for i, c in enumerate(segments): + contents = open(f.name, 'rb') + contents.seek(i * segment_length) + data = u.LengthWrapper(contents, segment_length, True) + read_data = b''.join(iter(data.read, '')) + s = (c * segment_length).encode() + + self.assertEqual(segment_length, len(data)) + self.assertEqual(segment_length, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + data.reset() + self.assertEqual(md5().hexdigest(), data.get_md5sum()) + read_data = b''.join(iter(data.read, '')) + self.assertEqual(segment_length, len(data)) + self.assertEqual(segment_length, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + +class TestGroupers(unittest.TestCase): + def test_n_at_a_time(self): + result = list(u.n_at_a_time(range(100), 9)) + self.assertEqual([9] * 11 + [1], list(map(len, result))) + + result = list(u.n_at_a_time(range(100), 10)) + self.assertEqual([10] * 10, list(map(len, result))) + + result = list(u.n_at_a_time(range(100), 11)) + self.assertEqual([11] * 9 + [1], list(map(len, result))) + + result = list(u.n_at_a_time(range(100), 12)) + self.assertEqual([12] * 8 + [4], list(map(len, result))) + + def test_n_groups(self): + result = list(u.n_groups(range(100), 9)) + self.assertEqual([12] * 8 + [4], list(map(len, result))) + + result = list(u.n_groups(range(100), 10)) + self.assertEqual([10] * 10, list(map(len, result))) + + result = list(u.n_groups(range(100), 11)) + self.assertEqual([10] * 10, list(map(len, result))) + + result = list(u.n_groups(range(100), 12)) + self.assertEqual([9] * 11 + [1], list(map(len, result))) + + +class TestApiResponeParser(unittest.TestCase): + + def test_utf8_default(self): + result = u.parse_api_response( + {}, u'{"test": "\u2603"}'.encode('utf8')) + self.assertEqual({'test': u'\u2603'}, result) + + result = u.parse_api_response( + {}, u'{"test": "\\u2603"}'.encode('utf8')) + self.assertEqual({'test': u'\u2603'}, result) + + def test_bad_json(self): + self.assertRaises(ValueError, u.parse_api_response, + {}, b'{"foo": "bar}') + + def test_bad_utf8(self): + self.assertRaises(UnicodeDecodeError, u.parse_api_response, + {}, b'{"foo": "b\xffr"}') + + def test_latin_1(self): + result = u.parse_api_response( + {'content-type': 'application/json; charset=iso8859-1'}, + b'{"t\xe9st": "\xff"}') + self.assertEqual({u't\xe9st': u'\xff'}, result) + + def test_gzipped_utf8(self): + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.close() + result = u.parse_api_response( + {'content-encoding': 'gzip'}, + buf.getvalue()) + self.assertEqual({'test': u'\u2603'}, result) + + +class TestGetBody(unittest.TestCase): + + def test_not_gzipped(self): + result = u.parse_api_response( + {}, u'{"test": "\\u2603"}'.encode('utf8')) + self.assertEqual({'test': u'\u2603'}, result) + + def test_gzipped_body(self): + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.close() + result = u.parse_api_response( + {'content-encoding': 'gzip'}, + buf.getvalue()) + self.assertEqual({'test': u'\u2603'}, result) + + +class JSONTracker(object): + def __init__(self, data): + self.data = data + self.calls = [] + + def __iter__(self): + for item in self.data: + self.calls.append(('read', item)) + yield item + + def write(self, s): + self.calls.append(('write', s)) + + +class TestJSONableIterable(unittest.TestCase): + def test_json_dump_iterencodes(self): + t = JSONTracker([1, 'fish', 2, 'fish']) + json.dump(u.JSONableIterable(t), t) + self.assertEqual(t.calls, [ + ('read', 1), + ('write', '[1'), + ('read', 'fish'), + ('write', ', "fish"'), + ('read', 2), + ('write', ', 2'), + ('read', 'fish'), + ('write', ', "fish"'), + ('write', ']'), + ]) + + def test_json_dump_empty_iter(self): + t = JSONTracker([]) + json.dump(u.JSONableIterable(t), t) + self.assertEqual(t.calls, [ + ('write', '[]'), + ]) diff -Nru python-swiftclient-3.8.1/test/unit/utils.py python-swiftclient-3.9.0/test/unit/utils.py --- python-swiftclient-3.8.1/test/unit/utils.py 1970-01-01 00:00:00.000000000 +0000 +++ python-swiftclient-3.9.0/test/unit/utils.py 2020-02-13 17:31:20.000000000 +0000 @@ -0,0 +1,582 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import functools +import sys +from requests import RequestException +from requests.structures import CaseInsensitiveDict +from time import sleep +import unittest +import mock +import six +import os +from six.moves import reload_module +from six.moves.urllib.parse import urlparse, ParseResult +from swiftclient import client as c +from swiftclient import shell as s +from swiftclient.utils import EMPTY_ETAG + + +def fake_get_auth_keystone(expected_os_options=None, exc=None, + storage_url='http://url/', token='token', + **kwargs): + def fake_get_auth_keystone(auth_url, + user, + key, + actual_os_options, **actual_kwargs): + if exc: + raise exc('test') + # TODO: some way to require auth_url, user and key? + if expected_os_options: + for key, value in actual_os_options.items(): + if value and value != expected_os_options.get(key): + return "", None + if 'required_kwargs' in kwargs: + for k, v in kwargs['required_kwargs'].items(): + if v != actual_kwargs.get(k): + return "", None + + if auth_url.startswith("https") and \ + auth_url.endswith("invalid-certificate") and \ + not actual_kwargs['insecure']: + from swiftclient import client as c + raise c.ClientException("invalid-certificate") + if auth_url.startswith("https") and \ + auth_url.endswith("self-signed-certificate") and \ + not actual_kwargs['insecure'] and \ + actual_kwargs['cacert'] is None: + from swiftclient import client as c + raise c.ClientException("unverified-certificate") + if auth_url.startswith("https") and \ + auth_url.endswith("client-certificate") and \ + not (actual_kwargs['cert'] and actual_kwargs['cert_key']): + from swiftclient import client as c + raise c.ClientException("noclient-certificate") + + return storage_url, token + return fake_get_auth_keystone + + +class StubResponse(object): + """ + Placeholder structure for use with fake_http_connect's code_iter to modify + response attributes (status, body, headers) on a per-request basis. + """ + + def __init__(self, status=200, body='', headers=None): + self.status = status + self.body = body + self.headers = headers or {} + + def __repr__(self): + return '%s(%r, %r, %r)' % (self.__class__.__name__, self.status, + self.body, self.headers) + + +def fake_http_connect(*code_iter, **kwargs): + """ + Generate a callable which yields a series of stubbed responses. Because + swiftclient will reuse an HTTP connection across pipelined requests it is + not always the case that this fake is used strictly for mocking an HTTP + connection, but rather each HTTP response (i.e. each call to requests + get_response). + """ + + class FakeConn(object): + + def __init__(self, status, etag=None, body='', timestamp='1', + headers=None): + self.status_code = self.status = status + self.reason = 'Fake' + self.scheme = 'http' + self.host = '1.2.3.4' + self.port = '1234' + self.sent = 0 + self.received = 0 + self.etag = etag + self.content = self.body = body + self.timestamp = timestamp + self.headers = headers or {} + self.request = None + + def getresponse(self): + if kwargs.get('raise_exc'): + raise Exception('test') + return self + + def getheaders(self): + if self.headers: + return self.headers.items() + headers = {'content-length': str(len(self.body)), + 'content-type': 'x-application/test', + 'x-timestamp': self.timestamp, + 'last-modified': self.timestamp, + 'x-object-meta-test': 'testing', + 'etag': + self.etag or '"%s"' % EMPTY_ETAG, + 'x-works': 'yes', + 'x-account-container-count': '12345'} + if not self.timestamp: + del headers['x-timestamp'] + try: + if next(container_ts_iter) is False: + headers['x-container-timestamp'] = '1' + except StopIteration: + pass + if 'slow' in kwargs: + headers['content-length'] = '4' + if 'headers' in kwargs: + headers.update(kwargs['headers']) + if 'auth_v1' in kwargs: + headers.update( + {'x-storage-url': 'storageURL', + 'x-auth-token': 'someauthtoken'}) + return headers.items() + + def read(self, amt=None): + if 'slow' in kwargs: + if self.sent < 4: + self.sent += 1 + sleep(0.1) + return ' ' + rv = self.body[:amt] + if amt is not None: + self.body = self.body[amt:] + else: + self.body = '' + return rv + + def send(self, amt=None): + if 'slow' in kwargs: + if self.received < 4: + self.received += 1 + sleep(0.1) + + def getheader(self, name, default=None): + return dict(self.getheaders()).get(name.lower(), default) + + def close(self): + pass + + timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) + etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) + x = kwargs.get('missing_container', [False] * len(code_iter)) + if not isinstance(x, (tuple, list)): + x = [x] * len(code_iter) + container_ts_iter = iter(x) + code_iter = iter(code_iter) + + def connect(*args, **ckwargs): + if 'give_content_type' in kwargs: + if len(args) >= 7 and 'Content-Type' in args[6]: + kwargs['give_content_type'](args[6]['Content-Type']) + else: + kwargs['give_content_type']('') + if 'give_connect' in kwargs: + kwargs['give_connect'](*args, **ckwargs) + status = next(code_iter) + if isinstance(status, StubResponse): + fake_conn = FakeConn(status.status, body=status.body, + headers=status.headers) + else: + etag = next(etag_iter) + timestamp = next(timestamps_iter) + fake_conn = FakeConn(status, etag, body=kwargs.get('body', ''), + timestamp=timestamp) + if fake_conn.status <= 0: + raise RequestException() + return fake_conn + + connect.code_iter = code_iter + return connect + + +class MockHttpTest(unittest.TestCase): + + def setUp(self): + super(MockHttpTest, self).setUp() + self.fake_connect = None + self.request_log = [] + + # Capture output, since the test-runner stdout/stderr monkey-patching + # won't cover the references to sys.stdout/sys.stderr in + # swiftclient.multithreading + self.capture_output = CaptureOutput() + if 'SWIFTCLIENT_DEBUG' not in os.environ: + self.capture_output.__enter__() + self.addCleanup(self.capture_output.__exit__) + + # since we're going to steal all stderr output globally; we should + # give the developer an escape hatch or risk scorn + def blowup_but_with_the_helpful(*args, **kwargs): + raise Exception( + "You tried to enter a debugger while stderr is " + "patched, you need to set SWIFTCLIENT_DEBUG=1 " + "and try again") + import pdb + pdb.set_trace = blowup_but_with_the_helpful + + def fake_http_connection(*args, **kwargs): + self.validateMockedRequestsConsumed() + self.request_log = [] + self.fake_connect = fake_http_connect(*args, **kwargs) + _orig_http_connection = c.http_connection + query_string = kwargs.get('query_string') + storage_url = kwargs.get('storage_url') + auth_token = kwargs.get('auth_token') + exc = kwargs.get('exc') + on_request = kwargs.get('on_request') + + def wrapper(url, proxy=None, cacert=None, insecure=False, + cert=None, cert_key=None, + ssl_compression=True, timeout=None): + if storage_url: + self.assertEqual(storage_url, url) + + parsed, _conn = _orig_http_connection(url, proxy=proxy) + + class RequestsWrapper(object): + def close(self): + pass + conn = RequestsWrapper() + + def request(method, path, *args, **kwargs): + try: + conn.resp = self.fake_connect() + except StopIteration: + self.fail('Unexpected %s request for %s' % ( + method, path)) + self.request_log.append((parsed, method, path, args, + kwargs, conn.resp)) + conn.host = conn.resp.host + conn.resp.request = RequestsWrapper() + conn.resp.request.url = '%s://%s%s' % ( + conn.resp.scheme, conn.resp.host, path) + conn.resp.has_been_read = False + _orig_read = conn.resp.read + + def read(*args, **kwargs): + conn.resp.has_been_read = True + return _orig_read(*args, **kwargs) + conn.resp.read = read + if on_request: + status = on_request(method, path, *args, **kwargs) + conn.resp.status = status + if auth_token: + headers = args[1] + self.assertEqual(auth_token, + headers.get('X-Auth-Token')) + if query_string: + self.assertTrue(path.endswith('?' + query_string)) + if path.endswith('invalid_cert') and not insecure: + from swiftclient import client as c + raise c.ClientException("invalid_certificate") + if exc: + raise exc + return conn.resp + + def putrequest(path, data=None, headers=None, **kwargs): + request('PUT', path, data, headers, **kwargs) + + conn.request = request + conn.putrequest = putrequest + + def getresponse(): + return conn.resp + conn.getresponse = getresponse + + return parsed, conn + return wrapper + self.fake_http_connection = fake_http_connection + + def iter_request_log(self): + for parsed, method, path, args, kwargs, resp in self.request_log: + parts = parsed._asdict() + parts['path'] = path + full_path = ParseResult(**parts).geturl() + args = list(args) + log = dict(zip(('body', 'headers'), args)) + log.update({ + 'method': method, + 'full_path': full_path, + 'parsed_path': urlparse(full_path), + 'path': path, + 'headers': CaseInsensitiveDict(log.get('headers')), + 'resp': resp, + 'status': resp.status, + }) + yield log + + orig_assertEqual = unittest.TestCase.assertEqual + + def assert_request_equal(self, expected, real_request): + method, path = expected[:2] + if urlparse(path).scheme: + match_path = real_request['full_path'] + else: + match_path = real_request['path'] + self.assertEqual((method, path), (real_request['method'], + match_path)) + if len(expected) > 2: + body = expected[2] + real_request['expected'] = body + err_msg = 'Body mismatch for %(method)s %(path)s, ' \ + 'expected %(expected)r, and got %(body)r' % real_request + self.orig_assertEqual(body, real_request['body'], err_msg) + + if len(expected) > 3: + headers = CaseInsensitiveDict(expected[3]) + for key, value in headers.items(): + real_request['key'] = key + real_request['expected_value'] = value + real_request['value'] = real_request['headers'].get(key) + err_msg = ( + 'Header mismatch on %(key)r, ' + 'expected %(expected_value)r and got %(value)r ' + 'for %(method)s %(path)s %(headers)r' % real_request) + self.orig_assertEqual(value, real_request['value'], + err_msg) + real_request['extra_headers'] = dict( + (key, value) for key, value in real_request['headers'].items() + if key not in headers) + if real_request['extra_headers']: + self.fail('Received unexpected headers for %(method)s ' + '%(path)s, got %(extra_headers)r' % real_request) + + def assertRequests(self, expected_requests): + """ + Make sure some requests were made like you expected, provide a list of + expected requests, typically in the form of [(method, path), ...] + or [(method, path, body, headers), ...] + """ + real_requests = self.iter_request_log() + for expected in expected_requests: + real_request = next(real_requests) + self.assert_request_equal(expected, real_request) + try: + real_request = next(real_requests) + except StopIteration: + pass + else: + self.fail('At least one extra request received: %r' % + real_request) + + def assert_request(self, expected_request): + """ + Make sure a request was made as expected. Provide the + expected request in the form of [(method, path), ...] + """ + real_requests = self.iter_request_log() + for real_request in real_requests: + try: + self.assert_request_equal(expected_request, real_request) + break + except AssertionError: + pass + else: + raise AssertionError( + "Expected request %s not found in actual requests %s" + % (expected_request, self.request_log) + ) + + def validateMockedRequestsConsumed(self): + if not self.fake_connect: + return + unused_responses = list(self.fake_connect.code_iter) + if unused_responses: + self.fail('Unused responses %r' % (unused_responses,)) + + def tearDown(self): + self.validateMockedRequestsConsumed() + super(MockHttpTest, self).tearDown() + # TODO: this nuke from orbit clean up seems to be encouraging + # un-hygienic mocking on the swiftclient.client module; which may lead + # to some unfortunate test order dependency bugs by way of the broken + # window theory if any other modules are similarly patched + reload_module(c) + + +class CaptureStreamPrinter(object): + """ + CaptureStreamPrinter is used for testing unicode writing for PY3. Anything + written here is encoded as utf-8 and written to the parent CaptureStream + """ + def __init__(self, captured_stream): + self._captured_stream = captured_stream + + def write(self, data): + # No encoding, just convert the raw bytes into a str for testing + # The below call also validates that we have a byte string. + self._captured_stream.write( + data if isinstance(data, six.binary_type) else data.encode('utf8')) + + +class CaptureStream(object): + + def __init__(self, stream): + self.stream = stream + self._buffer = six.BytesIO() + self._capture = CaptureStreamPrinter(self._buffer) + self.streams = [self._capture] + + @property + def buffer(self): + if six.PY3: + return self._buffer + else: + raise AttributeError( + 'Output stream has no attribute "buffer" in Python2') + + def flush(self): + pass + + def write(self, *args, **kwargs): + for stream in self.streams: + stream.write(*args, **kwargs) + + def writelines(self, *args, **kwargs): + for stream in self.streams: + stream.writelines(*args, **kwargs) + + def getvalue(self): + return self._buffer.getvalue() + + def clear(self): + self._buffer.truncate(0) + self._buffer.seek(0) + + +class CaptureOutput(object): + + def __init__(self, suppress_systemexit=False): + self._out = CaptureStream(sys.stdout) + self._err = CaptureStream(sys.stderr) + self.patchers = [] + + WrappedOutputManager = functools.partial(s.OutputManager, + print_stream=self._out, + error_stream=self._err) + + if suppress_systemexit: + self.patchers += [ + mock.patch('swiftclient.shell.OutputManager.get_error_count', + return_value=0) + ] + + self.patchers += [ + mock.patch('swiftclient.shell.OutputManager', + WrappedOutputManager), + mock.patch('sys.stdout', self._out), + mock.patch('sys.stderr', self._err), + ] + + def __enter__(self): + for patcher in self.patchers: + patcher.start() + return self + + def __exit__(self, *args, **kwargs): + for patcher in self.patchers: + patcher.stop() + + @property + def out(self): + return self._out.getvalue().decode('utf8') + + @property + def err(self): + return self._err.getvalue().decode('utf8') + + def clear(self): + self._out.clear() + self._err.clear() + + # act like the string captured by stdout + + def __str__(self): + return self.out + + def __len__(self): + return len(self.out) + + def __eq__(self, other): + return self.out == other + + def __ne__(self, other): + return not self.__eq__(other) + + def __getattr__(self, name): + return getattr(self.out, name) + + +class FakeKeystone(object): + ''' + Fake keystone client module. Returns given endpoint url and auth token. + ''' + def __init__(self, endpoint, token): + self.calls = [] + self.auth_version = None + self.endpoint = endpoint + self.token = token + + class _Client(object): + def __init__(self, endpoint, auth_token, **kwargs): + self.auth_token = auth_token + self.endpoint = endpoint + self.service_catalog = self.ServiceCatalog(endpoint) + + class ServiceCatalog(object): + def __init__(self, endpoint): + self.calls = [] + self.endpoint_url = endpoint + + def url_for(self, **kwargs): + self.calls.append(kwargs) + return self.endpoint_url + + def Client(self, **kwargs): + self.calls.append(kwargs) + self.client = self._Client( + endpoint=self.endpoint, auth_token=self.token, **kwargs) + return self.client + + class Unauthorized(Exception): + pass + + class AuthorizationFailure(Exception): + pass + + class EndpointNotFound(Exception): + pass + + +class FakeStream(object): + def __init__(self, size): + self.bytes_read = 0 + self.size = size + + def read(self, size=-1): + if self.bytes_read == self.size: + return b'' + + if size == -1 or size + self.bytes_read > self.size: + remaining = self.size - self.bytes_read + self.bytes_read = self.size + return b'A' * remaining + + self.bytes_read += size + return b'A' * size + + def __len__(self): + return self.size diff -Nru python-swiftclient-3.8.1/test-requirements.txt python-swiftclient-3.9.0/test-requirements.txt --- python-swiftclient-3.8.1/test-requirements.txt 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/test-requirements.txt 2020-02-13 17:31:20.000000000 +0000 @@ -4,3 +4,4 @@ keystoneauth1>=3.4.0 # Apache-2.0 mock>=1.2.0 # BSD stestr>=2.0.0 # Apache-2.0 +openstacksdk>=0.11.0 # Apache-2.0 diff -Nru python-swiftclient-3.8.1/tests/functional/test_swiftclient.py python-swiftclient-3.9.0/tests/functional/test_swiftclient.py --- python-swiftclient-3.8.1/tests/functional/test_swiftclient.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/tests/functional/test_swiftclient.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,557 +0,0 @@ -# Copyright (c) 2014 Christian Schwede -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import unittest -import time -from io import BytesIO - -import six -from six.moves import configparser - -import swiftclient - - -class TestFunctional(unittest.TestCase): - - def __init__(self, *args, **kwargs): - super(TestFunctional, self).__init__(*args, **kwargs) - self.skip_tests = False - self._get_config() - - self.test_data = b'42' * 10 - self.etag = '2704306ec982238d85d4b235c925d58e' - - self.containername = "functional-tests-container-%s" % int(time.time()) - self.containername_2 = self.containername + '_second' - self.containername_3 = self.containername + '_third' - self.objectname = "functional-tests-object-%s" % int(time.time()) - self.objectname_2 = self.objectname + '_second' - - def _get_config(self): - config_file = os.environ.get('SWIFT_TEST_CONFIG_FILE', - '/etc/swift/test.conf') - config = configparser.ConfigParser({'auth_version': '1'}) - config.read(config_file) - self.config = config - if config.has_section('func_test'): - if config.has_option('func_test', 'auth_uri'): - self.auth_url = config.get('func_test', 'auth_uri') - try: - self.auth_version = config.get('func_test', 'auth_version') - except configparser.NoOptionError: - last_piece = self.auth_url.rstrip('/').rsplit('/', 1)[1] - if last_piece.endswith('.0'): - last_piece = last_piece[:-2] - if last_piece in ('1', '2', '3'): - self.auth_version = last_piece - else: - raise - else: - auth_host = config.get('func_test', 'auth_host') - auth_port = config.getint('func_test', 'auth_port') - auth_ssl = config.getboolean('func_test', 'auth_ssl') - auth_prefix = config.get('func_test', 'auth_prefix') - self.auth_version = config.get('func_test', 'auth_version') - self.auth_url = "" - if auth_ssl: - self.auth_url += "https://" - else: - self.auth_url += "http://" - self.auth_url += "%s:%s%s" % ( - auth_host, auth_port, auth_prefix) - if self.auth_version == "1": - self.auth_url += 'v1.0' - - try: - self.account_username = config.get('func_test', - 'account_username') - except configparser.NoOptionError: - account = config.get('func_test', 'account') - username = config.get('func_test', 'username') - self.account_username = "%s:%s" % (account, username) - self.password = config.get('func_test', 'password') - else: - self.skip_tests = True - - def _get_connection(self): - """ - Subclasses may override to use different connection setup - """ - return swiftclient.Connection( - self.auth_url, self.account_username, self.password, - auth_version=self.auth_version) - - def setUp(self): - super(TestFunctional, self).setUp() - if self.skip_tests: - self.skipTest('SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG') - - self.conn = self._get_connection() - self.conn.put_container(self.containername) - self.conn.put_container(self.containername_2) - self.conn.put_object( - self.containername, self.objectname, self.test_data) - self.conn.put_object( - self.containername, self.objectname_2, self.test_data) - - def tearDown(self): - super(TestFunctional, self).tearDown() - for obj in [self.objectname, self.objectname_2]: - try: - self.conn.delete_object(self.containername, obj) - except swiftclient.ClientException: - pass - - for container in [self.containername, - self.containername_2, - self.containername_3, - self.containername + '_segments']: - try: - self.conn.delete_container(container) - except swiftclient.ClientException: - pass - self.conn.close() - - def _check_account_headers(self, headers): - headers_to_check = [ - 'content-length', - 'x-account-object-count', - 'x-timestamp', - 'x-trans-id', - 'date', - 'x-account-bytes-used', - 'x-account-container-count', - 'content-type', - 'accept-ranges', - ] - for h in headers_to_check: - self.assertIn(h, headers) - self.assertTrue(headers[h]) - - def test_stat_account(self): - headers = self.conn.head_account() - self._check_account_headers(headers) - - def test_list_account(self): - headers, containers = self.conn.get_account() - self._check_account_headers(headers) - - self.assertTrue(len(containers)) - test_container = [c - for c in containers - if c.get('name') == self.containername][0] - self.assertTrue(test_container.get('bytes') >= 0) - self.assertTrue(test_container.get('count') >= 0) - - # Check if list limit is working - headers, containers = self.conn.get_account(limit=1) - self.assertEqual(1, len(containers)) - - # Check full listing - headers, containers = self.conn.get_account(limit=1, full_listing=True) - self.assertTrue(len(containers) >= 2) # there might be more containers - - # Test marker - headers, containers = self.conn.get_account(marker=self.containername) - self.assertTrue(len(containers) >= 1) - self.assertEqual(self.containername_2, containers[0].get('name')) - - # Test prefix - _, containers = self.conn.get_account(prefix='dne') - self.assertEqual(0, len(containers)) - - # Test delimiter - _, containers = self.conn.get_account( - prefix=self.containername, delimiter='_') - self.assertEqual(2, len(containers)) - self.assertEqual(self.containername, containers[0].get('name')) - self.assertTrue( - self.containername_2.startswith(containers[1].get('subdir'))) - - def _check_container_headers(self, headers): - self.assertTrue(headers.get('content-length')) - self.assertTrue(headers.get('x-container-object-count')) - self.assertTrue(headers.get('x-timestamp')) - self.assertTrue(headers.get('x-trans-id')) - self.assertTrue(headers.get('date')) - self.assertTrue(headers.get('x-container-bytes-used')) - self.assertTrue(headers.get('x-container-object-count')) - self.assertTrue(headers.get('content-type')) - self.assertTrue(headers.get('accept-ranges')) - - def test_stat_container(self): - headers = self.conn.head_container(self.containername) - self._check_container_headers(headers) - - def test_list_container(self): - headers, objects = self.conn.get_container(self.containername) - self._check_container_headers(headers) - self.assertTrue(len(objects)) - test_object = [o - for o in objects - if o.get('name') == self.objectname][0] - self.assertEqual(len(self.test_data), test_object.get('bytes')) - self.assertEqual(self.etag, test_object.get('hash')) - self.assertEqual('application/octet-stream', - test_object.get('content_type')) - - # Check if list limit is working - headers, objects = self.conn.get_container(self.containername, limit=1) - self.assertEqual(1, len(objects)) - - # Check full listing - headers, objects = self.conn.get_container( - self.containername, limit=1, full_listing=True) - self.assertEqual(2, len(objects)) - - # Test marker - headers, objects = self.conn.get_container( - self.containername, marker=self.objectname) - self.assertEqual(1, len(objects)) - self.assertEqual(self.objectname_2, objects[0].get('name')) - - def test_create_container(self): - self.conn.put_container(self.containername_3) - self.assertTrue(self.conn.head_container(self.containername_3)) - - def test_delete(self): - self.conn.delete_object(self.containername, self.objectname) - self.conn.delete_object(self.containername, self.objectname_2) - self.conn.delete_container(self.containername) - - # Container HEAD will raise an exception if container doesn't exist - # which is only possible if previous requests succeeded - self.assertRaises( - swiftclient.ClientException, - self.conn.head_container, - self.containername) - - def test_upload_object(self): - # Object with content from string - self.conn.put_object( - self.containername, self.objectname, contents=self.test_data) - hdrs = self.conn.head_object(self.containername, self.objectname) - self.assertEqual(str(len(self.test_data)), - hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) - self.assertEqual('application/octet-stream', - hdrs.get('content-type')) - - # Same but with content_type - self.conn.put_object( - self.containername, self.objectname, - content_type='text/plain', contents=self.test_data) - hdrs = self.conn.head_object(self.containername, self.objectname) - self.assertEqual(str(len(self.test_data)), - hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) - self.assertEqual('text/plain', - hdrs.get('content-type')) - - # Same but with content-type in headers - self.conn.put_object( - self.containername, self.objectname, - headers={'Content-Type': 'text/plain'}, contents=self.test_data) - hdrs = self.conn.head_object(self.containername, self.objectname) - self.assertEqual(str(len(self.test_data)), - hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) - self.assertEqual('text/plain', - hdrs.get('content-type')) - - # content_type rewrites content-type in headers - self.conn.put_object( - self.containername, self.objectname, - content_type='image/jpeg', - headers={'Content-Type': 'text/plain'}, contents=self.test_data) - hdrs = self.conn.head_object(self.containername, self.objectname) - self.assertEqual(str(len(self.test_data)), - hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) - self.assertEqual('image/jpeg', - hdrs.get('content-type')) - - # Same but with content-length - self.conn.put_object( - self.containername, self.objectname, - contents=self.test_data, content_length=len(self.test_data)) - hdrs = self.conn.head_object(self.containername, self.objectname) - self.assertEqual(str(len(self.test_data)), - hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) - self.assertEqual('application/octet-stream', hdrs.get('content-type')) - - # Content from File-like object - fileobj = BytesIO(self.test_data) - self.conn.put_object( - self.containername, self.objectname, contents=fileobj) - hdrs = self.conn.head_object(self.containername, self.objectname) - self.assertEqual(str(len(self.test_data)), - hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) - self.assertEqual('application/octet-stream', hdrs.get('content-type')) - - # Content from File-like object, but read in chunks - fileobj = BytesIO(self.test_data) - self.conn.put_object( - self.containername, self.objectname, - contents=fileobj, content_length=len(self.test_data), - chunk_size=10) - hdrs = self.conn.head_object(self.containername, self.objectname) - self.assertEqual(str(len(self.test_data)), - hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) - self.assertEqual('application/octet-stream', hdrs.get('content-type')) - - # Wrong etag arg, should raise an exception - self.assertRaises( - swiftclient.ClientException, - self.conn.put_object, - self.containername, self.objectname, - contents=self.test_data, etag='invalid') - - def test_download_object(self): - # Download whole object - hdrs, body = self.conn.get_object(self.containername, self.objectname) - self.assertEqual(self.test_data, body) - - # Download in chunks, should return a generator - hdrs, body = self.conn.get_object( - self.containername, self.objectname, - resp_chunk_size=10) - downloaded_contents = b'' - while True: - try: - chunk = next(body) - except StopIteration: - break - downloaded_contents += chunk - self.assertEqual(self.test_data, downloaded_contents) - - # Download in chunks, should also work with read - hdrs, body = self.conn.get_object( - self.containername, self.objectname, - resp_chunk_size=10) - num_bytes = 5 - downloaded_contents = body.read(num_bytes) - self.assertEqual(num_bytes, len(downloaded_contents)) - downloaded_contents += body.read() - self.assertEqual(self.test_data, downloaded_contents) - - def test_put_object_using_generator(self): - # verify that put using a generator yielding empty strings does not - # cause connection to be closed - def data(): - yield b"should" - yield b"" - yield b" tolerate" - yield b"" - yield b" empty chunks" - - self.conn.put_object( - self.containername, self.objectname, data()) - hdrs, body = self.conn.get_object(self.containername, self.objectname) - self.assertEqual(b"should tolerate empty chunks", body) - - def test_download_object_retry_chunked(self): - resp_chunk_size = 2 - hdrs, body = self.conn.get_object(self.containername, - self.objectname, - resp_chunk_size=resp_chunk_size) - data = next(body) - self.assertEqual(self.test_data[:resp_chunk_size], data) - self.assertTrue(1, self.conn.attempts) - for chunk in body.resp: - # Flush remaining data from underlying response - # (simulate a dropped connection) - pass - # Trigger the retry - for chunk in body: - data += chunk - self.assertEqual(self.test_data, data) - self.assertEqual(2, self.conn.attempts) - - def test_download_object_retry_chunked_auth_failure(self): - resp_chunk_size = 2 - self.conn.token = 'invalid' - hdrs, body = self.conn.get_object(self.containername, - self.objectname, - resp_chunk_size=resp_chunk_size) - self.assertEqual(2, self.conn.attempts) - for chunk in body.resp: - # Flush remaining data from underlying response - # (simulate a dropped connection) - pass - - self.conn.token = 'invalid' - data = next(body) - self.assertEqual(4, self.conn.attempts) - - for chunk in body: - data += chunk - - self.assertEqual(self.test_data, data) - self.assertEqual(4, self.conn.attempts) - - def test_download_object_non_chunked(self): - hdrs, body = self.conn.get_object(self.containername, self.objectname) - data = body - self.assertEqual(self.test_data, data) - self.assertTrue(1, self.conn.attempts) - - hdrs, body = self.conn.get_object(self.containername, self.objectname, - resp_chunk_size=0) - data = body - self.assertEqual(self.test_data, data) - self.assertTrue(1, self.conn.attempts) - - def test_post_account(self): - self.conn.post_account({'x-account-meta-data': 'Something'}) - headers = self.conn.head_account() - self.assertEqual('Something', headers.get('x-account-meta-data')) - - def test_post_container(self): - self.conn.post_container( - self.containername, {'x-container-meta-color': 'Something'}) - - headers = self.conn.head_container(self.containername) - self.assertEqual('Something', headers.get('x-container-meta-color')) - - def test_post_object(self): - self.conn.post_object(self.containername, - self.objectname, - {'x-object-meta-color': 'Something', - 'x-object-meta-uni': b'\xd8\xaa'.decode('utf8'), - 'x-object-meta-int': 123, - 'x-object-meta-float': 45.67, - 'x-object-meta-bool': False}) - - headers = self.conn.head_object(self.containername, self.objectname) - self.assertEqual('Something', headers.get('x-object-meta-color')) - self.assertEqual(b'\xd8\xaa'.decode('utf-8'), - headers.get('x-object-meta-uni')) - self.assertEqual('123', headers.get('x-object-meta-int')) - self.assertEqual('45.67', headers.get('x-object-meta-float')) - self.assertEqual('False', headers.get('x-object-meta-bool')) - - def test_post_object_unicode_header_name(self): - self.conn.post_object(self.containername, - self.objectname, - {u'x-object-meta-\U0001f44d': u'\U0001f44d'}) - - # Note that we can't actually read this header back on py3; see - # https://bugs.python.org/issue37093 - # We'll have to settle for just testing that the POST doesn't blow up - # with a UnicodeDecodeError - if six.PY2: - headers = self.conn.head_object( - self.containername, self.objectname) - self.assertIn(u'x-object-meta-\U0001f44d', headers) - self.assertEqual(u'\U0001f44d', - headers.get(u'x-object-meta-\U0001f44d')) - - def test_copy_object(self): - self.conn.put_object( - self.containername, self.objectname, self.test_data) - self.conn.copy_object(self.containername, - self.objectname, - headers={'x-object-meta-color': 'Something'}) - - headers = self.conn.head_object(self.containername, self.objectname) - self.assertEqual('Something', headers.get('x-object-meta-color')) - - self.conn.copy_object(self.containername, - self.objectname, - headers={'x-object-meta-taste': 'Second'}) - - headers = self.conn.head_object(self.containername, self.objectname) - self.assertEqual('Something', headers.get('x-object-meta-color')) - self.assertEqual('Second', headers.get('x-object-meta-taste')) - - destination = "/%s/%s" % (self.containername, self.objectname_2) - self.conn.copy_object(self.containername, - self.objectname, - destination, - headers={'x-object-meta-taste': 'Second'}) - headers, data = self.conn.get_object(self.containername, - self.objectname_2) - self.assertEqual(self.test_data, data) - self.assertEqual('Something', headers.get('x-object-meta-color')) - self.assertEqual('Second', headers.get('x-object-meta-taste')) - - destination = "/%s/%s" % (self.containername, self.objectname_2) - self.conn.copy_object(self.containername, - self.objectname, - destination, - headers={'x-object-meta-color': 'Else'}, - fresh_metadata=True) - - headers = self.conn.head_object(self.containername, self.objectname_2) - self.assertEqual('Else', headers.get('x-object-meta-color')) - self.assertIsNone(headers.get('x-object-meta-taste')) - - def test_get_capabilities(self): - resp = self.conn.get_capabilities() - self.assertTrue(resp.get('swift')) - - -class TestUsingKeystone(TestFunctional): - """ - Repeat tests using os_options parameter to Connection. - """ - - def _get_connection(self): - account = username = password = None - if self.auth_version not in ('2', '3'): - self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS') - try: - account = self.config.get('func_test', 'account') - username = self.config.get('func_test', 'username') - password = self.config.get('func_test', 'password') - except Exception: - self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS' + - ' - NO CONFIG') - os_options = {'tenant_name': account} - return swiftclient.Connection( - self.auth_url, username, password, auth_version=self.auth_version, - os_options=os_options) - - -class TestUsingKeystoneV3(TestFunctional): - """ - Repeat tests using a keystone user with domain specified. - """ - - def _get_connection(self): - account = username = password = project_domain = user_domain = None - if self.auth_version != '3': - self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS') - try: - account = self.config.get('func_test', 'account4') - username = self.config.get('func_test', 'username4') - user_domain = self.config.get('func_test', 'domain4') - project_domain = self.config.get('func_test', 'domain4') - password = self.config.get('func_test', 'password4') - except Exception: - self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS' + - ' - NO CONFIG') - - os_options = {'project_name': account, - 'project_domain_name': project_domain, - 'user_domain_name': user_domain} - return swiftclient.Connection(self.auth_url, username, password, - auth_version=self.auth_version, - os_options=os_options) diff -Nru python-swiftclient-3.8.1/tests/sample.conf python-swiftclient-3.9.0/tests/sample.conf --- python-swiftclient-3.8.1/tests/sample.conf 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/tests/sample.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,32 +0,0 @@ -[func_test] -# sample config -auth_host = 127.0.0.1 -auth_port = 8080 -auth_ssl = no -auth_prefix = /auth/ -## sample config for Swift with Keystone v2 API -# For keystone v3 change auth_version to 3 and auth_prefix to /v3/ -#auth_version = 2 -#auth_host = localhost -#auth_port = 5000 -#auth_ssl = no -#auth_prefix = /v2.0/ - -# Primary functional test account (needs admin access to the account). -# By default the tests use a swiftclient.client.Connection instance with user -# attribute set to 'account:username' based on the options 'account' and -# 'username' specified below. This can be overridden for auth systems that -# expect a different form of user attribute by setting the option -# 'account_username'. -# account_username = test_tester -account = test -username = tester -password = testing - -# Another user is required for keystone v3 specific tests. -# Account must be in a non-default domain. -# (Suffix '4' is used to be consistent with swift functional test config). -#account4 = test4 -#username4 = tester4 -#password4 = testing4 -#domain4 = test-domain diff -Nru python-swiftclient-3.8.1/tests/unit/test_authv1.py python-swiftclient-3.9.0/tests/unit/test_authv1.py --- python-swiftclient-3.8.1/tests/unit/test_authv1.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/tests/unit/test_authv1.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,246 +0,0 @@ -# Copyright 2016 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. - -import datetime -import json -import mock -import unittest -from keystoneauth1 import plugin -from keystoneauth1 import loading -from keystoneauth1 import exceptions -from swiftclient import authv1 - - -class TestDataNoAccount(object): - options = dict( - auth_url='http://saio:8080/auth/v1.0', - username='test:tester', - password='testing') - storage_url = 'http://saio:8080/v1/AUTH_test' - expected_endpoint = storage_url - token = 'token' - - -class TestDataWithAccount(object): - options = dict( - auth_url='http://saio:8080/auth/v1.0', - username='test2:tester2', - project_name='SOME_other_account', - password='testing2') - storage_url = 'http://saio:8080/v1/AUTH_test2' - expected_endpoint = 'http://saio:8080/v1/SOME_other_account' - token = 'other_token' - - -class TestPluginLoading(TestDataNoAccount, unittest.TestCase): - def test_can_load(self): - loader = loading.get_plugin_loader('v1password') - self.assertIsInstance(loader, authv1.PasswordLoader) - - auth_plugin = loader.load_from_options(**self.options) - self.assertIsInstance(auth_plugin, authv1.PasswordPlugin) - - self.assertEqual(self.options['auth_url'], auth_plugin.auth_url) - self.assertEqual(self.options['username'], auth_plugin.user) - self.assertEqual(self.options.get('project_name'), auth_plugin.account) - self.assertEqual(self.options['password'], auth_plugin.key) - - def test_get_state(self): - auth_plugin = authv1.PasswordPlugin(**self.options) - self.assertIsNone(auth_plugin.get_auth_state()) - - with mock.patch('swiftclient.authv1.time.time', return_value=1234.56): - auth_plugin.auth_ref = authv1.AccessInfoV1( - self.options['auth_url'], - self.storage_url, - self.options.get('project_name'), - self.options['username'], - self.token, - 60) - - expected = json.dumps({ - 'auth_url': self.options['auth_url'], - 'username': self.options['username'], - 'account': self.options.get('project_name'), - 'issued': 1234.56, - 'storage_url': self.storage_url, - 'auth_token': self.token, - 'expires': 1234.56 + 60, - }, sort_keys=True) - self.assertEqual(expected, auth_plugin.auth_ref.get_state()) - self.assertEqual(expected, auth_plugin.get_auth_state()) - - def test_set_state(self): - auth_plugin = authv1.PasswordPlugin(**self.options) - self.assertIsNone(auth_plugin.auth_ref) - - auth_plugin.auth_ref = object() - auth_plugin.set_auth_state(None) - self.assertIsNone(auth_plugin.get_auth_state()) - - state = json.dumps({ - 'auth_url': self.options['auth_url'], - 'username': self.options['username'], - 'account': self.options.get('project_name'), - 'issued': 1234.56, - 'storage_url': self.storage_url, - 'auth_token': self.token, - 'expires': None, - }, sort_keys=True) - auth_plugin.set_auth_state(state) - self.assertIsInstance(auth_plugin.auth_ref, authv1.AccessInfoV1) - - self.assertEqual(self.options['username'], - auth_plugin.auth_ref.username) - self.assertEqual(self.options['auth_url'], - auth_plugin.auth_ref.auth_url) - self.assertEqual(self.storage_url, auth_plugin.auth_ref.storage_url) - self.assertEqual(self.options.get('project_name'), auth_plugin.account) - self.assertEqual(self.token, auth_plugin.auth_ref.auth_token) - self.assertEqual(1234.56, auth_plugin.auth_ref._issued) - self.assertIs(datetime.datetime, type(auth_plugin.auth_ref.issued)) - self.assertIsNone(auth_plugin.auth_ref._expires) - self.assertIsNone(auth_plugin.auth_ref.expires) - - -class TestPluginLoadingWithAccount(TestDataWithAccount, TestPluginLoading): - pass - - -class TestPlugin(TestDataNoAccount, unittest.TestCase): - def setUp(self): - self.mock_session = mock.MagicMock() - self.mock_response = self.mock_session.get.return_value - self.mock_response.status_code = 200 - self.mock_response.headers = { - 'X-Auth-Token': self.token, - 'X-Storage-Url': self.storage_url, - } - - def test_get_access(self): - auth_plugin = authv1.PasswordPlugin(**self.options) - with mock.patch('swiftclient.authv1.time.time', return_value=1234.56): - access = auth_plugin.get_access(self.mock_session) - - self.assertEqual(self.mock_session.get.mock_calls, [mock.call( - self.options['auth_url'], authenticated=False, log=False, headers={ - 'X-Auth-User': self.options['username'], - 'X-Auth-Key': self.options['password'], - })]) - - self.assertEqual(self.options['username'], access.username) - # `openstack token issue` requires a user_id property - self.assertEqual(self.options['username'], access.user_id) - self.assertEqual(self.storage_url, access.storage_url) - self.assertEqual(self.token, access.auth_token) - self.assertEqual(1234.56, access._issued) - self.assertIs(datetime.datetime, type(auth_plugin.auth_ref.issued)) - self.assertIsNone(access.expires) - - # `openstack catalog list/show` require a catalog property - catalog = access.service_catalog.catalog - self.assertEqual('swift', catalog[0].get('name')) - self.assertEqual('object-store', catalog[0].get('type')) - self.assertIn('endpoints', catalog[0]) - self.assertIn(self.storage_url, [ - e.get('publicURL') for e in catalog[0]['endpoints']]) - - def test_get_access_with_expiry(self): - auth_plugin = authv1.PasswordPlugin(**self.options) - self.mock_response.headers['X-Auth-Token-Expires'] = '78.9' - with mock.patch('swiftclient.authv1.time.time', - return_value=1234.56) as mock_time: - access = auth_plugin.get_access(self.mock_session) - self.assertEqual(1234.56 + 78.9, access._expires) - self.assertIs(datetime.datetime, - type(auth_plugin.auth_ref.expires)) - - self.assertIs(True, access.will_expire_soon(90)) - self.assertIs(False, access.will_expire_soon(60)) - self.assertEqual(3, len(mock_time.mock_calls)) - - def test_get_access_bad_expiry(self): - auth_plugin = authv1.PasswordPlugin(**self.options) - self.mock_response.headers['X-Auth-Token-Expires'] = 'foo' - access = auth_plugin.get_access(self.mock_session) - self.assertIsNone(access.expires) - - self.assertIs(False, access.will_expire_soon(60)) - self.assertIs(False, access.will_expire_soon(1e20)) - - def test_get_access_bad_status(self): - auth_plugin = authv1.PasswordPlugin(**self.options) - self.mock_response.status_code = 401 - self.assertRaises(exceptions.InvalidResponse, - auth_plugin.get_access, self.mock_session) - - def test_get_access_missing_token(self): - auth_plugin = authv1.PasswordPlugin(**self.options) - self.mock_response.headers.pop('X-Auth-Token') - self.assertRaises(exceptions.InvalidResponse, - auth_plugin.get_access, self.mock_session) - - def test_get_access_accepts_storage_token(self): - auth_plugin = authv1.PasswordPlugin(**self.options) - self.mock_response.headers.pop('X-Auth-Token') - self.mock_response.headers['X-Storage-Token'] = 'yet another token' - access = auth_plugin.get_access(self.mock_session) - self.assertEqual('yet another token', access.auth_token) - - def test_get_access_missing_url(self): - auth_plugin = authv1.PasswordPlugin(**self.options) - self.mock_response.headers.pop('X-Storage-Url') - self.assertRaises(exceptions.InvalidResponse, - auth_plugin.get_access, self.mock_session) - - def test_get_endpoint(self): - auth_plugin = authv1.PasswordPlugin(**self.options) - - object_store_endpoint = auth_plugin.get_endpoint( - self.mock_session, service_type='object-store') - self.assertEqual(object_store_endpoint, self.expected_endpoint) - - auth_endpoint = auth_plugin.get_endpoint( - self.mock_session, interface=plugin.AUTH_INTERFACE) - self.assertEqual(auth_endpoint, self.options['auth_url']) - - with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: - auth_plugin.get_endpoint(self.mock_session) - self.assertEqual('public endpoint for None service not found', - str(exc_mgr.exception)) - - with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: - auth_plugin.get_endpoint( - self.mock_session, service_type='identity', region_name='DFW') - self.assertEqual( - 'public endpoint for identity service in DFW region not found', - str(exc_mgr.exception)) - - with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: - auth_plugin.get_endpoint( - self.mock_session, service_type='image', service_name='glance') - self.assertEqual( - 'public endpoint for image service named glance not found', - str(exc_mgr.exception)) - - with self.assertRaises(exceptions.EndpointNotFound) as exc_mgr: - auth_plugin.get_endpoint( - self.mock_session, service_type='compute', service_name='nova', - region_name='IAD') - self.assertEqual('public endpoint for compute service named nova in ' - 'IAD region not found', str(exc_mgr.exception)) - - -class TestPluginWithAccount(TestDataWithAccount, TestPlugin): - pass diff -Nru python-swiftclient-3.8.1/tests/unit/test_command_helpers.py python-swiftclient-3.9.0/tests/unit/test_command_helpers.py --- python-swiftclient-3.8.1/tests/unit/test_command_helpers.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/tests/unit/test_command_helpers.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,249 +0,0 @@ -# Copyright (c) 2010-2013 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock -from six import StringIO -import unittest - -from swiftclient import command_helpers as h -from swiftclient.multithreading import OutputManager - - -class TestStatHelpers(unittest.TestCase): - - def setUp(self): - super(TestStatHelpers, self).setUp() - conn_attrs = { - 'url': 'http://storage/v1/a', - 'token': 'tk12345', - } - self.conn = mock.MagicMock(**conn_attrs) - self.options = {'human': False, 'verbose': 1} - self.stdout = StringIO() - self.stderr = StringIO() - self.output_manager = OutputManager(self.stdout, self.stderr) - - def assertOut(self, expected): - real = self.stdout.getvalue() - # commonly if we strip of blank lines we have a match - try: - self.assertEqual(expected.strip('\n'), - real.strip('\n')) - except AssertionError: - # could be anything, try to find typos line by line - expected_lines = [line.lstrip() for line in - expected.splitlines() if line.strip()] - real_lines = [line.lstrip() for line in - real.splitlines() if line.strip()] - for expected, real in zip(expected_lines, real_lines): - self.assertEqual(expected, real) - # not a typo, might be an indent thing, hopefully you can spot it - raise - - def test_stat_account_human(self): - self.options['human'] = True - # stub head_account - stub_headers = { - 'x-account-container-count': 42, - 'x-account-object-count': 1000000, - 'x-account-bytes-used': 2 ** 30, - } - self.conn.head_account.return_value = stub_headers - - with self.output_manager as output_manager: - items, headers = h.stat_account(self.conn, self.options) - h.print_account_stats(items, headers, output_manager) - expected = """ - Account: a -Containers: 42 - Objects: 976K - Bytes: 1.0G -""" - self.assertOut(expected) - - def test_stat_account_verbose(self): - self.options['verbose'] += 1 - # stub head_account - stub_headers = { - 'x-account-container-count': 42, - 'x-account-object-count': 1000000, - 'x-account-bytes-used': 2 ** 30, - } - self.conn.head_account.return_value = stub_headers - - with self.output_manager as output_manager: - items, headers = h.stat_account(self.conn, self.options) - h.print_account_stats(items, headers, output_manager) - expected = """ -StorageURL: http://storage/v1/a -Auth Token: tk12345 - Account: a -Containers: 42 - Objects: 1000000 - Bytes: 1073741824 -""" - self.assertOut(expected) - - def test_stat_account_policy_stat(self): - # stub head_account - stub_headers = { - 'x-account-container-count': 42, - 'x-account-object-count': 1000000, - 'x-account-bytes-used': 2 ** 30, - 'x-account-storage-policy-nada-object-count': 1000000, - 'x-account-storage-policy-nada-bytes-used': 2 ** 30, - } - self.conn.head_account.return_value = stub_headers - - with self.output_manager as output_manager: - items, headers = h.stat_account(self.conn, self.options) - h.print_account_stats(items, headers, output_manager) - expected = """ - Account: a - Containers: 42 - Objects: 1000000 - Bytes: 1073741824 -Objects in policy "nada": 1000000 - Bytes in policy "nada": 1073741824 -""" - self.assertOut(expected) - - def test_stat_account_policy_stat_with_container_counts(self): - # stub head_account - stub_headers = { - 'x-account-container-count': 42, - 'x-account-object-count': 1000000, - 'x-account-bytes-used': 2 ** 30, - 'x-account-storage-policy-nada-container-count': 10, - 'x-account-storage-policy-nada-object-count': 1000000, - 'x-account-storage-policy-nada-bytes-used': 2 ** 30, - } - self.conn.head_account.return_value = stub_headers - - with self.output_manager as output_manager: - items, headers = h.stat_account(self.conn, self.options) - h.print_account_stats(items, headers, output_manager) - expected = """ - Account: a - Containers: 42 - Objects: 1000000 - Bytes: 1073741824 -Containers in policy "nada": 10 - Objects in policy "nada": 1000000 - Bytes in policy "nada": 1073741824 -""" - self.assertOut(expected) - - def test_stat_container_human(self): - self.options['human'] = True - # stub head container request - stub_headers = { - 'x-container-object-count': 10 ** 6, - 'x-container-bytes-used': 2 ** 30, - } - self.conn.head_container.return_value = stub_headers - args = ('c',) - with self.output_manager as output_manager: - items, headers = h.stat_container(self.conn, self.options, *args) - h.print_container_stats(items, headers, output_manager) - expected = """ - Account: a -Container: c - Objects: 976K - Bytes: 1.0G - Read ACL: -Write ACL: - Sync To: - Sync Key: -""" - self.assertOut(expected) - - def test_stat_container_verbose(self): - self.options['verbose'] += 1 - # stub head container request - stub_headers = { - 'x-container-object-count': 10 ** 6, - 'x-container-bytes-used': 2 ** 30, - } - self.conn.head_container.return_value = stub_headers - args = ('c',) - with self.output_manager as output_manager: - items, headers = h.stat_container(self.conn, self.options, *args) - h.print_container_stats(items, headers, output_manager) - expected = """ - URL: http://storage/v1/a/c -Auth Token: tk12345 - Account: a - Container: c - Objects: 1000000 - Bytes: 1073741824 - Read ACL: - Write ACL: - Sync To: - Sync Key: -""" - self.assertOut(expected) - - def test_stat_object_human(self): - self.options['human'] = True - # stub head object request - stub_headers = { - 'content-length': 2 ** 20, - 'x-object-meta-color': 'blue', - 'etag': '68b329da9893e34099c7d8ad5cb9c940', - 'content-encoding': 'gzip', - } - self.conn.head_object.return_value = stub_headers - args = ('c', 'o') - with self.output_manager as output_manager: - items, headers = h.stat_object(self.conn, self.options, *args) - h.print_object_stats(items, headers, output_manager) - expected = """ - Account: a - Container: c - Object: o - Content Length: 1.0M - ETag: 68b329da9893e34099c7d8ad5cb9c940 - Meta Color: blue -Content-Encoding: gzip -""" - self.assertOut(expected) - - def test_stat_object_verbose(self): - self.options['verbose'] += 1 - # stub head object request - stub_headers = { - 'content-length': 2 ** 20, - 'x-object-meta-color': 'blue', - 'etag': '68b329da9893e34099c7d8ad5cb9c940', - 'content-encoding': 'gzip', - } - self.conn.head_object.return_value = stub_headers - args = ('c', 'o') - with self.output_manager as output_manager: - items, headers = h.stat_object(self.conn, self.options, *args) - h.print_object_stats(items, headers, output_manager) - expected = """ - URL: http://storage/v1/a/c/o - Auth Token: tk12345 - Account: a - Container: c - Object: o - Content Length: 1048576 - ETag: 68b329da9893e34099c7d8ad5cb9c940 - Meta Color: blue -Content-Encoding: gzip -""" - self.assertOut(expected) diff -Nru python-swiftclient-3.8.1/tests/unit/test_multithreading.py python-swiftclient-3.9.0/tests/unit/test_multithreading.py --- python-swiftclient-3.8.1/tests/unit/test_multithreading.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/tests/unit/test_multithreading.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,240 +0,0 @@ -# Copyright (c) 2010-2013 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import unittest -import threading -import six - -from concurrent.futures import as_completed -from six.moves.queue import Queue, Empty -from time import sleep - -from swiftclient import multithreading as mt -from .utils import CaptureStream - - -class ThreadTestCase(unittest.TestCase): - def setUp(self): - super(ThreadTestCase, self).setUp() - self.got_items = Queue() - self.got_args_kwargs = Queue() - self.starting_thread_count = threading.active_count() - - def _func(self, conn, item, *args, **kwargs): - self.got_items.put((conn, item)) - self.got_args_kwargs.put((args, kwargs)) - - if item == 'sleep': - sleep(1) - if item == 'go boom': - raise Exception('I went boom!') - - return 'success' - - def _create_conn(self): - return "This is a connection" - - def _create_conn_fail(self): - raise Exception("This is a failed connection") - - def assertQueueContains(self, queue, expected_contents): - got_contents = [] - try: - while True: - got_contents.append(queue.get(timeout=0.1)) - except Empty: - pass - if isinstance(expected_contents, set): - got_contents = set(got_contents) - self.assertEqual(expected_contents, got_contents) - - -class TestConnectionThreadPoolExecutor(ThreadTestCase): - def setUp(self): - super(TestConnectionThreadPoolExecutor, self).setUp() - self.input_queue = Queue() - self.stored_results = [] - - def tearDown(self): - super(TestConnectionThreadPoolExecutor, self).tearDown() - - def test_submit_good_connection(self): - ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 1) - with ctpe as pool: - # Try submitting a job that should succeed - f = pool.submit(self._func, "succeed") - f.result() - self.assertQueueContains( - self.got_items, - [("This is a connection", "succeed")] - ) - - # Now a job that fails - try: - f = pool.submit(self._func, "go boom") - f.result() - except Exception as e: - self.assertEqual('I went boom!', str(e)) - else: - self.fail('I never went boom!') - - # Has the connection been returned to the pool? - f = pool.submit(self._func, "succeed") - f.result() - self.assertQueueContains( - self.got_items, - [ - ("This is a connection", "go boom"), - ("This is a connection", "succeed") - ] - ) - - def test_submit_bad_connection(self): - ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn_fail, 1) - with ctpe as pool: - # Now a connection that fails - try: - f = pool.submit(self._func, "succeed") - f.result() - except Exception as e: - self.assertEqual('This is a failed connection', str(e)) - else: - self.fail('The connection did not fail') - - # Make sure we don't lock up on failed connections - try: - f = pool.submit(self._func, "go boom") - f.result() - except Exception as e: - self.assertEqual('This is a failed connection', str(e)) - else: - self.fail('The connection did not fail') - - def test_lazy_connections(self): - ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 10) - with ctpe as pool: - # Submit multiple jobs sequentially - should only use 1 conn - f = pool.submit(self._func, "succeed") - f.result() - f = pool.submit(self._func, "succeed") - f.result() - f = pool.submit(self._func, "succeed") - f.result() - - expected_connections = [(0, "This is a connection")] - expected_connections.extend([(x, None) for x in range(1, 10)]) - - self.assertQueueContains( - pool._connections, expected_connections - ) - - ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 10) - with ctpe as pool: - fs = [] - f1 = pool.submit(self._func, "sleep") - f2 = pool.submit(self._func, "sleep") - f3 = pool.submit(self._func, "sleep") - fs.extend([f1, f2, f3]) - - expected_connections = [ - (0, "This is a connection"), - (1, "This is a connection"), - (2, "This is a connection") - ] - expected_connections.extend([(x, None) for x in range(3, 10)]) - - for f in as_completed(fs): - f.result() - - self.assertQueueContains( - pool._connections, expected_connections - ) - - -class TestOutputManager(unittest.TestCase): - - def test_instantiation(self): - output_manager = mt.OutputManager() - - self.assertEqual(sys.stdout, output_manager.print_stream) - self.assertEqual(sys.stderr, output_manager.error_stream) - - def test_printers(self): - out_stream = CaptureStream(sys.stdout) - err_stream = CaptureStream(sys.stderr) - starting_thread_count = threading.active_count() - - with mt.OutputManager( - print_stream=out_stream, - error_stream=err_stream) as thread_manager: - - # Sanity-checking these gives power to the previous test which - # looked at the default values of thread_manager.print/error_stream - self.assertEqual(out_stream, thread_manager.print_stream) - self.assertEqual(err_stream, thread_manager.error_stream) - - # No printing has happened yet, so no new threads - self.assertEqual(starting_thread_count, - threading.active_count()) - - thread_manager.print_msg('one-argument') - thread_manager.print_msg('one %s, %d fish', 'fish', 88) - thread_manager.error('I have %d problems, but a %s is not one', - 99, u'\u062A\u062A') - thread_manager.print_msg('some\n%s\nover the %r', 'where', - u'\u062A\u062A') - thread_manager.error('one-error-argument') - thread_manager.error('Sometimes\n%.1f%% just\ndoes not\nwork!', - 3.14159) - thread_manager.print_raw( - u'some raw bytes: \u062A\u062A'.encode('utf-8')) - - thread_manager.print_items([ - ('key', 'value'), - ('object', u'O\u0308bject'), - ]) - - thread_manager.print_raw(b'\xffugly\xffraw') - - # Now we have a thread for error printing and a thread for - # normal print messages - self.assertEqual(starting_thread_count + 2, - threading.active_count()) - - # The threads should have been cleaned up - self.assertEqual(starting_thread_count, threading.active_count()) - - if six.PY3: - over_the = "over the '\u062a\u062a'\n" - else: - over_the = "over the u'\\u062a\\u062a'\n" - # We write to the CaptureStream so no decoding is performed - self.assertEqual(''.join([ - 'one-argument\n', - 'one fish, 88 fish\n', - 'some\n', 'where\n', - over_the, - u'some raw bytes: \u062a\u062a', - ' key: value\n', - u' object: O\u0308bject\n' - ]).encode('utf8') + b'\xffugly\xffraw', out_stream.getvalue()) - - self.assertEqual(''.join([ - u'I have 99 problems, but a \u062A\u062A is not one\n', - 'one-error-argument\n', - 'Sometimes\n', '3.1% just\n', 'does not\n', 'work!\n' - ]), err_stream.getvalue().decode('utf8')) - - self.assertEqual(3, thread_manager.error_count) diff -Nru python-swiftclient-3.8.1/tests/unit/test_service.py python-swiftclient-3.9.0/tests/unit/test_service.py --- python-swiftclient-3.8.1/tests/unit/test_service.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/tests/unit/test_service.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,2909 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2014 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import unicode_literals -import contextlib -import mock -import os -import six -import tempfile -import unittest -import time - -from concurrent.futures import Future -from hashlib import md5 -from mock import Mock, PropertyMock -from six.moves.queue import Queue, Empty as QueueEmptyError -from six import BytesIO -from time import sleep - -import swiftclient -import swiftclient.utils as utils -from swiftclient.client import Connection, ClientException -from swiftclient.service import ( - SwiftService, SwiftError, SwiftUploadObject -) - -from tests.unit import utils as test_utils - - -clean_os_environ = {} -environ_prefixes = ('ST_', 'OS_') -for key in os.environ: - if any(key.startswith(m) for m in environ_prefixes): - clean_os_environ[key] = '' - - -if six.PY2: - import __builtin__ as builtins -else: - import builtins - - -class TestSwiftPostObject(unittest.TestCase): - - def setUp(self): - super(TestSwiftPostObject, self).setUp() - self.spo = swiftclient.service.SwiftPostObject - - def test_create(self): - spo = self.spo('obj_name') - - self.assertEqual(spo.object_name, 'obj_name') - self.assertIsNone(spo.options) - - def test_create_with_invalid_name(self): - # empty strings are not allowed as names - self.assertRaises(SwiftError, self.spo, '') - - # names cannot be anything but strings - self.assertRaises(SwiftError, self.spo, 1) - - -class TestSwiftCopyObject(unittest.TestCase): - - def setUp(self): - super(TestSwiftCopyObject, self).setUp() - self.sco = swiftclient.service.SwiftCopyObject - - def test_create(self): - sco = self.sco('obj_name') - - self.assertEqual(sco.object_name, 'obj_name') - self.assertIsNone(sco.destination) - self.assertFalse(sco.fresh_metadata) - - sco = self.sco('obj_name', - {'destination': '/dest', 'fresh_metadata': True}) - - self.assertEqual(sco.object_name, 'obj_name') - self.assertEqual(sco.destination, '/dest/obj_name') - self.assertTrue(sco.fresh_metadata) - - sco = self.sco('obj_name', - {'destination': '/dest/new_obj/a', - 'fresh_metadata': False}) - - self.assertEqual(sco.object_name, 'obj_name') - self.assertEqual(sco.destination, '/dest/new_obj/a') - self.assertFalse(sco.fresh_metadata) - - def test_create_with_invalid_name(self): - # empty strings are not allowed as names - self.assertRaises(SwiftError, self.sco, '') - - # names cannot be anything but strings - self.assertRaises(SwiftError, self.sco, 1) - - -class TestSwiftReader(unittest.TestCase): - - def setUp(self): - super(TestSwiftReader, self).setUp() - self.sr = swiftclient.service._SwiftReader - self.md5_type = type(md5()) - - def test_create(self): - sr = self.sr('path', 'body', {}) - - self.assertEqual(sr._path, 'path') - self.assertEqual(sr._body, 'body') - self.assertIsNone(sr._content_length) - self.assertFalse(sr._expected_md5) - - self.assertIsNone(sr._actual_md5) - - def test_create_with_large_object_headers(self): - # md5 should not be initialized if large object headers are present - sr = self.sr('path', 'body', {'x-object-manifest': 'test', - 'etag': '"%s"' % ('0' * 32)}) - self.assertEqual(sr._path, 'path') - self.assertEqual(sr._body, 'body') - self.assertIsNone(sr._content_length) - self.assertFalse(sr._expected_md5) - self.assertIsNone(sr._actual_md5) - - sr = self.sr('path', 'body', {'x-static-large-object': 'test', - 'etag': '"%s"' % ('0' * 32)}) - self.assertEqual(sr._path, 'path') - self.assertEqual(sr._body, 'body') - self.assertIsNone(sr._content_length) - self.assertFalse(sr._expected_md5) - self.assertIsNone(sr._actual_md5) - - def test_create_with_content_range_header(self): - # md5 should not be initialized if large object headers are present - sr = self.sr('path', 'body', {'content-range': 'bytes 0-3/10', - 'etag': '"%s"' % ('0' * 32)}) - self.assertEqual(sr._path, 'path') - self.assertEqual(sr._body, 'body') - self.assertIsNone(sr._content_length) - self.assertFalse(sr._expected_md5) - self.assertIsNone(sr._actual_md5) - - def test_create_with_ignore_checksum(self): - # md5 should not be initialized if checksum is False - sr = self.sr('path', 'body', {}, False) - self.assertEqual(sr._path, 'path') - self.assertEqual(sr._body, 'body') - self.assertIsNone(sr._content_length) - self.assertFalse(sr._expected_md5) - self.assertIsNone(sr._actual_md5) - - def test_create_with_content_length(self): - sr = self.sr('path', 'body', {'content-length': 5}) - - self.assertEqual(sr._path, 'path') - self.assertEqual(sr._body, 'body') - self.assertEqual(sr._content_length, 5) - self.assertFalse(sr._expected_md5) - - self.assertIsNone(sr._actual_md5) - - # Check Contentlength raises error if it isn't an integer - self.assertRaises(SwiftError, self.sr, 'path', 'body', - {'content-length': 'notanint'}) - - def test_iterator_usage(self): - def _consume(sr): - for _ in sr: - pass - - sr = self.sr('path', BytesIO(b'body'), {}) - _consume(sr) - - # Check error is raised if expected etag doesn't match calculated md5. - # md5 for a SwiftReader that has done nothing is - # d41d8cd98f00b204e9800998ecf8427e i.e md5 of nothing - sr = self.sr('path', BytesIO(b'body'), - {'etag': md5(b'doesntmatch').hexdigest()}) - self.assertRaises(SwiftError, _consume, sr) - - sr = self.sr('path', BytesIO(b'body'), - {'etag': md5(b'body').hexdigest()}) - _consume(sr) - - # Should still work if etag was quoted - sr = self.sr('path', BytesIO(b'body'), - {'etag': '"%s"' % md5(b'body').hexdigest()}) - _consume(sr) - - # Check error is raised if SwiftReader doesn't read the same length - # as the content length it is created with - sr = self.sr('path', BytesIO(b'body'), {'content-length': 5}) - self.assertRaises(SwiftError, _consume, sr) - - sr = self.sr('path', BytesIO(b'body'), {'content-length': 4}) - _consume(sr) - - # Check that the iterator generates expected length and etag values - sr = self.sr('path', ['abc'.encode()] * 3, - {'content-length': 9, - 'etag': md5('abc'.encode() * 3).hexdigest()}) - _consume(sr) - self.assertEqual(sr._actual_read, 9) - self.assertEqual(sr._actual_md5.hexdigest(), - md5('abc'.encode() * 3).hexdigest()) - - -class _TestServiceBase(unittest.TestCase): - def _get_mock_connection(self, attempts=2): - m = Mock(spec=Connection) - type(m).attempts = PropertyMock(return_value=attempts) - type(m).auth_end_time = PropertyMock(return_value=4) - return m - - def _get_queue(self, q): - # Instead of blocking pull items straight from the queue. - # expects at least one item otherwise the test will fail. - try: - return q.get_nowait() - except QueueEmptyError: - self.fail('Expected item in queue but found none') - - def _get_expected(self, update=None): - expected = self.expected.copy() - if update: - expected.update(update) - - return expected - - -class TestServiceDelete(_TestServiceBase): - def setUp(self): - super(TestServiceDelete, self).setUp() - self.opts = {'leave_segments': False, 'yes_all': False} - self.exc = Exception('test_exc') - # Base response to be copied and updated to matched the expected - # response for each test - self.expected = { - 'action': None, # Should be string in the form delete_XX - 'container': 'test_c', - 'object': 'test_o', - 'attempts': 2, - 'response_dict': {}, - 'success': None # Should be a bool - } - - def test_delete_segment(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - expected_r = self._get_expected({ - 'action': 'delete_segment', - 'object': 'test_s', - 'success': True, - }) - - r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q) - - mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_s', response_dict={} - ) - self.assertEqual(expected_r, r) - self.assertEqual(expected_r, self._get_queue(mock_q)) - - def test_delete_segment_exception(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - mock_conn.delete_object = Mock(side_effect=self.exc) - expected_r = self._get_expected({ - 'action': 'delete_segment', - 'object': 'test_s', - 'success': False, - 'error': self.exc, - 'traceback': mock.ANY, - 'error_timestamp': mock.ANY - }) - - before = time.time() - r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q) - after = time.time() - - mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_s', response_dict={} - ) - self.assertEqual(expected_r, r) - self.assertEqual(expected_r, self._get_queue(mock_q)) - self.assertGreaterEqual(r['error_timestamp'], before) - self.assertLessEqual(r['error_timestamp'], after) - self.assertIn('Traceback', r['traceback']) - - def test_delete_object(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - mock_conn.head_object = Mock(return_value={}) - expected_r = self._get_expected({ - 'action': 'delete_object', - 'success': True - }) - - s = SwiftService() - r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) - - mock_conn.head_object.assert_called_once_with( - 'test_c', 'test_o', query_string='symlink=get', headers={}) - mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_o', query_string=None, response_dict={}, - headers={} - ) - self.assertEqual(expected_r, r) - - def test_delete_object_with_headers(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - mock_conn.head_object = Mock(return_value={}) - expected_r = self._get_expected({ - 'action': 'delete_object', - 'success': True - }) - opt_c = self.opts.copy() - opt_c['header'] = ['Skip-Middleware: Test'] - - s = SwiftService() - r = s._delete_object(mock_conn, 'test_c', 'test_o', opt_c, mock_q) - - mock_conn.head_object.assert_called_once_with( - 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'}, - query_string='symlink=get') - mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_o', query_string=None, response_dict={}, - headers={'Skip-Middleware': 'Test'} - ) - self.assertEqual(expected_r, r) - - def test_delete_object_exception(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - mock_conn.delete_object = Mock(side_effect=self.exc) - expected_r = self._get_expected({ - 'action': 'delete_object', - 'success': False, - 'error': self.exc, - 'traceback': mock.ANY, - 'error_timestamp': mock.ANY - }) - # _delete_object doesn't populate attempts or response dict if it hits - # an error. This may not be the correct behaviour. - del expected_r['response_dict'], expected_r['attempts'] - - before = time.time() - s = SwiftService() - r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) - after = time.time() - - mock_conn.head_object.assert_called_once_with( - 'test_c', 'test_o', query_string='symlink=get', headers={}) - mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_o', query_string=None, response_dict={}, - headers={} - ) - self.assertEqual(expected_r, r) - self.assertGreaterEqual(r['error_timestamp'], before) - self.assertLessEqual(r['error_timestamp'], after) - self.assertIn('Traceback', r['traceback']) - - def test_delete_object_slo_support(self): - # If SLO headers are present the delete call should include an - # additional query string to cause the right delete server side - mock_q = Queue() - mock_conn = self._get_mock_connection() - mock_conn.head_object = Mock( - return_value={'x-static-large-object': True} - ) - expected_r = self._get_expected({ - 'action': 'delete_object', - 'success': True - }) - - s = SwiftService() - r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) - - mock_conn.head_object.assert_called_once_with( - 'test_c', 'test_o', query_string='symlink=get', headers={}) - mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_o', - query_string='multipart-manifest=delete', - response_dict={}, - headers={} - ) - self.assertEqual(expected_r, r) - - def test_delete_object_dlo_support(self): - mock_q = Queue() - s = SwiftService() - mock_conn = self._get_mock_connection() - expected_r = self._get_expected({ - 'action': 'delete_object', - 'success': True, - 'dlo_segments_deleted': True - }) - # A DLO object is determined in _delete_object by heading the object - # and checking for the existence of a x-object-manifest header. - # Mock that here. - mock_conn.head_object = Mock( - return_value={'x-object-manifest': 'manifest_c/manifest_p'} - ) - mock_conn.get_container = Mock( - side_effect=[(None, [{'name': 'test_seg_1'}, - {'name': 'test_seg_2'}]), - (None, {})] - ) - - def get_mock_list_conn(options): - return mock_conn - - with mock.patch('swiftclient.service.get_conn', get_mock_list_conn): - r = s._delete_object( - mock_conn, 'test_c', 'test_o', self.opts, mock_q - ) - - self.assertEqual(expected_r, r) - expected = [ - mock.call('test_c', 'test_o', query_string=None, response_dict={}, - headers={}), - mock.call('manifest_c', 'test_seg_1', response_dict={}), - mock.call('manifest_c', 'test_seg_2', response_dict={})] - mock_conn.delete_object.assert_has_calls(expected, any_order=True) - - def test_delete_empty_container(self): - mock_conn = self._get_mock_connection() - expected_r = self._get_expected({ - 'action': 'delete_container', - 'success': True, - 'object': None - }) - - r = SwiftService._delete_empty_container(mock_conn, 'test_c', - self.opts) - - mock_conn.delete_container.assert_called_once_with( - 'test_c', response_dict={}, headers={} - ) - self.assertEqual(expected_r, r) - - def test_delete_empty_container_with_headers(self): - mock_conn = self._get_mock_connection() - expected_r = self._get_expected({ - 'action': 'delete_container', - 'success': True, - 'object': None - }) - opt_c = self.opts.copy() - opt_c['header'] = ['Skip-Middleware: Test'] - - r = SwiftService._delete_empty_container(mock_conn, 'test_c', opt_c) - - mock_conn.delete_container.assert_called_once_with( - 'test_c', response_dict={}, headers={'Skip-Middleware': 'Test'} - ) - self.assertEqual(expected_r, r) - - def test_delete_empty_container_exception(self): - mock_conn = self._get_mock_connection() - mock_conn.delete_container = Mock(side_effect=self.exc) - expected_r = self._get_expected({ - 'action': 'delete_container', - 'success': False, - 'object': None, - 'error': self.exc, - 'traceback': mock.ANY, - 'error_timestamp': mock.ANY - }) - - before = time.time() - s = SwiftService() - r = s._delete_empty_container(mock_conn, 'test_c', {}) - after = time.time() - - mock_conn.delete_container.assert_called_once_with( - 'test_c', response_dict={}, headers={} - ) - self.assertEqual(expected_r, r) - self.assertGreaterEqual(r['error_timestamp'], before) - self.assertLessEqual(r['error_timestamp'], after) - self.assertIn('Traceback', r['traceback']) - - @mock.patch.object(swiftclient.service.SwiftService, 'capabilities', - lambda *a: {'action': 'capabilities', - 'timestamp': time.time(), - 'success': True, - 'capabilities': { - 'bulk_delete': - {'max_deletes_per_request': 10}} - }) - def test_bulk_delete_page_size(self): - # make a list of 100 objects - obj_list = ['x%02d' % i for i in range(100)] - errors = [] - - # _bulk_delete_page_size uses 2x the number of threads to determine - # if if there are "many" object to delete or not - - # format is: [(thread_count, expected result), ...] - obj_threads_exp = [ - (10, 10), # something small - (49, 10), # just under the bounds - (50, 1), # cutover point - (51, 1), # just over bounds - (100, 1), # something big - ] - for thread_count, exp in obj_threads_exp: - s = SwiftService(options={'object_dd_threads': thread_count}) - res = s._bulk_delete_page_size(obj_list) - if res != exp: - msg = 'failed for thread_count %d: got %r expected %r' % \ - (thread_count, res, exp) - errors.append(msg) - if errors: - self.fail('_bulk_delete_page_size() failed\n' + '\n'.join(errors)) - - -class TestSwiftError(unittest.TestCase): - - def test_is_exception(self): - se = SwiftError(5) - self.assertIsInstance(se, Exception) - - def test_empty_swifterror_creation(self): - se = SwiftError(5) - - self.assertEqual(se.value, 5) - self.assertIsNone(se.container) - self.assertIsNone(se.obj) - self.assertIsNone(se.segment) - self.assertIsNone(se.exception) - - self.assertEqual(str(se), '5') - - def test_swifterror_creation(self): - test_exc = Exception('test exc') - se = SwiftError(5, 'con', 'obj', 'seg', test_exc) - - self.assertEqual(se.value, 5) - self.assertEqual(se.container, 'con') - self.assertEqual(se.obj, 'obj') - self.assertEqual(se.segment, 'seg') - self.assertEqual(se.exception, test_exc) - - self.assertEqual(str(se), '5 container:con object:obj segment:seg') - - -class TestServiceUtils(unittest.TestCase): - - def setUp(self): - super(TestServiceUtils, self).setUp() - with mock.patch.dict(swiftclient.service.environ, clean_os_environ): - swiftclient.service._default_global_options = \ - swiftclient.service._build_default_global_options() - self.opts = swiftclient.service._default_global_options.copy() - - def test_process_options_defaults(self): - # The only actions that should be taken on default options set is - # to change the auth version to v2.0 and create the os_options dict - opt_c = self.opts.copy() - - swiftclient.service.process_options(opt_c) - - self.assertIn('os_options', opt_c) - del opt_c['os_options'] - self.assertEqual(opt_c['auth_version'], '2.0') - opt_c['auth_version'] = '1.0' - - self.assertEqual(opt_c, self.opts) - - def test_process_options_auth_version(self): - # auth_version should be set to 2.0 - # if it isn't already set to 3.0 - # and the v1 command line arguments aren't present - opt_c = self.opts.copy() - - # Check v3 isn't changed - opt_c['auth_version'] = '3' - swiftclient.service.process_options(opt_c) - self.assertEqual(opt_c['auth_version'], '3') - - # Check v1 isn't changed if user, key and auth are set - opt_c = self.opts.copy() - opt_c['auth_version'] = '1' - opt_c['auth'] = True - opt_c['user'] = True - opt_c['key'] = True - swiftclient.service.process_options(opt_c) - self.assertEqual(opt_c['auth_version'], '1') - - def test_process_options_new_style_args(self): - # checks new style args are copied to old style - # when old style don't exist - opt_c = self.opts.copy() - - opt_c['auth'] = '' - opt_c['user'] = '' - opt_c['key'] = '' - opt_c['os_auth_url'] = 'os_auth' - opt_c['os_username'] = 'os_user' - opt_c['os_password'] = 'os_pass' - swiftclient.service.process_options(opt_c) - self.assertEqual(opt_c['auth_version'], '2.0') - self.assertEqual(opt_c['auth'], 'os_auth') - self.assertEqual(opt_c['user'], 'os_user') - self.assertEqual(opt_c['key'], 'os_pass') - - # Check old style args are left alone if they exist - opt_c = self.opts.copy() - opt_c['auth'] = 'auth' - opt_c['user'] = 'user' - opt_c['key'] = 'key' - opt_c['os_auth_url'] = 'os_auth' - opt_c['os_username'] = 'os_user' - opt_c['os_password'] = 'os_pass' - swiftclient.service.process_options(opt_c) - self.assertEqual(opt_c['auth_version'], '1.0') - self.assertEqual(opt_c['auth'], 'auth') - self.assertEqual(opt_c['user'], 'user') - self.assertEqual(opt_c['key'], 'key') - - def test_split_headers(self): - mock_headers = ['color:blue', 'SIZE: large'] - expected = {'Color': 'blue', 'Size': 'large'} - - actual = swiftclient.service.split_headers(mock_headers) - self.assertEqual(expected, actual) - - def test_split_headers_prefix(self): - mock_headers = ['color:blue', 'size:large'] - expected = {'Prefix-Color': 'blue', 'Prefix-Size': 'large'} - - actual = swiftclient.service.split_headers(mock_headers, 'prefix-') - self.assertEqual(expected, actual) - - def test_split_headers_list_of_tuples(self): - mock_headers = [('color', 'blue'), ('size', 'large')] - expected = {'Prefix-Color': 'blue', 'Prefix-Size': 'large'} - - actual = swiftclient.service.split_headers(mock_headers, 'prefix-') - self.assertEqual(expected, actual) - - def test_split_headers_dict(self): - expected = {'Color': 'blue', 'Size': 'large'} - - actual = swiftclient.service.split_headers(expected) - self.assertEqual(expected, actual) - - def test_split_headers_error(self): - self.assertRaises(SwiftError, swiftclient.service.split_headers, - ['notvalid']) - self.assertRaises(SwiftError, swiftclient.service.split_headers, - [('also', 'not', 'valid')]) - - -class TestSwiftUploadObject(unittest.TestCase): - - def setUp(self): - self.suo = swiftclient.service.SwiftUploadObject - super(TestSwiftUploadObject, self).setUp() - - def test_create_with_string(self): - suo = self.suo('source') - self.assertEqual(suo.source, 'source') - self.assertEqual(suo.object_name, 'source') - self.assertIsNone(suo.options) - - suo = self.suo('source', 'obj_name') - self.assertEqual(suo.source, 'source') - self.assertEqual(suo.object_name, 'obj_name') - self.assertIsNone(suo.options) - - suo = self.suo('source', 'obj_name', {'opt': '123'}) - self.assertEqual(suo.source, 'source') - self.assertEqual(suo.object_name, 'obj_name') - self.assertEqual(suo.options, {'opt': '123'}) - - def test_create_with_file(self): - with tempfile.TemporaryFile() as mock_file: - # Check error is raised if no object name is provided with a - # filelike object - self.assertRaises(SwiftError, self.suo, mock_file) - - # Check that empty strings are invalid object names - self.assertRaises(SwiftError, self.suo, mock_file, '') - - suo = self.suo(mock_file, 'obj_name') - self.assertEqual(suo.source, mock_file) - self.assertEqual(suo.object_name, 'obj_name') - self.assertIsNone(suo.options) - - suo = self.suo(mock_file, 'obj_name', {'opt': '123'}) - self.assertEqual(suo.source, mock_file) - self.assertEqual(suo.object_name, 'obj_name') - self.assertEqual(suo.options, {'opt': '123'}) - - def test_create_with_no_source(self): - suo = self.suo(None, 'obj_name') - self.assertIsNone(suo.source) - self.assertEqual(suo.object_name, 'obj_name') - self.assertIsNone(suo.options) - - # Check error is raised if source is None without an object name - self.assertRaises(SwiftError, self.suo, None) - - def test_create_with_invalid_source(self): - # Source can only be None, string or filelike object, - # check an error is raised with an invalid type. - self.assertRaises(SwiftError, self.suo, []) - - -class TestServiceList(_TestServiceBase): - def setUp(self): - super(TestServiceList, self).setUp() - self.opts = {'prefix': None, 'long': False, 'delimiter': ''} - self.exc = Exception('test_exc') - # Base response to be copied and updated to matched the expected - # response for each test - self.expected = { - 'action': None, # Should be list_X_part (account or container) - 'container': None, # Should be a string when listing a container - 'prefix': None, - 'success': None # Should be a bool - } - - def test_list_account(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - get_account_returns = [ - (None, [{'name': 'test_c'}]), - (None, []) - ] - mock_conn.get_account = Mock(side_effect=get_account_returns) - - expected_r = self._get_expected({ - 'action': 'list_account_part', - 'success': True, - 'listing': [{'name': 'test_c'}], - 'marker': '' - }) - - SwiftService._list_account_job( - mock_conn, self.opts, mock_q - ) - self.assertEqual(expected_r, self._get_queue(mock_q)) - self.assertIsNone(self._get_queue(mock_q)) - - long_opts = dict(self.opts, **{'long': True}) - mock_conn.head_container = Mock(return_value={'test_m': '1'}) - get_account_returns = [ - (None, [{'name': 'test_c'}]), - (None, []) - ] - mock_conn.get_account = Mock(side_effect=get_account_returns) - - expected_r_long = self._get_expected({ - 'action': 'list_account_part', - 'success': True, - 'listing': [{'name': 'test_c', 'meta': {'test_m': '1'}}], - 'marker': '', - }) - - SwiftService._list_account_job( - mock_conn, long_opts, mock_q - ) - self.assertEqual(expected_r_long, self._get_queue(mock_q)) - self.assertIsNone(self._get_queue(mock_q)) - - def test_list_account_with_headers(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - get_account_returns = [ - (None, [{'name': 'test_c'}]), - (None, []) - ] - mock_conn.get_account = Mock(side_effect=get_account_returns) - - expected_r = self._get_expected({ - 'action': 'list_account_part', - 'success': True, - 'listing': [{'name': 'test_c'}], - 'marker': '' - }) - opt_c = self.opts.copy() - opt_c['header'] = ['Skip-Middleware: True'] - SwiftService._list_account_job( - mock_conn, opt_c, mock_q - ) - self.assertEqual(expected_r, self._get_queue(mock_q)) - self.assertIsNone(self._get_queue(mock_q)) - self.assertEqual(mock_conn.get_account.mock_calls, [ - mock.call(headers={'Skip-Middleware': 'True'}, marker='', - prefix=None), - mock.call(headers={'Skip-Middleware': 'True'}, marker='test_c', - prefix=None)]) - - def test_list_account_exception(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - mock_conn.get_account = Mock(side_effect=self.exc) - expected_r = self._get_expected({ - 'action': 'list_account_part', - 'success': False, - 'error': self.exc, - 'marker': '', - 'traceback': mock.ANY, - 'error_timestamp': mock.ANY - }) - - SwiftService._list_account_job( - mock_conn, self.opts, mock_q) - - mock_conn.get_account.assert_called_once_with( - marker='', prefix=None, headers={} - ) - self.assertEqual(expected_r, self._get_queue(mock_q)) - self.assertIsNone(self._get_queue(mock_q)) - - def test_list_container(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - get_container_returns = [ - (None, [{'name': 'test_o'}]), - (None, []) - ] - mock_conn.get_container = Mock(side_effect=get_container_returns) - - expected_r = self._get_expected({ - 'action': 'list_container_part', - 'container': 'test_c', - 'success': True, - 'listing': [{'name': 'test_o'}], - 'marker': '' - }) - - SwiftService._list_container_job( - mock_conn, 'test_c', self.opts, mock_q - ) - self.assertEqual(expected_r, self._get_queue(mock_q)) - self.assertIsNone(self._get_queue(mock_q)) - - long_opts = dict(self.opts, **{'long': True}) - mock_conn.head_container = Mock(return_value={'test_m': '1'}) - get_container_returns = [ - (None, [{'name': 'test_o'}]), - (None, []) - ] - mock_conn.get_container = Mock(side_effect=get_container_returns) - - expected_r_long = self._get_expected({ - 'action': 'list_container_part', - 'container': 'test_c', - 'success': True, - 'listing': [{'name': 'test_o'}], - 'marker': '' - }) - - SwiftService._list_container_job( - mock_conn, 'test_c', long_opts, mock_q - ) - self.assertEqual(expected_r_long, self._get_queue(mock_q)) - self.assertIsNone(self._get_queue(mock_q)) - - def test_list_container_marker(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - - get_container_returns = [ - (None, [{'name': 'b'}, {'name': 'c'}]), - (None, []) - ] - mock_get_cont = Mock(side_effect=get_container_returns) - mock_conn.get_container = mock_get_cont - - expected_r = self._get_expected({ - 'action': 'list_container_part', - 'container': 'test_c', - 'success': True, - 'listing': [{'name': 'b'}, {'name': 'c'}], - 'marker': 'b' - }) - - _opts = self.opts.copy() - _opts['marker'] = 'b' - SwiftService._list_container_job(mock_conn, 'test_c', _opts, mock_q) - - # This does not test if the marker is propagated, because we always - # get the final call to the get_container with the final item 'c', - # even if marker wasn't set. This test just makes sure the whole - # stack works in a sane way. - mock_kw = mock_get_cont.call_args[1] - self.assertEqual(mock_kw['marker'], 'c') - - # This tests that the lower levels get the marker delivered. - self.assertEqual(expected_r, self._get_queue(mock_q)) - - self.assertIsNone(self._get_queue(mock_q)) - - def test_list_container_with_headers(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - get_container_returns = [ - (None, [{'name': 'test_o'}]), - (None, []) - ] - mock_conn.get_container = Mock(side_effect=get_container_returns) - - expected_r = self._get_expected({ - 'action': 'list_container_part', - 'container': 'test_c', - 'success': True, - 'listing': [{'name': 'test_o'}], - 'marker': '' - }) - - opt_c = self.opts.copy() - opt_c['header'] = ['Skip-Middleware: Test'] - - SwiftService._list_container_job( - mock_conn, 'test_c', opt_c, mock_q - ) - self.assertEqual(expected_r, self._get_queue(mock_q)) - self.assertIsNone(self._get_queue(mock_q)) - self.assertEqual(mock_conn.get_container.mock_calls, [ - mock.call('test_c', headers={'Skip-Middleware': 'Test'}, - delimiter='', marker='', prefix=None), - mock.call('test_c', headers={'Skip-Middleware': 'Test'}, - delimiter='', marker='test_o', prefix=None)]) - - def test_list_container_exception(self): - mock_q = Queue() - mock_conn = self._get_mock_connection() - mock_conn.get_container = Mock(side_effect=self.exc) - expected_r = self._get_expected({ - 'action': 'list_container_part', - 'container': 'test_c', - 'success': False, - 'error': self.exc, - 'marker': '', - 'error_timestamp': mock.ANY, - 'traceback': mock.ANY - }) - - SwiftService._list_container_job( - mock_conn, 'test_c', self.opts, mock_q - ) - - mock_conn.get_container.assert_called_once_with( - 'test_c', marker='', delimiter='', prefix=None, headers={} - ) - self.assertEqual(expected_r, self._get_queue(mock_q)) - self.assertIsNone(self._get_queue(mock_q)) - - @mock.patch('swiftclient.service.get_conn') - def test_list_queue_size(self, mock_get_conn): - mock_conn = self._get_mock_connection() - # Return more results than should fit in the results queue - get_account_returns = [ - (None, [{'name': 'container1'}]), - (None, [{'name': 'container2'}]), - (None, [{'name': 'container3'}]), - (None, [{'name': 'container4'}]), - (None, [{'name': 'container5'}]), - (None, [{'name': 'container6'}]), - (None, [{'name': 'container7'}]), - (None, [{'name': 'container8'}]), - (None, [{'name': 'container9'}]), - (None, [{'name': 'container10'}]), - (None, [{'name': 'container11'}]), - (None, [{'name': 'container12'}]), - (None, [{'name': 'container13'}]), - (None, [{'name': 'container14'}]), - (None, []) - ] - mock_conn.get_account = Mock(side_effect=get_account_returns) - mock_get_conn.return_value = mock_conn - - s = SwiftService(options=self.opts) - lg = s.list() - - # Start the generator - first_list_part = next(lg) - - # Wait for the number of calls to get_account to reach our expected - # value, then let it run some more to make sure the value remains - # stable - count = mock_conn.get_account.call_count - stable = 0 - while mock_conn.get_account.call_count != count or stable < 5: - if mock_conn.get_account.call_count == count: - stable += 1 - else: - count = mock_conn.get_account.call_count - stable = 0 - # The test requires a small sleep to allow other threads to - # execute - in this mocked environment we assume that if the call - # count to get_account has not changed in 0.25s then no more calls - # will be made. - sleep(0.05) - - stable_get_account_call_count = mock_conn.get_account.call_count - - # Collect all remaining results from the generator - list_results = [first_list_part] + list(lg) - - # Make sure the stable call count is correct - this should be 12 calls - # to get_account; - # 1 for first_list_part - # 10 for the values on the queue - # 1 for the value blocking whilst trying to place onto the queue - self.assertEqual(12, stable_get_account_call_count) - - # Make sure all the containers were listed and placed onto the queue - self.assertEqual(15, mock_conn.get_account.call_count) - - # Check the results were all returned - observed_listing = [] - for lir in list_results: - observed_listing.append( - [li['name'] for li in lir['listing']] - ) - expected_listing = [] - for gar in get_account_returns[:-1]: # The empty list is not returned - expected_listing.append( - [li['name'] for li in gar[1]] - ) - self.assertEqual(observed_listing, expected_listing) - - -class TestService(unittest.TestCase): - - def test_upload_with_bad_segment_size(self): - for bad in ('ten', '1234X', '100.3'): - options = {'segment_size': bad} - try: - service = SwiftService(options) - next(service.upload('c', 'o')) - self.fail('Expected SwiftError when segment_size=%s' % bad) - except SwiftError as exc: - self.assertEqual('Segment size should be an integer value', - exc.value) - - @mock.patch('swiftclient.service.stat') - @mock.patch('swiftclient.service.getmtime', return_value=1.0) - @mock.patch('swiftclient.service.getsize', return_value=4) - def test_upload_with_relative_path(self, *args, **kwargs): - service = SwiftService({}) - objects = [{'path': "./test", - 'strt_indx': 2}, - {'path': os.path.join(os.getcwd(), "test"), - 'strt_indx': 1}, - {'path': ".\\test", - 'strt_indx': 2}] - for obj in objects: - with mock.patch('swiftclient.service.Connection') as mock_conn, \ - mock.patch.object(builtins, 'open') as mock_open: - mock_open.return_value = six.StringIO('asdf') - mock_conn.return_value.head_object.side_effect = \ - ClientException('Not Found', http_status=404) - mock_conn.return_value.put_object.return_value =\ - 'd41d8cd98f00b204e9800998ecf8427e' - resp_iter = service.upload( - 'c', [SwiftUploadObject(obj['path'])]) - responses = [x for x in resp_iter] - for resp in responses: - self.assertIsNone(resp.get('error')) - self.assertIs(True, resp['success']) - self.assertEqual(2, len(responses)) - create_container_resp, upload_obj_resp = responses - self.assertEqual(create_container_resp['action'], - 'create_container') - self.assertEqual(upload_obj_resp['action'], - 'upload_object') - self.assertEqual(upload_obj_resp['object'], - obj['path'][obj['strt_indx']:]) - self.assertEqual(upload_obj_resp['path'], obj['path']) - self.assertTrue(mock_open.return_value.closed) - - @mock.patch('swiftclient.service.Connection') - def test_upload_stream(self, mock_conn): - service = SwiftService({}) - - stream = test_utils.FakeStream(2048) - segment_etag = md5(b'A' * 1024).hexdigest() - - mock_conn.return_value.head_object.side_effect = \ - ClientException('Not Found', http_status=404) - mock_conn.return_value.put_object.return_value = \ - segment_etag - options = {'use_slo': True, 'segment_size': 1024} - resp_iter = service.upload( - 'container', - [SwiftUploadObject(stream, object_name='streamed')], - options) - responses = [x for x in resp_iter] - for resp in responses: - self.assertFalse('error' in resp) - self.assertTrue(resp['success']) - self.assertEqual(5, len(responses)) - container_resp, segment_container_resp = responses[0:2] - segment_response = responses[2:4] - upload_obj_resp = responses[-1] - self.assertEqual(container_resp['action'], - 'create_container') - self.assertEqual(upload_obj_resp['action'], - 'upload_object') - self.assertEqual(upload_obj_resp['object'], - 'streamed') - self.assertTrue(upload_obj_resp['path'] is None) - self.assertTrue(upload_obj_resp['large_object']) - self.assertIn('manifest_response_dict', upload_obj_resp) - self.assertEqual(upload_obj_resp['manifest_response_dict'], {}) - for i, resp in enumerate(segment_response): - self.assertEqual(i, resp['segment_index']) - self.assertEqual(1024, resp['segment_size']) - self.assertEqual('d47b127bc2de2d687ddc82dac354c415', - resp['segment_etag']) - self.assertTrue(resp['segment_location'].endswith( - '/0000000%d' % i)) - self.assertTrue(resp['segment_location'].startswith( - '/container_segments/streamed')) - - @mock.patch('swiftclient.service.Connection') - def test_upload_stream_fits_in_one_segment(self, mock_conn): - service = SwiftService({}) - - stream = test_utils.FakeStream(2048) - whole_etag = md5(b'A' * 2048).hexdigest() - - mock_conn.return_value.head_object.side_effect = \ - ClientException('Not Found', http_status=404) - mock_conn.return_value.put_object.return_value = \ - whole_etag - options = {'use_slo': True, 'segment_size': 10240} - resp_iter = service.upload( - 'container', - [SwiftUploadObject(stream, object_name='streamed')], - options) - responses = [x for x in resp_iter] - for resp in responses: - self.assertNotIn('error', resp) - self.assertTrue(resp['success']) - self.assertEqual(3, len(responses)) - container_resp, segment_container_resp = responses[0:2] - upload_obj_resp = responses[-1] - self.assertEqual(container_resp['action'], - 'create_container') - self.assertEqual(upload_obj_resp['action'], - 'upload_object') - self.assertEqual(upload_obj_resp['object'], - 'streamed') - self.assertTrue(upload_obj_resp['path'] is None) - self.assertFalse(upload_obj_resp['large_object']) - self.assertNotIn('manifest_response_dict', upload_obj_resp) - - -class TestServiceUpload(_TestServiceBase): - - @contextlib.contextmanager - def assert_open_results_are_closed(self): - opened_files = [] - builtin_open = builtins.open - - def open_wrapper(*a, **kw): - opened_files.append((builtin_open(*a, **kw), a, kw)) - return opened_files[-1][0] - - with mock.patch.object(builtins, 'open', open_wrapper): - yield - for fp, args, kwargs in opened_files: - formatted_args = [repr(a) for a in args] - formatted_args.extend('%s=%r' % kv for kv in kwargs.items()) - formatted_args = ', '.join(formatted_args) - self.assertTrue(fp.closed, - 'Failed to close open(%s)' % formatted_args) - - def test_upload_object_job_file_with_unicode_path(self): - # Uploading a file results in the file object being wrapped in a - # LengthWrapper. This test sets the options in such a way that much - # of _upload_object_job is skipped bringing the critical path down - # to around 60 lines to ease testing. - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 30) - f.flush() - expected_r = { - 'action': 'upload_object', - 'attempts': 2, - 'container': 'test_c', - 'headers': {}, - 'large_object': True, - 'object': 'テスト/dummy.dat', - 'manifest_response_dict': {}, - 'segment_results': [{'action': 'upload_segment', - 'success': True}] * 3, - 'status': 'uploaded', - 'success': True, - } - expected_mtime = '%f' % os.path.getmtime(f.name) - - mock_conn = mock.Mock() - mock_conn.put_object.return_value = '' - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - - s = SwiftService() - with mock.patch.object(s, '_upload_segment_job') as mock_job: - mock_job.return_value = { - 'action': 'upload_segment', - 'success': True} - - r = s._upload_object_job(conn=mock_conn, - container='test_c', - source=f.name, - obj='テスト/dummy.dat', - options=dict(s._options, - segment_size=10, - leave_segments=True)) - - mtime = r['headers']['x-object-meta-mtime'] - self.assertEqual(expected_mtime, mtime) - del r['headers']['x-object-meta-mtime'] - - self.assertEqual( - 'test_c_segments/%E3%83%86%E3%82%B9%E3%83%88/dummy.dat/' + - '%s/30/10/' % mtime, r['headers']['x-object-manifest']) - del r['headers']['x-object-manifest'] - - self.assertEqual(r['path'], f.name) - del r['path'] - - self.assertEqual(r, expected_r) - self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with('test_c', 'テスト/dummy.dat', - '', - content_length=0, - headers={}, - response_dict={}) - - def test_upload_segment_job(self): - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 10) - f.write(b'b' * 10) - f.write(b'c' * 10) - f.flush() - - # run read() when put_object is called to calculate md5sum - def _consuming_conn(*a, **kw): - contents = a[2] - contents.read() # Force md5 calculation - return contents.get_md5sum() - - mock_conn = mock.Mock() - mock_conn.put_object.side_effect = _consuming_conn - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - expected_r = { - 'action': 'upload_segment', - 'for_container': 'test_c', - 'for_object': 'test_o', - 'segment_index': 2, - 'segment_size': 10, - 'segment_location': '/test_c_segments/test_s_1', - 'log_line': 'test_o segment 2', - 'success': True, - 'response_dict': {}, - 'segment_etag': md5(b'b' * 10).hexdigest(), - 'attempts': 2, - } - - s = SwiftService() - with self.assert_open_results_are_closed(): - r = s._upload_segment_job(conn=mock_conn, - path=f.name, - container='test_c', - segment_name='test_s_1', - segment_start=10, - segment_size=10, - segment_index=2, - obj_name='test_o', - options={'segment_container': None, - 'checksum': True}) - - self.assertEqual(r, expected_r) - - self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with( - 'test_c_segments', 'test_s_1', - mock.ANY, - content_length=10, - content_type='application/swiftclient-segment', - response_dict={}) - contents = mock_conn.put_object.call_args[0][2] - self.assertIsInstance(contents, utils.LengthWrapper) - self.assertEqual(len(contents), 10) - - def test_upload_stream_segment(self): - common_params = { - 'segment_container': 'segments', - 'segment_name': 'test_stream_2', - 'container': 'test_stream', - 'object': 'stream_object', - } - tests = [ - {'test_params': { - 'segment_size': 1024, - 'segment_index': 2, - 'content_size': 1024}, - 'put_object_args': { - 'container': 'segments', - 'object': 'test_stream_2'}, - 'expected': { - 'complete': False, - 'segment_etag': md5(b'A' * 1024).hexdigest()}}, - {'test_params': { - 'segment_size': 2048, - 'segment_index': 0, - 'content_size': 512}, - 'put_object_args': { - 'container': 'test_stream', - 'object': 'stream_object'}, - 'expected': { - 'complete': True, - 'segment_etag': md5(b'A' * 512).hexdigest()}}, - # 0-sized segment should not be uploaded - {'test_params': { - 'segment_size': 1024, - 'segment_index': 1, - 'content_size': 0}, - 'put_object_args': {}, - 'expected': { - 'complete': True}}, - # 0-sized objects should be uploaded - {'test_params': { - 'segment_size': 1024, - 'segment_index': 0, - 'content_size': 0}, - 'put_object_args': { - 'container': 'test_stream', - 'object': 'stream_object'}, - 'expected': { - 'complete': True, - 'segment_etag': md5(b'').hexdigest()}}, - # Test boundary conditions - {'test_params': { - 'segment_size': 1024, - 'segment_index': 1, - 'content_size': 1023}, - 'put_object_args': { - 'container': 'segments', - 'object': 'test_stream_2'}, - 'expected': { - 'complete': True, - 'segment_etag': md5(b'A' * 1023).hexdigest()}}, - {'test_params': { - 'segment_size': 2048, - 'segment_index': 0, - 'content_size': 2047}, - 'put_object_args': { - 'container': 'test_stream', - 'object': 'stream_object'}, - 'expected': { - 'complete': True, - 'segment_etag': md5(b'A' * 2047).hexdigest()}}, - {'test_params': { - 'segment_size': 1024, - 'segment_index': 2, - 'content_size': 1025}, - 'put_object_args': { - 'container': 'segments', - 'object': 'test_stream_2'}, - 'expected': { - 'complete': False, - 'segment_etag': md5(b'A' * 1024).hexdigest()}}, - ] - - for test_args in tests: - params = test_args['test_params'] - stream = test_utils.FakeStream(params['content_size']) - segment_size = params['segment_size'] - segment_index = params['segment_index'] - - def _fake_put_object(*args, **kwargs): - contents = args[2] - # Consume and compute md5 - return md5(contents).hexdigest() - - mock_conn = mock.Mock() - mock_conn.put_object.side_effect = _fake_put_object - - s = SwiftService() - resp = s._upload_stream_segment( - conn=mock_conn, - container=common_params['container'], - object_name=common_params['object'], - segment_container=common_params['segment_container'], - segment_name=common_params['segment_name'], - segment_size=segment_size, - segment_index=segment_index, - headers={}, - fd=stream) - expected_args = test_args['expected'] - put_args = test_args['put_object_args'] - expected_response = { - 'segment_size': min(len(stream), segment_size), - 'complete': expected_args['complete'], - 'success': True, - } - if len(stream) or segment_index == 0: - segment_location = '/%s/%s' % (put_args['container'], - put_args['object']) - expected_response.update( - {'segment_index': segment_index, - 'segment_location': segment_location, - 'segment_etag': expected_args['segment_etag'], - 'for_object': common_params['object']}) - mock_conn.put_object.assert_called_once_with( - put_args['container'], - put_args['object'], - mock.ANY, - content_length=min(len(stream), segment_size), - headers={'etag': expected_args['segment_etag']}, - response_dict=mock.ANY) - else: - self.assertEqual([], mock_conn.put_object.mock_calls) - expected_response.update( - {'segment_index': None, - 'segment_location': None, - 'segment_etag': None}) - self.assertEqual(expected_response, resp) - - def test_etag_mismatch_with_ignore_checksum(self): - def _consuming_conn(*a, **kw): - contents = a[2] - contents.read() # Force md5 calculation - return 'badresponseetag' - - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 10) - f.write(b'b' * 10) - f.write(b'c' * 10) - f.flush() - - mock_conn = mock.Mock() - mock_conn.put_object.side_effect = _consuming_conn - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - - s = SwiftService() - r = s._upload_segment_job(conn=mock_conn, - path=f.name, - container='test_c', - segment_name='test_s_1', - segment_start=10, - segment_size=10, - segment_index=2, - obj_name='test_o', - options={'segment_container': None, - 'checksum': False}) - - self.assertIsNone(r.get('error')) - self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with( - 'test_c_segments', 'test_s_1', - mock.ANY, - content_length=10, - content_type='application/swiftclient-segment', - response_dict={}) - contents = mock_conn.put_object.call_args[0][2] - # Check that md5sum is not calculated. - self.assertEqual(contents.get_md5sum(), '') - - def test_upload_segment_job_etag_mismatch(self): - def _consuming_conn(*a, **kw): - contents = a[2] - contents.read() # Force md5 calculation - return 'badresponseetag' - - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 10) - f.write(b'b' * 10) - f.write(b'c' * 10) - f.flush() - - mock_conn = mock.Mock() - mock_conn.put_object.side_effect = _consuming_conn - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - - s = SwiftService() - with self.assert_open_results_are_closed(): - r = s._upload_segment_job(conn=mock_conn, - path=f.name, - container='test_c', - segment_name='test_s_1', - segment_start=10, - segment_size=10, - segment_index=2, - obj_name='test_o', - options={'segment_container': None, - 'checksum': True}) - - self.assertIn('md5 mismatch', str(r.get('error'))) - - self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with( - 'test_c_segments', 'test_s_1', - mock.ANY, - content_length=10, - content_type='application/swiftclient-segment', - response_dict={}) - contents = mock_conn.put_object.call_args[0][2] - self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest()) - - def test_upload_object_job_file(self): - # Uploading a file results in the file object being wrapped in a - # LengthWrapper. This test sets the options in such a way that much - # of _upload_object_job is skipped bringing the critical path down - # to around 60 lines to ease testing. - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 30) - f.flush() - expected_r = { - 'action': 'upload_object', - 'attempts': 2, - 'container': 'test_c', - 'headers': {}, - 'large_object': False, - 'object': 'test_o', - 'response_dict': {}, - 'status': 'uploaded', - 'success': True, - } - expected_mtime = '%f' % os.path.getmtime(f.name) - - # run read() when put_object is called to calculate md5sum - # md5sum is verified in _upload_object_job. - def _consuming_conn(*a, **kw): - contents = a[2] - contents.read() # Force md5 calculation - return contents.get_md5sum() - - mock_conn = mock.Mock() - mock_conn.put_object.side_effect = _consuming_conn - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - - s = SwiftService() - with self.assert_open_results_are_closed(): - r = s._upload_object_job(conn=mock_conn, - container='test_c', - source=f.name, - obj='test_o', - options=dict(s._options, - leave_segments=True)) - - mtime = r['headers']['x-object-meta-mtime'] - self.assertEqual(expected_mtime, mtime) - del r['headers']['x-object-meta-mtime'] - - self.assertEqual(r['path'], f.name) - del r['path'] - - self.assertEqual(r, expected_r) - self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with('test_c', 'test_o', - mock.ANY, - content_length=30, - headers={}, - response_dict={}) - contents = mock_conn.put_object.call_args[0][2] - self.assertIsInstance(contents, utils.LengthWrapper) - self.assertEqual(len(contents), 30) - - @mock.patch('swiftclient.service.time', return_value=1400000000) - def test_upload_object_job_stream(self, time_mock): - # Streams are wrapped as ReadableToIterable - with tempfile.TemporaryFile() as f: - f.write(b'a' * 30) - f.flush() - f.seek(0) - expected_r = { - 'action': 'upload_object', - 'attempts': 2, - 'container': 'test_c', - 'headers': {}, - 'large_object': False, - 'object': 'test_o', - 'response_dict': {}, - 'status': 'uploaded', - 'success': True, - 'path': None, - } - expected_mtime = 1400000000 - - mock_conn = mock.Mock() - mock_conn.put_object.return_value = '' - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - - s = SwiftService() - r = s._upload_object_job(conn=mock_conn, - container='test_c', - source=f, - obj='test_o', - options=dict(s._options, - leave_segments=True)) - - mtime = float(r['headers']['x-object-meta-mtime']) - self.assertEqual(mtime, expected_mtime) - del r['headers']['x-object-meta-mtime'] - - self.assertEqual(r, expected_r) - self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with('test_c', 'test_o', - mock.ANY, - content_length=None, - headers={}, - response_dict={}) - contents = mock_conn.put_object.call_args[0][2] - self.assertIsInstance(contents, utils.ReadableToIterable) - self.assertEqual(contents.chunk_size, 65536) - # next retrieves the first chunk of the stream or len(chunk_size) - # or less, it also forces the md5 to be calculated. - self.assertEqual(next(contents), b'a' * 30) - self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest()) - - def test_upload_object_job_etag_mismatch(self): - # The etag test for both streams and files use the same code - # so only one test should be needed. - def _consuming_conn(*a, **kw): - contents = a[2] - contents.read() # Force md5 calculation - return 'badresponseetag' - - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 30) - f.flush() - - mock_conn = mock.Mock() - mock_conn.put_object.side_effect = _consuming_conn - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - - s = SwiftService() - r = s._upload_object_job(conn=mock_conn, - container='test_c', - source=f.name, - obj='test_o', - options=dict(s._options, - leave_segments=True)) - - self.assertIs(r['success'], False) - self.assertIn('md5 mismatch', str(r.get('error'))) - - self.assertEqual(mock_conn.put_object.call_count, 1) - expected_headers = {'x-object-meta-mtime': mock.ANY} - mock_conn.put_object.assert_called_with('test_c', 'test_o', - mock.ANY, - content_length=30, - headers=expected_headers, - response_dict={}) - - contents = mock_conn.put_object.call_args[0][2] - self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest()) - - def test_upload_object_job_identical_etag(self): - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 30) - f.flush() - - mock_conn = mock.Mock() - mock_conn.head_object.return_value = { - 'content-length': 30, - 'etag': md5(b'a' * 30).hexdigest()} - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - - s = SwiftService() - r = s._upload_object_job(conn=mock_conn, - container='test_c', - source=f.name, - obj='test_o', - options={'changed': False, - 'skip_identical': True, - 'leave_segments': True, - 'header': '', - 'segment_size': 0}) - - self.assertIsNone(r.get('error')) - self.assertIs(True, r['success']) - self.assertEqual(r.get('status'), 'skipped-identical') - self.assertEqual(mock_conn.put_object.call_count, 0) - self.assertEqual(mock_conn.head_object.call_count, 1) - mock_conn.head_object.assert_called_with('test_c', 'test_o') - - def test_upload_object_job_identical_slo_with_nesting(self): - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 30) - f.flush() - seg_etag = md5(b'a' * 10).hexdigest() - submanifest = "[%s]" % ",".join( - ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) - submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() - manifest = "[%s]" % ",".join([ - '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' - '"bytes":20,"hash":"%s"}' % submanifest_etag, - '{"bytes":10,"hash":"%s"}' % seg_etag]) - - mock_conn = mock.Mock() - mock_conn.head_object.return_value = { - 'x-static-large-object': True, - 'content-length': 30, - 'etag': md5(submanifest_etag.encode('ascii') + - seg_etag.encode('ascii')).hexdigest()} - mock_conn.get_object.side_effect = [ - ({}, manifest.encode('ascii')), - ({}, submanifest.encode('ascii'))] - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - - s = SwiftService() - r = s._upload_object_job(conn=mock_conn, - container='test_c', - source=f.name, - obj='test_o', - options={'changed': False, - 'skip_identical': True, - 'leave_segments': True, - 'header': '', - 'segment_size': 10}) - - self.assertIsNone(r.get('error')) - self.assertIs(True, r['success']) - self.assertEqual('skipped-identical', r.get('status')) - self.assertEqual(0, mock_conn.put_object.call_count) - self.assertEqual([mock.call('test_c', 'test_o')], - mock_conn.head_object.mock_calls) - self.assertEqual([ - mock.call('test_c', 'test_o', - query_string='multipart-manifest=get'), - mock.call('test_c_segments', 'test_sub_slo', - query_string='multipart-manifest=get'), - ], mock_conn.get_object.mock_calls) - - def test_upload_object_job_identical_dlo(self): - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 30) - f.flush() - segment_etag = md5(b'a' * 10).hexdigest() - - mock_conn = mock.Mock() - mock_conn.head_object.return_value = { - 'x-object-manifest': 'test_c_segments/test_o/prefix', - 'content-length': 30, - 'etag': md5(segment_etag.encode('ascii') * 3).hexdigest()} - mock_conn.get_container.side_effect = [ - (None, [{"bytes": 10, "hash": segment_etag, - "name": "test_o/prefix/00"}, - {"bytes": 10, "hash": segment_etag, - "name": "test_o/prefix/01"}]), - (None, [{"bytes": 10, "hash": segment_etag, - "name": "test_o/prefix/02"}]), - (None, {})] - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - - s = SwiftService() - with mock.patch('swiftclient.service.get_conn', - return_value=mock_conn): - r = s._upload_object_job(conn=mock_conn, - container='test_c', - source=f.name, - obj='test_o', - options={'changed': False, - 'skip_identical': True, - 'leave_segments': True, - 'header': '', - 'segment_size': 10}) - - self.assertIsNone(r.get('error')) - self.assertIs(True, r['success']) - self.assertEqual('skipped-identical', r.get('status')) - self.assertEqual(0, mock_conn.put_object.call_count) - self.assertEqual(1, mock_conn.head_object.call_count) - self.assertEqual(3, mock_conn.get_container.call_count) - mock_conn.head_object.assert_called_with('test_c', 'test_o') - expected = [ - mock.call('test_c_segments', prefix='test_o/prefix', - marker='', delimiter=None, headers={}), - mock.call('test_c_segments', prefix='test_o/prefix', - marker="test_o/prefix/01", delimiter=None, - headers={}), - mock.call('test_c_segments', prefix='test_o/prefix', - marker="test_o/prefix/02", delimiter=None, - headers={}), - ] - mock_conn.get_container.assert_has_calls(expected) - - def test_make_upload_objects(self): - check_names_pseudo_to_expected = { - (('/absolute/file/path',), ''): ['absolute/file/path'], - (('relative/file/path',), ''): ['relative/file/path'], - (('/absolute/file/path',), '/absolute/pseudo/dir'): [ - 'absolute/pseudo/dir/absolute/file/path'], - (('/absolute/file/path',), 'relative/pseudo/dir'): [ - 'relative/pseudo/dir/absolute/file/path'], - (('relative/file/path',), '/absolute/pseudo/dir'): [ - 'absolute/pseudo/dir/relative/file/path'], - (('relative/file/path',), 'relative/pseudo/dir'): [ - 'relative/pseudo/dir/relative/file/path'], - } - errors = [] - for (filenames, pseudo_folder), expected in \ - check_names_pseudo_to_expected.items(): - actual = SwiftService._make_upload_objects( - filenames, pseudo_folder=pseudo_folder) - try: - self.assertEqual(expected, [o.object_name for o in actual]) - except AssertionError as e: - msg = 'given (%r, %r) expected %r, got %s' % ( - filenames, pseudo_folder, expected, e) - errors.append(msg) - self.assertFalse(errors, "\nERRORS:\n%s" % '\n'.join(errors)) - - def test_create_dir_marker_job_unchanged(self): - mock_conn = mock.Mock() - mock_conn.head_object.return_value = { - 'content-type': 'application/directory', - 'content-length': '0', - 'x-object-meta-mtime': '1.234000', - 'etag': md5().hexdigest()} - - s = SwiftService() - with mock.patch('swiftclient.service.get_conn', - return_value=mock_conn): - with mock.patch('swiftclient.service.getmtime', - return_value=1.234): - r = s._create_dir_marker_job(conn=mock_conn, - container='test_c', - obj='test_o', - path='test', - options={'changed': True, - 'skip_identical': True, - 'leave_segments': True, - 'header': '', - 'segment_size': 10}) - self.assertEqual({ - 'action': 'create_dir_marker', - 'container': 'test_c', - 'object': 'test_o', - 'path': 'test', - 'headers': {'x-object-meta-mtime': '1.234000'}, - # NO response dict! - 'success': True, - }, r) - self.assertEqual([], mock_conn.put_object.mock_calls) - - def test_create_dir_marker_job_unchanged_old_type(self): - mock_conn = mock.Mock() - mock_conn.head_object.return_value = { - 'content-type': 'text/directory', - 'content-length': '0', - 'x-object-meta-mtime': '1.000000', - 'etag': md5().hexdigest()} - - s = SwiftService() - with mock.patch('swiftclient.service.get_conn', - return_value=mock_conn): - with mock.patch('swiftclient.service.time', - return_value=1.234): - r = s._create_dir_marker_job(conn=mock_conn, - container='test_c', - obj='test_o', - options={'changed': True, - 'skip_identical': True, - 'leave_segments': True, - 'header': '', - 'segment_size': 10}) - self.assertEqual({ - 'action': 'create_dir_marker', - 'container': 'test_c', - 'object': 'test_o', - 'path': None, - 'headers': {'x-object-meta-mtime': '1.000000'}, - # NO response dict! - 'success': True, - }, r) - self.assertEqual([], mock_conn.put_object.mock_calls) - - def test_create_dir_marker_job_overwrites_bad_type(self): - mock_conn = mock.Mock() - mock_conn.head_object.return_value = { - 'content-type': 'text/plain', - 'content-length': '0', - 'x-object-meta-mtime': '1.000000', - 'etag': md5().hexdigest()} - - s = SwiftService() - with mock.patch('swiftclient.service.get_conn', - return_value=mock_conn): - with mock.patch('swiftclient.service.time', - return_value=1.234): - r = s._create_dir_marker_job(conn=mock_conn, - container='test_c', - obj='test_o', - options={'changed': True, - 'skip_identical': True, - 'leave_segments': True, - 'header': '', - 'segment_size': 10}) - self.assertEqual({ - 'action': 'create_dir_marker', - 'container': 'test_c', - 'object': 'test_o', - 'path': None, - 'headers': {'x-object-meta-mtime': '1.000000'}, - 'response_dict': {}, - 'success': True, - }, r) - self.assertEqual([mock.call( - 'test_c', 'test_o', '', - content_length=0, - content_type='application/directory', - headers={'x-object-meta-mtime': '1.000000'}, - response_dict={})], mock_conn.put_object.mock_calls) - - def test_create_dir_marker_job_missing(self): - mock_conn = mock.Mock() - mock_conn.head_object.side_effect = \ - ClientException('Not Found', http_status=404) - - s = SwiftService() - with mock.patch('swiftclient.service.get_conn', - return_value=mock_conn): - with mock.patch('swiftclient.service.time', - return_value=1.234): - r = s._create_dir_marker_job(conn=mock_conn, - container='test_c', - obj='test_o', - options={'changed': True, - 'skip_identical': True, - 'leave_segments': True, - 'header': '', - 'segment_size': 10}) - self.assertEqual({ - 'action': 'create_dir_marker', - 'container': 'test_c', - 'object': 'test_o', - 'path': None, - 'headers': {'x-object-meta-mtime': '1.000000'}, - 'response_dict': {}, - 'success': True, - }, r) - self.assertEqual([mock.call( - 'test_c', 'test_o', '', - content_length=0, - content_type='application/directory', - headers={'x-object-meta-mtime': '1.000000'}, - response_dict={})], mock_conn.put_object.mock_calls) - - -class TestServiceDownload(_TestServiceBase): - - def setUp(self): - super(TestServiceDownload, self).setUp() - self.opts = swiftclient.service._default_local_options.copy() - self.opts['no_download'] = True - self.obj_content = b'c' * 10 - self.obj_etag = md5(self.obj_content).hexdigest() - self.obj_len = len(self.obj_content) - self.exc = Exception('test_exc') - # Base response to be copied and updated to matched the expected - # response for each test - self.expected = { - 'action': 'download_object', # Should always be download_object - 'container': 'test_c', - 'object': 'test_o', - 'attempts': 2, - 'response_dict': {}, - 'path': 'test_o', - 'pseudodir': False, - 'success': None # Should be a bool - } - - def _readbody(self): - yield self.obj_content - - @mock.patch('swiftclient.service.SwiftService.list') - @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') - @mock.patch('swiftclient.service.interruptable_as_completed') - def test_download_container_job(self, as_comp, sub_page, service_list): - """ - Check that paged downloads work correctly - """ - obj_count = [0] - - def make_counting_generator(object_to_yield, total_count): - # maintain a counter of objects yielded - count = [0] - - def counting_generator(): - while count[0] < 10: - yield object_to_yield - count[0] += 1 - total_count[0] += 1 - return counting_generator() - - obj_count_on_sub_page_call = [] - sub_page_call_count = [0] - - def fake_sub_page(*args): - # keep a record of obj_count when this function is called - obj_count_on_sub_page_call.append(obj_count[0]) - sub_page_call_count[0] += 1 - if sub_page_call_count[0] < 3: - return range(0, 10) - return None - - sub_page.side_effect = fake_sub_page - - r = Mock(spec=Future) - r.result.return_value = self._get_expected({ - 'success': True, - 'start_time': 1, - 'finish_time': 2, - 'headers_receipt': 3, - 'auth_end_time': 4, - 'read_length': len(b'objcontent'), - }) - - as_comp.side_effect = [ - make_counting_generator(r, obj_count), - make_counting_generator(r, obj_count) - ] - - s = SwiftService() - down_gen = s._download_container('test_c', self.opts) - results = list(down_gen) - self.assertEqual(20, len(results)) - self.assertEqual(2, as_comp.call_count) - self.assertEqual(3, sub_page_call_count[0]) - self.assertEqual([0, 7, 17], obj_count_on_sub_page_call) - - @mock.patch('swiftclient.service.SwiftService.list') - @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') - @mock.patch('swiftclient.service.interruptable_as_completed') - def test_download_container_job_error( - self, as_comp, sub_page, service_list): - """ - Check that paged downloads work correctly - """ - class BoomError(Exception): - def __init__(self, value): - self.value = value - - def __str__(self): - return repr(self.value) - - def _make_result(): - r = Mock(spec=Future) - r.result.return_value = self._get_expected({ - 'success': True, - 'start_time': 1, - 'finish_time': 2, - 'headers_receipt': 3, - 'auth_end_time': 4, - 'read_length': len(b'objcontent'), - }) - return r - - as_comp.side_effect = [ - - ] - # We need Futures here because the error will cause a call to .cancel() - sub_page_effects = [ - [_make_result() for _ in range(0, 10)], - BoomError('Go Boom') - ] - sub_page.side_effect = sub_page_effects - # ...but we must also mock the returns to as_completed - as_comp.side_effect = [ - [_make_result() for _ in range(0, 10)] - ] - - s = SwiftService() - self.assertRaises( - BoomError, - lambda: list(s._download_container('test_c', self.opts)) - ) - # This was an unknown error, so make sure we attempt to cancel futures - for spe in sub_page_effects[0]: - spe.cancel.assert_called_once_with() - self.assertEqual(1, as_comp.call_count) - - # Now test ClientException - sub_page_effects = [ - [_make_result() for _ in range(0, 10)], - ClientException('Go Boom') - ] - sub_page.side_effect = sub_page_effects - as_comp.reset_mock() - as_comp.side_effect = [ - [_make_result() for _ in range(0, 10)], - ] - self.assertRaises( - ClientException, - lambda: list(s._download_container('test_c', self.opts)) - ) - # This was a ClientException, so make sure we don't cancel futures - for spe in sub_page_effects[0]: - self.assertFalse(spe.cancel.called) - self.assertEqual(1, as_comp.call_count) - - def test_download_object_job(self): - mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') - mock_conn.get_object.side_effect = [ - ({'content-type': 'text/plain', - 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, - objcontent) - ] - expected_r = self._get_expected({ - 'success': True, - 'start_time': 1, - 'finish_time': 2, - 'headers_receipt': 3, - 'auth_end_time': 4, - 'read_length': len(b'objcontent'), - }) - - with mock.patch.object(builtins, 'open') as mock_open: - written_content = Mock() - mock_open.return_value = written_content - s = SwiftService() - _opts = self.opts.copy() - _opts['no_download'] = False - actual_r = s._download_object_job( - mock_conn, 'test_c', 'test_o', _opts) - actual_r = dict( # Need to override the times we got from the call - actual_r, - **{ - 'start_time': 1, - 'finish_time': 2, - 'headers_receipt': 3 - } - ) - mock_open.assert_called_once_with('test_o', 'wb', 65536) - written_content.write.assert_called_once_with(b'objcontent') - - mock_conn.get_object.assert_called_once_with( - 'test_c', 'test_o', resp_chunk_size=65536, headers={}, - response_dict={} - ) - self.assertEqual(expected_r, actual_r) - - def test_download_object_job_with_mtime(self): - mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') - mock_conn.get_object.side_effect = [ - ({'content-type': 'text/plain', - 'etag': '2cbbfe139a744d6abbe695e17f3c1991', - 'x-object-meta-mtime': '1454113727.682512'}, - objcontent) - ] - expected_r = self._get_expected({ - 'success': True, - 'start_time': 1, - 'finish_time': 2, - 'headers_receipt': 3, - 'auth_end_time': 4, - 'read_length': len(b'objcontent'), - }) - - with mock.patch.object(builtins, 'open') as mock_open, \ - mock.patch('swiftclient.service.utime') as mock_utime: - written_content = Mock() - mock_open.return_value = written_content - s = SwiftService() - _opts = self.opts.copy() - _opts['no_download'] = False - actual_r = s._download_object_job( - mock_conn, 'test_c', 'test_o', _opts) - actual_r = dict( # Need to override the times we got from the call - actual_r, - **{ - 'start_time': 1, - 'finish_time': 2, - 'headers_receipt': 3 - } - ) - mock_open.assert_called_once_with('test_o', 'wb', 65536) - mock_utime.assert_called_once_with( - 'test_o', (1454113727.682512, 1454113727.682512)) - written_content.write.assert_called_once_with(b'objcontent') - - mock_conn.get_object.assert_called_once_with( - 'test_c', 'test_o', resp_chunk_size=65536, headers={}, - response_dict={} - ) - self.assertEqual(expected_r, actual_r) - - def test_download_object_job_bad_mtime(self): - mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') - mock_conn.get_object.side_effect = [ - ({'content-type': 'text/plain', - 'etag': '2cbbfe139a744d6abbe695e17f3c1991', - 'x-object-meta-mtime': 'foo'}, - objcontent) - ] - expected_r = self._get_expected({ - 'success': True, - 'start_time': 1, - 'finish_time': 2, - 'headers_receipt': 3, - 'auth_end_time': 4, - 'read_length': len(b'objcontent'), - }) - - with mock.patch.object(builtins, 'open') as mock_open, \ - mock.patch('swiftclient.service.utime') as mock_utime: - written_content = Mock() - mock_open.return_value = written_content - s = SwiftService() - _opts = self.opts.copy() - _opts['no_download'] = False - actual_r = s._download_object_job( - mock_conn, 'test_c', 'test_o', _opts) - actual_r = dict( # Need to override the times we got from the call - actual_r, - **{ - 'start_time': 1, - 'finish_time': 2, - 'headers_receipt': 3 - } - ) - mock_open.assert_called_once_with('test_o', 'wb', 65536) - self.assertEqual(0, len(mock_utime.mock_calls)) - written_content.write.assert_called_once_with(b'objcontent') - - mock_conn.get_object.assert_called_once_with( - 'test_c', 'test_o', resp_chunk_size=65536, headers={}, - response_dict={} - ) - self.assertEqual(expected_r, actual_r) - - def test_download_object_job_ignore_mtime(self): - mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') - mock_conn.get_object.side_effect = [ - ({'content-type': 'text/plain', - 'etag': '2cbbfe139a744d6abbe695e17f3c1991', - 'x-object-meta-mtime': '1454113727.682512'}, - objcontent) - ] - expected_r = self._get_expected({ - 'success': True, - 'start_time': 1, - 'finish_time': 2, - 'headers_receipt': 3, - 'auth_end_time': 4, - 'read_length': len(b'objcontent'), - }) - - with mock.patch.object(builtins, 'open') as mock_open, \ - mock.patch('swiftclient.service.utime') as mock_utime: - written_content = Mock() - mock_open.return_value = written_content - s = SwiftService() - _opts = self.opts.copy() - _opts['no_download'] = False - _opts['ignore_mtime'] = True - actual_r = s._download_object_job( - mock_conn, 'test_c', 'test_o', _opts) - actual_r = dict( # Need to override the times we got from the call - actual_r, - **{ - 'start_time': 1, - 'finish_time': 2, - 'headers_receipt': 3 - } - ) - mock_open.assert_called_once_with('test_o', 'wb', 65536) - self.assertEqual([], mock_utime.mock_calls) - written_content.write.assert_called_once_with(b'objcontent') - - mock_conn.get_object.assert_called_once_with( - 'test_c', 'test_o', resp_chunk_size=65536, headers={}, - response_dict={} - ) - self.assertEqual(expected_r, actual_r) - - def test_download_object_job_exception(self): - mock_conn = self._get_mock_connection() - mock_conn.get_object = Mock(side_effect=self.exc) - expected_r = self._get_expected({ - 'success': False, - 'error': self.exc, - 'error_timestamp': mock.ANY, - 'traceback': mock.ANY - }) - - s = SwiftService() - actual_r = s._download_object_job( - mock_conn, 'test_c', 'test_o', self.opts) - - mock_conn.get_object.assert_called_once_with( - 'test_c', 'test_o', resp_chunk_size=65536, headers={}, - response_dict={} - ) - self.assertEqual(expected_r, actual_r) - - def test_download(self): - with mock.patch('swiftclient.service.Connection') as mock_conn: - header = {'content-length': self.obj_len, - 'etag': self.obj_etag} - mock_conn.get_object.return_value = header, self._readbody() - - resp = SwiftService()._download_object_job(mock_conn, - 'c', - 'test', - self.opts) - - self.assertIsNone(resp.get('error')) - self.assertIs(True, resp['success']) - self.assertEqual(resp['action'], 'download_object') - self.assertEqual(resp['object'], 'test') - self.assertEqual(resp['path'], 'test') - - @mock.patch('swiftclient.service.interruptable_as_completed') - @mock.patch('swiftclient.service.SwiftService._download_container') - @mock.patch('swiftclient.service.SwiftService._download_object_job') - def test_download_with_objects_empty(self, mock_down_obj, - mock_down_cont, mock_as_comp): - fake_future = Future() - fake_future.set_result(1) - mock_as_comp.return_value = [fake_future] - service = SwiftService() - next(service.download('c', [], self.opts), None) - mock_down_obj.assert_not_called() - mock_down_cont.assert_not_called() - - next(service.download('c', options=self.opts), None) - self.assertTrue(mock_down_cont.called) - - def test_download_with_output_dir(self): - with mock.patch('swiftclient.service.Connection') as mock_conn: - header = {'content-length': self.obj_len, - 'etag': self.obj_etag} - mock_conn.get_object.return_value = header, self._readbody() - - options = self.opts.copy() - options['out_directory'] = 'temp_dir' - resp = SwiftService()._download_object_job(mock_conn, - 'c', - 'example/test', - options) - - self.assertIsNone(resp.get('error')) - self.assertIs(True, resp['success']) - self.assertEqual(resp['action'], 'download_object') - self.assertEqual(resp['object'], 'example/test') - self.assertEqual(resp['path'], 'temp_dir/example/test') - - def test_download_with_remove_prefix(self): - with mock.patch('swiftclient.service.Connection') as mock_conn: - header = {'content-length': self.obj_len, - 'etag': self.obj_etag} - mock_conn.get_object.return_value = header, self._readbody() - - options = self.opts.copy() - options['prefix'] = 'example/' - options['remove_prefix'] = True - resp = SwiftService()._download_object_job(mock_conn, - 'c', - 'example/test', - options) - - self.assertIsNone(resp.get('error')) - self.assertIs(True, resp['success']) - self.assertEqual(resp['action'], 'download_object') - self.assertEqual(resp['object'], 'example/test') - self.assertEqual(resp['path'], 'test') - - def test_download_with_remove_prefix_and_remove_slashes(self): - with mock.patch('swiftclient.service.Connection') as mock_conn: - header = {'content-length': self.obj_len, - 'etag': self.obj_etag} - mock_conn.get_object.return_value = header, self._readbody() - - options = self.opts.copy() - options['prefix'] = 'example' - options['remove_prefix'] = True - resp = SwiftService()._download_object_job(mock_conn, - 'c', - 'example/test', - options) - - self.assertIsNone(resp.get('error')) - self.assertIs(True, resp['success']) - self.assertEqual(resp['action'], 'download_object') - self.assertEqual(resp['object'], 'example/test') - self.assertEqual(resp['path'], 'test') - - def test_download_with_output_dir_and_remove_prefix(self): - with mock.patch('swiftclient.service.Connection') as mock_conn: - header = {'content-length': self.obj_len, - 'etag': self.obj_etag} - mock_conn.get_object.return_value = header, self._readbody() - - options = self.opts.copy() - options['prefix'] = 'example' - options['out_directory'] = 'new/dir' - options['remove_prefix'] = True - resp = SwiftService()._download_object_job(mock_conn, - 'c', - 'example/test', - options) - - self.assertIsNone(resp.get('error')) - self.assertIs(True, resp['success']) - self.assertEqual(resp['action'], 'download_object') - self.assertEqual(resp['object'], 'example/test') - self.assertEqual(resp['path'], 'new/dir/test') - - def test_download_object_job_skip_identical(self): - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 30) - f.flush() - - err = swiftclient.ClientException('Object GET failed', - http_status=304) - - def fake_get(*args, **kwargs): - kwargs['response_dict']['headers'] = {} - raise err - - mock_conn = mock.Mock() - mock_conn.get_object.side_effect = fake_get - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - expected_r = { - 'action': 'download_object', - 'container': 'test_c', - 'object': 'test_o', - 'success': False, - 'error': err, - 'response_dict': {'headers': {}}, - 'path': 'test_o', - 'pseudodir': False, - 'attempts': 2, - 'traceback': mock.ANY, - 'error_timestamp': mock.ANY - } - - s = SwiftService() - r = s._download_object_job(conn=mock_conn, - container='test_c', - obj='test_o', - options={'out_file': f.name, - 'out_directory': None, - 'prefix': None, - 'remove_prefix': False, - 'header': {}, - 'yes_all': False, - 'skip_identical': True}) - self.assertEqual(r, expected_r) - - self.assertEqual(mock_conn.get_object.call_count, 1) - mock_conn.get_object.assert_called_with( - 'test_c', - 'test_o', - resp_chunk_size=65536, - headers={'If-None-Match': md5(b'a' * 30).hexdigest()}, - query_string='multipart-manifest=get', - response_dict=expected_r['response_dict']) - - def test_download_object_job_skip_identical_dlo(self): - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 30) - f.flush() - on_disk_md5 = md5(b'a' * 30).hexdigest() - segment_md5 = md5(b'a' * 10).hexdigest() - - mock_conn = mock.Mock() - mock_conn.get_object.return_value = ( - {'x-object-manifest': 'test_c_segments/test_o/prefix'}, [b'']) - mock_conn.get_container.side_effect = [ - (None, [{'name': 'test_o/prefix/1', - 'bytes': 10, 'hash': segment_md5}, - {'name': 'test_o/prefix/2', - 'bytes': 10, 'hash': segment_md5}]), - (None, [{'name': 'test_o/prefix/3', - 'bytes': 10, 'hash': segment_md5}]), - (None, [])] - - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - expected_r = { - 'action': 'download_object', - 'container': 'test_c', - 'object': 'test_o', - 'success': False, - 'response_dict': {}, - 'path': 'test_o', - 'pseudodir': False, - 'attempts': 2, - 'traceback': mock.ANY, - 'error_timestamp': mock.ANY - } - - s = SwiftService() - with mock.patch('swiftclient.service.get_conn', - return_value=mock_conn): - r = s._download_object_job(conn=mock_conn, - container='test_c', - obj='test_o', - options={'out_file': f.name, - 'out_directory': None, - 'prefix': None, - 'remove_prefix': False, - 'header': {}, - 'yes_all': False, - 'skip_identical': True}) - - err = r.pop('error') - self.assertEqual("Large object is identical", err.msg) - self.assertEqual(304, err.http_status) - - self.assertEqual(r, expected_r) - - self.assertEqual(mock_conn.get_object.call_count, 1) - mock_conn.get_object.assert_called_with( - 'test_c', - 'test_o', - resp_chunk_size=65536, - headers={'If-None-Match': on_disk_md5}, - query_string='multipart-manifest=get', - response_dict=expected_r['response_dict']) - self.assertEqual(mock_conn.get_container.mock_calls, [ - mock.call('test_c_segments', - delimiter=None, - prefix='test_o/prefix', - marker='', - headers={}), - mock.call('test_c_segments', - delimiter=None, - prefix='test_o/prefix', - marker='test_o/prefix/2', - headers={}), - mock.call('test_c_segments', - delimiter=None, - prefix='test_o/prefix', - marker='test_o/prefix/3', - headers={})]) - - def test_download_object_job_skip_identical_nested_slo(self): - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 30) - f.flush() - on_disk_md5 = md5(b'a' * 30).hexdigest() - - seg_etag = md5(b'a' * 10).hexdigest() - submanifest = "[%s]" % ",".join( - ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) - submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() - manifest = "[%s]" % ",".join([ - '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' - '"bytes":20,"hash":"%s"}' % submanifest_etag, - '{"bytes":10,"hash":"%s"}' % seg_etag]) - - mock_conn = mock.Mock() - mock_conn.get_object.side_effect = [ - ({'x-static-large-object': True, - 'content-length': 30, - 'etag': md5(submanifest_etag.encode('ascii') + - seg_etag.encode('ascii')).hexdigest()}, - [manifest.encode('ascii')]), - ({'x-static-large-object': True, - 'content-length': 20, - 'etag': submanifest_etag}, - submanifest.encode('ascii'))] - - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - expected_r = { - 'action': 'download_object', - 'container': 'test_c', - 'object': 'test_o', - 'success': False, - 'response_dict': {}, - 'path': 'test_o', - 'pseudodir': False, - 'attempts': 2, - 'traceback': mock.ANY, - 'error_timestamp': mock.ANY - } - - s = SwiftService() - with mock.patch('swiftclient.service.get_conn', - return_value=mock_conn): - r = s._download_object_job(conn=mock_conn, - container='test_c', - obj='test_o', - options={'out_file': f.name, - 'out_directory': None, - 'prefix': None, - 'remove_prefix': False, - 'header': {}, - 'yes_all': False, - 'skip_identical': True}) - - err = r.pop('error') - self.assertEqual("Large object is identical", err.msg) - self.assertEqual(304, err.http_status) - - self.assertEqual(r, expected_r) - self.assertEqual(mock_conn.get_object.mock_calls, [ - mock.call('test_c', - 'test_o', - resp_chunk_size=65536, - headers={'If-None-Match': on_disk_md5}, - query_string='multipart-manifest=get', - response_dict={}), - mock.call('test_c_segments', - 'test_sub_slo', - query_string='multipart-manifest=get')]) - - def test_download_object_job_skip_identical_diff_dlo(self): - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 30) - f.write(b'b') - f.flush() - on_disk_md5 = md5(b'a' * 30 + b'b').hexdigest() - segment_md5 = md5(b'a' * 10).hexdigest() - - mock_conn = mock.Mock() - mock_conn.get_object.side_effect = [ - ({'x-object-manifest': 'test_c_segments/test_o/prefix'}, - [b'']), - ({'x-object-manifest': 'test_c_segments/test_o/prefix'}, - [b'a' * 30])] - mock_conn.get_container.side_effect = [ - (None, [{'name': 'test_o/prefix/1', - 'bytes': 10, 'hash': segment_md5}, - {'name': 'test_o/prefix/2', - 'bytes': 10, 'hash': segment_md5}]), - (None, [{'name': 'test_o/prefix/3', - 'bytes': 10, 'hash': segment_md5}]), - (None, [])] - - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - type(mock_conn).auth_end_time = mock.PropertyMock(return_value=14) - expected_r = { - 'action': 'download_object', - 'container': 'test_c', - 'object': 'test_o', - 'success': True, - 'response_dict': {}, - 'path': 'test_o', - 'pseudodir': False, - 'read_length': 30, - 'attempts': 2, - 'start_time': 0, - 'headers_receipt': 1, - 'finish_time': 2, - 'auth_end_time': mock_conn.auth_end_time, - } - - options = self.opts.copy() - options['out_file'] = f.name - options['skip_identical'] = True - s = SwiftService() - with mock.patch('swiftclient.service.time', side_effect=range(3)): - with mock.patch('swiftclient.service.get_conn', - return_value=mock_conn): - r = s._download_object_job( - conn=mock_conn, - container='test_c', - obj='test_o', - options=options) - - self.assertEqual(r, expected_r) - - self.assertEqual(mock_conn.get_container.mock_calls, [ - mock.call('test_c_segments', - delimiter=None, - prefix='test_o/prefix', - marker='', - headers={}), - mock.call('test_c_segments', - delimiter=None, - prefix='test_o/prefix', - marker='test_o/prefix/2', - headers={}), - mock.call('test_c_segments', - delimiter=None, - prefix='test_o/prefix', - marker='test_o/prefix/3', - headers={})]) - self.assertEqual(mock_conn.get_object.mock_calls, [ - mock.call('test_c', - 'test_o', - resp_chunk_size=65536, - headers={'If-None-Match': on_disk_md5}, - query_string='multipart-manifest=get', - response_dict={}), - mock.call('test_c', - 'test_o', - resp_chunk_size=65536, - headers={'If-None-Match': on_disk_md5}, - response_dict={})]) - - def test_download_object_job_skip_identical_diff_nested_slo(self): - with tempfile.NamedTemporaryFile() as f: - f.write(b'a' * 29) - f.flush() - on_disk_md5 = md5(b'a' * 29).hexdigest() - - seg_etag = md5(b'a' * 10).hexdigest() - submanifest = "[%s]" % ",".join( - ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) - submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() - manifest = "[%s]" % ",".join([ - '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' - '"bytes":20,"hash":"%s"}' % submanifest_etag, - '{"bytes":10,"hash":"%s"}' % seg_etag]) - - mock_conn = mock.Mock() - mock_conn.get_object.side_effect = [ - ({'x-static-large-object': True, - 'content-length': 30, - 'etag': md5(submanifest_etag.encode('ascii') + - seg_etag.encode('ascii')).hexdigest()}, - [manifest.encode('ascii')]), - ({'x-static-large-object': True, - 'content-length': 20, - 'etag': submanifest_etag}, - submanifest.encode('ascii')), - ({'x-static-large-object': True, - 'content-length': 30, - 'etag': md5(submanifest_etag.encode('ascii') + - seg_etag.encode('ascii')).hexdigest()}, - [b'a' * 30])] - - type(mock_conn).attempts = mock.PropertyMock(return_value=2) - type(mock_conn).auth_end_time = mock.PropertyMock(return_value=14) - expected_r = { - 'action': 'download_object', - 'container': 'test_c', - 'object': 'test_o', - 'success': True, - 'response_dict': {}, - 'path': 'test_o', - 'pseudodir': False, - 'read_length': 30, - 'attempts': 2, - 'start_time': 0, - 'headers_receipt': 1, - 'finish_time': 2, - 'auth_end_time': mock_conn.auth_end_time, - } - - options = self.opts.copy() - options['out_file'] = f.name - options['skip_identical'] = True - s = SwiftService() - with mock.patch('swiftclient.service.time', side_effect=range(3)): - with mock.patch('swiftclient.service.get_conn', - return_value=mock_conn): - r = s._download_object_job( - conn=mock_conn, - container='test_c', - obj='test_o', - options=options) - - self.assertEqual(r, expected_r) - self.assertEqual(mock_conn.get_object.mock_calls, [ - mock.call('test_c', - 'test_o', - resp_chunk_size=65536, - headers={'If-None-Match': on_disk_md5}, - query_string='multipart-manifest=get', - response_dict={}), - mock.call('test_c_segments', - 'test_sub_slo', - query_string='multipart-manifest=get'), - mock.call('test_c', - 'test_o', - resp_chunk_size=65536, - headers={'If-None-Match': on_disk_md5}, - response_dict={})]) - - -class TestServicePost(_TestServiceBase): - - def setUp(self): - super(TestServicePost, self).setUp() - self.opts = swiftclient.service._default_local_options.copy() - - @mock.patch('swiftclient.service.MultiThreadingManager') - @mock.patch('swiftclient.service.ResultsIterator') - def test_object_post(self, res_iter, thread_manager): - """ - Check post method translates strings and objects to _post_object_job - calls correctly - """ - tm_instance = Mock() - thread_manager.return_value = tm_instance - - self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) - spo = swiftclient.service.SwiftPostObject( - "test_spo", - {'meta': ["meta1:test2"], "header": ["hdr1:test2"]}) - - SwiftService().post('test_c', ['test_o', spo], self.opts) - - calls = [ - mock.call( - SwiftService._post_object_job, 'test_c', 'test_o', - { - "X-Object-Meta-Meta1": "test1", - "Hdr1": "test1"}, - {}), - mock.call( - SwiftService._post_object_job, 'test_c', 'test_spo', - { - "X-Object-Meta-Meta1": "test2", - "Hdr1": "test2"}, - {}), - ] - tm_instance.object_uu_pool.submit.assert_has_calls(calls) - self.assertEqual( - tm_instance.object_uu_pool.submit.call_count, len(calls)) - - res_iter.assert_called_with( - [tm_instance.object_uu_pool.submit()] * len(calls)) - - -class TestServiceCopy(_TestServiceBase): - - def setUp(self): - super(TestServiceCopy, self).setUp() - self.opts = swiftclient.service._default_local_options.copy() - - @mock.patch('swiftclient.service.MultiThreadingManager') - @mock.patch('swiftclient.service.interruptable_as_completed') - def test_object_copy(self, inter_compl, thread_manager): - """ - Check copy method translates strings and objects to _copy_object_job - calls correctly - """ - tm_instance = Mock() - thread_manager.return_value = tm_instance - - self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) - sco = swiftclient.service.SwiftCopyObject( - "test_sco", - options={'meta': ["meta1:test2"], "header": ["hdr1:test2"], - 'destination': "/cont_new/test_sco"}) - - res = SwiftService().copy('test_c', ['test_o', sco], self.opts) - res = list(res) - - calls = [ - mock.call( - SwiftService._create_container_job, 'cont_new', headers={}), - ] - tm_instance.container_pool.submit.assert_has_calls(calls, - any_order=True) - self.assertEqual( - tm_instance.container_pool.submit.call_count, len(calls)) - - calls = [ - mock.call( - SwiftService._copy_object_job, 'test_c', 'test_o', - None, - { - "X-Object-Meta-Meta1": "test1", - "Hdr1": "test1"}, - False), - mock.call( - SwiftService._copy_object_job, 'test_c', 'test_sco', - '/cont_new/test_sco', - { - "X-Object-Meta-Meta1": "test2", - "Hdr1": "test2"}, - False), - ] - tm_instance.object_uu_pool.submit.assert_has_calls(calls) - self.assertEqual( - tm_instance.object_uu_pool.submit.call_count, len(calls)) - - inter_compl.assert_called_with( - [tm_instance.object_uu_pool.submit()] * len(calls)) - - def test_object_copy_fail_dest(self): - """ - Destination in incorrect format and destination with object - used when multiple objects are copied raises SwiftError - """ - with self.assertRaises(SwiftError): - list(SwiftService().copy('test_c', ['test_o'], - {'destination': 'cont'})) - with self.assertRaises(SwiftError): - list(SwiftService().copy('test_c', ['test_o', 'test_o2'], - {'destination': '/cont/obj'})) diff -Nru python-swiftclient-3.8.1/tests/unit/test_shell.py python-swiftclient-3.9.0/tests/unit/test_shell.py --- python-swiftclient-3.8.1/tests/unit/test_shell.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/tests/unit/test_shell.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,3402 +0,0 @@ -# Copyright (c) 2014 Christian Schwede -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import unicode_literals - -import contextlib -from genericpath import getmtime -import getpass -import hashlib -import json -import logging -import mock -import os -import tempfile -import unittest -import textwrap -from time import localtime, mktime, strftime, strptime - -import six -import sys - -import swiftclient -from swiftclient.service import SwiftError -import swiftclient.shell -import swiftclient.utils - -from os.path import basename, dirname -from .utils import ( - CaptureOutput, fake_get_auth_keystone, - FakeKeystone, StubResponse, MockHttpTest) -from swiftclient.utils import ( - EMPTY_ETAG, EXPIRES_ISO8601_FORMAT, - SHORT_EXPIRES_ISO8601_FORMAT, TIME_ERRMSG) - -try: - from requests.packages.urllib3.exceptions import InsecureRequestWarning -except ImportError: - InsecureRequestWarning = None - -if six.PY2: - BUILTIN_OPEN = '__builtin__.open' -else: - BUILTIN_OPEN = 'builtins.open' - -mocked_os_environ = { - 'ST_AUTH': 'http://localhost:8080/auth/v1.0', - 'ST_USER': 'test:tester', - 'ST_KEY': 'testing' -} -clean_os_environ = {} -environ_prefixes = ('ST_', 'OS_') -for key in os.environ: - if any(key.startswith(m) for m in environ_prefixes): - clean_os_environ[key] = '' - - -def _make_args(cmd, opts, os_opts, separator='-', flags=None, cmd_args=None): - """ - Construct command line arguments for given options. - """ - args = [""] - flags = flags or [] - for k, v in opts.items(): - args.append("--" + k.replace("_", "-")) - if v is not None: - args.append(v) - for k, v in os_opts.items(): - args.append("--os" + separator + k.replace("_", separator)) - if v is not None: - args.append(v) - for flag in flags: - args.append('--%s' % flag) - if cmd: - args.append(cmd) - if cmd_args: - args.extend(cmd_args) - return args - - -def _make_env(opts, os_opts): - """ - Construct a dict of environment variables for given options. - """ - env = {} - for k, v in opts.items(): - key = 'ST_' + k.upper().replace('-', '_') - env[key] = v - for k, v in os_opts.items(): - key = 'OS_' + k.upper().replace('-', '_') - env[key] = v - return env - - -def _make_cmd(cmd, opts, os_opts, use_env=False, flags=None, cmd_args=None): - flags = flags or [] - if use_env: - # set up fake environment variables and make a minimal command line - env = _make_env(opts, os_opts) - args = _make_args(cmd, {}, {}, separator='-', flags=flags, - cmd_args=cmd_args) - else: - # set up empty environment and make full command line - env = {} - args = _make_args(cmd, opts, os_opts, separator='-', flags=flags, - cmd_args=cmd_args) - return args, env - - -@contextlib.contextmanager -def patch_disable_warnings(): - if InsecureRequestWarning is None: - # If InsecureRequestWarning isn't available, disbale_warnings won't - # be either; they both came in with - # https://github.com/requests/requests/commit/811ee4e and left again - # in https://github.com/requests/requests/commit/8e17600 - yield None - else: - with mock.patch('requests.packages.urllib3.disable_warnings') \ - as patched: - yield patched - - -@mock.patch.dict(os.environ, mocked_os_environ) -class TestShell(unittest.TestCase): - def setUp(self): - super(TestShell, self).setUp() - tmpfile = tempfile.NamedTemporaryFile(delete=False) - self.tmpfile = tmpfile.name - - def tearDown(self): - try: - os.remove(self.tmpfile) - except OSError: - pass - super(TestShell, self).tearDown() - - @mock.patch('swiftclient.service.Connection') - def test_stat_account(self, connection): - argv = ["", "stat"] - return_headers = { - 'x-account-container-count': '1', - 'x-account-object-count': '2', - 'x-account-bytes-used': '3', - 'content-length': 0, - 'date': ''} - connection.return_value.head_account.return_value = return_headers - connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' - with CaptureOutput() as output: - swiftclient.shell.main(argv) - - self.assertEqual(output.out, - ' Account: AUTH_account\n' - 'Containers: 1\n' - ' Objects: 2\n' - ' Bytes: 3\n') - - @mock.patch('swiftclient.service.Connection') - def test_stat_account_with_headers(self, connection): - argv = ["", "stat", "-H", "Skip-Middleware: Test"] - return_headers = { - 'x-account-container-count': '1', - 'x-account-object-count': '2', - 'x-account-bytes-used': '3', - 'content-length': 0, - 'date': ''} - connection.return_value.head_account.return_value = return_headers - connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' - with CaptureOutput() as output: - swiftclient.shell.main(argv) - - self.assertEqual(output.out, - ' Account: AUTH_account\n' - 'Containers: 1\n' - ' Objects: 2\n' - ' Bytes: 3\n') - self.assertEqual(connection.return_value.head_account.mock_calls, [ - mock.call(headers={'Skip-Middleware': 'Test'})]) - - @mock.patch('swiftclient.service.Connection') - def test_stat_container(self, connection): - return_headers = { - 'x-container-object-count': '1', - 'x-container-bytes-used': '2', - 'x-container-read': 'test2:tester2', - 'x-container-write': 'test3:tester3', - 'x-container-sync-to': 'other', - 'x-container-sync-key': 'secret', - } - argv = ["", "stat", "container"] - connection.return_value.head_container.return_value = return_headers - connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' - with CaptureOutput() as output: - swiftclient.shell.main(argv) - - self.assertEqual(output.out, - ' Account: AUTH_account\n' - 'Container: container\n' - ' Objects: 1\n' - ' Bytes: 2\n' - ' Read ACL: test2:tester2\n' - 'Write ACL: test3:tester3\n' - ' Sync To: other\n' - ' Sync Key: secret\n') - - @mock.patch('swiftclient.service.Connection') - def test_stat_container_with_headers(self, connection): - return_headers = { - 'x-container-object-count': '1', - 'x-container-bytes-used': '2', - 'x-container-read': 'test2:tester2', - 'x-container-write': 'test3:tester3', - 'x-container-sync-to': 'other', - 'x-container-sync-key': 'secret', - } - argv = ["", "stat", "container", "-H", "Skip-Middleware: Test"] - connection.return_value.head_container.return_value = return_headers - connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' - with CaptureOutput() as output: - swiftclient.shell.main(argv) - - self.assertEqual(output.out, - ' Account: AUTH_account\n' - 'Container: container\n' - ' Objects: 1\n' - ' Bytes: 2\n' - ' Read ACL: test2:tester2\n' - 'Write ACL: test3:tester3\n' - ' Sync To: other\n' - ' Sync Key: secret\n') - self.assertEqual(connection.return_value.head_container.mock_calls, [ - mock.call('container', headers={'Skip-Middleware': 'Test'})]) - - @mock.patch('swiftclient.service.Connection') - def test_stat_object(self, connection): - return_headers = { - 'x-object-manifest': 'manifest', - 'etag': 'md5', - 'last-modified': 'yesterday', - 'content-type': 'text/plain', - 'content-length': 42, - } - argv = ["", "stat", "container", "object"] - connection.return_value.head_object.return_value = return_headers - connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' - - with CaptureOutput() as output: - swiftclient.shell.main(argv) - - self.assertEqual(output.out, - ' Account: AUTH_account\n' - ' Container: container\n' - ' Object: object\n' - ' Content Type: text/plain\n' - 'Content Length: 42\n' - ' Last Modified: yesterday\n' - ' ETag: md5\n' - ' Manifest: manifest\n') - - @mock.patch('swiftclient.service.Connection') - def test_stat_object_with_headers(self, connection): - return_headers = { - 'x-object-manifest': 'manifest', - 'etag': 'md5', - 'last-modified': 'yesterday', - 'content-type': 'text/plain', - 'content-length': 42, - } - argv = ["", "stat", "container", "object", - "-H", "Skip-Middleware: Test"] - connection.return_value.head_object.return_value = return_headers - connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' - - with CaptureOutput() as output: - swiftclient.shell.main(argv) - - self.assertEqual(output.out, - ' Account: AUTH_account\n' - ' Container: container\n' - ' Object: object\n' - ' Content Type: text/plain\n' - 'Content Length: 42\n' - ' Last Modified: yesterday\n' - ' ETag: md5\n' - ' Manifest: manifest\n') - self.assertEqual(connection.return_value.head_object.mock_calls, [ - mock.call('container', 'object', - headers={'Skip-Middleware': 'Test'})]) - - @mock.patch('swiftclient.service.Connection') - def test_list_json(self, connection): - connection.return_value.get_account.side_effect = [ - [None, [{'name': 'container'}]], - [None, [{'name': u'\u263A', 'some-custom-key': 'and value'}]], - [None, []], - ] - - argv = ["", "list", "--json"] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - calls = [mock.call(marker='', prefix=None, headers={}), - mock.call(marker='container', prefix=None, headers={})] - connection.return_value.get_account.assert_has_calls(calls) - - listing = [{'name': 'container'}, - {'name': u'\u263A', 'some-custom-key': 'and value'}] - expected = json.dumps(listing, sort_keys=True, indent=2) + '\n' - self.assertEqual(output.out, expected) - - @mock.patch('swiftclient.service.Connection') - def test_list_account(self, connection): - # Test account listing - connection.return_value.get_account.side_effect = [ - [None, [{'name': 'container'}]], - [None, []], - ] - - argv = ["", "list"] - - with CaptureOutput() as output: - swiftclient.shell.main(argv) - calls = [mock.call(marker='', prefix=None, headers={}), - mock.call(marker='container', prefix=None, headers={})] - connection.return_value.get_account.assert_has_calls(calls) - - self.assertEqual(output.out, 'container\n') - - @mock.patch('swiftclient.service.Connection') - def test_list_account_with_headers(self, connection): - # Test account listing - connection.return_value.get_account.side_effect = [ - [None, [{'name': 'container'}]], - [None, []], - ] - - argv = ["", "list", '-H', 'Skip-Custom-Middleware: True'] - - with CaptureOutput() as output: - swiftclient.shell.main(argv) - calls = [mock.call(marker='', prefix=None, - headers={'Skip-Custom-Middleware': 'True'}), - mock.call(marker='container', prefix=None, - headers={'Skip-Custom-Middleware': 'True'})] - connection.return_value.get_account.assert_has_calls(calls) - - self.assertEqual(output.out, 'container\n') - - @mock.patch('swiftclient.service.Connection') - def test_list_account_long(self, connection): - # Test account listing - connection.return_value.get_account.side_effect = [ - [None, [{'name': 'container', 'bytes': 0, 'count': 0}]], - [None, []], - ] - - argv = ["", "list", "--lh"] - with CaptureOutput() as output: - swiftclient.shell.main(argv) - calls = [mock.call(marker='', prefix=None, headers={}), - mock.call(marker='container', prefix=None, headers={})] - connection.return_value.get_account.assert_has_calls(calls) - - self.assertEqual(output.out, - ' 0 0 1970-01-01 00:00:01 container\n' - ' 0 0\n') - - # Now test again, this time without returning metadata - connection.return_value.head_container.return_value = {} - - # Test account listing - connection.return_value.get_account.side_effect = [ - [None, [{'name': 'container', 'bytes': 0, 'count': 0}]], - [None, []], - ] - - argv = ["", "list", "--lh"] - with CaptureOutput() as output: - swiftclient.shell.main(argv) - calls = [mock.call(marker='', prefix=None, headers={}), - mock.call(marker='container', prefix=None, headers={})] - connection.return_value.get_account.assert_has_calls(calls) - - self.assertEqual(output.out, - ' 0 0 ????-??-?? ??:??:?? container\n' - ' 0 0\n') - - def test_list_account_totals_error(self): - # No --lh provided: expect info message about incorrect --totals use - argv = ["", "list", "--totals"] - - with CaptureOutput() as output: - self.assertRaises(SystemExit, swiftclient.shell.main, argv) - self.assertEqual(output.err, - "Listing totals only works with -l or --lh.\n") - - @mock.patch('swiftclient.service.Connection') - def test_list_account_totals(self, connection): - - # Test account listing, only total count and size - connection.return_value.get_account.side_effect = [ - [None, [{'name': 'container1', 'bytes': 1, 'count': 2}, - {'name': 'container2', 'bytes': 2, 'count': 4}]], - [None, []], - ] - - argv = ["", "list", "--lh", "--totals"] - with CaptureOutput() as output: - swiftclient.shell.main(argv) - calls = [mock.call(marker='', prefix=None, headers={})] - connection.return_value.get_account.assert_has_calls(calls) - self.assertEqual(output.out, ' 6 3\n') - - @mock.patch('swiftclient.service.Connection') - def test_list_container(self, connection): - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object_a'}]], - [None, []], - ] - argv = ["", "list", "container"] - with CaptureOutput() as output: - swiftclient.shell.main(argv) - calls = [ - mock.call('container', marker='', - delimiter=None, prefix=None, headers={}), - mock.call('container', marker='object_a', - delimiter=None, prefix=None, headers={})] - connection.return_value.get_container.assert_has_calls(calls) - - self.assertEqual(output.out, 'object_a\n') - - # Test container listing with --long - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object_a', 'bytes': 0, - 'content_type': 'type/content', - 'last_modified': '123T456'}]], - [None, []], - ] - argv = ["", "list", "container", "--long"] - with CaptureOutput() as output: - swiftclient.shell.main(argv) - calls = [ - mock.call('container', marker='', - delimiter=None, prefix=None, headers={}), - mock.call('container', marker='object_a', - delimiter=None, prefix=None, headers={})] - connection.return_value.get_container.assert_has_calls(calls) - - self.assertEqual(output.out, - ' 0 123 456' - ' type/content object_a\n' - ' 0\n') - - @mock.patch('swiftclient.service.Connection') - def test_list_container_with_headers(self, connection): - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object_a'}]], - [None, []], - ] - argv = ["", "list", "container", "-H", "Skip-Middleware: Test"] - with CaptureOutput() as output: - swiftclient.shell.main(argv) - calls = [ - mock.call('container', marker='', - delimiter=None, prefix=None, - headers={'Skip-Middleware': 'Test'}), - mock.call('container', marker='object_a', - delimiter=None, prefix=None, - headers={'Skip-Middleware': 'Test'})] - connection.return_value.get_container.assert_has_calls(calls) - - self.assertEqual(output.out, 'object_a\n') - - @mock.patch('swiftclient.service.makedirs') - @mock.patch('swiftclient.service.Connection') - def test_download(self, connection, makedirs): - objcontent = six.BytesIO(b'objcontent') - connection.return_value.get_object.side_effect = [ - ({'content-type': 'text/plain', - 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, - objcontent), - ({'content-type': 'text/plain', - 'etag': EMPTY_ETAG}, - '') - ] - - # Test downloading whole container - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object'}]], - [None, [{'name': 'pseudo/'}]], - [None, []], - ] - connection.return_value.auth_end_time = 0 - connection.return_value.attempts = 0 - - with mock.patch(BUILTIN_OPEN) as mock_open: - argv = ["", "download", "container"] - swiftclient.shell.main(argv) - calls = [mock.call('container', 'object', - headers={}, resp_chunk_size=65536, - response_dict={}), - mock.call('container', 'pseudo/', - headers={}, resp_chunk_size=65536, - response_dict={})] - connection.return_value.get_object.assert_has_calls( - calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb', 65536) - self.assertEqual([mock.call('pseudo')], makedirs.mock_calls) - makedirs.reset_mock() - - # Test downloading single object - objcontent = six.BytesIO(b'objcontent') - connection.return_value.get_object.side_effect = [ - ({'content-type': 'text/plain', - 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, - objcontent) - ] - with mock.patch(BUILTIN_OPEN) as mock_open: - argv = ["", "download", "container", "object"] - swiftclient.shell.main(argv) - connection.return_value.get_object.assert_called_with( - 'container', 'object', headers={}, resp_chunk_size=65536, - response_dict={}) - mock_open.assert_called_with('object', 'wb', 65536) - self.assertEqual([], makedirs.mock_calls) - - # Test downloading without md5 checks - objcontent = six.BytesIO(b'objcontent') - connection.return_value.get_object.side_effect = [ - ({'content-type': 'text/plain', - 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, - objcontent) - ] - with mock.patch(BUILTIN_OPEN) as mock_open, mock.patch( - 'swiftclient.service._SwiftReader') as sr: - argv = ["", "download", "container", "object", "--ignore-check"] - swiftclient.shell.main(argv) - connection.return_value.get_object.assert_called_with( - 'container', 'object', headers={}, resp_chunk_size=65536, - response_dict={}) - mock_open.assert_called_with('object', 'wb', 65536) - sr.assert_called_once_with('object', mock.ANY, mock.ANY, False) - self.assertEqual([], makedirs.mock_calls) - - # Test downloading single object to stdout - objcontent = six.BytesIO(b'objcontent') - connection.return_value.get_object.side_effect = [ - ({'content-type': 'text/plain', - 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, - objcontent) - ] - with CaptureOutput() as output: - argv = ["", "download", "--output", "-", "container", "object"] - swiftclient.shell.main(argv) - self.assertEqual('objcontent', output.out) - - @mock.patch('swiftclient.service.shuffle') - @mock.patch('swiftclient.service.Connection') - def test_download_shuffle(self, connection, mock_shuffle): - # Test that the container and object lists are shuffled - mock_shuffle.side_effect = lambda l: l - connection.return_value.get_object.return_value = [ - {'content-type': 'text/plain', - 'etag': EMPTY_ETAG}, - ''] - - connection.return_value.get_container.side_effect = [ - (None, [{'name': 'object'}]), - (None, [{'name': 'pseudo/'}]), - (None, []), - ] - connection.return_value.auth_end_time = 0 - connection.return_value.attempts = 0 - connection.return_value.get_account.side_effect = [ - (None, [{'name': 'container'}]), - (None, []) - ] - - with mock.patch(BUILTIN_OPEN) as mock_open: - with mock.patch('swiftclient.service.makedirs') as mock_mkdir: - argv = ["", "download", "--all"] - swiftclient.shell.main(argv) - self.assertEqual(3, mock_shuffle.call_count) - mock_shuffle.assert_any_call(['container']) - mock_shuffle.assert_any_call(['object']) - mock_shuffle.assert_any_call(['pseudo/']) - mock_open.assert_called_once_with('container/object', 'wb', 65536) - self.assertEqual([ - mock.call('container'), - mock.call('container/pseudo'), - ], mock_mkdir.mock_calls) - - # Test that the container and object lists are not shuffled - mock_shuffle.reset_mock() - - connection.return_value.get_container.side_effect = [ - (None, [{'name': 'object'}]), - (None, [{'name': 'pseudo/'}]), - (None, []), - ] - connection.return_value.get_account.side_effect = [ - (None, [{'name': 'container'}]), - (None, []) - ] - - with mock.patch(BUILTIN_OPEN) as mock_open: - with mock.patch('swiftclient.service.makedirs') as mock_mkdir: - argv = ["", "download", "--all", "--no-shuffle"] - swiftclient.shell.main(argv) - self.assertEqual(0, mock_shuffle.call_count) - mock_open.assert_called_once_with('container/object', 'wb', 65536) - self.assertEqual([ - mock.call('container'), - mock.call('container/pseudo'), - ], mock_mkdir.mock_calls) - - @mock.patch('swiftclient.service.Connection') - def test_download_no_content_type(self, connection): - connection.return_value.get_object.return_value = [ - {'etag': EMPTY_ETAG}, - ''] - - # Test downloading whole container - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object'}]], - [None, [{'name': 'pseudo/'}]], - [None, []], - ] - connection.return_value.auth_end_time = 0 - connection.return_value.attempts = 0 - - with mock.patch(BUILTIN_OPEN) as mock_open: - with mock.patch('swiftclient.service.makedirs') as mock_mkdir: - argv = ["", "download", "container"] - swiftclient.shell.main(argv) - calls = [mock.call('container', 'object', - headers={}, resp_chunk_size=65536, - response_dict={}), - mock.call('container', 'pseudo/', - headers={}, resp_chunk_size=65536, - response_dict={})] - connection.return_value.get_object.assert_has_calls( - calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb', 65536) - self.assertEqual([ - mock.call('pseudo'), - ], mock_mkdir.mock_calls) - - @mock.patch('swiftclient.shell.walk') - @mock.patch('swiftclient.service.Connection') - def test_upload(self, connection, walk): - connection.return_value.head_object.return_value = { - 'content-length': '0'} - connection.return_value.put_object.return_value = EMPTY_ETAG - connection.return_value.attempts = 0 - argv = ["", "upload", "container", self.tmpfile, - "-H", "X-Storage-Policy:one", - "--meta", "Color:Blue"] - swiftclient.shell.main(argv) - connection.return_value.put_container.assert_called_once_with( - 'container', - {'X-Storage-Policy': 'one'}, - response_dict={}) - - connection.return_value.put_object.assert_called_with( - 'container', - self.tmpfile.lstrip('/'), - mock.ANY, - content_length=0, - headers={'x-object-meta-mtime': mock.ANY, - 'X-Storage-Policy': 'one', - 'X-Object-Meta-Color': 'Blue'}, - response_dict={}) - - # upload to pseudo-folder (via param) - argv = ["", "upload", "container/pseudo-folder/nested", self.tmpfile, - "-H", "X-Storage-Policy:one"] - swiftclient.shell.main(argv) - connection.return_value.put_container.assert_called_with( - 'container', - {'X-Storage-Policy': 'one'}, - response_dict={}) - - connection.return_value.put_object.assert_called_with( - 'container', - 'pseudo-folder/nested' + self.tmpfile, - mock.ANY, - content_length=0, - headers={'x-object-meta-mtime': mock.ANY, - 'X-Storage-Policy': 'one'}, - response_dict={}) - - # Upload whole directory - argv = ["", "upload", "container", "/tmp"] - _tmpfile = self.tmpfile - _tmpfile_dir = dirname(_tmpfile) - _tmpfile_base = basename(_tmpfile) - walk.return_value = [(_tmpfile_dir, [], [_tmpfile_base])] - swiftclient.shell.main(argv) - connection.return_value.put_object.assert_called_with( - 'container', - self.tmpfile.lstrip('/'), - mock.ANY, - content_length=0, - headers={'x-object-meta-mtime': mock.ANY}, - response_dict={}) - - # Upload in segments - connection.return_value.head_container.return_value = { - 'x-storage-policy': 'one'} - argv = ["", "upload", "container", self.tmpfile, "-S", "10"] - with open(self.tmpfile, "wb") as fh: - fh.write(b'12345678901234567890') - swiftclient.shell.main(argv) - expected_calls = [mock.call('container', - {'X-Storage-Policy': mock.ANY}, - response_dict={}), - mock.call('container_segments', - {'X-Storage-Policy': mock.ANY}, - response_dict={})] - connection.return_value.put_container.has_calls(expected_calls) - connection.return_value.put_object.assert_called_with( - 'container', - self.tmpfile.lstrip('/'), - '', - content_length=0, - headers={'x-object-manifest': mock.ANY, - 'x-object-meta-mtime': mock.ANY}, - response_dict={}) - - # upload in segments to pseudo-folder (via param) - connection.reset_mock() - connection.return_value.head_container.return_value = { - 'x-storage-policy': 'one'} - argv = ["", "upload", "container/pseudo-folder/nested", - self.tmpfile, "-S", "10", "--use-slo"] - with open(self.tmpfile, "wb") as fh: - fh.write(b'12345678901234567890') - swiftclient.shell.main(argv) - expected_calls = [mock.call('container', - {}, - response_dict={}), - mock.call('container_segments', - {'X-Storage-Policy': 'one'}, - response_dict={})] - connection.return_value.put_container.assert_has_calls(expected_calls) - connection.return_value.put_object.assert_called_with( - 'container', - 'pseudo-folder/nested' + self.tmpfile, - mock.ANY, - headers={ - 'x-object-meta-mtime': mock.ANY, - }, - query_string='multipart-manifest=put', - response_dict=mock.ANY) - - @mock.patch('swiftclient.service.SwiftService.upload') - def test_upload_object_with_account_readonly(self, upload): - argv = ["", "upload", "container", self.tmpfile] - upload.return_value = [ - {"success": False, - "headers": {}, - "container": 'container', - "action": 'create_container', - "error": swiftclient.ClientException( - 'Container PUT failed', - http_status=403, - http_reason='Forbidden', - http_response_content=b'

Forbidden

') - }] - - with CaptureOutput() as output: - swiftclient.shell.main(argv) - self.assertTrue(output.err != '') - warning_msg = "Warning: failed to create container 'container': " \ - "403 Forbidden" - self.assertTrue(output.err.startswith(warning_msg)) - - @mock.patch('swiftclient.service.Connection') - def test_upload_delete_slo_segments(self, connection): - # Upload delete existing segments - connection.return_value.head_container.return_value = { - 'x-storage-policy': 'one'} - connection.return_value.attempts = 0 - argv = ["", "upload", "container", self.tmpfile] - connection.return_value.head_object.side_effect = [ - {'x-static-large-object': 'true', # For the upload call - 'content-length': '2'}, - {'x-static-large-object': 'false', # For the 1st delete call - 'content-length': '2'}, - {'x-static-large-object': 'false', # For the 2nd delete call - 'content-length': '2'} - ] - connection.return_value.get_object.return_value = ( - {}, - b'[{"name": "container1/old_seg1"},' - b' {"name": "container2/old_seg2"}]' - ) - connection.return_value.put_object.return_value = EMPTY_ETAG - # create the delete_object child mock here in attempt to fix - # https://bugs.launchpad.net/python-swiftclient/+bug/1480223 - connection.return_value.delete_object.return_value = None - swiftclient.shell.main(argv) - connection.return_value.put_object.assert_called_with( - 'container', - self.tmpfile.lstrip('/'), - mock.ANY, - content_length=0, - headers={'x-object-meta-mtime': mock.ANY}, - response_dict={}) - expected_delete_calls = [ - mock.call( - 'container1', 'old_seg1', - response_dict={} - ), - mock.call( - 'container2', 'old_seg2', - response_dict={} - ) - ] - self.assertEqual( - sorted(expected_delete_calls), - sorted(connection.return_value.delete_object.mock_calls) - ) - - @mock.patch('swiftclient.service.Connection') - def test_upload_over_symlink_to_slo(self, connection): - # Upload delete existing segments - connection.return_value.head_container.return_value = { - 'x-storage-policy': 'one'} - connection.return_value.attempts = 0 - connection.return_value.head_object.side_effect = [ - {'x-static-large-object': 'true', - 'content-location': '/v1/a/c/manifest', - 'content-length': '2'}, - ] - connection.return_value.get_object.return_value = ( - {'content-location': '/v1/a/c/manifest'}, - b'[{"name": "container1/old_seg1"},' - b' {"name": "container2/old_seg2"}]' - ) - connection.return_value.put_object.return_value = EMPTY_ETAG - connection.return_value.delete_object.return_value = None - argv = ["", "upload", "container", self.tmpfile] - swiftclient.shell.main(argv) - connection.return_value.put_object.assert_called_with( - 'container', - self.tmpfile.lstrip('/'), - mock.ANY, - content_length=0, - headers={'x-object-meta-mtime': mock.ANY}, - response_dict={}) - self.assertEqual([], connection.return_value.delete_object.mock_calls) - - @mock.patch('swiftclient.service.Connection') - def test_upload_leave_slo_segments(self, connection): - # Test upload overwriting a manifest respects --leave-segments - connection.return_value.head_container.return_value = { - 'x-storage-policy': 'one'} - connection.return_value.attempts = 0 - argv = ["", "upload", "container", self.tmpfile, "--leave-segments"] - connection.return_value.head_object.side_effect = [ - {'x-static-large-object': 'true', # For the upload call - 'content-length': '2'}] - connection.return_value.put_object.return_value = ( - 'd41d8cd98f00b204e9800998ecf8427e') - swiftclient.shell.main(argv) - connection.return_value.put_object.assert_called_with( - 'container', - self.tmpfile.lstrip('/'), - mock.ANY, - content_length=0, - headers={'x-object-meta-mtime': mock.ANY}, - response_dict={}) - self.assertFalse(connection.return_value.delete_object.mock_calls) - - @mock.patch('swiftclient.service.Connection') - def test_reupload_leaves_slo_segments(self, connection): - with open(self.tmpfile, "wb") as fh: - fh.write(b'12345678901234567890') - mtime = '{:.6f}'.format(os.path.getmtime(self.tmpfile)) - expected_segments = [ - 'container_segments/{}/slo/{}/20/10/{:08d}'.format( - self.tmpfile[1:], mtime, i) - for i in range(2) - ] - - # Test re-upload overwriting a manifest doesn't remove - # segments it just wrote - connection.return_value.head_container.return_value = { - 'x-storage-policy': 'one'} - connection.return_value.attempts = 0 - argv = ["", "upload", "container", self.tmpfile, - "--use-slo", "-S", "10"] - connection.return_value.head_object.side_effect = [ - {'x-static-large-object': 'true', # For the upload call - 'content-length': '20'}] - connection.return_value.get_object.return_value = ( - {}, - # we've already *got* the expected manifest! - json.dumps([ - {'name': seg} for seg in expected_segments - ]).encode('ascii') - ) - connection.return_value.put_object.return_value = ( - 'd41d8cd98f00b204e9800998ecf8427e') - swiftclient.shell.main(argv) - connection.return_value.put_object.assert_called_with( - 'container', - self.tmpfile[1:], # drop leading / - mock.ANY, - headers={'x-object-meta-mtime': mtime}, - query_string='multipart-manifest=put', - response_dict={}) - self.assertFalse(connection.return_value.delete_object.mock_calls) - - @mock.patch('swiftclient.service.Connection') - def test_upload_delete_dlo_segments(self, connection): - # Upload delete existing segments - connection.return_value.head_container.return_value = { - 'x-storage-policy': 'one'} - connection.return_value.attempts = 0 - argv = ["", "upload", "container", self.tmpfile] - connection.return_value.head_object.side_effect = [ - {'x-object-manifest': 'container1/prefix', - 'content-length': '0'}, - {}, - {} - ] - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'prefix_a', 'bytes': 0, - 'last_modified': '123T456'}]], - # Have multiple pages worth of DLO segments - [None, [{'name': 'prefix_b', 'bytes': 0, - 'last_modified': '123T456'}]], - [None, []] - ] - connection.return_value.put_object.return_value = EMPTY_ETAG - # create the delete_object child mock here in attempt to fix - # https://bugs.launchpad.net/python-swiftclient/+bug/1480223 - connection.return_value.delete_object.return_value = None - swiftclient.shell.main(argv) - connection.return_value.put_object.assert_called_with( - 'container', - self.tmpfile.lstrip('/'), - mock.ANY, - content_length=0, - headers={'x-object-meta-mtime': mock.ANY}, - response_dict={}) - expected_delete_calls = [ - mock.call( - 'container1', 'prefix_a', - response_dict={} - ), - mock.call( - 'container1', 'prefix_b', - response_dict={} - ) - ] - self.assertEqual( - sorted(expected_delete_calls), - sorted(connection.return_value.delete_object.mock_calls) - ) - - @mock.patch('swiftclient.service.Connection') - def test_upload_leave_dlo_segments(self, connection): - # Upload delete existing segments - connection.return_value.head_container.return_value = { - 'x-storage-policy': 'one'} - connection.return_value.attempts = 0 - argv = ["", "upload", "container", self.tmpfile, "--leave-segments"] - connection.return_value.head_object.side_effect = [ - {'x-object-manifest': 'container1/prefix', - 'content-length': '0'}] - connection.return_value.put_object.return_value = ( - 'd41d8cd98f00b204e9800998ecf8427e') - swiftclient.shell.main(argv) - connection.return_value.put_object.assert_called_with( - 'container', - self.tmpfile.lstrip('/'), - mock.ANY, - content_length=0, - headers={'x-object-meta-mtime': mock.ANY}, - response_dict={}) - self.assertFalse(connection.return_value.delete_object.mock_calls) - - @mock.patch('swiftclient.service.Connection') - def test_upload_segments_to_same_container(self, connection): - # Upload in segments to same container - connection.return_value.head_object.return_value = { - 'content-length': '0'} - connection.return_value.attempts = 0 - connection.return_value.put_object.return_value = EMPTY_ETAG - argv = ["", "upload", "container", self.tmpfile, "-S", "10", - "-C", "container"] - with open(self.tmpfile, "wb") as fh: - fh.write(b'12345678901234567890') - swiftclient.shell.main(argv) - connection.return_value.put_container.assert_called_once_with( - 'container', {}, response_dict={}) - connection.return_value.put_object.assert_called_with( - 'container', - self.tmpfile.lstrip('/'), - '', - content_length=0, - headers={'x-object-manifest': mock.ANY, - 'x-object-meta-mtime': mock.ANY}, - response_dict={}) - - @mock.patch('swiftclient.shell.io.open') - @mock.patch('swiftclient.service.SwiftService.upload') - def test_upload_from_stdin(self, upload_mock, io_open_mock): - def fake_open(fd, mode): - mock_io = mock.Mock() - mock_io.fileno.return_value = fd - return mock_io - - io_open_mock.side_effect = fake_open - - argv = ["", "upload", "container", "-", "--object-name", "foo"] - swiftclient.shell.main(argv) - upload_mock.assert_called_once_with("container", mock.ANY) - # This is a little convoluted: we want to examine the first call ([0]), - # the argv list([1]), the second parameter ([1]), and the first - # element. This is because the upload method takes a container and a - # list of SwiftUploadObjects. - swift_upload_obj = upload_mock.mock_calls[0][1][1][0] - self.assertEqual(sys.stdin.fileno(), swift_upload_obj.source.fileno()) - io_open_mock.assert_called_once_with(sys.stdin.fileno(), mode='rb') - - @mock.patch('swiftclient.service.SwiftService.upload') - def test_upload_from_stdin_no_name(self, upload_mock): - argv = ["", "upload", "container", "-"] - with CaptureOutput() as out: - self.assertRaises(SystemExit, swiftclient.shell.main, argv) - self.assertEqual(0, len(upload_mock.mock_calls)) - self.assertTrue(out.err.find('object-name must be specified') >= 0) - - @mock.patch('swiftclient.service.SwiftService.upload') - def test_upload_from_stdin_and_others(self, upload_mock): - argv = ["", "upload", "container", "-", "foo", "--object-name", "bar"] - with CaptureOutput() as out: - self.assertRaises(SystemExit, swiftclient.shell.main, argv) - self.assertEqual(0, len(upload_mock.mock_calls)) - self.assertTrue(out.err.find( - 'upload from stdin cannot be used') >= 0) - - @mock.patch.object(swiftclient.service.SwiftService, - '_bulk_delete_page_size', lambda *a: 0) - @mock.patch('swiftclient.service.Connection') - def test_delete_bad_threads(self, mock_connection): - mock_connection.return_value.get_container.return_value = (None, []) - mock_connection.return_value.attempts = 0 - - def check_bad(argv): - args, env = _make_cmd( - 'delete', {}, {}, cmd_args=['cont'] + argv) - with mock.patch.dict(os.environ, env): - with CaptureOutput() as output: - self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertIn( - 'ERROR: option %s should be a positive integer.' % argv[0], - output.err) - - def check_good(argv): - args, env = _make_cmd( - 'delete', {}, {}, cmd_args=['cont'] + argv) - with mock.patch.dict(os.environ, env): - with CaptureOutput() as output: - swiftclient.shell.main(args) - self.assertEqual('', output.err) - check_bad(["--object-threads", "-1"]) - check_bad(["--object-threads", "0"]) - check_bad(["--container-threads", "-1"]) - check_bad(["--container-threads", "0"]) - check_good(["--object-threads", "1"]) - check_good(["--container-threads", "1"]) - - @mock.patch.object(swiftclient.service.SwiftService, - '_bulk_delete_page_size', lambda *a: 1) - @mock.patch('swiftclient.service.Connection') - def test_delete_account(self, connection): - connection.return_value.get_account.side_effect = [ - [None, [{'name': 'container'}, {'name': 'container2'}]], - [None, [{'name': 'empty_container'}]], - [None, []], - ] - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]], - [None, []], - [None, [{'name': 'object'}]], - [None, []], - [None, []], - ] - connection.return_value.attempts = 0 - argv = ["", "delete", "--all"] - connection.return_value.head_object.return_value = {} - connection.return_value.delete_object.return_value = None - swiftclient.shell.main(argv) - connection.return_value.delete_object.assert_has_calls([ - mock.call('container', 'object', query_string=None, - response_dict={}, headers={}), - mock.call('container', 'obj\xe9ct2', query_string=None, - response_dict={}, headers={}), - mock.call('container2', 'object', query_string=None, - response_dict={}, headers={})], any_order=True) - self.assertEqual(3, connection.return_value.delete_object.call_count, - 'Expected 3 calls but found\n%r' - % connection.return_value.delete_object.mock_calls) - self.assertEqual( - connection.return_value.delete_container.mock_calls, [ - mock.call('container', response_dict={}, headers={}), - mock.call('container2', response_dict={}, headers={}), - mock.call('empty_container', response_dict={}, headers={})]) - - @mock.patch.object(swiftclient.service.SwiftService, - '_bulk_delete_page_size', lambda *a: 10) - @mock.patch('swiftclient.service.Connection') - def test_delete_bulk_account(self, connection): - connection.return_value.get_account.side_effect = [ - [None, [{'name': 'container'}, {'name': 'container2'}]], - [None, [{'name': 'empty_container'}]], - [None, []], - ] - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, - {'name': 'object3'}]], - [None, []], - [None, [{'name': 'object'}]], - [None, []], - [None, []], - ] - connection.return_value.attempts = 0 - argv = ["", "delete", "--all", "--object-threads", "2"] - connection.return_value.post_account.return_value = {}, ( - b'{"Number Not Found": 0, "Response Status": "200 OK", ' - b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') - swiftclient.shell.main(argv) - self.assertEqual( - 3, len(connection.return_value.post_account.mock_calls), - 'Expected 3 calls but found\n%r' - % connection.return_value.post_account.mock_calls) - # POSTs for same container are made in parallel so expect any order - for expected in [ - mock.call(query_string='bulk-delete', - data=b'/container/object\n/container/obj%C3%A9ct2\n', - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}), - mock.call(query_string='bulk-delete', - data=b'/container/object3\n', - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={})]: - self.assertIn(expected, - connection.return_value.post_account.mock_calls[:2]) - # POSTs for different containers are made sequentially so expect order - self.assertEqual( - mock.call(query_string='bulk-delete', - data=b'/container2/object\n', - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}), - connection.return_value.post_account.mock_calls[2]) - self.assertEqual( - connection.return_value.delete_container.mock_calls, [ - mock.call('container', response_dict={}, headers={}), - mock.call('container2', response_dict={}, headers={}), - mock.call('empty_container', response_dict={}, headers={})]) - - @mock.patch('swiftclient.service.Connection') - def test_delete_bulk_account_with_capabilities(self, connection): - connection.return_value.get_capabilities.return_value = { - 'bulk_delete': { - 'max_deletes_per_request': 10000, - 'max_failed_deletes': 1000, - }, - } - connection.return_value.get_account.side_effect = [ - [None, [{'name': 'container'}]], - [None, [{'name': 'container2'}]], - [None, [{'name': 'empty_container'}]], - [None, []], - ] - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, - {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], - [None, []], - [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, - {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], - [None, []], - [None, []], - ] - connection.return_value.attempts = 0 - argv = ["", "delete", "--all", "--object-threads", "1"] - connection.return_value.post_account.return_value = {}, ( - b'{"Number Not Found": 0, "Response Status": "200 OK", ' - b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') - swiftclient.shell.main(argv) - self.assertEqual( - connection.return_value.post_account.mock_calls, [ - mock.call(query_string='bulk-delete', - data=b''.join([ - b'/container/object\n', - b'/container/obj%C3%A9ct2\n', - b'/container/z_object\n', - b'/container/z_obj%C3%A9ct2\n' - ]), - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}), - mock.call(query_string='bulk-delete', - data=b''.join([ - b'/container2/object\n', - b'/container2/obj%C3%A9ct2\n', - b'/container2/z_object\n', - b'/container2/z_obj%C3%A9ct2\n' - ]), - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={})]) - self.assertEqual( - connection.return_value.delete_container.mock_calls, [ - mock.call('container', response_dict={}, headers={}), - mock.call('container2', response_dict={}, headers={}), - mock.call('empty_container', response_dict={}, headers={})]) - self.assertEqual(connection.return_value.get_capabilities.mock_calls, - [mock.call(None)]) # only one /info request - - @mock.patch('swiftclient.service.Connection') - def test_delete_bulk_account_with_capabilities_and_pages(self, connection): - connection.return_value.get_capabilities.return_value = { - 'bulk_delete': { - 'max_deletes_per_request': 2, - 'max_failed_deletes': 1000, - }, - } - connection.return_value.get_account.side_effect = [ - [None, [{'name': 'container'}]], - [None, [{'name': 'container2'}]], - [None, [{'name': 'empty_container'}]], - [None, []], - ] - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, - {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], - [None, []], - [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, - {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], - [None, []], - [None, []], - ] - connection.return_value.attempts = 0 - argv = ["", "delete", "--all", "--object-threads", "1"] - connection.return_value.post_account.return_value = {}, ( - b'{"Number Not Found": 0, "Response Status": "200 OK", ' - b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') - swiftclient.shell.main(argv) - # check that each bulk call was only called with 2 objects - self.assertEqual( - connection.return_value.post_account.mock_calls, [ - mock.call(query_string='bulk-delete', - data=b''.join([ - b'/container/object\n', - b'/container/obj%C3%A9ct2\n', - ]), - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}), - mock.call(query_string='bulk-delete', - data=b''.join([ - b'/container/z_object\n', - b'/container/z_obj%C3%A9ct2\n' - ]), - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}), - mock.call(query_string='bulk-delete', - data=b''.join([ - b'/container2/object\n', - b'/container2/obj%C3%A9ct2\n', - ]), - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}), - mock.call(query_string='bulk-delete', - data=b''.join([ - b'/container2/z_object\n', - b'/container2/z_obj%C3%A9ct2\n' - ]), - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={})]) - self.assertEqual( - connection.return_value.delete_container.mock_calls, [ - mock.call('container', response_dict={}, headers={}), - mock.call('container2', response_dict={}, headers={}), - mock.call('empty_container', response_dict={}, headers={})]) - self.assertEqual(connection.return_value.get_capabilities.mock_calls, - [mock.call(None)]) # only one /info request - - @mock.patch.object(swiftclient.service.SwiftService, - '_bulk_delete_page_size', lambda *a: 1) - @mock.patch('swiftclient.service.Connection') - def test_delete_container(self, connection): - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object'}]], - [None, []], - ] - connection.return_value.attempts = 0 - argv = ["", "delete", "container"] - connection.return_value.head_object.return_value = {} - swiftclient.shell.main(argv) - connection.return_value.delete_container.assert_called_with( - 'container', response_dict={}, headers={}) - connection.return_value.delete_object.assert_called_with( - 'container', 'object', query_string=None, response_dict={}, - headers={}) - - @mock.patch.object(swiftclient.service.SwiftService, - '_bulk_delete_page_size', lambda *a: 1) - @mock.patch('swiftclient.service.Connection') - def test_delete_container_headers(self, connection): - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object'}]], - [None, []], - ] - connection.return_value.attempts = 0 - argv = ["", "delete", "container", "-H", "Skip-Middleware: Test"] - connection.return_value.head_object.return_value = {} - swiftclient.shell.main(argv) - connection.return_value.delete_container.assert_called_with( - 'container', response_dict={}, - headers={'Skip-Middleware': 'Test'}) - connection.return_value.delete_object.assert_called_with( - 'container', 'object', query_string=None, response_dict={}, - headers={'Skip-Middleware': 'Test'}) - - @mock.patch.object(swiftclient.service.SwiftService, - '_bulk_delete_page_size', lambda *a: 10) - @mock.patch('swiftclient.service.Connection') - def test_delete_bulk_container(self, connection): - connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object'}]], - [None, []], - ] - connection.return_value.attempts = 0 - argv = ["", "delete", "container"] - connection.return_value.post_account.return_value = {}, ( - b'{"Number Not Found": 0, "Response Status": "200 OK", ' - b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') - swiftclient.shell.main(argv) - connection.return_value.post_account.assert_called_with( - query_string='bulk-delete', data=b'/container/object\n', - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}) - connection.return_value.delete_container.assert_called_with( - 'container', response_dict={}, headers={}) - - def test_delete_verbose_output_utf8(self): - container = 't\u00e9st_c' - base_argv = ['', '--verbose', 'delete'] - - # simulate container having an object with utf-8 code points in name, - # just returning the object delete result - res = {'success': True, 'response_dict': {}, 'attempts': 2, - 'container': container, 'action': 'delete_object', - 'object': 'obj_t\u00east_o'} - - with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: - with CaptureOutput() as out: - mock_func.return_value = [res] - swiftclient.shell.main(base_argv + [container.encode('utf-8')]) - - mock_func.assert_called_once_with(container=container) - self.assertTrue(out.out.find( - 'obj_t\u00east_o [after 2 attempts]') >= 0, out) - - # simulate empty container - res = {'success': True, 'response_dict': {}, 'attempts': 2, - 'container': container, 'action': 'delete_container'} - - with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: - with CaptureOutput() as out: - mock_func.return_value = [res] - swiftclient.shell.main(base_argv + [container.encode('utf-8')]) - - mock_func.assert_called_once_with(container=container) - self.assertTrue(out.out.find( - 't\u00e9st_c [after 2 attempts]') >= 0, out) - - @mock.patch.object(swiftclient.service.SwiftService, - '_bulk_delete_page_size', lambda *a: 1) - @mock.patch('swiftclient.service.Connection') - def test_delete_per_object(self, connection): - argv = ["", "delete", "container", "object"] - connection.return_value.head_object.return_value = {} - connection.return_value.attempts = 0 - swiftclient.shell.main(argv) - connection.return_value.delete_object.assert_called_with( - 'container', 'object', query_string=None, response_dict={}, - headers={}) - - @mock.patch.object(swiftclient.service.SwiftService, - '_bulk_delete_page_size', lambda *a: 10) - @mock.patch('swiftclient.service.Connection') - def test_delete_bulk_object(self, connection): - argv = ["", "delete", "container", "object"] - connection.return_value.post_account.return_value = {}, ( - b'{"Number Not Found": 0, "Response Status": "200 OK", ' - b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') - connection.return_value.attempts = 0 - swiftclient.shell.main(argv) - connection.return_value.post_account.assert_called_with( - query_string='bulk-delete', data=b'/container/object\n', - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}) - - def test_delete_verbose_output(self): - del_obj_res = {'success': True, 'response_dict': {}, 'attempts': 2, - 'container': 't\xe9st_c', 'action': 'delete_object', - 'object': 't\xe9st_o'} - - del_seg_res = del_obj_res.copy() - del_seg_res.update({'action': 'delete_segment'}) - - del_con_res = del_obj_res.copy() - del_con_res.update({'action': 'delete_container', 'object': None}) - - test_exc = Exception('t\xe9st_exc') - error_res = del_obj_res.copy() - error_res.update({'success': False, 'error': test_exc, 'object': None}) - - mock_delete = mock.Mock() - base_argv = ['', '--verbose', 'delete'] - - with mock.patch('swiftclient.shell.SwiftService.delete', mock_delete): - with CaptureOutput() as out: - mock_delete.return_value = [del_obj_res] - swiftclient.shell.main(base_argv + ['t\xe9st_c', 't\xe9st_o']) - - mock_delete.assert_called_once_with(container='t\xe9st_c', - objects=['t\xe9st_o']) - self.assertTrue(out.out.find( - 't\xe9st_o [after 2 attempts]') >= 0) - - with CaptureOutput() as out: - mock_delete.return_value = [del_seg_res] - swiftclient.shell.main(base_argv + ['t\xe9st_c', 't\xe9st_o']) - - mock_delete.assert_called_with(container='t\xe9st_c', - objects=['t\xe9st_o']) - self.assertTrue(out.out.find( - 't\xe9st_c/t\xe9st_o [after 2 attempts]') >= 0) - - with CaptureOutput() as out: - mock_delete.return_value = [del_con_res] - swiftclient.shell.main(base_argv + ['t\xe9st_c']) - - mock_delete.assert_called_with(container='t\xe9st_c') - self.assertTrue(out.out.find( - 't\xe9st_c [after 2 attempts]') >= 0) - - with CaptureOutput() as out: - mock_delete.return_value = [error_res] - self.assertRaises(SystemExit, - swiftclient.shell.main, - base_argv + ['t\xe9st_c']) - - mock_delete.assert_called_with(container='t\xe9st_c') - self.assertTrue(out.err.find( - 'Error Deleting: t\xe9st_c: t\xe9st_exc') >= 0) - - @mock.patch('swiftclient.service.Connection') - def test_post_account(self, connection): - argv = ["", "post"] - swiftclient.shell.main(argv) - connection.return_value.post_account.assert_called_with( - headers={}, response_dict={}) - - @mock.patch('swiftclient.service.Connection') - def test_post_account_bad_auth(self, connection): - argv = ["", "post"] - connection.return_value.post_account.side_effect = \ - swiftclient.ClientException( - 'bad auth', http_response_headers={'X-Trans-Id': 'trans_id'}) - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - - self.assertEqual(output.err, - 'bad auth\nFailed Transaction ID: trans_id\n') - - # do it again with a unicode token - connection.return_value.post_account.side_effect = \ - swiftclient.ClientException( - 'bad auth', http_response_headers={ - 'X-Trans-Id': 'non\u2011utf8'}) - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - - self.assertEqual(output.err, - 'bad auth\n' - 'Failed Transaction ID: non\u2011utf8\n') - - # do it again with a wonky token - connection.return_value.post_account.side_effect = \ - swiftclient.ClientException( - 'bad auth', http_response_headers={ - 'X-Trans-Id': b'non\xffutf8'}) - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - - self.assertEqual(output.err, - 'bad auth\nFailed Transaction ID: non%FFutf8\n') - - @mock.patch('swiftclient.service.Connection') - def test_post_account_not_found(self, connection): - argv = ["", "post"] - connection.return_value.post_account.side_effect = \ - swiftclient.ClientException('test', http_status=404) - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - - self.assertEqual(output.err, 'Account not found\n') - - @mock.patch('swiftclient.service.Connection') - def test_post_container(self, connection): - argv = ["", "post", "container"] - swiftclient.shell.main(argv) - connection.return_value.post_container.assert_called_with( - 'container', headers={}, response_dict={}) - - @mock.patch('swiftclient.service.Connection') - def test_post_container_bad_auth(self, connection): - argv = ["", "post", "container"] - connection.return_value.post_container.side_effect = \ - swiftclient.ClientException('bad auth') - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - - self.assertEqual(output.err, 'bad auth\n') - - @mock.patch('swiftclient.service.Connection') - def test_post_container_not_found_causes_put(self, connection): - argv = ["", "post", "container"] - connection.return_value.post_container.side_effect = \ - swiftclient.ClientException('test', http_status=404) - swiftclient.shell.main(argv) - self.assertEqual('container', - connection.return_value.put_container.call_args[0][0]) - - def test_post_container_with_bad_name(self): - argv = ["", "post", "conta/iner"] - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - self.assertTrue(output.err != '') - self.assertTrue(output.err.startswith('WARNING: / in')) - - @mock.patch('swiftclient.service.Connection') - def test_post_container_with_options(self, connection): - argv = ["", "post", "container", - "--read-acl", "test2:tester2", - "--write-acl", "test3:tester3 test4", - "--sync-to", "othersite", - "--sync-key", "secret", - ] - swiftclient.shell.main(argv) - connection.return_value.post_container.assert_called_with( - 'container', headers={ - 'X-Container-Write': 'test3:tester3 test4', - 'X-Container-Read': 'test2:tester2', - 'X-Container-Sync-Key': 'secret', - 'X-Container-Sync-To': 'othersite'}, response_dict={}) - - @mock.patch('swiftclient.service.Connection') - def test_post_object(self, connection): - argv = ["", "post", "container", "object", - "--meta", "Color:Blue", - "--header", "content-type:text/plain" - ] - swiftclient.shell.main(argv) - connection.return_value.post_object.assert_called_with( - 'container', 'object', headers={ - 'Content-Type': 'text/plain', - 'X-Object-Meta-Color': 'Blue'}, response_dict={}) - - @mock.patch('swiftclient.service.Connection') - def test_post_object_bad_auth(self, connection): - argv = ["", "post", "container", "object"] - connection.return_value.post_object.side_effect = \ - swiftclient.ClientException("bad auth") - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - - self.assertEqual(output.err, 'bad auth\n') - - def test_post_object_too_many_args(self): - argv = ["", "post", "container", "object", "bad_arg"] - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - - self.assertTrue(output.err != '') - self.assertTrue(output.err.startswith('Usage')) - - @mock.patch('swiftclient.service.Connection') - def test_copy_object_no_destination(self, connection): - argv = ["", "copy", "container", "object", - "--meta", "Color:Blue", - "--header", "content-type:text/plain" - ] - with CaptureOutput() as output: - swiftclient.shell.main(argv) - connection.return_value.copy_object.assert_called_with( - 'container', 'object', destination=None, fresh_metadata=False, - headers={ - 'Content-Type': 'text/plain', - 'X-Object-Meta-Color': 'Blue'}, response_dict={}) - self.assertEqual(output.out, 'container/object copied to \n') - - @mock.patch('swiftclient.service.Connection') - def test_copy_object(self, connection): - argv = ["", "copy", "container", "object", - "--meta", "Color:Blue", - "--header", "content-type:text/plain", - "--destination", "/c/o" - ] - with CaptureOutput() as output: - swiftclient.shell.main(argv) - connection.return_value.copy_object.assert_called_with( - 'container', 'object', destination="/c/o", - fresh_metadata=False, - headers={ - 'Content-Type': 'text/plain', - 'X-Object-Meta-Color': 'Blue'}, response_dict={}) - self.assertEqual( - output.out, - 'created container c\ncontainer/object copied to /c/o\n' - ) - - @mock.patch('swiftclient.service.Connection') - def test_copy_object_fresh_metadata(self, connection): - argv = ["", "copy", "container", "object", - "--meta", "Color:Blue", "--fresh-metadata", - "--header", "content-type:text/plain", - "--destination", "/c/o" - ] - swiftclient.shell.main(argv) - connection.return_value.copy_object.assert_called_with( - 'container', 'object', destination="/c/o", fresh_metadata=True, - headers={ - 'Content-Type': 'text/plain', - 'X-Object-Meta-Color': 'Blue'}, response_dict={}) - - @mock.patch('swiftclient.service.Connection') - def test_copy_two_objects(self, connection): - argv = ["", "copy", "container", "object", "object2", - "--meta", "Color:Blue"] - connection.return_value.copy_object.return_value = None - swiftclient.shell.main(argv) - calls = [ - mock.call( - 'container', 'object', destination=None, - fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, - response_dict={}), - mock.call( - 'container', 'object2', destination=None, - fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, - response_dict={}) - ] - connection.return_value.copy_object.assert_has_calls( - calls, any_order=True) - self.assertEqual(len(connection.return_value.copy_object.mock_calls), - len(calls)) - - @mock.patch('swiftclient.service.Connection') - def test_copy_two_objects_destination(self, connection): - argv = ["", "copy", "container", "object", "object2", - "--meta", "Color:Blue", "--destination", "/c"] - connection.return_value.copy_object.return_value = None - swiftclient.shell.main(argv) - calls = [ - mock.call( - 'container', 'object', destination="/c/object", - fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, - response_dict={}), - mock.call( - 'container', 'object2', destination="/c/object2", - fresh_metadata=False, headers={'X-Object-Meta-Color': 'Blue'}, - response_dict={}) - ] - connection.return_value.copy_object.assert_has_calls( - calls, any_order=True) - self.assertEqual(len(connection.return_value.copy_object.mock_calls), - len(calls)) - - @mock.patch('swiftclient.service.Connection') - def test_copy_two_objects_bad_destination(self, connection): - argv = ["", "copy", "container", "object", "object2", - "--meta", "Color:Blue", "--destination", "/c/o"] - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - - self.assertEqual( - output.err, - 'Combination of multiple objects and destination ' - 'including object is invalid\n') - - @mock.patch('swiftclient.service.Connection') - def test_copy_object_bad_auth(self, connection): - argv = ["", "copy", "container", "object"] - connection.return_value.copy_object.side_effect = \ - swiftclient.ClientException("bad auth") - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - - self.assertEqual(output.err, 'bad auth\n') - - def test_copy_object_not_enough_args(self): - argv = ["", "copy", "container"] - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - - self.assertTrue(output.err != '') - self.assertTrue(output.err.startswith('Usage')) - - def test_copy_bad_container(self): - argv = ["", "copy", "cont/ainer", "object"] - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - swiftclient.shell.main(argv) - - self.assertTrue(output.err != '') - self.assertTrue(output.err.startswith('WARN')) - - @mock.patch('swiftclient.shell.generate_temp_url', return_value='') - def test_temp_url(self, temp_url): - argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", - "secret_key"] - swiftclient.shell.main(argv) - temp_url.assert_called_with( - '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False, ip_range=None) - - @mock.patch('swiftclient.shell.generate_temp_url', return_value='') - def test_temp_url_prefix_based(self, temp_url): - argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/", - "secret_key", "--prefix-based"] - swiftclient.shell.main(argv) - temp_url.assert_called_with( - '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=True, ip_range=None) - - @mock.patch('swiftclient.shell.generate_temp_url', return_value='') - def test_temp_url_iso8601_in(self, temp_url): - dates = ('1970-01-01T00:01:00Z', '1970-01-01T00:01:00', - '1970-01-01') - for d in dates: - argv = ["", "tempurl", "GET", d, "/v1/AUTH_account/c/", - "secret_key"] - swiftclient.shell.main(argv) - temp_url.assert_called_with( - '/v1/AUTH_account/c/', d, 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False, ip_range=None) - - @mock.patch('swiftclient.shell.generate_temp_url', return_value='') - def test_temp_url_iso8601_out(self, temp_url): - argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/", - "secret_key", "--iso8601"] - swiftclient.shell.main(argv) - temp_url.assert_called_with( - '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=True, prefix=False, ip_range=None) - - @mock.patch('swiftclient.shell.generate_temp_url', return_value='') - def test_absolute_expiry_temp_url(self, temp_url): - argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", - "secret_key", "--absolute"] - swiftclient.shell.main(argv) - temp_url.assert_called_with( - '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True, - iso8601=False, prefix=False, ip_range=None) - - @mock.patch('swiftclient.shell.generate_temp_url', return_value='') - def test_temp_url_with_ip_range(self, temp_url): - argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", - "secret_key", "--ip-range", "1.2.3.4"] - swiftclient.shell.main(argv) - temp_url.assert_called_with( - '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False, ip_range='1.2.3.4') - - def test_temp_url_output(self): - argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", - "secret_key", "--absolute"] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" - expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig - self.assertEqual(expected, output.out) - - argv = ["", "tempurl", "GET", "60", "http://saio:8080/v1/a/c/o", - "secret_key", "--absolute"] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - expected = "http://saio:8080%s" % expected - self.assertEqual(expected, output.out) - - argv = ["", "tempurl", "GET", "60", "/v1/a/c/", - "secret_key", "--absolute", "--prefix"] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' - expected = ("/v1/a/c/?temp_url_sig=%s&temp_url_expires=60" - "&temp_url_prefix=\n" % sig) - self.assertEqual(expected, output.out) - - argv = ["", "tempurl", "GET", "60", "/v1/a/c/", - "secret_key", "--absolute", "--prefix", '--iso8601'] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' - expires = '1970-01-01T00:01:00Z' - expected = ("/v1/a/c/?temp_url_sig=%s&temp_url_expires=%s" - "&temp_url_prefix=\n" % (sig, expires)) - self.assertEqual(expected, output.out) - - dates = ("1970-01-01T00:01:00Z", - strftime(EXPIRES_ISO8601_FORMAT[:-1], localtime(60))) - for d in dates: - argv = ["", "tempurl", "GET", d, "/v1/a/c/o", - "secret_key"] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" - expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig - self.assertEqual(expected, output.out) - - ts = str(int( - mktime(strptime('2005-05-01', SHORT_EXPIRES_ISO8601_FORMAT)))) - - argv = ["", "tempurl", "GET", ts, "/v1/a/c/", - "secret_key", "--absolute"] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - expected = output.out - - argv = ["", "tempurl", "GET", '2005-05-01', "/v1/a/c/", - "secret_key", "--absolute"] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - self.assertEqual(expected, output.out) - - argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", - "secret_key", "--absolute", "--ip-range", "1.2.3.4"] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - sig = "6a6ec8efa4be53904ecba8d055d841e24a937c98" - expected = ( - "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60" - "&temp_url_ip_range=1.2.3.4\n" % sig - ) - self.assertEqual(expected, output.out) - - def test_temp_url_error_output(self): - expected = 'path must be full path to an object e.g. /v1/a/c/o\n' - for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o', - 'http://saio/v1/a/c', 'http://v1/a/c/o'): - argv = ["", "tempurl", "GET", "60", bad_path, - "secret_key", "--absolute"] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - self.assertEqual(expected, output.err, - 'Expected %r but got %r for path %r' % - (expected, output.err, bad_path)) - - expected = 'path must at least contain /v1/a/c/\n' - argv = ["", "tempurl", "GET", "60", '/v1/a/c', - "secret_key", "--absolute", '--prefix-based'] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - self.assertEqual(expected, output.err, - 'Expected %r but got %r for path %r' % - (expected, output.err, '/v1/a/c')) - - expected = TIME_ERRMSG + '\n' - for bad_time in ('not_an_int', '-1', '2015-05', '2015-05-01T01:00'): - argv = ["", "tempurl", "GET", bad_time, '/v1/a/c/o', - "secret_key", "--absolute"] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - self.assertEqual(expected, output.err, - 'Expected %r but got %r for time %r' % - (expected, output.err, bad_time)) - - @mock.patch('swiftclient.service.Connection') - def test_capabilities(self, connection): - argv = ["", "capabilities"] - connection.return_value.get_capabilities.return_value = {'swift': None} - swiftclient.shell.main(argv) - connection.return_value.get_capabilities.assert_called_with(None) - - @mock.patch('swiftclient.service.Connection') - def test_capabilities_json(self, connection): - capabilities = { - 'slo': {'min_segment_size': 1000000}, - 'some': [{'arbitrary': 'nested'}, {'crazy': 'structure'}], - 'swift': {'version': '2.5.0'}} - - connection.return_value.get_capabilities.return_value = capabilities - argv = ["", "capabilities", "--json"] - with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) - expected = json.dumps(capabilities, sort_keys=True, indent=2) + '\n' - self.assertEqual(expected, output.out) - connection.return_value.get_capabilities.assert_called_with(None) - - def test_human_readable_upload_segment_size(self): - def _check_expected(x, expected): - actual = x.call_args_list[-1][1]["options"]["segment_size"] - self.assertEqual(int(actual), expected) - - mock_swift = mock.MagicMock(spec=swiftclient.shell.SwiftService) - with mock.patch("swiftclient.shell.SwiftService", mock_swift): - with CaptureOutput(suppress_systemexit=True) as output: - # Test new behaviour with both upper and lower case - # trailing characters - argv = ["", "upload", "-S", "1B", "container", "object"] - swiftclient.shell.main(argv) - _check_expected(mock_swift, 1) - - argv = ["", "upload", "-S", "1K", "container", "object"] - swiftclient.shell.main(argv) - _check_expected(mock_swift, 1024) - - argv = ["", "upload", "-S", "1m", "container", "object"] - swiftclient.shell.main(argv) - _check_expected(mock_swift, 1048576) - - argv = ["", "upload", "-S", "1G", "container", "object"] - swiftclient.shell.main(argv) - _check_expected(mock_swift, 1073741824) - - # Test old behaviour is not affected - argv = ["", "upload", "-S", "12345", "container", "object"] - swiftclient.shell.main(argv) - _check_expected(mock_swift, 12345) - - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - # Test invalid states - argv = ["", "upload", "-S", "1234X", "container", "object"] - swiftclient.shell.main(argv) - self.assertEqual(output.err, "Invalid segment size\n") - output.clear() - - with self.assertRaises(SystemExit): - argv = ["", "upload", "-S", "K1234", "container", "object"] - swiftclient.shell.main(argv) - self.assertEqual(output.err, "Invalid segment size\n") - output.clear() - - with self.assertRaises(SystemExit): - argv = ["", "upload", "-S", "K", "container", "object"] - swiftclient.shell.main(argv) - self.assertEqual(output.err, "Invalid segment size\n") - - def test_negative_upload_segment_size(self): - with CaptureOutput() as output: - with self.assertRaises(SystemExit): - argv = ["", "upload", "-S", "-40", "container", "object"] - swiftclient.shell.main(argv) - self.assertEqual(output.err, "segment-size should be positive\n") - output.clear() - with self.assertRaises(SystemExit): - argv = ["", "upload", "-S=-40K", "container", "object"] - swiftclient.shell.main(argv) - self.assertEqual(output.err, "segment-size should be positive\n") - output.clear() - with self.assertRaises(SystemExit): - argv = ["", "upload", "-S=-40M", "container", "object"] - swiftclient.shell.main(argv) - self.assertEqual(output.err, "segment-size should be positive\n") - output.clear() - with self.assertRaises(SystemExit): - argv = ["", "upload", "-S=-40G", "container", "object"] - swiftclient.shell.main(argv) - self.assertEqual(output.err, "segment-size should be positive\n") - output.clear() - - -class TestSubcommandHelp(unittest.TestCase): - - def test_subcommand_help(self): - for command in swiftclient.shell.commands: - help_var = 'st_%s_help' % command - options_var = 'st_%s_options' % command - self.assertTrue(hasattr(swiftclient.shell, help_var)) - with CaptureOutput() as out: - argv = ['', command, '--help'] - self.assertRaises(SystemExit, swiftclient.shell.main, argv) - expected = 'Usage: swift %s %s\n%s' % ( - command, vars(swiftclient.shell).get(options_var, "\n"), - vars(swiftclient.shell)[help_var]) - self.assertEqual(out.strip('\n'), expected) - - def test_no_help(self): - with CaptureOutput() as out: - argv = ['', 'bad_command', '--help'] - self.assertRaises(SystemExit, swiftclient.shell.main, argv) - expected = 'no such command: bad_command' - self.assertEqual(out.strip('\n'), expected) - - -@mock.patch.dict(os.environ, mocked_os_environ) -class TestDebugAndInfoOptions(unittest.TestCase): - @mock.patch('logging.basicConfig') - @mock.patch('swiftclient.service.Connection') - def test_option_after_posarg(self, connection, mock_logging): - argv = ["", "stat", "--info"] - swiftclient.shell.main(argv) - mock_logging.assert_called_with(level=logging.INFO) - - argv = ["", "stat", "--debug"] - swiftclient.shell.main(argv) - mock_logging.assert_called_with(level=logging.DEBUG) - - @mock.patch('logging.basicConfig') - @mock.patch('swiftclient.service.Connection') - def test_debug_trumps_info(self, connection, mock_logging): - argv_scenarios = (["", "stat", "--info", "--debug"], - ["", "stat", "--debug", "--info"], - ["", "--info", "stat", "--debug"], - ["", "--debug", "stat", "--info"], - ["", "--info", "--debug", "stat"], - ["", "--debug", "--info", "stat"]) - for argv in argv_scenarios: - mock_logging.reset_mock() - swiftclient.shell.main(argv) - try: - mock_logging.assert_called_once_with(level=logging.DEBUG) - except AssertionError: - self.fail('Unexpected call(s) %r for args %r' - % (mock_logging.call_args_list, argv)) - - -class TestBase(unittest.TestCase): - """ - Provide some common methods to subclasses - """ - def _remove_swift_env_vars(self): - self._environ_vars = {} - keys = list(os.environ.keys()) - for k in keys: - if (k in ('ST_KEY', 'ST_USER', 'ST_AUTH') or - k.startswith('OS_')): - self._environ_vars[k] = os.environ.pop(k) - - def _replace_swift_env_vars(self): - os.environ.update(self._environ_vars) - - -class TestParsing(TestBase): - - def setUp(self): - super(TestParsing, self).setUp() - self._remove_swift_env_vars() - - def tearDown(self): - self._replace_swift_env_vars() - super(TestParsing, self).tearDown() - - def _make_fake_command(self, result): - def fake_command(parser, args, thread_manager): - result[0], result[1] = swiftclient.shell.parse_args(parser, args) - return fake_command - - def _verify_opts(self, actual_opts, expected_opts, expected_os_opts=None, - expected_os_opts_dict=None): - """ - Check parsed options are correct. - - :param expected_opts: v1 style options. - :param expected_os_opts: openstack style options. - :param expected_os_opts_dict: openstack options that should be found in - the os_options dict. - """ - expected_os_opts = expected_os_opts or {} - expected_os_opts_dict = expected_os_opts_dict or {} - # check the expected opts are set - for key, v in expected_opts.items(): - actual = actual_opts.get(key) - self.assertEqual(v, actual, 'Expected %s for key %s, found %s' % - (v, key, actual)) - - for key, v in expected_os_opts.items(): - actual = actual_opts.get("os_" + key) - self.assertEqual(v, actual, 'Expected %s for key %s, found %s' % - (v, key, actual)) - - # check the os_options dict values are set - self.assertIn('os_options', actual_opts) - actual_os_opts_dict = actual_opts['os_options'] - expected_os_opts_keys = ['project_name', 'region_name', - 'tenant_name', - 'user_domain_name', 'endpoint_type', - 'object_storage_url', 'project_domain_id', - 'user_id', 'user_domain_id', 'tenant_id', - 'service_type', 'project_id', 'auth_token', - 'project_domain_name'] - for key in expected_os_opts_keys: - self.assertIn(key, actual_os_opts_dict) - cli_key = key - if key == 'object_storage_url': - # exceptions to the pattern... - cli_key = 'storage_url' - if cli_key in expected_os_opts_dict: - expect = expected_os_opts_dict[cli_key] - else: - expect = None - actual = actual_os_opts_dict[key] - self.assertEqual(expect, actual, 'Expected %s for %s, got %s' - % (expect, key, actual)) - for key in actual_os_opts_dict: - self.assertIn(key, expected_os_opts_keys) - - # check that equivalent keys have equal values - equivalents = [('os_username', 'user'), - ('os_auth_url', 'auth'), - ('os_password', 'key')] - for pair in equivalents: - self.assertEqual(actual_opts.get(pair[0]), - actual_opts.get(pair[1])) - - def test_minimum_required_args_v3(self): - opts = {"auth_version": "3"} - os_opts = {"password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v3"} - - # username with domain is sufficient in args because keystone will - # assume user is in default domain - args = _make_args("stat", opts, os_opts, '-') - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - self._verify_opts(result[0], opts, os_opts, {}) - - # check its ok to have user_id instead of username - os_opts = {"password": "secret", - "auth_url": "http://example.com:5000/v3"} - os_opts_dict = {"user_id": "user_ID"} - all_os_opts = os_opts.copy() - all_os_opts.update(os_opts_dict) - - args = _make_args("stat", opts, all_os_opts, '-') - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - self._verify_opts(result[0], opts, os_opts, os_opts_dict) - - # check no user credentials required if token and url supplied - os_opts = {} - os_opts_dict = {"storage_url": "http://example.com:8080/v1", - "auth_token": "0123abcd"} - - args = _make_args("stat", opts, os_opts_dict, '-') - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - self._verify_opts(result[0], opts, os_opts, os_opts_dict) - - def test_sloppy_versions(self): - os_opts = {"password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v3", - "identity-api-version": "3.0"} - - # check os_identity_api_version=3.0 is mapped to auth_version=3 - args = _make_args("stat", {}, os_opts, '-') - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch.dict(os.environ, {}): - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - expected_opts = {'auth_version': '3'} # NB: not '3.0' - expected_os_opts = {"password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v3"} - self._verify_opts(result[0], expected_opts, expected_os_opts, {}) - - # check os_identity_api_version=2 is mapped to auth_version=2.0 - # A somewhat contrived scenario - we need to pass in the v1 style opts - # to prevent auth version defaulting to 2.0 due to lack of v1 style - # options. That way we can actually verify that the sloppy 2 was - # interpreted and mapped to 2.0 - os_opts = {"password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v2.0", - "identity-api-version": "2"} - opts = {"key": "secret", - "user": "user", - "auth": "http://example.com:5000/v2.0"} - args = _make_args("stat", opts, os_opts, '-') - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch.dict(os.environ, {}): - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - expected_opts = {'auth_version': '2.0'} # NB: not '2' - expected_os_opts = {"password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v2.0"} - self._verify_opts(result[0], expected_opts, expected_os_opts, {}) - - def test_os_identity_api_version(self): - os_opts = {"password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v3", - "identity-api-version": "3"} - - # check os_identity_api_version is sufficient in place of auth_version - args = _make_args("stat", {}, os_opts, '-') - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch.dict(os.environ, {}): - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - expected_opts = {'auth_version': '3'} - expected_os_opts = {"password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v3"} - self._verify_opts(result[0], expected_opts, expected_os_opts, {}) - - # check again using environment variables - args = _make_args("stat", {}, {}) - env = _make_env({}, os_opts) - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch.dict(os.environ, env): - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - self._verify_opts(result[0], expected_opts, expected_os_opts, {}) - - # check that last of auth-version, os-identity-api-version is preferred - args = _make_args("stat", {}, os_opts, '-') + ['--auth-version', '2.0'] - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch.dict(os.environ, {}): - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - expected_opts = {'auth_version': '2.0'} - self._verify_opts(result[0], expected_opts, expected_os_opts, {}) - - # now put auth_version ahead of os-identity-api-version - args = _make_args("stat", {"auth_version": "2.0"}, os_opts, '-') - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch.dict(os.environ, {}): - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - expected_opts = {'auth_version': '3'} - self._verify_opts(result[0], expected_opts, expected_os_opts, {}) - - # check that OS_AUTH_VERSION overrides OS_IDENTITY_API_VERSION - args = _make_args("stat", {}, {}) - env = _make_env({}, os_opts) - env.update({'OS_AUTH_VERSION': '2.0'}) - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch.dict(os.environ, env): - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - expected_opts = {'auth_version': '2.0'} - self._verify_opts(result[0], expected_opts, expected_os_opts, {}) - - # check that ST_AUTH_VERSION overrides OS_IDENTITY_API_VERSION - args = _make_args("stat", {}, {}) - env = _make_env({}, os_opts) - env.update({'ST_AUTH_VERSION': '2.0'}) - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch.dict(os.environ, env): - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - self._verify_opts(result[0], expected_opts, expected_os_opts, {}) - - # check that ST_AUTH_VERSION overrides OS_AUTH_VERSION - args = _make_args("stat", {}, {}) - env = _make_env({}, os_opts) - env.update({'ST_AUTH_VERSION': '2.0', 'OS_AUTH_VERSION': '3'}) - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch.dict(os.environ, env): - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - self._verify_opts(result[0], expected_opts, expected_os_opts, {}) - - def test_args_v3(self): - opts = {"auth_version": "3"} - os_opts = {"password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v3"} - os_opts_dict = {"user_id": "user_ID", - "project_id": "project_ID", - "tenant_id": "tenant_ID", - "project_domain_id": "project_domain_ID", - "user_domain_id": "user_domain_ID", - "tenant_name": "tenant", - "project_name": "project", - "project_domain_name": "project_domain", - "user_domain_name": "user_domain", - "auth_token": "token", - "storage_url": "http://example.com:8080/v1", - "region_name": "region", - "service_type": "service", - "endpoint_type": "endpoint"} - all_os_opts = os_opts.copy() - all_os_opts.update(os_opts_dict) - - # check using hyphen separator - args = _make_args("stat", opts, all_os_opts, '-') - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - self._verify_opts(result[0], opts, os_opts, os_opts_dict) - - # check using underscore separator - args = _make_args("stat", opts, all_os_opts, '_') - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - self._verify_opts(result[0], opts, os_opts, os_opts_dict) - - # check using environment variables - args = _make_args("stat", {}, {}) - env = _make_env(opts, all_os_opts) - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch.dict(os.environ, env): - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - self._verify_opts(result[0], opts, os_opts, os_opts_dict) - - # check again using OS_AUTH_VERSION instead of ST_AUTH_VERSION - env = _make_env({}, all_os_opts) - env.update({'OS_AUTH_VERSION': '3'}) - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch.dict(os.environ, env): - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - self._verify_opts(result[0], opts, os_opts, os_opts_dict) - - def test_command_args_v3(self): - result = [None, None] - fake_command = self._make_fake_command(result) - opts = {"auth_version": "3"} - os_opts = {"password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v3"} - args = _make_args("stat", opts, os_opts) - with mock.patch('swiftclient.shell.st_stat', fake_command): - swiftclient.shell.main(args) - self.assertEqual(['stat'], result[1]) - with mock.patch('swiftclient.shell.st_stat', fake_command): - args = args + ["container_name"] - swiftclient.shell.main(args) - self.assertEqual(["stat", "container_name"], result[1]) - - def test_insufficient_args_v3(self): - opts = {"auth_version": "3"} - os_opts = {"password": "secret", - "auth_url": "http://example.com:5000/v3"} - args = _make_args("stat", opts, os_opts) - with self.assertRaises(SystemExit) as cm: - swiftclient.shell.main(args) - self.assertIn( - 'Auth version 3 requires either OS_USERNAME or OS_USER_ID', - str(cm.exception)) - - os_opts = {"username": "user", - "auth_url": "http://example.com:5000/v3"} - args = _make_args("stat", opts, os_opts) - with self.assertRaises(SystemExit) as cm: - swiftclient.shell.main(args) - self.assertIn('Auth version 3 requires OS_PASSWORD', str(cm.exception)) - - os_opts = {"username": "user", - "password": "secret"} - args = _make_args("stat", opts, os_opts) - with self.assertRaises(SystemExit) as cm: - swiftclient.shell.main(args) - self.assertIn('Auth version 3 requires OS_AUTH_URL', str(cm.exception)) - - def test_password_prompt(self): - def do_test(opts, os_opts, auth_version): - args = _make_args("stat", opts, os_opts) - result = [None, None] - fake_command = self._make_fake_command(result) - with mock.patch('swiftclient.shell.st_stat', fake_command): - with mock.patch('getpass.getpass', - return_value='input_pwd') as mock_getpass: - swiftclient.shell.main(args) - mock_getpass.assert_called_once_with() - self.assertEqual('input_pwd', result[0]['key']) - self.assertEqual('input_pwd', result[0]['os_password']) - - # ctrl-D - with self.assertRaises(SystemExit) as cm: - with mock.patch('swiftclient.shell.st_stat', fake_command): - with mock.patch('getpass.getpass', - side_effect=EOFError) as mock_getpass: - swiftclient.shell.main(args) - mock_getpass.assert_called_once_with() - self.assertIn( - 'Auth version %s requires' % auth_version, str(cm.exception)) - - # force getpass to think it needs to use raw input - with self.assertRaises(SystemExit) as cm: - with mock.patch('getpass.getpass', getpass.fallback_getpass): - swiftclient.shell.main(args) - self.assertIn( - 'Input stream incompatible', str(cm.exception)) - - opts = {"prompt": None, "user": "bob", "key": "secret", - "auth": "http://example.com:8080/auth/v1.0"} - do_test(opts, {}, '1.0') - os_opts = {"username": "user", - "password": "secret", - "auth_url": "http://example.com:5000/v3"} - opts = {"auth_version": "2.0", "prompt": None} - do_test(opts, os_opts, '2.0') - opts = {"auth_version": "3", "prompt": None} - do_test(opts, os_opts, '3') - - def test_no_tenant_name_or_id_v2(self): - os_opts = {"password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v3", - "tenant_name": "", - "tenant_id": ""} - - with CaptureOutput() as output: - args = _make_args("stat", {}, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertEqual(output.err.strip(), 'No tenant specified') - - with CaptureOutput() as output: - args = _make_args("stat", {}, os_opts, cmd_args=["testcontainer"]) - self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertEqual(output.err.strip(), 'No tenant specified') - - def test_no_tenant_name_or_id_v3(self): - os_opts = {"password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v3", - "tenant_name": "", - "tenant_id": ""} - - with CaptureOutput() as output: - args = _make_args("stat", {"auth_version": "3"}, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertEqual(output.err.strip(), - 'No project name or project id specified.') - - with CaptureOutput() as output: - args = _make_args("stat", {"auth_version": "3"}, - os_opts, cmd_args=["testcontainer"]) - self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertEqual(output.err.strip(), - 'No project name or project id specified.') - - def test_insufficient_env_vars_v3(self): - args = _make_args("stat", {}, {}) - opts = {"auth_version": "3"} - os_opts = {"password": "secret", - "auth_url": "http://example.com:5000/v3"} - env = _make_env(opts, os_opts) - with mock.patch.dict(os.environ, env): - self.assertRaises(SystemExit, swiftclient.shell.main, args) - - os_opts = {"username": "user", - "auth_url": "http://example.com:5000/v3"} - env = _make_env(opts, os_opts) - with mock.patch.dict(os.environ, env): - self.assertRaises(SystemExit, swiftclient.shell.main, args) - - os_opts = {"username": "user", - "password": "secret"} - env = _make_env(opts, os_opts) - with mock.patch.dict(os.environ, env): - self.assertRaises(SystemExit, swiftclient.shell.main, args) - - def test_help(self): - # --help returns condensed help message - opts = {"help": None} - os_opts = {} - args = _make_args(None, opts, os_opts) - with CaptureOutput() as out: - self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertTrue(out.find('[--key ]') > 0) - self.assertEqual(-1, out.find('--os-username=')) - - # --help returns condensed help message, overrides --os-help - opts = {"help": None} - os_opts = {"help": None} - args = _make_args("", opts, os_opts) - with CaptureOutput() as out: - self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertTrue(out.find('[--key ]') > 0) - self.assertEqual(-1, out.find('--os-username=')) - - # --os-password, --os-username and --os-auth_url should be ignored - # because --help overrides it - opts = {"help": None} - os_opts = {"help": None, - "password": "secret", - "username": "user", - "auth_url": "http://example.com:5000/v3"} - args = _make_args("", opts, os_opts) - with CaptureOutput() as out: - self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertTrue(out.find('[--key ]') > 0) - self.assertEqual(-1, out.find('--os-username=')) - - # --os-help return os options help - opts = {} - args = _make_args("", opts, os_opts) - with CaptureOutput() as out: - self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertTrue(out.find('[--key ]') > 0) - self.assertTrue(out.find('--os-username=') > 0) - - -class TestKeystoneOptions(MockHttpTest): - """ - Tests to check that options are passed from the command line or - environment variables through to the keystone client interface. - """ - all_os_opts = {'password': 'secret', - 'username': 'user', - 'auth-url': 'http://example.com:5000/v3', - 'user-domain-name': 'userdomain', - 'user-id': 'userid', - 'user-domain-id': 'userdomainid', - 'tenant-name': 'tenantname', - 'tenant-id': 'tenantid', - 'project-name': 'projectname', - 'project-id': 'projectid', - 'project-domain-id': 'projectdomainid', - 'project-domain-name': 'projectdomain', - 'cacert': 'foo', - 'cert': 'minnie', - 'key': 'mickey'} - catalog_opts = {'service-type': 'my-object-store', - 'endpoint-type': 'public', - 'region-name': 'my-region'} - flags = ['insecure', 'debug'] - - # options that are given default values in code if missing from CLI - defaults = {'auth-version': '2.0', - 'service-type': 'object-store', - 'endpoint-type': 'publicURL'} - - def _build_os_opts(self, keys): - os_opts = {} - for k in keys: - os_opts[k] = self.all_os_opts.get(k, self.catalog_opts.get(k)) - return os_opts - - def _test_options_passed_to_keystone(self, cmd, opts, os_opts, - flags=None, use_env=False, - cmd_args=None, no_auth=False): - flags = flags or [] - if use_env: - # set up fake environment variables and make a minimal command line - env = _make_env(opts, os_opts) - args = _make_args(cmd, {}, {}, separator='-', flags=flags, - cmd_args=cmd_args) - else: - # set up empty environment and make full command line - env = {} - args = _make_args(cmd, opts, os_opts, separator='-', flags=flags, - cmd_args=cmd_args) - ks_endpoint = 'http://example.com:8080/v1/AUTH_acc' - ks_token = 'fake_auth_token' - # check correct auth version gets used - key = 'auth-version' - fake_ks = FakeKeystone(endpoint=ks_endpoint, token=ks_token) - if no_auth: - fake_ks2 = fake_ks3 = None - elif opts.get(key, self.defaults.get(key)) == '2.0': - fake_ks2 = fake_ks - fake_ks3 = None - else: - fake_ks2 = None - fake_ks3 = fake_ks - # fake_conn will check that storage_url and auth_token are as expected - endpoint = os_opts.get('storage-url', ks_endpoint) - token = os_opts.get('auth-token', ks_token) - fake_conn = self.fake_http_connection(204, headers={}, - storage_url=endpoint, - auth_token=token) - - with mock.patch('swiftclient.client.ksclient_v2', fake_ks2), \ - mock.patch('swiftclient.client.ksclient_v3', fake_ks3), \ - mock.patch('swiftclient.client.http_connection', fake_conn), \ - mock.patch.dict(os.environ, env, clear=True), \ - patch_disable_warnings() as mock_disable_warnings: - try: - swiftclient.shell.main(args) - except SystemExit as e: - self.fail('Unexpected SystemExit: %s' % e) - except SwiftError as err: - self.fail('Unexpected SwiftError: %s' % err) - - if InsecureRequestWarning is not None: - if 'insecure' in flags: - self.assertEqual([mock.call(InsecureRequestWarning)], - mock_disable_warnings.mock_calls) - else: - self.assertEqual([], mock_disable_warnings.mock_calls) - - if no_auth: - # We patched out both keystoneclient versions to be None; - # they *can't* have been used and if we tried to, we would - # have raised ClientExceptions - return - - # check args passed to keystone Client __init__ - self.assertEqual(len(fake_ks.calls), 1) - actual_args = fake_ks.calls[0] - for key in self.all_os_opts.keys(): - expected = os_opts.get(key, self.defaults.get(key)) - key = key.replace('-', '_') - self.assertIn(key, actual_args) - self.assertEqual(expected, actual_args[key], - 'Expected %s for key %s, found %s' - % (expected, key, actual_args[key])) - for flag in flags: - self.assertIn(flag, actual_args) - self.assertTrue(actual_args[flag]) - - check_attr = True - # check args passed to ServiceCatalog.url_for() method - self.assertEqual(len(fake_ks.client.service_catalog.calls), 1) - actual_args = fake_ks.client.service_catalog.calls[0] - for key in self.catalog_opts.keys(): - expected = os_opts.get(key, self.defaults.get(key)) - key = key.replace('-', '_') - if key == 'region_name': - key = 'filter_value' - if expected is None: - check_attr = False - self.assertNotIn(key, actual_args) - self.assertNotIn('attr', actual_args) - continue - self.assertIn(key, actual_args) - self.assertEqual(expected, actual_args[key], - 'Expected %s for key %s, found %s' - % (expected, key, actual_args[key])) - if check_attr: - key, v = 'attr', 'region' - self.assertIn(key, actual_args) - self.assertEqual(v, actual_args[key], - 'Expected %s for key %s, found %s' - % (v, key, actual_args[key])) - - def _test_options(self, opts, os_opts, flags=None, no_auth=False): - # repeat test for different commands using env and command line options - for cmd in ('stat', 'post'): - self._test_options_passed_to_keystone(cmd, opts, os_opts, - flags=flags, no_auth=no_auth) - self._test_options_passed_to_keystone(cmd, opts, os_opts, - flags=flags, use_env=True, - no_auth=no_auth) - - def test_all_args_passed_to_keystone(self): - # check that all possible command line args are passed to keystone - opts = {'auth-version': '3'} - os_opts = dict(self.all_os_opts) - os_opts.update(self.catalog_opts) - self._test_options(opts, os_opts, flags=self.flags) - - opts = {'auth-version': '2.0'} - self._test_options(opts, os_opts, flags=self.flags) - - opts = {} - self.defaults['auth-version'] = '3' - self._test_options(opts, os_opts, flags=self.flags) - - for o in ('user-domain-name', 'user-domain-id', - 'project-domain-name', 'project-domain-id'): - os_opts.pop(o) - self.defaults['auth-version'] = '2.0' - self._test_options(opts, os_opts, flags=self.flags) - - def test_catalog_options_and_flags_not_required_v3(self): - # check that all possible command line args are passed to keystone - opts = {'auth-version': '3'} - os_opts = dict(self.all_os_opts) - self._test_options(opts, os_opts, flags=None) - - def test_ok_option_combinations_v3(self): - opts = {'auth-version': '3'} - keys = ('username', 'password', 'tenant-name', 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - keys = ('user-id', 'password', 'tenant-name', 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - keys = ('user-id', 'password', 'tenant-id', 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - keys = ('user-id', 'password', 'project-name', 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - keys = ('user-id', 'password', 'project-id', 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - def test_ok_option_combinations_v2(self): - opts = {'auth-version': '2.0'} - keys = ('username', 'password', 'tenant-name', 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - keys = ('username', 'password', 'tenant-id', 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - # allow auth_version to default to 2.0 - opts = {} - keys = ('username', 'password', 'tenant-name', 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - keys = ('username', 'password', 'tenant-id', 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - # ...except when it should be 3 - self.defaults['auth-version'] = '3' - keys = ('username', 'user-domain-name', 'password', 'project-name', - 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - keys = ('username', 'user-domain-id', 'password', 'project-name', - 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - keys = ('username', 'project-domain-name', 'password', 'project-name', - 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - keys = ('username', 'project-domain-id', 'password', 'project-name', - 'auth-url') - os_opts = self._build_os_opts(keys) - self._test_options(opts, os_opts) - - def test_url_and_token_provided_on_command_line(self): - endpoint = 'http://alternate.com:8080/v1/AUTH_another' - token = 'alternate_auth_token' - os_opts = {'auth-token': token, - 'storage-url': endpoint} - opts = {'auth-version': '3'} - self._test_options(opts, os_opts, no_auth=True) - - opts = {'auth-version': '2.0'} - self._test_options(opts, os_opts, no_auth=True) - - def test_url_provided_on_command_line(self): - endpoint = 'http://alternate.com:8080/v1/AUTH_another' - os_opts = {'username': 'username', - 'password': 'password', - 'project-name': 'projectname', - 'auth-url': 'http://example.com:5000/v3', - 'storage-url': endpoint} - opts = {'auth-version': '3'} - self._test_options(opts, os_opts) - - opts = {'auth-version': '2.0'} - self._test_options(opts, os_opts) - - -@mock.patch.dict(os.environ, clean_os_environ) -class TestAuth(MockHttpTest): - - def test_pre_authed_request(self): - url = 'https://swift.storage.example.com/v1/AUTH_test' - token = 'AUTH_tk5b6b12' - - pre_auth_env = { - 'OS_STORAGE_URL': url, - 'OS_AUTH_TOKEN': token, - } - fake_conn = self.fake_http_connection(200) - with mock.patch('swiftclient.client.http_connection', new=fake_conn): - with mock.patch.dict(os.environ, pre_auth_env): - argv = ['', 'stat'] - swiftclient.shell.main(argv) - self.assertRequests([ - ('HEAD', url, '', {'x-auth-token': token}), - ]) - - # and again with re-auth - pre_auth_env.update(mocked_os_environ) - pre_auth_env['OS_AUTH_TOKEN'] = 'expired' - fake_conn = self.fake_http_connection(401, 200, 200, headers={ - 'x-auth-token': token + '_new', - 'x-storage-url': url + '_not_used', - }) - with mock.patch.multiple('swiftclient.client', - http_connection=fake_conn, - sleep=mock.DEFAULT): - with mock.patch.dict(os.environ, pre_auth_env): - argv = ['', 'stat'] - swiftclient.shell.main(argv) - self.assertRequests([ - ('HEAD', url, '', { - 'x-auth-token': 'expired', - }), - ('GET', mocked_os_environ['ST_AUTH'], '', { - 'x-auth-user': mocked_os_environ['ST_USER'], - 'x-auth-key': mocked_os_environ['ST_KEY'], - }), - ('HEAD', url, '', { - 'x-auth-token': token + '_new', - }), - ]) - - def test_os_pre_authed_request(self): - url = 'https://swift.storage.example.com/v1/AUTH_test' - token = 'AUTH_tk5b6b12' - - pre_auth_env = { - 'OS_STORAGE_URL': url, - 'OS_AUTH_TOKEN': token, - } - fake_conn = self.fake_http_connection(200) - with mock.patch('swiftclient.client.http_connection', new=fake_conn): - with mock.patch.dict(os.environ, pre_auth_env): - argv = ['', 'stat'] - swiftclient.shell.main(argv) - self.assertRequests([ - ('HEAD', url, '', {'x-auth-token': token}), - ]) - - # and again with re-auth - os_environ = { - 'OS_AUTH_URL': 'https://keystone.example.com/v2.0/', - 'OS_TENANT_NAME': 'demo', - 'OS_USERNAME': 'demo', - 'OS_PASSWORD': 'admin', - } - os_environ.update(pre_auth_env) - os_environ['OS_AUTH_TOKEN'] = 'expired' - - fake_conn = self.fake_http_connection(401, 200) - fake_keystone = fake_get_auth_keystone(storage_url=url + '_not_used', - token=token + '_new') - with mock.patch.multiple('swiftclient.client', - http_connection=fake_conn, - get_auth_keystone=fake_keystone, - sleep=mock.DEFAULT): - with mock.patch.dict(os.environ, os_environ): - argv = ['', 'stat'] - swiftclient.shell.main(argv) - self.assertRequests([ - ('HEAD', url, '', { - 'x-auth-token': 'expired', - }), - ('HEAD', url, '', { - 'x-auth-token': token + '_new', - }), - ]) - - def test_auth(self): - headers = { - 'x-auth-token': 'AUTH_tk5b6b12', - 'x-storage-url': 'https://swift.storage.example.com/v1/AUTH_test', - } - mock_resp = self.fake_http_connection(200, headers=headers) - with mock.patch('swiftclient.client.http_connection', new=mock_resp): - stdout = six.StringIO() - with mock.patch('sys.stdout', new=stdout): - argv = [ - '', - 'auth', - '--auth', 'https://swift.storage.example.com/auth/v1.0', - '--user', 'test:tester', '--key', 'testing', - ] - swiftclient.shell.main(argv) - - expected = """ - export OS_STORAGE_URL=https://swift.storage.example.com/v1/AUTH_test - export OS_AUTH_TOKEN=AUTH_tk5b6b12 - """ - self.assertEqual(textwrap.dedent(expected).lstrip(), - stdout.getvalue()) - - def test_auth_verbose(self): - with mock.patch('swiftclient.client.http_connection') as mock_conn: - stdout = six.StringIO() - with mock.patch('sys.stdout', new=stdout): - argv = [ - '', - 'auth', - '--auth', 'https://swift.storage.example.com/auth/v1.0', - '--user', 'test:tester', '--key', 'te$tin&', - '--verbose', - ] - swiftclient.shell.main(argv) - - expected = """ - export ST_AUTH=https://swift.storage.example.com/auth/v1.0 - export ST_USER=test:tester - export ST_KEY='te$tin&' - """ - self.assertEqual(textwrap.dedent(expected).lstrip(), - stdout.getvalue()) - self.assertEqual([], mock_conn.mock_calls) - - def test_auth_v2(self): - os_options = {'tenant_name': 'demo'} - with mock.patch('swiftclient.client.get_auth_keystone', - new=fake_get_auth_keystone(os_options)): - stdout = six.StringIO() - with mock.patch('sys.stdout', new=stdout): - argv = [ - '', - 'auth', '-V2', - '--auth', 'https://keystone.example.com/v2.0/', - '--os-tenant-name', 'demo', - '--os-username', 'demo', '--os-password', 'admin', - ] - swiftclient.shell.main(argv) - - expected = """ - export OS_STORAGE_URL=http://url/ - export OS_AUTH_TOKEN=token - """ - self.assertEqual(textwrap.dedent(expected).lstrip(), - stdout.getvalue()) - - def test_auth_verbose_v2(self): - with mock.patch('swiftclient.client.get_auth_keystone') \ - as mock_keystone: - stdout = six.StringIO() - with mock.patch('sys.stdout', new=stdout): - argv = [ - '', - 'auth', '-V2', - '--auth', 'https://keystone.example.com/v2.0/', - '--os-tenant-name', 'demo', - '--os-username', 'demo', '--os-password', '$eKr3t', - '--verbose', - ] - swiftclient.shell.main(argv) - - expected = """ - export OS_IDENTITY_API_VERSION=2.0 - export OS_AUTH_VERSION=2.0 - export OS_AUTH_URL=https://keystone.example.com/v2.0/ - export OS_PASSWORD='$eKr3t' - export OS_TENANT_NAME=demo - export OS_USERNAME=demo - """ - self.assertEqual(textwrap.dedent(expected).lstrip(), - stdout.getvalue()) - self.assertEqual([], mock_keystone.mock_calls) - - -class TestCrossAccountObjectAccess(TestBase, MockHttpTest): - """ - Tests to verify use of --os-storage-url will actually - result in the object request being sent despite account - read/write access and container write access being denied. - """ - def setUp(self): - super(TestCrossAccountObjectAccess, self).setUp() - self._remove_swift_env_vars() - temp_file = tempfile.NamedTemporaryFile(delete=False) - temp_file.file.write(b'01234567890123456789') - temp_file.file.flush() - self.obj = temp_file.name - self.url = 'http://alternate.com:8080/v1' - - # account tests will attempt to access - self.account = 'AUTH_alice' - - # keystone returns endpoint for another account - self.fake_ks = FakeKeystone( - endpoint='http://example.com:8080/v1/AUTH_bob', - token='bob_token') - - self.cont = 'c1' - self.cont_path = '/v1/%s/%s' % (self.account, self.cont) - self.obj_path = '%s%s' % (self.cont_path, self.obj) - - self.os_opts = {'username': 'bob', - 'password': 'password', - 'project-name': 'proj_bob', - 'auth-url': 'http://example.com:5000/v3', - 'storage-url': '%s/%s' % (self.url, self.account)} - self.opts = {'auth-version': '3'} - - def tearDown(self): - try: - os.remove(self.obj) - except OSError: - pass - self._replace_swift_env_vars() - super(TestCrossAccountObjectAccess, self).tearDown() - - def _make_cmd(self, cmd, cmd_args=None): - return _make_cmd(cmd, self.opts, self.os_opts, cmd_args=cmd_args) - - def _fake_cross_account_auth(self, read_ok, write_ok): - def on_request(method, path, *args, **kwargs): - """ - Modify response code to 200 if cross account permissions match. - """ - status = 403 - if (path.startswith('/v1/%s/%s' % (self.account, self.cont)) and - read_ok and method in ('GET', 'HEAD')): - status = 200 - elif (path.startswith('/v1/%s/%s%s' - % (self.account, self.cont, self.obj)) and - write_ok and method in ('PUT', 'POST', 'DELETE')): - status = 200 - return status - return on_request - - @mock.patch.object(swiftclient.service.SwiftService, - '_bulk_delete_page_size', lambda *a: 1) - @mock.patch('swiftclient.service.Connection') - def test_upload_bad_threads(self, mock_connection): - mock_connection.return_value.put_object.return_value = EMPTY_ETAG - mock_connection.return_value.attempts = 0 - - def check_bad(argv): - args, env = self._make_cmd( - 'upload', cmd_args=[self.cont, self.obj] + argv) - with mock.patch.dict(os.environ, env): - with CaptureOutput() as output: - self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertIn( - 'ERROR: option %s should be a positive integer.' % argv[0], - output.err) - - def check_good(argv): - args, env = self._make_cmd( - 'upload', - cmd_args=[self.cont, self.obj, '--leave-segments'] + argv) - with mock.patch.dict(os.environ, env): - with CaptureOutput() as output: - swiftclient.shell.main(args) - self.assertEqual('', output.err) - check_bad(["--object-threads", "-1"]) - check_bad(["--object-threads", "0"]) - check_bad(["--segment-threads", "-1"]) - check_bad(["--segment-threads", "0"]) - check_good(["--object-threads", "1"]) - check_good(["--segment-threads", "1"]) - - def test_upload_with_read_write_access(self): - req_handler = self._fake_cross_account_auth(True, True) - fake_conn = self.fake_http_connection(403, 403, - on_request=req_handler) - - args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, - '--leave-segments']) - with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): - with mock.patch('swiftclient.client.http_connection', fake_conn): - with mock.patch.dict(os.environ, env): - with CaptureOutput() as out: - try: - swiftclient.shell.main(args) - except SystemExit as e: - self.fail('Unexpected SystemExit: %s' % e) - - self.assertRequests([('PUT', self.cont_path), - ('PUT', self.obj_path)]) - self.assertEqual(self.obj[1:], out.strip()) - expected_err = "Warning: failed to create container '%s': 403 Fake" \ - % self.cont - self.assertEqual(expected_err, out.err.strip()) - - def test_upload_with_write_only_access(self): - req_handler = self._fake_cross_account_auth(False, True) - fake_conn = self.fake_http_connection(403, 403, - on_request=req_handler) - args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, - '--leave-segments']) - with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): - with mock.patch('swiftclient.client.http_connection', fake_conn): - with mock.patch.dict(os.environ, env): - with CaptureOutput() as out: - try: - swiftclient.shell.main(args) - except SystemExit as e: - self.fail('Unexpected SystemExit: %s' % e) - self.assertRequests([('PUT', self.cont_path), - ('PUT', self.obj_path)]) - self.assertEqual(self.obj[1:], out.strip()) - expected_err = "Warning: failed to create container '%s': 403 Fake" \ - % self.cont - self.assertEqual(expected_err, out.err.strip()) - - def test_segment_upload_with_write_only_access(self): - req_handler = self._fake_cross_account_auth(False, True) - fake_conn = self.fake_http_connection(403, 403, 403, 403, - on_request=req_handler) - - args, env = self._make_cmd('upload', - cmd_args=[self.cont, self.obj, - '--leave-segments', - '--segment-size=10', - '--segment-container=%s' - % self.cont]) - with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): - with mock.patch('swiftclient.client.http_connection', fake_conn): - with mock.patch.dict(os.environ, env): - with CaptureOutput() as out: - try: - swiftclient.shell.main(args) - except SystemExit as e: - self.fail('Unexpected SystemExit: %s' % e) - - segment_time = getmtime(self.obj) - segment_path_0 = '%s/%f/20/10/00000000' % (self.obj_path, segment_time) - segment_path_1 = '%s/%f/20/10/00000001' % (self.obj_path, segment_time) - # Note that the order of segment PUTs cannot be asserted, so test for - # existence in request log individually - self.assert_request(('PUT', self.cont_path)) - self.assert_request(('PUT', segment_path_0)) - self.assert_request(('PUT', segment_path_1)) - self.assert_request(('PUT', self.obj_path)) - self.assertIn(self.obj[1:], out.out) - expected_err = "Warning: failed to create container '%s': 403 Fake" \ - % self.cont - self.assertEqual(expected_err, out.err.strip()) - - def test_segment_upload_with_write_only_access_segments_container(self): - fake_conn = self.fake_http_connection( - 403, # PUT c1 - # HEAD c1 to get storage policy - StubResponse(200, headers={'X-Storage-Policy': 'foo'}), - 403, # PUT c1_segments - 201, # PUT c1_segments/...00 - 201, # PUT c1_segments/...01 - 201, # PUT c1/... - ) - - args, env = self._make_cmd('upload', - cmd_args=[self.cont, self.obj, - '--leave-segments', - '--segment-size=10']) - with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): - with mock.patch('swiftclient.client.http_connection', fake_conn): - with mock.patch.dict(os.environ, env): - with CaptureOutput() as out: - try: - swiftclient.shell.main(args) - except SystemExit as e: - self.fail('Unexpected SystemExit: %s' % e) - - segment_time = getmtime(self.obj) - segment_path_0 = '%s_segments%s/%f/20/10/00000000' % ( - self.cont_path, self.obj, segment_time) - segment_path_1 = '%s_segments%s/%f/20/10/00000001' % ( - self.cont_path, self.obj, segment_time) - # Note that the order of segment PUTs cannot be asserted, so test for - # existence in request log individually - self.assert_request(('PUT', self.cont_path)) - self.assert_request(('PUT', self.cont_path + '_segments', '', { - 'X-Auth-Token': 'bob_token', - 'X-Storage-Policy': 'foo', - 'Content-Length': '0', - })) - self.assert_request(('PUT', segment_path_0)) - self.assert_request(('PUT', segment_path_1)) - self.assert_request(('PUT', self.obj_path)) - self.assertIn(self.obj[1:], out.out) - expected_err = ("Warning: failed to create container '%s': 403 Fake\n" - "Warning: failed to create container '%s': 403 Fake" - ) % (self.cont, self.cont + '_segments') - self.assertEqual(expected_err, out.err.strip()) - - def test_upload_with_no_access(self): - fake_conn = self.fake_http_connection(403, 403) - - args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, - '--leave-segments']) - with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): - with mock.patch('swiftclient.client.http_connection', fake_conn): - with mock.patch.dict(os.environ, env): - with CaptureOutput() as out: - try: - swiftclient.shell.main(args) - self.fail('Expected SystemExit') - except SystemExit: - pass - - self.assertRequests([('PUT', self.cont_path), - ('PUT', self.obj_path)]) - expected_err = 'Object PUT failed: http://1.2.3.4%s 403 Fake' \ - % self.obj_path - self.assertIn(expected_err, out.err) - self.assertEqual('', out) - - @mock.patch.object(swiftclient.service.SwiftService, - '_bulk_delete_page_size', lambda *a: 1) - @mock.patch('swiftclient.service.Connection') - def test_download_bad_threads(self, mock_connection): - mock_connection.return_value.get_object.return_value = [{}, ''] - mock_connection.return_value.attempts = 0 - - def check_bad(argv): - args, env = self._make_cmd( - 'download', cmd_args=[self.cont, self.obj] + argv) - with mock.patch.dict(os.environ, env): - with CaptureOutput() as output: - self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertIn( - 'ERROR: option %s should be a positive integer.' % argv[0], - output.err) - - def check_good(argv): - args, env = self._make_cmd( - 'download', - cmd_args=[self.cont, self.obj, '--no-download'] + argv) - with mock.patch.dict(os.environ, env): - with CaptureOutput() as output: - swiftclient.shell.main(args) - self.assertEqual('', output.err) - check_bad(["--object-threads", "-1"]) - check_bad(["--object-threads", "0"]) - check_bad(["--container-threads", "-1"]) - check_bad(["--container-threads", "0"]) - check_good(["--object-threads", "1"]) - check_good(["--container-threads", "1"]) - - def test_download_with_read_write_access(self): - req_handler = self._fake_cross_account_auth(True, True) - fake_conn = self.fake_http_connection(403, on_request=req_handler, - etags=[EMPTY_ETAG]) - - args, env = self._make_cmd('download', cmd_args=[self.cont, - self.obj.lstrip('/'), - '--no-download']) - with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): - with mock.patch('swiftclient.client.http_connection', fake_conn): - with mock.patch.dict(os.environ, env): - with CaptureOutput() as out: - try: - swiftclient.shell.main(args) - except SystemExit as e: - self.fail('Unexpected SystemExit: %s' % e) - - self.assertRequests([('GET', self.obj_path)]) - self.assertTrue(out.out.startswith(self.obj.lstrip('/'))) - self.assertEqual('', out.err) - - def test_download_with_read_only_access(self): - req_handler = self._fake_cross_account_auth(True, False) - fake_conn = self.fake_http_connection(403, on_request=req_handler, - etags=[EMPTY_ETAG]) - - args, env = self._make_cmd('download', cmd_args=[self.cont, - self.obj.lstrip('/'), - '--no-download']) - with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): - with mock.patch('swiftclient.client.http_connection', fake_conn): - with mock.patch.dict(os.environ, env): - with CaptureOutput() as out: - try: - swiftclient.shell.main(args) - except SystemExit as e: - self.fail('Unexpected SystemExit: %s' % e) - - self.assertRequests([('GET', self.obj_path)]) - self.assertTrue(out.out.startswith(self.obj.lstrip('/'))) - self.assertEqual('', out.err) - - def test_download_with_no_access(self): - fake_conn = self.fake_http_connection(403) - args, env = self._make_cmd('download', cmd_args=[self.cont, - self.obj.lstrip('/'), - '--no-download']) - with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): - with mock.patch('swiftclient.client.http_connection', fake_conn): - with mock.patch.dict(os.environ, env): - with CaptureOutput() as out: - try: - swiftclient.shell.main(args) - self.fail('Expected SystemExit') - except SystemExit: - pass - - self.assertRequests([('GET', self.obj_path)]) - path = '%s%s' % (self.cont, self.obj) - expected_err = "Error downloading object '%s'" % path - self.assertTrue(out.err.startswith(expected_err)) - self.assertEqual('', out) - - def test_list_with_read_access(self): - req_handler = self._fake_cross_account_auth(True, False) - resp_body = b'{}' - resp = StubResponse(403, resp_body, { - 'etag': hashlib.md5(resp_body).hexdigest()}) - fake_conn = self.fake_http_connection(resp, on_request=req_handler) - - args, env = self._make_cmd('download', cmd_args=[self.cont]) - with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): - with mock.patch('swiftclient.client.http_connection', fake_conn): - with mock.patch.dict(os.environ, env): - with CaptureOutput() as out: - try: - swiftclient.shell.main(args) - except SystemExit as e: - self.fail('Unexpected SystemExit: %s' % e) - - self.assertRequests([('GET', '%s?format=json' % self.cont_path)]) - self.assertEqual('', out) - self.assertEqual('', out.err) - - def test_list_with_no_access(self): - fake_conn = self.fake_http_connection(403) - - args, env = self._make_cmd('download', cmd_args=[self.cont]) - with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): - with mock.patch('swiftclient.client.http_connection', fake_conn): - with mock.patch.dict(os.environ, env): - with CaptureOutput() as out: - try: - swiftclient.shell.main(args) - self.fail('Expected SystemExit') - except SystemExit: - pass - - self.assertRequests([('GET', '%s?format=json' % self.cont_path)]) - self.assertEqual('', out) - self.assertTrue(out.err.startswith('Container GET failed:')) - - -class TestCrossAccountObjectAccessUsingEnv(TestCrossAccountObjectAccess): - """ - Repeat super-class tests using environment variables rather than command - line to set options. - """ - - def _make_cmd(self, cmd, cmd_args=None): - return _make_cmd(cmd, self.opts, self.os_opts, cmd_args=cmd_args, - use_env=True) diff -Nru python-swiftclient-3.8.1/tests/unit/test_swiftclient.py python-swiftclient-3.9.0/tests/unit/test_swiftclient.py --- python-swiftclient-3.8.1/tests/unit/test_swiftclient.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/tests/unit/test_swiftclient.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,3328 +0,0 @@ -# Copyright (c) 2010-2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import gzip -import json -import logging -import mock -import six -import socket -import string -import unittest -import warnings -import tempfile -from hashlib import md5 -from six import binary_type -from six.moves.urllib.parse import urlparse -from requests.exceptions import RequestException - -from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, - FakeKeystone) - -from swiftclient.utils import EMPTY_ETAG -from swiftclient.exceptions import ClientException -from swiftclient import client as c -import swiftclient.utils -import swiftclient - - -class TestClientException(unittest.TestCase): - - def test_is_exception(self): - self.assertTrue(issubclass(c.ClientException, Exception)) - - def test_format(self): - exc = c.ClientException('something failed') - self.assertIn('something failed', str(exc)) - test_kwargs = ( - 'scheme', - 'host', - 'port', - 'path', - 'query', - 'status', - 'reason', - 'device', - 'response_content', - ) - for value in test_kwargs: - kwargs = { - 'http_%s' % value: value, - } - exc = c.ClientException('test', **kwargs) - self.assertIn(value, str(exc)) - - def test_attrs(self): - test_kwargs = ( - 'scheme', - 'host', - 'port', - 'path', - 'query', - 'status', - 'reason', - 'device', - 'response_content', - 'response_headers', - ) - for value in test_kwargs: - key = 'http_%s' % value - kwargs = {key: value} - exc = c.ClientException('test', **kwargs) - self.assertIs(True, hasattr(exc, key)) - self.assertEqual(getattr(exc, key), value) - - -class MockHttpResponse(object): - def __init__(self, status=0, headers=None, verify=False): - self.status = status - self.status_code = status - self.reason = "OK" - self.buffer = [] - self.requests_params = None - self.verify = verify - self.md5sum = md5() - self.headers = {'etag': '"%s"' % EMPTY_ETAG} - if headers: - self.headers.update(headers) - self.closed = False - - class Raw(object): - def __init__(self, headers): - self.headers = headers - - def read(self, **kw): - return "" - - def getheader(self, name, default): - return self.headers.get(name, default) - - self.raw = Raw(headers) - - def read(self): - return "" - - def close(self): - self.closed = True - - def getheader(self, name, default): - return self.headers.get(name, default) - - def getheaders(self): - return dict(self.headers).items() - - def fake_response(self): - return self - - def _fake_request(self, *arg, **kwarg): - self.status = 200 - self.requests_params = kwarg - if self.verify: - for chunk in kwarg['data']: - self.md5sum.update(chunk) - - # This simulate previous httplib implementation that would do a - # putrequest() and then use putheader() to send header. - for k, v in kwarg['headers'].items(): - self.buffer.append((k, v)) - return self.fake_response() - - -class TestHttpHelpers(MockHttpTest): - - def test_quote(self): - value = b'bytes\xff' - self.assertEqual('bytes%FF', c.quote(value)) - value = 'native string' - self.assertEqual('native%20string', c.quote(value)) - value = u'unicode string' - self.assertEqual('unicode%20string', c.quote(value)) - value = u'unicode:\xe9\u20ac' - self.assertEqual('unicode%3A%C3%A9%E2%82%AC', c.quote(value)) - - def test_parse_header_string(self): - value = b'bytes' - self.assertEqual(u'bytes', c.parse_header_string(value)) - value = u'unicode:\xe9\u20ac' - self.assertEqual(u'unicode:\xe9\u20ac', c.parse_header_string(value)) - value = 'native%20string' - self.assertEqual(u'native string', c.parse_header_string(value)) - - value = b'encoded%20bytes%E2%82%AC' - self.assertEqual(u'encoded bytes\u20ac', c.parse_header_string(value)) - value = 'encoded%20unicode%E2%82%AC' - self.assertEqual(u'encoded unicode\u20ac', - c.parse_header_string(value)) - - value = b'bad%20bytes%ff%E2%82%AC' - self.assertEqual(u'bad%20bytes%ff%E2%82%AC', - c.parse_header_string(value)) - value = u'bad%20unicode%ff\u20ac' - self.assertEqual(u'bad%20unicode%ff\u20ac', - c.parse_header_string(value)) - - value = b'really%20bad\xffbytes' - self.assertEqual(u'really%2520bad%FFbytes', - c.parse_header_string(value)) - - def test_http_connection(self): - url = 'http://www.test.com' - _junk, conn = c.http_connection(url) - self.assertIs(type(conn), c.HTTPConnection) - url = 'https://www.test.com' - _junk, conn = c.http_connection(url) - self.assertIs(type(conn), c.HTTPConnection) - url = 'ftp://www.test.com' - self.assertRaises(c.ClientException, c.http_connection, url) - - def test_encode_meta_headers(self): - headers = {'abc': '123', - u'x-container-meta-\u0394': 123, - u'x-account-meta-\u0394': 12.3, - u'x-object-meta-\u0394': True} - - r = swiftclient.encode_meta_headers(headers) - - self.assertEqual(len(headers), len(r)) - # ensure non meta headers are not encoded - self.assertIs(type(r.get('abc')), binary_type) - del r['abc'] - - for k, v in r.items(): - self.assertIs(type(k), binary_type) - self.assertIs(type(v), binary_type) - self.assertIn(v, (b'123', b'12.3', b'True')) - - def test_set_user_agent_default(self): - _junk, conn = c.http_connection('http://www.example.com') - req_headers = {} - - def my_request_handler(*a, **kw): - req_headers.update(kw.get('headers', {})) - conn._request = my_request_handler - - # test the default - conn.request('GET', '/') - ua = req_headers.get('user-agent', 'XXX-MISSING-XXX') - self.assertTrue(ua.startswith('python-swiftclient-')) - - def test_set_user_agent_per_request_override(self): - _junk, conn = c.http_connection('http://www.example.com') - req_headers = {} - - def my_request_handler(*a, **kw): - req_headers.update(kw.get('headers', {})) - conn._request = my_request_handler - - # test if it's actually set - conn.request('GET', '/', headers={'User-Agent': 'Me'}) - ua = req_headers.get('user-agent', 'XXX-MISSING-XXX') - self.assertEqual(ua, b'Me', req_headers) - - def test_set_user_agent_default_override(self): - _junk, conn = c.http_connection( - 'http://www.example.com', - default_user_agent='a-new-default') - req_headers = {} - - def my_request_handler(*a, **kw): - req_headers.update(kw.get('headers', {})) - conn._request = my_request_handler - - # test setting a default - conn._request = my_request_handler - conn.request('GET', '/') - ua = req_headers.get('user-agent', 'XXX-MISSING-XXX') - self.assertEqual(ua, 'a-new-default') - - -class TestGetAuth(MockHttpTest): - - def test_ok(self): - c.http_connection = self.fake_http_connection(200) - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf') - self.assertIsNone(url) - self.assertIsNone(token) - - def test_invalid_auth(self): - self.assertRaises(c.ClientException, c.get_auth, - 'http://www.tests.com', 'asdf', 'asdf', - auth_version="foo") - - def test_auth_v1(self): - c.http_connection = self.fake_http_connection(200, auth_v1=True) - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - auth_version="1.0") - self.assertEqual(url, 'storageURL') - self.assertEqual(token, 'someauthtoken') - - def test_auth_v1_insecure(self): - c.http_connection = self.fake_http_connection(200, 200, auth_v1=True) - url, token = c.get_auth('http://www.test.com/invalid_cert', - 'asdf', 'asdf', - auth_version='1.0', - insecure=True) - self.assertEqual(url, 'storageURL') - self.assertEqual(token, 'someauthtoken') - - with self.assertRaises(c.ClientException) as exc_context: - c.get_auth('http://www.test.com/invalid_cert', - 'asdf', 'asdf', auth_version='1.0') - # TODO: this test is really on validating the mock and not the - # the full plumbing into the requests's 'verify' option - self.assertIn('invalid_certificate', str(exc_context.exception)) - - def test_auth_v1_timeout(self): - # this test has some overlap with - # TestConnection.test_timeout_passed_down but is required to check that - # get_auth does the right thing when it is not passed a timeout arg - orig_http_connection = c.http_connection - timeouts = [] - - def fake_request_handler(*a, **kw): - if 'timeout' in kw: - timeouts.append(kw['timeout']) - else: - timeouts.append(None) - return MockHttpResponse( - status=200, - headers={ - 'x-auth-token': 'a_token', - 'x-storage-url': 'http://files.example.com/v1/AUTH_user'}) - - def fake_connection(*a, **kw): - url, conn = orig_http_connection(*a, **kw) - conn._request = fake_request_handler - return url, conn - - with mock.patch('swiftclient.client.http_connection', fake_connection): - c.get_auth('http://www.test.com', 'asdf', 'asdf', - auth_version="1.0", timeout=42.0) - c.get_auth('http://www.test.com', 'asdf', 'asdf', - auth_version="1.0", timeout=None) - c.get_auth('http://www.test.com', 'asdf', 'asdf', - auth_version="1.0") - - self.assertEqual(timeouts, [42.0, None, None]) - - def test_auth_v2_timeout(self): - # this test has some overlap with - # TestConnection.test_timeout_passed_down but is required to check that - # get_auth does the right thing when it is not passed a timeout arg - fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client.ksclient_v2', fake_ks): - c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=dict(tenant_name='tenant'), - auth_version="2.0", timeout=42.0) - c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=dict(tenant_name='tenant'), - auth_version="2.0", timeout=None) - c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=dict(tenant_name='tenant'), - auth_version="2.0") - self.assertEqual(3, len(fake_ks.calls)) - timeouts = [call['timeout'] for call in fake_ks.calls] - self.assertEqual([42.0, None, None], timeouts) - - def test_auth_v2_with_tenant_name(self): - os_options = {'tenant_name': 'asdf'} - req_args = {'auth_version': '2.0'} - ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) - with mock.patch('swiftclient.client.get_auth_keystone', ks): - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - def test_auth_v2_with_tenant_id(self): - os_options = {'tenant_id': 'asdf'} - req_args = {'auth_version': '2.0'} - ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) - with mock.patch('swiftclient.client.get_auth_keystone', ks): - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - def test_auth_v2_with_project_name(self): - os_options = {'project_name': 'asdf'} - req_args = {'auth_version': '2.0'} - ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) - with mock.patch('swiftclient.client.get_auth_keystone', ks): - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - def test_auth_v2_with_project_id(self): - os_options = {'project_id': 'asdf'} - req_args = {'auth_version': '2.0'} - - ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) - with mock.patch('swiftclient.client.get_auth_keystone', ks): - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - def test_auth_v2_no_tenant_name_or_tenant_id(self): - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone({})): - self.assertRaises(c.ClientException, c.get_auth, - 'http://www.tests.com', 'asdf', 'asdf', - os_options={}, - auth_version='2.0') - - def test_auth_v2_with_tenant_name_none_and_tenant_id_none(self): - os_options = {'tenant_name': None, - 'tenant_id': None} - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone(os_options)): - self.assertRaises(c.ClientException, c.get_auth, - 'http://www.tests.com', 'asdf', 'asdf', - os_options=os_options, - auth_version='2.0') - - def test_auth_v2_with_tenant_user_in_user(self): - tenant_option = {'tenant_name': 'foo'} - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone(tenant_option)): - url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf', - os_options={}, - auth_version="2.0") - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - def test_auth_v2_tenant_name_no_os_options(self): - tenant_option = {'tenant_name': 'asdf'} - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone(tenant_option)): - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - tenant_name='asdf', - os_options={}, - auth_version="2.0") - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - def test_auth_v2_with_os_options(self): - os_options = {'service_type': 'object-store', - 'endpoint_type': 'internalURL', - 'tenant_name': 'asdf'} - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone(os_options)): - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - def test_auth_v2_with_tenant_user_in_user_no_os_options(self): - tenant_option = {'tenant_name': 'foo'} - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone(tenant_option)): - url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf', - auth_version="2.0") - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - def test_auth_v2_with_os_region_name(self): - os_options = {'region_name': 'good-region', - 'tenant_name': 'asdf'} - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone(os_options)): - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - def test_auth_v2_no_endpoint(self): - os_options = {'region_name': 'unknown_region', - 'tenant_name': 'asdf'} - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone(os_options, c.ClientException)): - self.assertRaises(c.ClientException, c.get_auth, - 'http://www.tests.com', 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') - - def test_auth_v2_ks_exception(self): - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone({}, c.ClientException)): - self.assertRaises(c.ClientException, c.get_auth, - 'http://www.tests.com', 'asdf', 'asdf', - os_options={}, - auth_version='2.0') - - def test_auth_v2_cacert(self): - os_options = {'tenant_name': 'foo'} - auth_url_secure = 'https://www.tests.com' - auth_url_insecure = 'https://www.tests.com/self-signed-certificate' - - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone(os_options, None)): - url, token = c.get_auth(auth_url_secure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - insecure=False) - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - url, token = c.get_auth(auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - cacert='ca.pem', insecure=False) - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - self.assertRaises(c.ClientException, c.get_auth, - auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') - self.assertRaises(c.ClientException, c.get_auth, - auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - insecure=False) - - def test_auth_v2_insecure(self): - os_options = {'tenant_name': 'foo'} - auth_url_secure = 'https://www.tests.com' - auth_url_insecure = 'https://www.tests.com/invalid-certificate' - - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone(os_options, None)): - url, token = c.get_auth(auth_url_secure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - url, token = c.get_auth(auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - insecure=True) - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - self.assertRaises(c.ClientException, c.get_auth, - auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') - self.assertRaises(c.ClientException, c.get_auth, - auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - insecure=False) - - def test_auth_v2_cert(self): - os_options = {'tenant_name': 'foo'} - auth_url_no_sslauth = 'https://www.tests.com' - auth_url_sslauth = 'https://www.tests.com/client-certificate' - - with mock.patch('swiftclient.client.get_auth_keystone', - fake_get_auth_keystone(os_options, None)): - url, token = c.get_auth(auth_url_no_sslauth, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - url, token = c.get_auth(auth_url_sslauth, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - cert='minnie', cert_key='mickey') - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - self.assertRaises(c.ClientException, c.get_auth, - auth_url_sslauth, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') - self.assertRaises(c.ClientException, c.get_auth, - auth_url_sslauth, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - cert='minnie') - - def test_auth_v3_with_tenant_name(self): - # check the correct auth version is passed to get_auth_keystone - os_options = {'tenant_name': 'asdf'} - req_args = {'auth_version': '3'} - - ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) - with mock.patch('swiftclient.client.get_auth_keystone', ks): - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="3") - - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - def test_get_keystone_client_2_0(self): - # check the correct auth version is passed to get_auth_keystone - os_options = {'tenant_name': 'asdf'} - req_args = {'auth_version': '2.0'} - - ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) - with mock.patch('swiftclient.client.get_auth_keystone', ks): - url, token = c.get_keystoneclient_2_0('http://www.test.com', - 'asdf', 'asdf', - os_options=os_options) - - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - def test_get_auth_keystone_versionless(self): - fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - - with mock.patch('swiftclient.client.ksclient_v3', fake_ks): - c.get_auth_keystone('http://authurl', 'user', 'key', {}) - self.assertEqual(1, len(fake_ks.calls)) - self.assertEqual('http://authurl/v3', fake_ks.calls[0].get('auth_url')) - - def test_get_auth_keystone_versionless_auth_version_set(self): - fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - - with mock.patch('swiftclient.client.ksclient_v2', fake_ks): - c.get_auth_keystone('http://auth_url', 'user', 'key', - {}, auth_version='2.0') - self.assertEqual(1, len(fake_ks.calls)) - self.assertEqual('http://auth_url/v2.0', - fake_ks.calls[0].get('auth_url')) - - def test_get_auth_keystone_versionful(self): - fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - - with mock.patch('swiftclient.client.ksclient_v3', fake_ks): - c.get_auth_keystone('http://auth_url/v3', 'user', 'key', - {}, auth_version='3') - self.assertEqual(1, len(fake_ks.calls)) - self.assertEqual('http://auth_url/v3', - fake_ks.calls[0].get('auth_url')) - - def test_get_auth_keystone_devstack_versionful(self): - fake_ks = FakeKeystone( - endpoint='http://storage.example.com/v1/AUTH_user', token='secret') - with mock.patch('swiftclient.client.ksclient_v3', fake_ks): - c.get_auth_keystone('https://192.168.8.8/identity/v3', - 'user', 'key', {}, auth_version='3') - self.assertEqual(1, len(fake_ks.calls)) - self.assertEqual('https://192.168.8.8/identity/v3', - fake_ks.calls[0].get('auth_url')) - - def test_get_auth_keystone_devstack_versionless(self): - fake_ks = FakeKeystone( - endpoint='http://storage.example.com/v1/AUTH_user', token='secret') - with mock.patch('swiftclient.client.ksclient_v3', fake_ks): - c.get_auth_keystone('https://192.168.8.8/identity', - 'user', 'key', {}, auth_version='3') - self.assertEqual(1, len(fake_ks.calls)) - self.assertEqual('https://192.168.8.8/identity/v3', - fake_ks.calls[0].get('auth_url')) - - def test_auth_keystone_url_some_junk_nonsense(self): - fake_ks = FakeKeystone( - endpoint='http://storage.example.com/v1/AUTH_user', - token='secret') - with mock.patch('swiftclient.client.ksclient_v3', fake_ks): - c.get_auth_keystone('http://blah.example.com/v2moo', - 'user', 'key', {}, auth_version='3') - self.assertEqual(1, len(fake_ks.calls)) - # v2 looks sorta version-y, but it's not an exact match, so this is - # probably about just as bad as anything else we might guess at - self.assertEqual('http://blah.example.com/v2moo/v3', - fake_ks.calls[0].get('auth_url')) - - def test_auth_with_session(self): - mock_session = mock.MagicMock() - mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' - mock_session.get_token.return_value = 'token' - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - session=mock_session) - self.assertEqual(url, 'http://storagehost/v1/acct') - self.assertTrue(token) - - -class TestGetAccount(MockHttpTest): - - def test_no_content(self): - c.http_connection = self.fake_http_connection(204) - value = c.get_account('http://www.test.com/v1/acct', 'asdf')[1] - self.assertEqual(value, []) - self.assertRequests([ - ('GET', '/v1/acct?format=json', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'asdf'}), - ]) - - def test_param_marker(self): - c.http_connection = self.fake_http_connection( - 204, - query_string="format=json&marker=marker") - c.get_account('http://www.test.com/v1/acct', 'asdf', marker='marker') - self.assertRequests([ - ('GET', '/v1/acct?format=json&marker=marker', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'asdf'}), - ]) - - def test_param_limit(self): - c.http_connection = self.fake_http_connection( - 204, - query_string="format=json&limit=10") - c.get_account('http://www.test.com/v1/acct', 'asdf', limit=10) - self.assertRequests([ - ('GET', '/v1/acct?format=json&limit=10', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'asdf'}), - ]) - - def test_param_prefix(self): - c.http_connection = self.fake_http_connection( - 204, - query_string="format=json&prefix=asdf/") - c.get_account('http://www.test.com/v1/acct', 'asdf', prefix='asdf/') - self.assertRequests([ - ('GET', '/v1/acct?format=json&prefix=asdf/', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'asdf'}), - ]) - - def test_param_end_marker(self): - c.http_connection = self.fake_http_connection( - 204, - query_string="format=json&end_marker=end_marker") - c.get_account('http://www.test.com/v1/acct', 'asdf', - end_marker='end_marker') - self.assertRequests([ - ('GET', '/v1/acct?format=json&end_marker=end_marker', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'asdf'}), - ]) - - def test_param_delimiter(self): - c.http_connection = self.fake_http_connection( - 204, - query_string="format=json&delimiter=-") - c.get_account('http://www.test.com/v1/acct', 'asdf', - delimiter='-') - self.assertRequests([ - ('GET', '/v1/acct?format=json&delimiter=-', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'asdf'}), - ]) - - -class TestHeadAccount(MockHttpTest): - - def test_ok(self): - c.http_connection = self.fake_http_connection(200, headers={ - 'x-account-meta-color': 'blue', - }) - resp_headers = c.head_account('http://www.tests.com', 'asdf') - self.assertEqual(resp_headers['x-account-meta-color'], 'blue') - self.assertRequests([ - ('HEAD', 'http://www.tests.com', '', {'x-auth-token': 'asdf'}) - ]) - - def test_server_error(self): - body = 'c' * 65 - headers = {'foo': 'bar'} - c.http_connection = self.fake_http_connection( - StubResponse(500, body, headers)) - with self.assertRaises(c.ClientException) as exc_context: - c.head_account('http://www.tests.com', 'asdf') - e = exc_context.exception - self.assertEqual(e.http_response_content, body) - self.assertEqual(e.http_status, 500) - self.assertRequests([ - ('HEAD', 'http://www.tests.com', '', {'x-auth-token': 'asdf'}) - ]) - # TODO: this is a fairly brittle test of the __repr__ on the - # ClientException which should probably be in a targeted test - new_body = "[first 60 chars of response] " + body[0:60] - self.assertEqual(e.__str__()[-89:], new_body) - - -class TestPostAccount(MockHttpTest): - - def test_ok(self): - c.http_connection = self.fake_http_connection(200, headers={ - 'X-Account-Meta-Color': 'blue', - }, body='foo') - headers = {'x-account-meta-shape': 'square'} - resp_headers, body = c.post_account( - 'http://www.tests.com/path/to/account', 'asdf', - headers, query_string='bar=baz', - data='some data') - self.assertEqual('blue', resp_headers.get('x-account-meta-color')) - self.assertEqual('foo', body) - self.assertRequests([ - ('POST', 'http://www.tests.com/path/to/account?bar=baz', - 'some data', {'x-auth-token': 'asdf', - 'x-account-meta-shape': 'square'}) - ]) - # Check that we didn't mutate the request ehader dict - self.assertEqual(headers, {'x-account-meta-shape': 'square'}) - - def test_server_error(self): - body = 'c' * 65 - c.http_connection = self.fake_http_connection(500, body=body) - with self.assertRaises(c.ClientException) as exc_mgr: - c.post_account('http://www.tests.com', 'asdf', {}) - self.assertEqual(exc_mgr.exception.http_response_content, body) - self.assertEqual(exc_mgr.exception.http_status, 500) - self.assertRequests([ - ('POST', 'http://www.tests.com', None, {'x-auth-token': 'asdf'}) - ]) - # TODO: this is a fairly brittle test of the __repr__ on the - # ClientException which should probably be in a targeted test - new_body = "[first 60 chars of response] " + body[0:60] - self.assertEqual(exc_mgr.exception.__str__()[-89:], new_body) - - -class TestGetContainer(MockHttpTest): - - def test_no_content(self): - c.http_connection = self.fake_http_connection(204) - value = c.get_container('http://www.test.com/v1/acct', 'token', - 'container')[1] - self.assertEqual(value, []) - self.assertRequests([ - ('GET', '/v1/acct/container?format=json', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'token'}), - ]) - - def test_param_marker(self): - c.http_connection = self.fake_http_connection( - 204, - query_string="format=json&marker=marker") - c.get_container('http://www.test.com/v1/acct', 'token', 'container', - marker='marker') - self.assertRequests([ - ('GET', '/v1/acct/container?format=json&marker=marker', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'token'}), - ]) - - def test_param_limit(self): - c.http_connection = self.fake_http_connection( - 204, - query_string="format=json&limit=10") - c.get_container('http://www.test.com/v1/acct', 'token', 'container', - limit=10) - self.assertRequests([ - ('GET', '/v1/acct/container?format=json&limit=10', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'token'}), - ]) - - def test_param_prefix(self): - c.http_connection = self.fake_http_connection( - 204, - query_string="format=json&prefix=asdf/") - c.get_container('http://www.test.com/v1/acct', 'token', 'container', - prefix='asdf/') - self.assertRequests([ - ('GET', '/v1/acct/container?format=json&prefix=asdf/', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'token'}), - ]) - - def test_param_delimiter(self): - c.http_connection = self.fake_http_connection( - 204, - query_string="format=json&delimiter=/") - c.get_container('http://www.test.com/v1/acct', 'token', 'container', - delimiter='/') - self.assertRequests([ - ('GET', '/v1/acct/container?format=json&delimiter=/', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'token'}), - ]) - - def test_param_end_marker(self): - c.http_connection = self.fake_http_connection( - 204, - query_string="format=json&end_marker=end_marker") - c.get_container('http://www.test.com/v1/acct', 'token', 'container', - end_marker='end_marker') - self.assertRequests([ - ('GET', '/v1/acct/container?format=json&end_marker=end_marker', - '', {'x-auth-token': 'token', 'accept-encoding': 'gzip'}), - ]) - - def test_param_path(self): - c.http_connection = self.fake_http_connection( - 204, - query_string="format=json&path=asdf") - c.get_container('http://www.test.com/v1/acct', 'token', 'container', - path='asdf') - self.assertRequests([ - ('GET', '/v1/acct/container?format=json&path=asdf', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'token'}), - ]) - - def test_request_headers(self): - c.http_connection = self.fake_http_connection( - 204, query_string="format=json") - conn = c.http_connection('http://www.test.com') - headers = {'x-client-key': 'client key'} - c.get_container('url_is_irrelevant', 'TOKEN', 'container', - http_conn=conn, headers=headers) - self.assertRequests([ - ('GET', '/container?format=json', '', { - 'x-auth-token': 'TOKEN', - 'x-client-key': 'client key', - 'accept-encoding': 'gzip', - }), - ]) - - def test_query_string(self): - c.http_connection = self.fake_http_connection( - 200, query_string="format=json&hello=20", body=b'[]') - c.get_container('http://www.test.com', 'asdf', 'asdf', - query_string="hello=20") - self.assertRequests([ - ('GET', '/asdf?format=json&hello=20', '', { - 'accept-encoding': 'gzip', - 'x-auth-token': 'asdf'}), - ]) - - -class TestHeadContainer(MockHttpTest): - - def test_head_ok(self): - fake_conn = self.fake_http_connection( - 200, headers={'x-container-meta-color': 'blue'}) - with mock.patch('swiftclient.client.http_connection', - new=fake_conn): - resp = c.head_container('https://example.com/v1/AUTH_test', - 'token', 'container') - self.assertEqual(resp['x-container-meta-color'], 'blue') - self.assertRequests([ - ('HEAD', 'https://example.com/v1/AUTH_test/container', '', - {'x-auth-token': 'token'}), - ]) - - def test_server_error(self): - body = 'c' * 60 - headers = {'foo': 'bar'} - c.http_connection = self.fake_http_connection( - StubResponse(500, body, headers)) - with self.assertRaises(c.ClientException) as exc_context: - c.head_container('http://www.test.com', 'asdf', 'container') - e = exc_context.exception - self.assertRequests([ - ('HEAD', '/container', '', {'x-auth-token': 'asdf'}), - ]) - self.assertEqual(e.http_status, 500) - self.assertEqual(e.http_response_content, body) - self.assertEqual(e.http_response_headers, headers) - - -class TestPutContainer(MockHttpTest): - - def test_ok(self): - c.http_connection = self.fake_http_connection(200) - value = c.put_container('http://www.test.com', 'token', 'container') - self.assertIsNone(value) - self.assertRequests([ - ('PUT', '/container', '', { - 'x-auth-token': 'token', - 'content-length': '0'}), - ]) - - def test_server_error(self): - body = 'c' * 60 - headers = {'foo': 'bar'} - c.http_connection = self.fake_http_connection( - StubResponse(500, body, headers)) - with self.assertRaises(c.ClientException) as exc_context: - c.put_container('http://www.test.com', 'token', 'container') - self.assertEqual(exc_context.exception.http_response_content, body) - self.assertEqual(exc_context.exception.http_response_headers, headers) - self.assertRequests([ - ('PUT', '/container', '', { - 'x-auth-token': 'token', - 'content-length': '0'}), - ]) - - def test_query_string(self): - c.http_connection = self.fake_http_connection(200, - query_string="hello=20") - c.put_container('http://www.test.com', 'asdf', 'asdf', - query_string="hello=20") - for req in self.iter_request_log(): - self.assertEqual(req['method'], 'PUT') - self.assertEqual(req['parsed_path'].path, '/asdf') - self.assertEqual(req['parsed_path'].query, 'hello=20') - self.assertEqual(req['headers']['x-auth-token'], 'asdf') - - -class TestDeleteContainer(MockHttpTest): - - def test_ok(self): - c.http_connection = self.fake_http_connection(200) - value = c.delete_container('http://www.test.com', 'token', 'container') - self.assertIsNone(value) - self.assertRequests([ - ('DELETE', '/container', '', { - 'x-auth-token': 'token'}), - ]) - - def test_query_string(self): - c.http_connection = self.fake_http_connection(200, - query_string="hello=20") - c.delete_container('http://www.test.com', 'token', 'container', - query_string="hello=20") - self.assertRequests([ - ('DELETE', 'http://www.test.com/container?hello=20', '', { - 'x-auth-token': 'token'}) - ]) - - -class TestGetObject(MockHttpTest): - - def test_server_error(self): - body = 'c' * 60 - headers = {'foo': 'bar'} - c.http_connection = self.fake_http_connection( - StubResponse(500, body, headers)) - with self.assertRaises(c.ClientException) as exc_context: - c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') - self.assertEqual(exc_context.exception.http_response_content, body) - self.assertEqual(exc_context.exception.http_response_headers, headers) - - def test_query_string(self): - c.http_connection = self.fake_http_connection(200, - query_string="hello=20") - c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf', - query_string="hello=20") - self.assertRequests([ - ('GET', '/asdf/asdf?hello=20', '', { - 'x-auth-token': 'asdf'}), - ]) - - def test_get_object_as_string(self): - c.http_connection = self.fake_http_connection(200, body='abcde') - __, resp = c.get_object('http://storage.example.com', 'TOKEN', - 'container_name', 'object_name') - self.assertEqual(resp, 'abcde') - - def test_request_headers(self): - c.http_connection = self.fake_http_connection(200) - conn = c.http_connection('http://www.test.com') - headers = {'Range': 'bytes=1-2'} - c.get_object('url_is_irrelevant', 'TOKEN', 'container', 'object', - http_conn=conn, headers=headers) - self.assertRequests([ - ('GET', '/container/object', '', { - 'x-auth-token': 'TOKEN', - 'range': 'bytes=1-2', - }), - ]) - - def test_response_headers(self): - c.http_connection = self.fake_http_connection( - 200, headers={'X-Utf-8-Header': b't%c3%a9st', - 'X-Non-Utf-8-Header': b'%ff', - 'X-Binary-Header': b'\xff'}) - conn = c.http_connection('http://www.test.com') - headers, data = c.get_object('url_is_irrelevant', 'TOKEN', - 'container', 'object', http_conn=conn) - self.assertEqual(u't\xe9st', headers.get('x-utf-8-header', '')) - self.assertEqual(u'%ff', headers.get('x-non-utf-8-header', '')) - self.assertEqual(u'%FF', headers.get('x-binary-header', '')) - - def test_chunk_size_read_method(self): - conn = c.Connection('http://auth.url/', 'some_user', 'some_key') - with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: - mock_get_auth.return_value = ('http://auth.url/', 'tToken') - c.http_connection = self.fake_http_connection(200, body='abcde') - __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=3) - self.assertTrue(hasattr(resp, 'read')) - self.assertEqual(resp.read(3), 'abc') - self.assertEqual(resp.read(None), 'de') - self.assertEqual(resp.read(), '') - - def test_chunk_size_iter(self): - conn = c.Connection('http://auth.url/', 'some_user', 'some_key') - with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: - mock_get_auth.return_value = ('http://auth.url/', 'tToken') - c.http_connection = self.fake_http_connection(200, body='abcde') - __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=3) - self.assertTrue(hasattr(resp, 'next')) - self.assertEqual(next(resp), 'abc') - self.assertEqual(next(resp), 'de') - self.assertRaises(StopIteration, next, resp) - - def test_chunk_size_read_and_iter(self): - conn = c.Connection('http://auth.url/', 'some_user', 'some_key') - with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: - mock_get_auth.return_value = ('http://auth.url/', 'tToken') - c.http_connection = self.fake_http_connection(200, body='abcdef') - __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) - self.assertTrue(hasattr(resp, 'read')) - self.assertEqual(resp.read(3), 'abc') - self.assertEqual(next(resp), 'de') - self.assertEqual(resp.read(), 'f') - self.assertRaises(StopIteration, next, resp) - self.assertEqual(resp.read(), '') - - def test_chunk_size_iter_chunked_no_retry(self): - conn = c.Connection('http://auth.url/', 'some_user', 'some_key') - with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: - mock_get_auth.return_value = ('http://auth.url/', 'tToken') - c.http_connection = self.fake_http_connection( - 200, body='abcdef', headers={'Transfer-Encoding': 'chunked'}) - __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) - self.assertEqual(next(resp), 'ab') - # simulate a dropped connection - resp.resp.read() - self.assertRaises(StopIteration, next, resp) - - def test_chunk_size_iter_retry(self): - conn = c.Connection('http://auth.url/', 'some_user', 'some_key') - with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: - mock_get_auth.return_value = ('http://auth.url', 'tToken') - c.http_connection = self.fake_http_connection( - StubResponse(200, 'abcdef', {'etag': 'some etag', - 'content-length': '6'}), - StubResponse(206, 'cdef', {'etag': 'some etag', - 'content-length': '4', - 'content-range': 'bytes 2-5/6'}), - StubResponse(206, 'ef', {'etag': 'some etag', - 'content-length': '2', - 'content-range': 'bytes 4-5/6'}), - ) - __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) - self.assertEqual(next(resp), 'ab') - self.assertEqual(1, conn.attempts) - # simulate a dropped connection - resp.resp.read() - self.assertEqual(next(resp), 'cd') - self.assertEqual(2, conn.attempts) - # simulate a dropped connection - resp.resp.read() - self.assertEqual(next(resp), 'ef') - self.assertEqual(3, conn.attempts) - self.assertRaises(StopIteration, next, resp) - self.assertRequests([ - ('GET', '/asdf/asdf', '', { - 'x-auth-token': 'tToken', - }), - ('GET', '/asdf/asdf', '', { - 'range': 'bytes=2-', - 'if-match': 'some etag', - 'x-auth-token': 'tToken', - }), - ('GET', '/asdf/asdf', '', { - 'range': 'bytes=4-', - 'if-match': 'some etag', - 'x-auth-token': 'tToken', - }), - ]) - - def test_chunk_size_iter_retry_no_range_support(self): - conn = c.Connection('http://auth.url/', 'some_user', 'some_key') - with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: - mock_get_auth.return_value = ('http://auth.url', 'tToken') - c.http_connection = self.fake_http_connection(*[ - StubResponse(200, 'abcdef', {'etag': 'some etag', - 'content-length': '6'}) - ] * 3) - __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) - self.assertEqual(next(resp), 'ab') - self.assertEqual(1, conn.attempts) - # simulate a dropped connection - resp.resp.read() - self.assertEqual(next(resp), 'cd') - self.assertEqual(2, conn.attempts) - # simulate a dropped connection - resp.resp.read() - self.assertEqual(next(resp), 'ef') - self.assertEqual(3, conn.attempts) - self.assertRaises(StopIteration, next, resp) - self.assertRequests([ - ('GET', '/asdf/asdf', '', { - 'x-auth-token': 'tToken', - }), - ('GET', '/asdf/asdf', '', { - 'range': 'bytes=2-', - 'if-match': 'some etag', - 'x-auth-token': 'tToken', - }), - ('GET', '/asdf/asdf', '', { - 'range': 'bytes=4-', - 'if-match': 'some etag', - 'x-auth-token': 'tToken', - }), - ]) - - def test_chunk_size_iter_retry_bad_range_response(self): - conn = c.Connection('http://auth.url/', 'some_user', 'some_key') - with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: - mock_get_auth.return_value = ('http://auth.url', 'tToken') - c.http_connection = self.fake_http_connection( - StubResponse(200, 'abcdef', {'etag': 'some etag', - 'content-length': '6'}), - StubResponse(206, 'abcdef', {'etag': 'some etag', - 'content-length': '6', - 'content-range': 'chunk 1-2/3'}) - ) - __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) - self.assertEqual(next(resp), 'ab') - self.assertEqual(1, conn.attempts) - # simulate a dropped connection - resp.resp.read() - self.assertRaises(c.ClientException, next, resp) - self.assertRequests([ - ('GET', '/asdf/asdf', '', { - 'x-auth-token': 'tToken', - }), - ('GET', '/asdf/asdf', '', { - 'range': 'bytes=2-', - 'if-match': 'some etag', - 'x-auth-token': 'tToken', - }), - ]) - - def test_get_object_with_resp_chunk_size_zero(self): - def get_connection(self): - def get_auth(): - return 'http://auth.test.com', 'token' - - conn = c.Connection('http://www.test.com', 'asdf', 'asdf') - self.assertIs(type(conn), c.Connection) - conn.get_auth = get_auth - self.assertEqual(conn.attempts, 0) - return conn - - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - conn = get_connection(self) - conn.get_object('container1', 'obj1', resp_chunk_size=0) - self.assertEqual(conn.attempts, 1) - - -class TestHeadObject(MockHttpTest): - - def test_server_error(self): - body = 'c' * 60 - headers = {'foo': 'bar'} - c.http_connection = self.fake_http_connection( - StubResponse(500, body, headers)) - with self.assertRaises(c.ClientException) as exc_context: - c.head_object('http://www.test.com', 'asdf', 'asdf', 'asdf') - self.assertEqual(exc_context.exception.http_response_content, body) - self.assertEqual(exc_context.exception.http_response_headers, headers) - - def test_request_headers(self): - c.http_connection = self.fake_http_connection(204) - conn = c.http_connection('http://www.test.com') - headers = {'x-client-key': 'client key'} - c.head_object('url_is_irrelevant', 'TOKEN', 'container', - 'asdf', http_conn=conn, headers=headers) - self.assertRequests([ - ('HEAD', '/container/asdf', '', { - 'x-auth-token': 'TOKEN', - 'x-client-key': 'client key', - }), - ]) - - def test_query_string(self): - c.http_connection = self.fake_http_connection(204) - conn = c.http_connection('http://www.test.com') - query_string = 'foo=bar' - c.head_object('url_is_irrelevant', 'token', 'container', 'key', - http_conn=conn, query_string=query_string) - self.assertRequests([ - ('HEAD', '/container/key?foo=bar', '', {'x-auth-token': 'token'}) - ]) - - -class TestPutObject(MockHttpTest): - - @mock.patch('swiftclient.requests.__version__', '2.2.0') - def test_ok(self): - c.http_connection = self.fake_http_connection(200) - args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4) - value = c.put_object(*args) - self.assertIsInstance(value, six.string_types) - self.assertEqual(value, EMPTY_ETAG) - self.assertRequests([ - ('PUT', '/container/obj', 'body', { - 'x-auth-token': 'TOKEN', - 'content-length': '4', - 'content-type': '' - }), - ]) - - def test_unicode_ok(self): - conn = c.http_connection(u'http://www.test.com/') - mock_file = six.StringIO(u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') - args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - mock_file) - text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - headers = {'X-Header1': text, - 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:fg:lp'} - - resp = MockHttpResponse() - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - value = c.put_object(*args, headers=headers, http_conn=conn) - self.assertIsInstance(value, six.string_types) - # Test for RFC-2616 encoded symbols - self.assertIn(("a-b", b".x:yz mn:fg:lp"), - resp.buffer) - # Test unicode header - self.assertIn(('x-header1', text.encode('utf8')), - resp.buffer) - - def test_chunk_warning(self): - conn = c.http_connection('http://www.test.com/') - mock_file = six.StringIO('asdf') - args = ('asdf', 'asdf', 'asdf', 'asdf', mock_file) - resp = MockHttpResponse() - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - with warnings.catch_warnings(record=True) as w: - c.put_object(*args, chunk_size=20, headers={}, http_conn=conn) - self.assertEqual(len(w), 0) - - body = 'c' * 60 - c.http_connection = self.fake_http_connection(200, body=body) - args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') - with warnings.catch_warnings(record=True) as w: - c.put_object(*args, chunk_size=20) - self.assertEqual(len(w), 1) - self.assertTrue(issubclass(w[-1].category, UserWarning)) - - @mock.patch('swiftclient.requests.__version__', '2.2.0') - def test_server_error(self): - body = 'c' * 60 - headers = {'foo': 'bar'} - c.http_connection = self.fake_http_connection( - StubResponse(500, body, headers)) - args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') - with self.assertRaises(c.ClientException) as exc_context: - c.put_object(*args) - e = exc_context.exception - self.assertEqual(e.http_response_content, body) - self.assertEqual(e.http_response_headers, headers) - self.assertEqual(e.http_status, 500) - self.assertRequests([ - ('PUT', '/asdf/asdf', 'asdf', { - 'x-auth-token': 'asdf', - 'content-type': ''}), - ]) - - def test_query_string(self): - c.http_connection = self.fake_http_connection(200, - query_string="hello=20") - c.put_object('http://www.test.com', 'asdf', 'asdf', 'asdf', - query_string="hello=20") - for req in self.iter_request_log(): - self.assertEqual(req['method'], 'PUT') - self.assertEqual(req['parsed_path'].path, '/asdf/asdf') - self.assertEqual(req['parsed_path'].query, 'hello=20') - self.assertEqual(req['headers']['x-auth-token'], 'asdf') - - def test_raw_upload(self): - # Raw upload happens when content_length is passed to put_object - conn = c.http_connection(u'http://www.test.com/') - resp = MockHttpResponse(status=200) - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - raw_data = b'asdf' * 256 - raw_data_len = len(raw_data) - - for kwarg in ({'headers': {'Content-Length': str(raw_data_len)}}, - {'content_length': raw_data_len}): - with tempfile.TemporaryFile() as mock_file: - mock_file.write(raw_data) - mock_file.seek(0) - - c.put_object(url='http://www.test.com', http_conn=conn, - contents=mock_file, **kwarg) - - req_data = resp.requests_params['data'] - self.assertIs(type(req_data), swiftclient.utils.LengthWrapper) - self.assertEqual(raw_data_len, len(req_data.read())) - - def test_chunk_upload(self): - # Chunked upload happens when no content_length is passed to put_object - conn = c.http_connection(u'http://www.test.com/') - resp = MockHttpResponse(status=200) - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - raw_data = b'asdf' * 256 - chunk_size = 16 - - with tempfile.TemporaryFile() as mock_file: - mock_file.write(raw_data) - mock_file.seek(0) - - c.put_object(url='http://www.test.com', http_conn=conn, - contents=mock_file, chunk_size=chunk_size) - req_data = resp.requests_params['data'] - self.assertTrue(hasattr(req_data, '__iter__')) - data = b'' - for chunk in req_data: - self.assertEqual(chunk_size, len(chunk)) - data += chunk - self.assertEqual(data, raw_data) - - def test_iter_upload(self): - def data(): - for chunk in ('foo', '', 'bar'): - yield chunk - conn = c.http_connection(u'http://www.test.com/') - resp = MockHttpResponse(status=200) - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - - c.put_object(url='http://www.test.com', http_conn=conn, - contents=data()) - req_headers = resp.requests_params['headers'] - self.assertNotIn('Content-Length', req_headers) - req_data = resp.requests_params['data'] - self.assertTrue(hasattr(req_data, '__iter__')) - # If we emit an empty chunk, requests will go ahead and send it, - # causing the server to close the connection. So make sure we don't - # do that. - self.assertEqual(['foo', 'bar'], list(req_data)) - - def test_md5_mismatch(self): - conn = c.http_connection('http://www.test.com') - resp = MockHttpResponse(status=200, verify=True, - headers={'etag': '"badresponseetag"'}) - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - raw_data = b'asdf' * 256 - raw_data_md5 = md5(raw_data).hexdigest() - chunk_size = 16 - - with tempfile.TemporaryFile() as mock_file: - mock_file.write(raw_data) - mock_file.seek(0) - - contents = swiftclient.utils.ReadableToIterable(mock_file, - md5=True) - - etag = c.put_object(url='http://www.test.com', - http_conn=conn, - contents=contents, - chunk_size=chunk_size) - - self.assertNotEqual(etag, contents.get_md5sum()) - self.assertEqual(etag, 'badresponseetag') - self.assertEqual(raw_data_md5, contents.get_md5sum()) - - def test_md5_match(self): - conn = c.http_connection('http://www.test.com') - raw_data = b'asdf' * 256 - raw_data_md5 = md5(raw_data).hexdigest() - resp = MockHttpResponse(status=200, verify=True, - headers={'etag': '"' + raw_data_md5 + '"'}) - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - chunk_size = 16 - - with tempfile.TemporaryFile() as mock_file: - mock_file.write(raw_data) - mock_file.seek(0) - contents = swiftclient.utils.ReadableToIterable(mock_file, - md5=True) - - etag = c.put_object(url='http://www.test.com', - http_conn=conn, - contents=contents, - chunk_size=chunk_size) - - self.assertEqual(raw_data_md5, contents.get_md5sum()) - self.assertEqual(etag, contents.get_md5sum()) - - def test_params(self): - conn = c.http_connection(u'http://www.test.com/') - resp = MockHttpResponse(status=200) - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - - c.put_object(url='http://www.test.com', http_conn=conn, - etag='1234-5678', content_type='text/plain') - request_header = resp.requests_params['headers'] - self.assertEqual(request_header['etag'], b'1234-5678') - self.assertEqual(request_header['content-type'], b'text/plain') - - @mock.patch('swiftclient.requests.__version__', '2.2.0') - def test_no_content_type_old_requests(self): - conn = c.http_connection(u'http://www.test.com/') - resp = MockHttpResponse(status=200) - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - - c.put_object(url='http://www.test.com', http_conn=conn) - request_header = resp.requests_params['headers'] - self.assertEqual(request_header['content-type'], b'') - - @mock.patch('swiftclient.requests.__version__', '2.4.0') - def test_no_content_type_new_requests(self): - conn = c.http_connection(u'http://www.test.com/') - resp = MockHttpResponse(status=200) - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - - c.put_object(url='http://www.test.com', http_conn=conn) - request_header = resp.requests_params['headers'] - self.assertNotIn('content-type', request_header) - - def test_content_type_in_headers(self): - conn = c.http_connection(u'http://www.test.com/') - resp = MockHttpResponse(status=200) - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - - # title-case header - hdrs = {'Content-Type': 'text/Plain'} - c.put_object(url='http://www.test.com', http_conn=conn, headers=hdrs) - request_header = resp.requests_params['headers'] - self.assertEqual(request_header['content-type'], b'text/Plain') - - # method param overrides headers - c.put_object(url='http://www.test.com', http_conn=conn, headers=hdrs, - content_type='image/jpeg') - request_header = resp.requests_params['headers'] - self.assertEqual(request_header['content-type'], b'image/jpeg') - - -class TestPostObject(MockHttpTest): - - def test_ok(self): - c.http_connection = self.fake_http_connection(200) - delete_at = 2.1 # not str! we don't know what other devs will use! - args = ('http://www.test.com', 'token', 'container', 'obj', - {'X-Object-Meta-Test': 'mymeta', - 'X-Delete-At': delete_at}) - c.post_object(*args) - self.assertRequests([ - ('POST', '/container/obj', '', { - 'x-auth-token': 'token', - 'X-Object-Meta-Test': 'mymeta', - 'X-Delete-At': delete_at}), - ]) - # Check that the request header dict didn't get mutated - self.assertEqual(args[-1], { - 'X-Object-Meta-Test': 'mymeta', - 'X-Delete-At': delete_at, - }) - - def test_unicode_ok(self): - conn = c.http_connection(u'http://www.test.com/') - args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') - text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - headers = {'X-Header1': text, - b'X-Header2': 'value', - 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:kl:qr', - 'X-Object-Meta-Header-not-encoded': text, - b'X-Object-Meta-Header-encoded': 'value'} - - resp = MockHttpResponse() - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - c.post_object(*args, headers=headers, http_conn=conn) - # Test for RFC-2616 encoded symbols - self.assertIn(('a-b', b".x:yz mn:kl:qr"), resp.buffer) - # Test unicode header - self.assertIn(('x-header1', text.encode('utf8')), - resp.buffer) - self.assertIn((b'x-object-meta-header-not-encoded', - text.encode('utf8')), resp.buffer) - self.assertIn((b'x-object-meta-header-encoded', b'value'), - resp.buffer) - self.assertIn((b'x-header2', b'value'), resp.buffer) - - def test_server_error(self): - body = 'c' * 60 - headers = {'foo': 'bar'} - c.http_connection = self.fake_http_connection( - StubResponse(500, body, headers)) - args = ('http://www.test.com', 'token', 'container', 'obj', {}) - with self.assertRaises(c.ClientException) as exc_context: - c.post_object(*args) - self.assertEqual(exc_context.exception.http_response_content, body) - self.assertEqual(exc_context.exception.http_response_headers, headers) - self.assertRequests([ - ('POST', 'http://www.test.com/container/obj', '', { - 'x-auth-token': 'token', - }), - ]) - - -class TestCopyObject(MockHttpTest): - - def test_server_error(self): - c.http_connection = self.fake_http_connection(500) - self.assertRaises( - c.ClientException, c.copy_object, - 'http://www.test.com/v1/AUTH', 'asdf', 'asdf', 'asdf') - - def test_ok(self): - c.http_connection = self.fake_http_connection(200) - c.copy_object( - 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', - destination='/container2/obj') - self.assertRequests([ - ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { - 'X-Auth-Token': 'token', - 'Destination': '/container2/obj', - }), - ]) - - def test_service_token(self): - c.http_connection = self.fake_http_connection(200) - c.copy_object('http://www.test.com/v1/AUTH', None, 'container', - 'obj', destination='/container2/obj', - service_token="TOKEN") - self.assertRequests([ - ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { - 'X-Service-Token': 'TOKEN', - 'Destination': '/container2/obj', - - }), - ]) - - def test_headers(self): - c.http_connection = self.fake_http_connection(200) - c.copy_object( - 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', - destination='/container2/obj', - headers={'some-hdr': 'a', 'other-hdr': 'b'}) - self.assertRequests([ - ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { - 'X-Auth-Token': 'token', - 'Destination': '/container2/obj', - 'some-hdr': 'a', - 'other-hdr': 'b', - }), - ]) - - def test_fresh_metadata_default(self): - c.http_connection = self.fake_http_connection(200) - c.copy_object( - 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', - '/container2/obj', {'x-fresh-metadata': 'hdr-value'}) - self.assertRequests([ - ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { - 'X-Auth-Token': 'token', - 'Destination': '/container2/obj', - 'X-Fresh-Metadata': 'hdr-value', - }), - ]) - - def test_fresh_metadata_true(self): - c.http_connection = self.fake_http_connection(200) - c.copy_object( - 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', - destination='/container2/obj', - headers={'x-fresh-metadata': 'hdr-value'}, - fresh_metadata=True) - self.assertRequests([ - ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { - 'X-Auth-Token': 'token', - 'Destination': '/container2/obj', - 'X-Fresh-Metadata': 'true', - }), - ]) - - def test_fresh_metadata_false(self): - c.http_connection = self.fake_http_connection(200) - c.copy_object( - 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj', - destination='/container2/obj', - headers={'x-fresh-metadata': 'hdr-value'}, - fresh_metadata=False) - self.assertRequests([ - ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { - 'x-auth-token': 'token', - 'Destination': '/container2/obj', - 'X-Fresh-Metadata': 'false', - }), - ]) - - def test_no_destination(self): - c.http_connection = self.fake_http_connection(200) - c.copy_object( - 'http://www.test.com/v1/AUTH', 'token', 'container', 'obj') - self.assertRequests([ - ('COPY', 'http://www.test.com/v1/AUTH/container/obj', '', { - 'x-auth-token': 'token', - 'Destination': '/container/obj', - }), - ]) - - -class TestDeleteObject(MockHttpTest): - - def test_ok(self): - c.http_connection = self.fake_http_connection(200) - c.delete_object('http://www.test.com', 'token', 'container', 'obj') - self.assertRequests([ - ('DELETE', 'http://www.test.com/container/obj', '', { - 'x-auth-token': 'token', - }), - ]) - - def test_server_error(self): - body = 'c' * 60 - headers = {'foo': 'bar'} - c.http_connection = self.fake_http_connection( - StubResponse(500, body, headers)) - with self.assertRaises(c.ClientException) as exc_context: - c.delete_object('http://www.test.com', 'asdf', 'asdf', 'asdf') - self.assertEqual(exc_context.exception.http_response_content, body) - self.assertEqual(exc_context.exception.http_response_headers, headers) - - def test_query_string(self): - c.http_connection = self.fake_http_connection(200, - query_string="hello=20") - c.delete_object('http://www.test.com', 'token', 'container', 'obj', - query_string="hello=20") - self.assertRequests([ - ('DELETE', 'http://www.test.com/container/obj?hello=20', '', { - 'x-auth-token': 'token', - }), - ]) - - -class TestGetCapabilities(MockHttpTest): - - def test_ok(self): - conn = self.fake_http_connection(200, body=b'{}') - http_conn = conn('http://www.test.com/info') - info = c.get_capabilities(http_conn) - self.assertRequests([ - ('GET', '/info', '', {'Accept-Encoding': 'gzip'}), - ]) - self.assertEqual(info, {}) - self.assertTrue(http_conn[1].resp.has_been_read) - - def test_server_error(self): - body = 'c' * 60 - headers = {'foo': 'bar'} - conn = self.fake_http_connection( - StubResponse(500, body, headers)) - http_conn = conn('http://www.test.com/info') - with self.assertRaises(c.ClientException) as exc_context: - c.get_capabilities(http_conn) - self.assertEqual(exc_context.exception.http_response_content, body) - self.assertEqual(exc_context.exception.http_response_headers, headers) - - def test_conn_get_capabilities_with_auth(self): - auth_headers = { - 'x-auth-token': 'token', - 'x-storage-url': 'http://storage.example.com/v1/AUTH_test' - } - auth_v1_response = StubResponse(headers=auth_headers) - stub_info = {'swift': {'fake': True}} - info_response = StubResponse(body=b'{"swift":{"fake":true}}') - fake_conn = self.fake_http_connection(auth_v1_response, info_response) - - conn = c.Connection('http://auth.example.com/auth/v1.0', - 'user', 'key') - with mock.patch('swiftclient.client.http_connection', - new=fake_conn): - info = conn.get_capabilities() - self.assertEqual(info, stub_info) - self.assertRequests([ - ('GET', '/auth/v1.0', '', { - 'x-auth-user': 'user', - 'x-auth-key': 'key'}), - ('GET', 'http://storage.example.com/info', '', { - 'accept-encoding': 'gzip'}), - ]) - - def test_conn_get_capabilities_with_os_auth(self): - fake_keystone = fake_get_auth_keystone( - storage_url='http://storage.example.com/v1/AUTH_test') - stub_info = {'swift': {'fake': True}} - info_response = StubResponse(body=b'{"swift":{"fake":true}}') - fake_conn = self.fake_http_connection(info_response) - - os_options = {'project_id': 'test'} - conn = c.Connection('http://keystone.example.com/v3.0', - 'user', 'key', os_options=os_options, - auth_version=3) - with mock.patch.multiple('swiftclient.client', - get_auth_keystone=fake_keystone, - http_connection=fake_conn): - info = conn.get_capabilities() - self.assertEqual(info, stub_info) - self.assertRequests([ - ('GET', 'http://storage.example.com/info'), - ]) - - def test_conn_get_capabilities_with_url_param(self): - stub_info = {'swift': {'fake': True}} - info_response = StubResponse(body=b'{"swift":{"fake":true}}') - fake_conn = self.fake_http_connection(info_response) - - conn = c.Connection('http://auth.example.com/auth/v1.0', - 'user', 'key') - with mock.patch('swiftclient.client.http_connection', - new=fake_conn): - info = conn.get_capabilities( - 'http://other-storage.example.com/info') - self.assertEqual(info, stub_info) - self.assertRequests([ - ('GET', 'http://other-storage.example.com/info'), - ]) - - def test_conn_get_capabilities_with_preauthurl_param(self): - stub_info = {'swift': {'fake': True}} - info_response = StubResponse(body=b'{"swift":{"fake":true}}') - fake_conn = self.fake_http_connection(info_response) - - storage_url = 'http://storage.example.com/v1/AUTH_test' - conn = c.Connection('http://auth.example.com/auth/v1.0', - 'user', 'key', preauthurl=storage_url) - with mock.patch('swiftclient.client.http_connection', - new=fake_conn): - info = conn.get_capabilities() - self.assertEqual(info, stub_info) - self.assertRequests([ - ('GET', 'http://storage.example.com/info'), - ]) - - def test_conn_get_capabilities_with_os_options(self): - stub_info = {'swift': {'fake': True}} - info_response = StubResponse(body=b'{"swift":{"fake":true}}') - fake_conn = self.fake_http_connection(info_response) - - storage_url = 'http://storage.example.com/v1/AUTH_test' - os_options = { - 'project_id': 'test', - 'object_storage_url': storage_url, - } - conn = c.Connection('http://keystone.example.com/v3.0', - 'user', 'key', os_options=os_options, - auth_version=3) - with mock.patch('swiftclient.client.http_connection', - new=fake_conn): - info = conn.get_capabilities() - self.assertEqual(info, stub_info) - self.assertRequests([ - ('GET', 'http://storage.example.com/info'), - ]) - - -class TestHTTPConnection(MockHttpTest): - - def test_bad_url_scheme(self): - url = u'www.test.com' - with self.assertRaises(c.ClientException) as exc_context: - c.http_connection(url) - exc = exc_context.exception - expected = u'Unsupported scheme "" in url "www.test.com"' - self.assertEqual(expected, str(exc)) - - url = u'://www.test.com' - with self.assertRaises(c.ClientException) as exc_context: - c.http_connection(url) - exc = exc_context.exception - expected = u'Unsupported scheme "" in url "://www.test.com"' - self.assertEqual(expected, str(exc)) - - url = u'blah://www.test.com' - with self.assertRaises(c.ClientException) as exc_context: - c.http_connection(url) - exc = exc_context.exception - expected = u'Unsupported scheme "blah" in url "blah://www.test.com"' - self.assertEqual(expected, str(exc)) - - def test_ok_url_scheme(self): - for scheme in ('http', 'https', 'HTTP', 'HTTPS'): - url = u'%s://www.test.com' % scheme - parsed_url, conn = c.http_connection(url) - self.assertEqual(scheme.lower(), parsed_url.scheme) - self.assertEqual(u'%s://www.test.com' % scheme, conn.url) - - def test_ok_proxy(self): - conn = c.http_connection(u'http://www.test.com/', - proxy='http://localhost:8080') - self.assertEqual(conn[1].requests_args['proxies']['http'], - 'http://localhost:8080') - - def test_bad_proxy(self): - try: - c.http_connection(u'http://www.test.com/', proxy='localhost:8080') - except c.ClientException as e: - self.assertEqual(e.msg, "Proxy's missing scheme") - - def test_cacert(self): - conn = c.http_connection(u'http://www.test.com/', - cacert='/dev/urandom') - self.assertEqual(conn[1].requests_args['verify'], '/dev/urandom') - - def test_insecure(self): - conn = c.http_connection(u'http://www.test.com/', insecure=True) - self.assertEqual(conn[1].requests_args['verify'], False) - - def test_cert(self): - conn = c.http_connection(u'http://www.test.com/', cert='minnie') - self.assertEqual(conn[1].requests_args['cert'], 'minnie') - - def test_cert_key(self): - conn = c.http_connection( - u'http://www.test.com/', cert='minnie', cert_key='mickey') - self.assertEqual(conn[1].requests_args['cert'], ('minnie', 'mickey')) - - def test_response_connection_released(self): - _parsed_url, conn = c.http_connection(u'http://www.test.com/') - conn.resp = MockHttpResponse() - conn.resp.raw = mock.Mock() - conn.resp.raw.read.side_effect = ["Chunk", ""] - resp = conn.getresponse() - self.assertFalse(resp.closed) - self.assertEqual("Chunk", resp.read()) - self.assertFalse(resp.read()) - self.assertTrue(resp.closed) - - @unittest.skipIf(six.PY3, 'python2 specific test') - def test_response_python2_headers(self): - '''Test utf-8 headers in Python 2. - ''' - _, conn = c.http_connection(u'http://www.test.com/') - conn.resp = MockHttpResponse( - status=200, - headers={ - '\xd8\xaa-unicode': '\xd8\xaa-value', - 'empty-header': '' - } - ) - - resp = conn.getresponse() - self.assertEqual( - '\xd8\xaa-value', resp.getheader('\xd8\xaa-unicode')) - self.assertEqual( - '\xd8\xaa-value', resp.getheader('\xd8\xaa-UNICODE')) - self.assertEqual('', resp.getheader('empty-header')) - self.assertEqual( - dict([('\xd8\xaa-unicode', '\xd8\xaa-value'), - ('empty-header', ''), - ('etag', '"%s"' % EMPTY_ETAG)]), - dict(resp.getheaders())) - - @unittest.skipIf(six.PY2, 'python3 specific test') - def test_response_python3_headers(self): - '''Test latin1-encoded headers in Python 3. - ''' - _, conn = c.http_connection(u'http://www.test.com/') - conn.resp = MockHttpResponse( - status=200, - headers={ - b'\xd8\xaa-unicode'.decode('iso-8859-1'): - b'\xd8\xaa-value'.decode('iso-8859-1'), - 'empty-header': '' - } - ) - - resp = conn.getresponse() - self.assertEqual( - '\u062a-value', resp.getheader('\u062a-unicode')) - self.assertEqual( - '\u062a-value', resp.getheader('\u062a-UNICODE')) - self.assertEqual('', resp.getheader('empty-header')) - self.assertEqual( - dict([('\u062a-unicode', '\u062a-value'), - ('empty-header', ''), - ('etag', ('"%s"' % EMPTY_ETAG))]), - dict(resp.getheaders())) - - -class TestConnection(MockHttpTest): - - def test_instance(self): - conn = c.Connection('http://www.test.com', 'asdf', 'asdf') - self.assertEqual(conn.retries, 5) - - def test_instance_kwargs(self): - args = {'user': 'ausername', - 'key': 'secretpass', - 'authurl': 'http://www.test.com', - 'tenant_name': 'atenant'} - conn = c.Connection(**args) - self.assertEqual(type(conn), c.Connection) - - def test_instance_kwargs_token(self): - args = {'preauthtoken': 'atoken123', - 'preauthurl': 'http://www.test.com:8080/v1/AUTH_123456'} - conn = c.Connection(**args) - self.assertEqual(conn.url, args['preauthurl']) - self.assertEqual(conn.token, args['preauthtoken']) - - def test_instance_kwargs_os_token(self): - storage_url = 'http://storage.example.com/v1/AUTH_test' - token = 'token' - args = { - 'os_options': { - 'object_storage_url': storage_url, - 'auth_token': token, - } - } - conn = c.Connection(**args) - self.assertEqual(conn.url, storage_url) - self.assertEqual(conn.token, token) - - def test_instance_kwargs_token_precedence(self): - storage_url = 'http://storage.example.com/v1/AUTH_test' - token = 'token' - args = { - 'preauthurl': storage_url, - 'preauthtoken': token, - 'os_options': { - 'auth_token': 'less-specific-token', - 'object_storage_url': 'less-specific-storage-url', - } - } - conn = c.Connection(**args) - self.assertEqual(conn.url, storage_url) - self.assertEqual(conn.token, token) - - def test_storage_url_override(self): - static_url = 'http://overridden.storage.url' - conn = c.Connection('http://auth.url/', 'some_user', 'some_key', - os_options={ - 'object_storage_url': static_url}) - method_signatures = ( - (conn.head_account, []), - (conn.get_account, []), - (conn.head_container, ('asdf',)), - (conn.get_container, ('asdf',)), - (conn.put_container, ('asdf',)), - (conn.delete_container, ('asdf',)), - (conn.head_object, ('asdf', 'asdf')), - (conn.get_object, ('asdf', 'asdf')), - (conn.put_object, ('asdf', 'asdf', 'asdf')), - (conn.post_object, ('asdf', 'asdf', {})), - (conn.delete_object, ('asdf', 'asdf')), - ) - - with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: - mock_get_auth.return_value = ('http://auth.storage.url', 'tToken') - - for method, args in method_signatures: - c.http_connection = self.fake_http_connection( - 200, body=b'[]', storage_url=static_url) - method(*args) - self.assertEqual(len(self.request_log), 1) - for request in self.iter_request_log(): - self.assertEqual(request['parsed_path'].netloc, - 'overridden.storage.url') - self.assertEqual(request['headers']['x-auth-token'], - 'tToken') - - def test_get_capabilities(self): - conn = c.Connection() - with mock.patch('swiftclient.client.get_capabilities') as get_cap: - conn.get_capabilities('http://storage2.test.com') - parsed = get_cap.call_args[0][0][0] - self.assertEqual(parsed.path, '/info') - self.assertEqual(parsed.netloc, 'storage2.test.com') - conn.get_auth = lambda: ('http://storage.test.com/v1/AUTH_test', - 'token') - conn.get_capabilities() - parsed = get_cap.call_args[0][0][0] - self.assertEqual(parsed.path, '/info') - self.assertEqual(parsed.netloc, 'storage.test.com') - - def test_retry(self): - def quick_sleep(*args): - pass - c.sleep = quick_sleep - conn = c.Connection('http://www.test.com', 'asdf', 'asdf') - code_iter = [500] * (conn.retries + 1) - c.http_connection = self.fake_http_connection(*code_iter) - - self.assertRaises(c.ClientException, conn.head_account) - self.assertEqual(conn.attempts, conn.retries + 1) - - def test_retry_on_ratelimit(self): - - def quick_sleep(*args): - pass - c.sleep = quick_sleep - - # test retries - conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf', - retry_on_ratelimit=True) - code_iter = [200] + [498] * (conn.retries + 1) - auth_resp_headers = { - 'x-auth-token': 'asdf', - 'x-storage-url': 'http://storage/v1/test', - } - c.http_connection = self.fake_http_connection( - *code_iter, headers=auth_resp_headers) - with self.assertRaises(c.ClientException) as exc_context: - conn.head_account() - self.assertIn('Account HEAD failed', str(exc_context.exception)) - self.assertEqual(conn.attempts, conn.retries + 1) - - # test default no-retry - c.http_connection = self.fake_http_connection( - 200, 498, - headers=auth_resp_headers) - conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf') - with self.assertRaises(c.ClientException) as exc_context: - conn.head_account() - self.assertIn('Account HEAD failed', str(exc_context.exception)) - self.assertEqual(conn.attempts, 1) - - def test_retry_with_socket_error(self): - def quick_sleep(*args): - pass - c.sleep = quick_sleep - conn = c.Connection('http://www.test.com', 'asdf', 'asdf') - with mock.patch('swiftclient.client.http_connection') as \ - fake_http_connection, \ - mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: - mock_auth.return_value = ('http://mock.com', 'mock_token') - fake_http_connection.side_effect = socket.error - self.assertRaises(socket.error, conn.head_account) - self.assertEqual(mock_auth.call_count, 1) - self.assertEqual(conn.attempts, conn.retries + 1) - - def test_retry_with_force_auth_retry_exceptions(self): - def quick_sleep(*args): - pass - - def do_test(exception): - c.sleep = quick_sleep - conn = c.Connection( - 'http://www.test.com', 'asdf', 'asdf', - force_auth_retry=True) - with mock.patch('swiftclient.client.http_connection') as \ - fake_http_connection, \ - mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: - mock_auth.return_value = ('http://mock.com', 'mock_token') - fake_http_connection.side_effect = exception - self.assertRaises(exception, conn.head_account) - self.assertEqual(mock_auth.call_count, conn.retries + 1) - self.assertEqual(conn.attempts, conn.retries + 1) - - do_test(socket.error) - do_test(RequestException) - - def test_retry_with_force_auth_retry_client_exceptions(self): - def quick_sleep(*args): - pass - - def do_test(http_status, count): - - def mock_http_connection(*args, **kwargs): - raise ClientException('fake', http_status=http_status) - - c.sleep = quick_sleep - conn = c.Connection( - 'http://www.test.com', 'asdf', 'asdf', - force_auth_retry=True) - with mock.patch('swiftclient.client.http_connection') as \ - fake_http_connection, \ - mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: - mock_auth.return_value = ('http://mock.com', 'mock_token') - fake_http_connection.side_effect = mock_http_connection - self.assertRaises(ClientException, conn.head_account) - self.assertEqual(mock_auth.call_count, count) - self.assertEqual(conn.attempts, count) - - # sanity, in case of 401, the auth will be called only twice because of - # retried_auth mechanism - do_test(401, 2) - # others will be tried until retry limits - do_test(408, 6) - do_test(500, 6) - do_test(503, 6) - - def test_resp_read_on_server_error(self): - conn = c.Connection('http://www.test.com', 'asdf', 'asdf', retries=0) - - def get_auth(*args, **kwargs): - return 'http://www.new.com', 'new' - conn.get_auth = get_auth - self.url, self.token = conn.get_auth() - - method_signatures = ( - (conn.head_account, []), - (conn.get_account, []), - (conn.head_container, ('asdf',)), - (conn.get_container, ('asdf',)), - (conn.put_container, ('asdf',)), - (conn.delete_container, ('asdf',)), - (conn.head_object, ('asdf', 'asdf')), - (conn.get_object, ('asdf', 'asdf')), - (conn.put_object, ('asdf', 'asdf', 'asdf')), - (conn.post_object, ('asdf', 'asdf', {})), - (conn.delete_object, ('asdf', 'asdf')), - ) - - for method, args in method_signatures: - c.http_connection = self.fake_http_connection(500) - self.assertRaises(c.ClientException, method, *args) - requests = list(self.iter_request_log()) - self.assertEqual(len(requests), 1) - for req in requests: - msg = '%s did not read resp on server error' % method.__name__ - self.assertTrue(req['resp'].has_been_read, msg) - - def test_reauth(self): - c.http_connection = self.fake_http_connection(401, 200) - - def get_auth(*args, **kwargs): - # this mock, and by extension this test are not - # representative of the unit under test. The real get_auth - # method will always return the os_option dict's - # object_storage_url which will be overridden by the - # preauthurl parameter to Connection if it is provided. - return 'http://www.new.com', 'new' - - def swap_sleep(*args): - self.swap_sleep_called = True - c.get_auth = get_auth - c.sleep = swap_sleep - self.swap_sleep_called = False - - conn = c.Connection('http://www.test.com', 'asdf', 'asdf', - preauthurl='http://www.old.com', - preauthtoken='old', - ) - - self.assertEqual(conn.attempts, 0) - self.assertEqual(conn.url, 'http://www.old.com') - self.assertEqual(conn.token, 'old') - - conn.head_account() - - self.assertTrue(self.swap_sleep_called) - self.assertEqual(conn.attempts, 2) - self.assertEqual(conn.url, 'http://www.new.com') - self.assertEqual(conn.token, 'new') - - def test_reauth_preauth(self): - conn = c.Connection( - 'http://auth.example.com', 'user', 'password', - preauthurl='http://storage.example.com/v1/AUTH_test', - preauthtoken='expired') - auth_v1_response = StubResponse(200, headers={ - 'x-auth-token': 'token', - 'x-storage-url': 'http://storage.example.com/v1/AUTH_user', - }) - fake_conn = self.fake_http_connection(401, auth_v1_response, 200) - with mock.patch.multiple('swiftclient.client', - http_connection=fake_conn, - sleep=mock.DEFAULT): - conn.head_account() - self.assertRequests([ - ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'expired'}), - ('GET', 'http://auth.example.com', '', { - 'x-auth-user': 'user', - 'x-auth-key': 'password'}), - ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'token'}), - ]) - - def test_reauth_os_preauth(self): - os_preauth_options = { - 'tenant_name': 'demo', - 'object_storage_url': 'http://storage.example.com/v1/AUTH_test', - 'auth_token': 'expired', - } - conn = c.Connection('http://auth.example.com', 'user', 'password', - os_options=os_preauth_options, auth_version=2) - fake_keystone = fake_get_auth_keystone(os_preauth_options) - fake_conn = self.fake_http_connection(401, 200) - with mock.patch.multiple('swiftclient.client', - get_auth_keystone=fake_keystone, - http_connection=fake_conn, - sleep=mock.DEFAULT): - conn.head_account() - self.assertRequests([ - ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'expired'}), - ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'token'}), - ]) - - def test_session_no_invalidate(self): - mock_session = mock.MagicMock() - mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' - mock_session.get_token.return_value = 'expired' - mock_session.invalidate.return_value = False - conn = c.Connection(session=mock_session) - fake_conn = self.fake_http_connection(401) - with mock.patch.multiple('swiftclient.client', - http_connection=fake_conn, - sleep=mock.DEFAULT): - self.assertRaises(c.ClientException, conn.head_account) - self.assertEqual(mock_session.get_token.mock_calls, [mock.call()]) - self.assertEqual(mock_session.invalidate.mock_calls, [mock.call()]) - - def test_session_can_invalidate(self): - mock_session = mock.MagicMock() - mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' - mock_session.get_token.side_effect = ['expired', 'token'] - mock_session.invalidate.return_value = True - conn = c.Connection(session=mock_session) - fake_conn = self.fake_http_connection(401, 200) - with mock.patch.multiple('swiftclient.client', - http_connection=fake_conn, - sleep=mock.DEFAULT): - conn.head_account() - self.assertRequests([ - ('HEAD', '/v1/acct', '', {'x-auth-token': 'expired'}), - ('HEAD', '/v1/acct', '', {'x-auth-token': 'token'}), - ]) - self.assertEqual(mock_session.get_token.mock_calls, [ - mock.call(), mock.call()]) - self.assertEqual(mock_session.invalidate.mock_calls, [mock.call()]) - - def test_preauth_token_with_no_storage_url_requires_auth(self): - conn = c.Connection( - 'http://auth.example.com', 'user', 'password', - preauthtoken='expired') - auth_v1_response = StubResponse(200, headers={ - 'x-auth-token': 'token', - 'x-storage-url': 'http://storage.example.com/v1/AUTH_user', - }) - fake_conn = self.fake_http_connection(auth_v1_response, 200) - with mock.patch.multiple('swiftclient.client', - http_connection=fake_conn, - sleep=mock.DEFAULT): - conn.head_account() - self.assertRequests([ - ('GET', 'http://auth.example.com', '', { - 'x-auth-user': 'user', - 'x-auth-key': 'password'}), - ('HEAD', '/v1/AUTH_user', '', {'x-auth-token': 'token'}), - ]) - - def test_os_preauth_token_with_no_storage_url_requires_auth(self): - os_preauth_options = { - 'tenant_name': 'demo', - 'auth_token': 'expired', - } - conn = c.Connection('http://auth.example.com', 'user', 'password', - os_options=os_preauth_options, auth_version=2) - storage_url = 'http://storage.example.com/v1/AUTH_user' - fake_keystone = fake_get_auth_keystone(storage_url=storage_url) - fake_conn = self.fake_http_connection(200) - with mock.patch.multiple('swiftclient.client', - get_auth_keystone=fake_keystone, - http_connection=fake_conn, - sleep=mock.DEFAULT): - conn.head_account() - self.assertRequests([ - ('HEAD', '/v1/AUTH_user', '', {'x-auth-token': 'token'}), - ]) - - def test_preauth_url_trumps_auth_url(self): - storage_url = 'http://storage.example.com/v1/AUTH_pre_url' - conn = c.Connection( - 'http://auth.example.com', 'user', 'password', - preauthurl=storage_url) - auth_v1_response = StubResponse(200, headers={ - 'x-auth-token': 'post_token', - 'x-storage-url': 'http://storage.example.com/v1/AUTH_post_url', - }) - fake_conn = self.fake_http_connection(auth_v1_response, 200) - with mock.patch.multiple('swiftclient.client', - http_connection=fake_conn, - sleep=mock.DEFAULT): - conn.head_account() - self.assertRequests([ - ('GET', 'http://auth.example.com', '', { - 'x-auth-user': 'user', - 'x-auth-key': 'password'}), - ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), - ]) - - def test_os_preauth_url_trumps_auth_url(self): - storage_url = 'http://storage.example.com/v1/AUTH_pre_url' - os_preauth_options = { - 'tenant_name': 'demo', - 'object_storage_url': storage_url, - } - conn = c.Connection('http://auth.example.com', 'user', 'password', - os_options=os_preauth_options, auth_version=2) - fake_keystone = fake_get_auth_keystone( - storage_url='http://storage.example.com/v1/AUTH_post_url', - token='post_token') - fake_conn = self.fake_http_connection(200) - with mock.patch.multiple('swiftclient.client', - get_auth_keystone=fake_keystone, - http_connection=fake_conn, - sleep=mock.DEFAULT): - conn.head_account() - self.assertRequests([ - ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), - ]) - - def test_preauth_url_trumps_os_preauth_url(self): - storage_url = 'http://storage.example.com/v1/AUTH_pre_url' - os_storage_url = 'http://storage.example.com/v1/AUTH_os_pre_url' - os_preauth_options = { - 'tenant_name': 'demo', - 'object_storage_url': os_storage_url, - } - orig_os_preauth_options = dict(os_preauth_options) - conn = c.Connection('http://auth.example.com', 'user', 'password', - os_options=os_preauth_options, auth_version=2, - preauthurl=storage_url, tenant_name='not_demo') - fake_keystone = fake_get_auth_keystone( - storage_url='http://storage.example.com/v1/AUTH_post_url', - token='post_token') - fake_conn = self.fake_http_connection(200) - with mock.patch.multiple('swiftclient.client', - get_auth_keystone=fake_keystone, - http_connection=fake_conn, - sleep=mock.DEFAULT): - conn.head_account() - self.assertRequests([ - ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), - ]) - - # check that Connection has not modified our os_options - self.assertEqual(orig_os_preauth_options, os_preauth_options) - - def test_get_auth_sets_url_and_token(self): - with mock.patch('swiftclient.client.get_auth') as mock_get_auth: - mock_get_auth.return_value = ( - "https://storage.url/v1/AUTH_storage_acct", "AUTH_token" - ) - conn = c.Connection("https://auth.url/auth/v2.0", - "user", "passkey", tenant_name="tenant") - conn.get_auth() - self.assertEqual("https://storage.url/v1/AUTH_storage_acct", conn.url) - self.assertEqual("AUTH_token", conn.token) - - def test_timeout_passed_down(self): - # We want to avoid mocking http_connection(), and most especially - # avoid passing it down in argument. However, we cannot simply - # instantiate C=Connection(), then shim C.http_conn. Doing so would - # avoid some of the code under test (where _retry() invokes - # http_connection()), and would miss get_auth() completely. - # So, with regret, we do mock http_connection(), but with a very - # light shim that swaps out _request() as originally intended. - - orig_http_connection = c.http_connection - - timeouts = [] - - def my_request_handler(*a, **kw): - if 'timeout' in kw: - timeouts.append(kw['timeout']) - else: - timeouts.append(None) - return MockHttpResponse( - status=200, - headers={ - 'x-auth-token': 'a_token', - 'x-storage-url': 'http://files.example.com/v1/AUTH_user'}) - - def shim_connection(*a, **kw): - url, conn = orig_http_connection(*a, **kw) - conn._request = my_request_handler - return url, conn - - # v1 auth - conn = c.Connection( - 'http://auth.example.com', 'user', 'password', timeout=33.0) - with mock.patch.multiple('swiftclient.client', - http_connection=shim_connection, - sleep=mock.DEFAULT): - conn.head_account() - - # 1 call is through get_auth, 1 call is HEAD for account - self.assertEqual(timeouts, [33.0, 33.0]) - - # v2 auth - timeouts = [] - os_options = {'tenant_name': 'tenant', 'auth_token': 'meta-token'} - conn = c.Connection( - 'http://auth.example.com', 'user', 'password', timeout=33.0, - os_options=os_options, auth_version=2.0) - fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client.ksclient_v2', fake_ks): - with mock.patch.multiple('swiftclient.client', - http_connection=shim_connection, - sleep=mock.DEFAULT): - conn.head_account() - - # check timeout is passed to keystone client - self.assertEqual(1, len(fake_ks.calls)) - self.assertEqual(33.0, fake_ks.calls[0].get('timeout')) - # check timeout passed to HEAD for account - self.assertEqual(timeouts, [33.0]) - - # check token passed to keystone client - self.assertIn('token', fake_ks.calls[0]) - self.assertEqual('meta-token', fake_ks.calls[0].get('token')) - - def test_reset_stream(self): - - class LocalContents(object): - - def __init__(self, tell_value=0): - self.data = six.BytesIO(string.ascii_letters.encode() * 10) - self.data.seek(tell_value) - self.reads = [] - self.seeks = [] - self.tells = [] - - def tell(self): - self.tells.append(self.data.tell()) - return self.tells[-1] - - def seek(self, position, mode=0): - self.seeks.append((position, mode)) - self.data.seek(position, mode) - - def read(self, size=-1): - read_data = self.data.read(size) - self.reads.append((size, read_data)) - return read_data - - class LocalConnection(object): - - def __init__(self, parsed_url=None): - self.reason = "" - if parsed_url: - self.host = parsed_url.netloc - self.port = parsed_url.netloc - - def putrequest(self, *args, **kwargs): - self.send('PUT', *args, **kwargs) - - def putheader(self, *args, **kwargs): - return - - def endheaders(self, *args, **kwargs): - return - - def send(self, *args, **kwargs): - data = kwargs.get('data') - if data is not None: - if hasattr(data, 'read'): - data.read() - else: - for datum in data: - pass - raise socket.error('oops') - - def request(self, *args, **kwargs): - return - - def getresponse(self, *args, **kwargs): - self.status = 200 - return self - - def getheader(self, *args, **kwargs): - return 'header' - - def getheaders(self): - return [('key1', 'value1'), ('key2', 'value2')] - - def read(self, *args, **kwargs): - return '' - - def close(self): - pass - - def local_http_connection(url, proxy=None, cacert=None, - insecure=False, cert=None, cert_key=None, - ssl_compression=True, timeout=None): - parsed = urlparse(url) - return parsed, LocalConnection() - - with mock.patch.object(c, 'http_connection', local_http_connection): - conn = c.Connection('http://www.example.com', 'asdf', 'asdf', - retries=1, starting_backoff=.0001) - - contents = LocalContents() - exc = None - try: - conn.put_object('c', 'o', contents) - except socket.error as err: - exc = err - self.assertEqual(contents.tells, [0]) - self.assertEqual(contents.seeks, [(0, 0)]) - # four reads: two in the initial pass, two in the retry - self.assertEqual(4, len(contents.reads)) - self.assertEqual((65536, b''), contents.reads[1]) - self.assertEqual((65536, b''), contents.reads[3]) - self.assertEqual(str(exc), 'oops') - - contents = LocalContents(tell_value=123) - exc = None - try: - conn.put_object('c', 'o', contents) - except socket.error as err: - exc = err - self.assertEqual(contents.tells, [123]) - self.assertEqual(contents.seeks, [(123, 0)]) - # four reads: two in the initial pass, two in the retry - self.assertEqual(4, len(contents.reads)) - self.assertEqual((65536, b''), contents.reads[1]) - self.assertEqual((65536, b''), contents.reads[3]) - self.assertEqual(str(exc), 'oops') - - contents = LocalContents(tell_value=123) - wrapped_contents = swiftclient.utils.LengthWrapper( - contents, 6, md5=True) - exc = None - try: - conn.put_object('c', 'o', wrapped_contents) - except socket.error as err: - exc = err - self.assertEqual(contents.tells, [123]) - self.assertEqual(contents.seeks, [(123, 0)]) - self.assertEqual(contents.reads, [(6, b'tuvwxy')] * 2) - self.assertEqual(str(exc), 'oops') - self.assertEqual(md5(b'tuvwxy').hexdigest(), - wrapped_contents.get_md5sum()) - - contents = LocalContents() - contents.tell = None - exc = None - try: - conn.put_object('c', 'o', contents) - except c.ClientException as err: - exc = err - self.assertEqual(contents.seeks, []) - self.assertEqual(str(exc), "put_object('c', 'o', ...) failure " - "and no ability to reset contents for reupload.") - - def test_get_container(self): - headers = {'X-Favourite-Pet': 'Aardvark'} - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200, body=b'{}')): - with mock.patch('swiftclient.client.get_auth', - lambda *a, **k: ('http://url:8080/v1/a', 'token')): - conn = c.Connection() - conn.get_container('c1', prefix='p', limit=5, - headers=headers) - self.assertEqual(1, len(self.request_log), self.request_log) - self.assertRequests([ - ('GET', '/v1/a/c1?format=json&limit=5&prefix=p', '', { - 'x-auth-token': 'token', - 'X-Favourite-Pet': 'Aardvark', - 'accept-encoding': 'gzip', - }), - ]) - self.assertEqual(conn.attempts, 1) - - def test_head_container(self): - headers = {'X-Favourite-Pet': 'Aardvark'} - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200, body=b'{}')): - with mock.patch('swiftclient.client.get_auth', - lambda *a, **k: ('http://url:8080/v1/a', 'token')): - conn = c.Connection() - conn.head_container('c1', headers=headers) - self.assertEqual(1, len(self.request_log), self.request_log) - self.assertRequests([ - ('HEAD', '/v1/a/c1', '', { - 'x-auth-token': 'token', - 'X-Favourite-Pet': 'Aardvark', - }), - ]) - self.assertEqual(conn.attempts, 1) - - def test_head_object(self): - headers = {'X-Favourite-Pet': 'Aardvark'} - query_string = 'foo=bar' - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - with mock.patch('swiftclient.client.get_auth', - lambda *a, **k: ('http://url:8080/v1/a', 'token')): - conn = c.Connection() - conn.head_object('c1', 'o1', - headers=headers, query_string=query_string) - self.assertEqual(1, len(self.request_log), self.request_log) - self.assertRequests([ - ('HEAD', '/v1/a/c1/o1?foo=bar', '', { - 'x-auth-token': 'token', - 'X-Favourite-Pet': 'Aardvark', - }), - ]) - self.assertEqual(conn.attempts, 1) - - -class TestResponseDict(MockHttpTest): - """ - Verify handling of optional response_dict argument. - """ - calls = [('post_account', {}), - ('post_container', 'c', {}), - ('put_container', 'c'), - ('delete_container', 'c'), - ('post_object', 'c', 'o', {}), - ('put_object', 'c', 'o', 'body'), - ('copy_object', 'c', 'o'), - ('delete_object', 'c', 'o')] - - def fake_get_auth(*args, **kwargs): - return 'http://url', 'token' - - def test_response_dict_with_auth_error(self): - def bad_get_auth(*args, **kwargs): - raise c.ClientException('test') - - for call in self.calls: - resp_dict = {'test': 'should be untouched'} - with mock.patch('swiftclient.client.get_auth', - bad_get_auth): - conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') - self.assertRaises(c.ClientException, getattr(conn, call[0]), - *call[1:], response_dict=resp_dict) - - self.assertEqual({'test': 'should be untouched'}, resp_dict) - - def test_response_dict_with_request_error(self): - for call in self.calls: - resp_dict = {'test': 'should be untouched'} - with mock.patch('swiftclient.client.get_auth', - self.fake_get_auth): - exc = c.ClientException('test') - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200, exc=exc)): - conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') - self.assertRaises(c.ClientException, - getattr(conn, call[0]), - *call[1:], - response_dict=resp_dict) - - self.assertEqual('should be untouched', resp_dict.get('test')) - self.assertEqual([{}], resp_dict.get('response_dicts')) - - def test_response_dict(self): - # test response_dict is populated and - # new list of response_dicts is created - for call in self.calls: - resp_dict = {'test': 'should be untouched'} - with mock.patch('swiftclient.client.get_auth', - self.fake_get_auth): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') - getattr(conn, call[0])(*call[1:], response_dict=resp_dict) - - self.assertEqual('should be untouched', - resp_dict.pop('test', None)) - self.assertEqual('Fake', resp_dict.get('reason')) - self.assertEqual(200, resp_dict.get('status')) - self.assertIn('headers', resp_dict) - self.assertEqual('yes', resp_dict['headers'].get('x-works')) - children = resp_dict.pop('response_dicts', []) - self.assertEqual(1, len(children)) - self.assertEqual(resp_dict, children[0]) - - def test_response_dict_with_existing(self): - # check response_dict is populated and new dict is appended - # to existing response_dicts list - for call in self.calls: - resp_dict = {'test': 'should be untouched', - 'response_dicts': [{'existing': 'response dict'}]} - with mock.patch('swiftclient.client.get_auth', - self.fake_get_auth): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') - getattr(conn, call[0])(*call[1:], response_dict=resp_dict) - - self.assertEqual('should be untouched', - resp_dict.pop('test', None)) - self.assertEqual('Fake', resp_dict.get('reason')) - self.assertEqual(200, resp_dict.get('status')) - self.assertIn('headers', resp_dict) - self.assertEqual('yes', resp_dict['headers'].get('x-works')) - children = resp_dict.pop('response_dicts', []) - self.assertEqual(2, len(children)) - self.assertEqual({'existing': 'response dict'}, children[0]) - self.assertEqual(resp_dict, children[1]) - - -class TestLogging(MockHttpTest): - """ - Make sure all the lines in http_log are covered. - """ - - def setUp(self): - super(TestLogging, self).setUp() - self.swiftclient_logger = logging.getLogger("swiftclient") - self.log_level = self.swiftclient_logger.getEffectiveLevel() - self.swiftclient_logger.setLevel(logging.INFO) - - def tearDown(self): - self.swiftclient_logger.setLevel(self.log_level) - super(TestLogging, self).tearDown() - - def test_put_ok(self): - c.http_connection = self.fake_http_connection(200) - args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') - value = c.put_object(*args) - self.assertIsInstance(value, six.string_types) - - def test_head_error(self): - c.http_connection = self.fake_http_connection(500) - self.assertRaises(c.ClientException, c.head_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') - - def test_get_error(self): - c.http_connection = self.fake_http_connection(404) - with self.assertRaises(c.ClientException) as exc_context: - c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') - self.assertEqual(exc_context.exception.http_status, 404) - - def test_content_encoding_gzip_body_is_logged_decoded(self): - buf = six.BytesIO() - gz = gzip.GzipFile(fileobj=buf, mode='w') - data = {"test": u"\u2603"} - decoded_body = json.dumps(data).encode('utf-8') - gz.write(decoded_body) - gz.close() - # stub a gzip encoded body - body = buf.getvalue() - headers = {'content-encoding': 'gzip'} - # ... and make a content-encoding gzip error response - stub_response = StubResponse(500, body, headers) - with mock.patch('swiftclient.client.logger.info') as mock_log: - # ... if the client gets such a response - c.http_connection = self.fake_http_connection(stub_response) - with self.assertRaises(c.ClientException) as exc_context: - c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') - self.assertEqual(exc_context.exception.http_status, 500) - # it will log the decoded body - self.assertEqual([ - mock.call('REQ: %s', u'curl -i http://www.test.com/asdf/asdf ' - '-X GET -H "X-Auth-Token: ..."'), - mock.call('RESP STATUS: %s %s', 500, 'Fake'), - mock.call('RESP HEADERS: %s', {'content-encoding': 'gzip'}), - mock.call('RESP BODY: %s', decoded_body) - ], mock_log.mock_calls) - - def test_redact_token(self): - with mock.patch('swiftclient.client.logger.debug') as mock_log: - token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' - token_encoded = token_value.encode('utf8') - unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c') - unicode_token_encoded = unicode_token_value.encode('utf8') - set_cookie_value = 'X-Auth-Token=%s' % token_value - set_cookie_encoded = set_cookie_value.encode('utf8') - c.http_log( - ['GET'], - {'headers': { - 'X-Auth-Token': token_encoded, - 'X-Storage-Token': unicode_token_encoded - }}, - MockHttpResponse( - status=200, - headers={ - 'X-Auth-Token': token_encoded, - 'X-Storage-Token': unicode_token_encoded, - 'Etag': b'mock_etag', - 'Set-Cookie': set_cookie_encoded - } - ), - '' - ) - out = [] - for _, args, kwargs in mock_log.mock_calls: - for arg in args: - out.append(u'%s' % arg) - output = u''.join(out) - self.assertIn('X-Auth-Token', output) - self.assertIn(token_value[:16] + '...', output) - self.assertIn('X-Storage-Token', output) - self.assertIn(unicode_token_value[:8] + '...', output) - self.assertIn('Set-Cookie', output) - self.assertIn(set_cookie_value[:16] + '...', output) - self.assertNotIn(token_value, output) - self.assertNotIn(unicode_token_value, output) - self.assertNotIn(set_cookie_value, output) - - def test_show_token(self): - with mock.patch('swiftclient.client.logger.debug') as mock_log: - token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' - token_encoded = token_value.encode('utf8') - unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c') - c.logger_settings['redact_sensitive_headers'] = False - unicode_token_encoded = unicode_token_value.encode('utf8') - c.http_log( - ['GET'], - {'headers': { - 'X-Auth-Token': token_encoded, - 'X-Storage-Token': unicode_token_encoded - }}, - MockHttpResponse( - status=200, - headers=[ - ('X-Auth-Token', token_encoded), - ('X-Storage-Token', unicode_token_encoded), - ('Etag', b'mock_etag') - ] - ), - '' - ) - out = [] - for _, args, kwargs in mock_log.mock_calls: - for arg in args: - out.append(u'%s' % arg) - output = u''.join(out) - self.assertIn('X-Auth-Token', output) - self.assertIn(token_value, output) - self.assertIn('X-Storage-Token', output) - self.assertIn(unicode_token_value, output) - - @mock.patch('swiftclient.client.logger.debug') - def test_unicode_path(self, mock_log): - path = u'http://swift/v1/AUTH_account-\u062a'.encode('utf-8') - c.http_log(['GET', path], {}, - MockHttpResponse(status=200, headers=[]), '') - request_log_line = mock_log.mock_calls[0] - self.assertEqual('REQ: %s', request_log_line[1][0]) - self.assertEqual(u'curl -i -X GET %s' % path.decode('utf-8'), - request_log_line[1][1]) - - -class TestCloseConnection(MockHttpTest): - - def test_close_none(self): - c.http_connection = self.fake_http_connection() - conn = c.Connection('http://www.test.com', 'asdf', 'asdf') - self.assertIsNone(conn.http_conn) - conn.close() - self.assertIsNone(conn.http_conn) - # Can re-close - conn.close() - self.assertIsNone(conn.http_conn) - - def test_close_ok(self): - url = 'http://www.test.com' - conn = c.Connection(url, 'asdf', 'asdf') - self.assertIsNone(conn.http_conn) - conn.http_conn = c.http_connection(url) - self.assertEqual(type(conn.http_conn), tuple) - self.assertEqual(len(conn.http_conn), 2) - http_conn_obj = conn.http_conn[1] - self.assertIsInstance(http_conn_obj, c.HTTPConnection) - self.assertTrue(hasattr(http_conn_obj, 'close')) - conn.close() - - -class TestServiceToken(MockHttpTest): - - def setUp(self): - super(TestServiceToken, self).setUp() - self.os_options = { - 'object_storage_url': 'http://storage_url.com', - 'service_username': 'service_username', - 'service_project_name': 'service_project_name', - 'service_key': 'service_key'} - - def get_connection(self): - conn = c.Connection('http://www.test.com', 'asdf', 'asdf', - os_options=self.os_options) - - self.assertIs(type(conn), c.Connection) - conn.get_auth = self.get_auth - conn.get_service_auth = self.get_service_auth - - self.assertEqual(conn.attempts, 0) - self.assertIsNone(conn.service_token) - - self.assertIs(type(conn), c.Connection) - return conn - - def get_auth(self): - # The real get_auth function will always return the os_option - # dict's object_storage_url which will be overridden by the - # preauthurl parameter to Connection if it is provided. - return self.os_options.get('object_storage_url'), 'token' - - def get_service_auth(self): - # The real get_auth function will always return the os_option - # dict's object_storage_url which will be overridden by the - # preauthurl parameter to Connection if it is provided. - return self.os_options.get('object_storage_url'), 'stoken' - - def test_service_token_reauth(self): - get_auth_call_list = [] - - def get_auth(url, user, key, **kwargs): - # The real get_auth function will always return the os_option - # dict's object_storage_url which will be overridden by the - # preauthurl parameter to Connection if it is provided. - args = {'url': url, 'user': user, 'key': key, 'kwargs': kwargs} - get_auth_call_list.append(args) - return_dict = {'asdf': 'new', 'service_username': 'newserv'} - storage_url = kwargs['os_options'].get('object_storage_url') - return storage_url, return_dict[user] - - def swap_sleep(*args): - self.swap_sleep_called = True - c.get_auth = get_auth - - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(401, 200)): - with mock.patch('swiftclient.client.sleep', swap_sleep): - self.swap_sleep_called = False - - conn = c.Connection('http://www.test.com', 'asdf', 'asdf', - preauthurl='http://www.old.com', - preauthtoken='old', - os_options=self.os_options) - - self.assertEqual(conn.attempts, 0) - self.assertEqual(conn.url, 'http://www.old.com') - self.assertEqual(conn.token, 'old') - - conn.head_account() - - self.assertTrue(self.swap_sleep_called) - self.assertEqual(conn.attempts, 2) - # The original 'preauth' storage URL *must* be preserved - self.assertEqual(conn.url, 'http://www.old.com') - self.assertEqual(conn.token, 'new') - self.assertEqual(conn.service_token, 'newserv') - - # Check get_auth was called with expected args - auth_args = get_auth_call_list[0] - auth_kwargs = get_auth_call_list[0]['kwargs'] - self.assertEqual('asdf', auth_args['user']) - self.assertEqual('asdf', auth_args['key']) - self.assertEqual('service_key', - auth_kwargs['os_options']['service_key']) - self.assertEqual('service_username', - auth_kwargs['os_options']['service_username']) - self.assertEqual('service_project_name', - auth_kwargs['os_options']['service_project_name']) - - auth_args = get_auth_call_list[1] - auth_kwargs = get_auth_call_list[1]['kwargs'] - self.assertEqual('service_username', auth_args['user']) - self.assertEqual('service_key', auth_args['key']) - self.assertEqual('service_project_name', - auth_kwargs['os_options']['tenant_name']) - - def test_service_token_reauth_retries_0(self): - get_auth_call_list = [] - - def get_auth(url, user, key, **kwargs): - # The real get_auth function will always return the os_option - # dict's object_storage_url which will be overridden by the - # preauthurl parameter to Connection if it is provided. - args = {'url': url, 'user': user, 'key': key, 'kwargs': kwargs} - get_auth_call_list.append(args) - return_dict = {'asdf': 'new', 'service_username': 'newserv'} - storage_url = kwargs['os_options'].get('object_storage_url') - return storage_url, return_dict[user] - - def swap_sleep(*args): - self.swap_sleep_called = True - c.get_auth = get_auth - - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(401, 200)): - with mock.patch('swiftclient.client.sleep', swap_sleep): - self.swap_sleep_called = False - - conn = c.Connection('http://www.test.com', 'asdf', 'asdf', - preauthurl='http://www.old.com', - preauthtoken='old', - os_options=self.os_options, - retries=0) - - self.assertEqual(conn.attempts, 0) - self.assertEqual(conn.url, 'http://www.old.com') - self.assertEqual(conn.token, 'old') - - conn.head_account() - - self.assertTrue(self.swap_sleep_called) - self.assertEqual(conn.attempts, 2) - # The original 'preauth' storage URL *must* be preserved - self.assertEqual(conn.url, 'http://www.old.com') - self.assertEqual(conn.token, 'new') - self.assertEqual(conn.service_token, 'newserv') - - # Check get_auth was called with expected args - auth_args = get_auth_call_list[0] - auth_kwargs = get_auth_call_list[0]['kwargs'] - self.assertEqual('asdf', auth_args['user']) - self.assertEqual('asdf', auth_args['key']) - self.assertEqual('service_key', - auth_kwargs['os_options']['service_key']) - self.assertEqual('service_username', - auth_kwargs['os_options']['service_username']) - self.assertEqual('service_project_name', - auth_kwargs['os_options']['service_project_name']) - - auth_args = get_auth_call_list[1] - auth_kwargs = get_auth_call_list[1]['kwargs'] - self.assertEqual('service_username', auth_args['user']) - self.assertEqual('service_key', auth_args['key']) - self.assertEqual('service_project_name', - auth_kwargs['os_options']['tenant_name']) - - # Ensure this is not an endless loop - it fails after the second 401 - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(401, 401, 401, 401)): - with mock.patch('swiftclient.client.sleep', swap_sleep): - self.swap_sleep_called = False - - conn = c.Connection('http://www.test.com', 'asdf', 'asdf', - preauthurl='http://www.old.com', - preauthtoken='old', - os_options=self.os_options, - retries=0) - - self.assertEqual(conn.attempts, 0) - self.assertRaises(c.ClientException, conn.head_account) - self.assertEqual(conn.attempts, 2) - unused_responses = list(self.fake_connect.code_iter) - self.assertEqual(unused_responses, [401, 401]) - - def test_service_token_get_account(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - with mock.patch('swiftclient.client.parse_api_response'): - conn = self.get_connection() - conn.get_account() - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('GET', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com/?format=json', - actual['full_path']) - self.assertEqual(conn.attempts, 1) - - def test_service_token_head_account(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - conn = self.get_connection() - conn.head_account() - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('HEAD', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com', actual['full_path']) - - self.assertEqual(conn.attempts, 1) - - def test_service_token_post_account(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(201)): - conn = self.get_connection() - conn.post_account(headers={}) - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('POST', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com', actual['full_path']) - self.assertEqual(conn.attempts, 1) - - def test_service_token_delete_container(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(204)): - conn = self.get_connection() - conn.delete_container('container1') - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('DELETE', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com/container1', - actual['full_path']) - self.assertEqual(conn.attempts, 1) - - def test_service_token_get_container(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - with mock.patch('swiftclient.client.parse_api_response'): - conn = self.get_connection() - conn.get_container('container1') - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('GET', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com/container1?format=json', - actual['full_path']) - self.assertEqual(conn.attempts, 1) - - def test_service_token_get_container_full_listing(self): - # verify service token is sent with each request for a full listing - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200, 200)): - with mock.patch('swiftclient.client.parse_api_response') as resp: - resp.side_effect = ([{"name": "obj1"}], []) - conn = self.get_connection() - conn.get_container('container1', full_listing=True) - self.assertEqual(2, len(self.request_log), self.request_log) - expected_urls = iter(( - 'http://storage_url.com/container1?format=json', - 'http://storage_url.com/container1?format=json&marker=obj1' - )) - for actual in self.iter_request_log(): - self.assertEqual('GET', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual(next(expected_urls), - actual['full_path']) - self.assertEqual(conn.attempts, 1) - - def test_service_token_head_container(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - conn = self.get_connection() - conn.head_container('container1') - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('HEAD', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com/container1', - actual['full_path']) - self.assertEqual(conn.attempts, 1) - - def test_service_token_post_container(self): - headers = {'X-Container-Meta-Color': 'blue'} - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(201)): - conn = self.get_connection() - conn.post_container('container1', headers) - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('POST', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com/container1', - actual['full_path']) - self.assertEqual(conn.attempts, 1) - # Check that we didn't mutate the request header dict - self.assertEqual(headers, {'X-Container-Meta-Color': 'blue'}) - - def test_service_token_put_container(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - conn = self.get_connection() - conn.put_container('container1') - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('PUT', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com/container1', - actual['full_path']) - self.assertEqual(conn.attempts, 1) - - def test_service_token_get_object(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - conn = self.get_connection() - conn.get_object('container1', 'obj1') - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('GET', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com/container1/obj1', - actual['full_path']) - self.assertEqual(conn.attempts, 1) - - def test_service_token_head_object(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - conn = self.get_connection() - conn.head_object('container1', 'obj1') - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('HEAD', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com/container1/obj1', - actual['full_path']) - self.assertEqual(conn.attempts, 1) - - def test_service_token_put_object(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(200)): - conn = self.get_connection() - conn.put_object('container1', 'obj1', 'a_string') - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('PUT', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com/container1/obj1', - actual['full_path']) - self.assertEqual(conn.attempts, 1) - - def test_service_token_post_object(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(202)): - conn = self.get_connection() - conn.post_object('container1', 'obj1', {}) - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('POST', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com/container1/obj1', - actual['full_path']) - self.assertEqual(conn.attempts, 1) - - def test_service_token_delete_object(self): - with mock.patch('swiftclient.client.http_connection', - self.fake_http_connection(202)): - conn = self.get_connection() - conn.delete_object('container1', 'obj1', query_string='a_string') - self.assertEqual(1, len(self.request_log), self.request_log) - for actual in self.iter_request_log(): - self.assertEqual('DELETE', actual['method']) - actual_hdrs = actual['headers'] - self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) - self.assertEqual('token', actual_hdrs['X-Auth-Token']) - self.assertEqual('http://storage_url.com/container1/obj1?a_string', - actual['full_path']) - self.assertEqual(conn.attempts, 1) diff -Nru python-swiftclient-3.8.1/tests/unit/test_utils.py python-swiftclient-3.9.0/tests/unit/test_utils.py --- python-swiftclient-3.8.1/tests/unit/test_utils.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/tests/unit/test_utils.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,679 +0,0 @@ -# Copyright (c) 2010-2013 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import gzip -import json -import unittest -import mock -import six -import tempfile -from time import gmtime, localtime, mktime, strftime, strptime -from hashlib import md5, sha1 - -from swiftclient import utils as u - - -class TestConfigTrueValue(unittest.TestCase): - - def test_TRUE_VALUES(self): - for v in u.TRUE_VALUES: - self.assertEqual(v, v.lower()) - - @mock.patch.object(u, 'TRUE_VALUES', 'hello world'.split()) - def test_config_true_value(self): - for val in 'hello world HELLO WORLD'.split(): - self.assertIs(True, u.config_true_value(val)) - self.assertIs(True, u.config_true_value(True)) - self.assertIs(False, u.config_true_value('foo')) - self.assertIs(False, u.config_true_value(False)) - - -class TestPrtBytes(unittest.TestCase): - - def test_zero_bytes(self): - bytes_ = 0 - raw = '0' - human = '0' - self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) - self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) - - def test_one_byte(self): - bytes_ = 1 - raw = '1' - human = '1' - self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) - self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) - - def test_less_than_one_k(self): - bytes_ = (2 ** 10) - 1 - raw = '1023' - human = '1023' - self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) - self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) - - def test_one_k(self): - bytes_ = 2 ** 10 - raw = '1024' - human = '1.0K' - self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) - self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) - - def test_a_decimal_k(self): - bytes_ = (3 * 2 ** 10) + 512 - raw = '3584' - human = '3.5K' - self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) - self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) - - def test_a_bit_less_than_one_meg(self): - bytes_ = (2 ** 20) - (2 ** 10) - raw = '1047552' - human = '1023K' - self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) - self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) - - def test_just_a_hair_less_than_one_meg(self): - bytes_ = (2 ** 20) - (2 ** 10) + 1 - raw = '1047553' - human = '1.0M' - self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) - self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) - - def test_one_meg(self): - bytes_ = 2 ** 20 - raw = '1048576' - human = '1.0M' - self.assertEqual(raw, u.prt_bytes(bytes_, False).lstrip()) - self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) - - def test_ten_meg(self): - bytes_ = 10 * 2 ** 20 - human = '10M' - self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) - - def test_bit_less_than_ten_meg(self): - bytes_ = (10 * 2 ** 20) - (100 * 2 ** 10) - human = '9.9M' - self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) - - def test_just_a_hair_less_than_ten_meg(self): - bytes_ = (10 * 2 ** 20) - 1 - human = '10.0M' - self.assertEqual(human, u.prt_bytes(bytes_, True).lstrip()) - - def test_a_yotta(self): - bytes_ = 42 * 2 ** 80 - self.assertEqual('42Y', u.prt_bytes(bytes_, True).lstrip()) - - def test_overflow(self): - bytes_ = 2 ** 90 - self.assertEqual('1024Y', u.prt_bytes(bytes_, True).lstrip()) - - -class TestTempURL(unittest.TestCase): - url = '/v1/AUTH_account/c/o' - seconds = 3600 - key = 'correcthorsebatterystaple' - method = 'GET' - expected_url = url + ('?temp_url_sig=temp_url_signature' - '&temp_url_expires=1400003600') - expected_body = '\n'.join([ - method, - '1400003600', - url, - ]).encode('utf-8') - - @mock.patch('hmac.HMAC') - @mock.patch('time.time', return_value=1400000000) - def test_generate_temp_url(self, time_mock, hmac_mock): - hmac_mock().hexdigest.return_value = 'temp_url_signature' - url = u.generate_temp_url(self.url, self.seconds, - self.key, self.method) - key = self.key - if not isinstance(key, six.binary_type): - key = key.encode('utf-8') - self.assertEqual(url, self.expected_url) - self.assertEqual(hmac_mock.mock_calls, [ - mock.call(), - mock.call(key, self.expected_body, sha1), - mock.call().hexdigest(), - ]) - self.assertIsInstance(url, type(self.url)) - - @mock.patch('hmac.HMAC') - @mock.patch('time.time', return_value=1400000000) - def test_generate_temp_url_ip_range(self, time_mock, hmac_mock): - hmac_mock().hexdigest.return_value = 'temp_url_signature' - ip_ranges = [ - '1.2.3.4', '1.2.3.4/24', '2001:db8::', - b'1.2.3.4', b'1.2.3.4/24', b'2001:db8::', - ] - path = '/v1/AUTH_account/c/o/' - expected_url = path + ('?temp_url_sig=temp_url_signature' - '&temp_url_expires=1400003600' - '&temp_url_ip_range=') - for ip_range in ip_ranges: - hmac_mock.reset_mock() - url = u.generate_temp_url(path, self.seconds, - self.key, self.method, - ip_range=ip_range) - key = self.key - if not isinstance(key, six.binary_type): - key = key.encode('utf-8') - - if isinstance(ip_range, six.binary_type): - ip_range_expected_url = ( - expected_url + ip_range.decode('utf-8') - ) - expected_body = '\n'.join([ - 'ip=' + ip_range.decode('utf-8'), - self.method, - '1400003600', - path, - ]).encode('utf-8') - else: - ip_range_expected_url = expected_url + ip_range - expected_body = '\n'.join([ - 'ip=' + ip_range, - self.method, - '1400003600', - path, - ]).encode('utf-8') - - self.assertEqual(url, ip_range_expected_url) - - self.assertEqual(hmac_mock.mock_calls, [ - mock.call(key, expected_body, sha1), - mock.call().hexdigest(), - ]) - self.assertIsInstance(url, type(path)) - - @mock.patch('hmac.HMAC') - def test_generate_temp_url_iso8601_argument(self, hmac_mock): - hmac_mock().hexdigest.return_value = 'temp_url_signature' - url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', - self.key, self.method) - self.assertEqual(url, self.expected_url) - - # Don't care about absolute arg. - url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', - self.key, self.method, absolute=True) - self.assertEqual(url, self.expected_url) - - lt = localtime() - expires = strftime(u.EXPIRES_ISO8601_FORMAT[:-1], lt) - - if not isinstance(self.expected_url, six.string_types): - expected_url = self.expected_url.replace( - b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) - else: - expected_url = self.expected_url.replace( - '1400003600', str(int(mktime(lt)))) - url = u.generate_temp_url(self.url, expires, - self.key, self.method) - self.assertEqual(url, expected_url) - - expires = strftime(u.SHORT_EXPIRES_ISO8601_FORMAT, lt) - lt = strptime(expires, u.SHORT_EXPIRES_ISO8601_FORMAT) - - if not isinstance(self.expected_url, six.string_types): - expected_url = self.expected_url.replace( - b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) - else: - expected_url = self.expected_url.replace( - '1400003600', str(int(mktime(lt)))) - url = u.generate_temp_url(self.url, expires, - self.key, self.method) - self.assertEqual(url, expected_url) - - @mock.patch('hmac.HMAC') - @mock.patch('time.time', return_value=1400000000) - def test_generate_temp_url_iso8601_output(self, time_mock, hmac_mock): - hmac_mock().hexdigest.return_value = 'temp_url_signature' - url = u.generate_temp_url(self.url, self.seconds, - self.key, self.method, - iso8601=True) - key = self.key - if not isinstance(key, six.binary_type): - key = key.encode('utf-8') - - expires = strftime(u.EXPIRES_ISO8601_FORMAT, gmtime(1400003600)) - if not isinstance(self.url, six.string_types): - self.assertTrue(url.endswith(bytes(expires, 'utf-8'))) - else: - self.assertTrue(url.endswith(expires)) - self.assertEqual(hmac_mock.mock_calls, [ - mock.call(), - mock.call(key, self.expected_body, sha1), - mock.call().hexdigest(), - ]) - self.assertIsInstance(url, type(self.url)) - - @mock.patch('hmac.HMAC') - @mock.patch('time.time', return_value=1400000000) - def test_generate_temp_url_prefix(self, time_mock, hmac_mock): - hmac_mock().hexdigest.return_value = 'temp_url_signature' - prefixes = ['', 'o', 'p0/p1/'] - for p in prefixes: - hmac_mock.reset_mock() - path = '/v1/AUTH_account/c/' + p - expected_url = path + ('?temp_url_sig=temp_url_signature' - '&temp_url_expires=1400003600' - '&temp_url_prefix=' + p) - expected_body = '\n'.join([ - self.method, - '1400003600', - 'prefix:' + path, - ]).encode('utf-8') - url = u.generate_temp_url(path, self.seconds, - self.key, self.method, prefix=True) - key = self.key - if not isinstance(key, six.binary_type): - key = key.encode('utf-8') - self.assertEqual(url, expected_url) - self.assertEqual(hmac_mock.mock_calls, [ - mock.call(key, expected_body, sha1), - mock.call().hexdigest(), - ]) - - self.assertIsInstance(url, type(path)) - - def test_generate_temp_url_invalid_path(self): - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url(b'/v1/a/c/\xff', self.seconds, self.key, - self.method) - self.assertEqual(exc_manager.exception.args[0], - 'path must be representable as UTF-8') - - @mock.patch('hmac.HMAC.hexdigest', return_value="temp_url_signature") - def test_generate_absolute_expiry_temp_url(self, hmac_mock): - if isinstance(self.expected_url, six.binary_type): - expected_url = self.expected_url.replace( - b'1400003600', b'2146636800') - else: - expected_url = self.expected_url.replace( - u'1400003600', u'2146636800') - url = u.generate_temp_url(self.url, 2146636800, self.key, self.method, - absolute=True) - self.assertEqual(url, expected_url) - - def test_generate_temp_url_bad_time(self): - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url(self.url, 'not_an_int', self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) - - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url(self.url, -1, self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) - - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url(self.url, 1.1, self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) - - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url(self.url, '-1', self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) - - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url(self.url, '1.1', self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url(self.url, '2015-05', self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) - - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url( - self.url, '2015-05-01T01:00', self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) - - def test_generate_temp_url_bad_path(self): - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url('/v1/a/c', 60, self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], - 'path must be full path to an object e.g. /v1/a/c/o') - - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url('v1/a/c/o', 60, self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], - 'path must be full path to an object e.g. /v1/a/c/o') - - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url('blah/v1/a/c/o', 60, self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], - 'path must be full path to an object e.g. /v1/a/c/o') - - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url('/v1//c/o', 60, self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], - 'path must be full path to an object e.g. /v1/a/c/o') - - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url('/v1/a/c/', 60, self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], - 'path must be full path to an object e.g. /v1/a/c/o') - - with self.assertRaises(ValueError) as exc_manager: - u.generate_temp_url('/v1/a/c', 60, self.key, self.method, - prefix=True) - self.assertEqual(exc_manager.exception.args[0], - 'path must at least contain /v1/a/c/') - - -class TestTempURLUnicodePathAndKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3' - key = u'k\u00e9y' - expected_url = (u'%s?temp_url_sig=temp_url_signature' - u'&temp_url_expires=1400003600') % url - expected_body = u'\n'.join([ - u'GET', - u'1400003600', - url, - ]).encode('utf-8') - - -class TestTempURLUnicodePathBytesKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3' - key = u'k\u00e9y'.encode('utf-8') - expected_url = (u'%s?temp_url_sig=temp_url_signature' - u'&temp_url_expires=1400003600') % url - expected_body = '\n'.join([ - u'GET', - u'1400003600', - url, - ]).encode('utf-8') - - -class TestTempURLBytesPathUnicodeKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') - key = u'k\u00e9y' - expected_url = url + (b'?temp_url_sig=temp_url_signature' - b'&temp_url_expires=1400003600') - expected_body = b'\n'.join([ - b'GET', - b'1400003600', - url, - ]) - - -class TestTempURLBytesPathAndKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') - key = u'k\u00e9y'.encode('utf-8') - expected_url = url + (b'?temp_url_sig=temp_url_signature' - b'&temp_url_expires=1400003600') - expected_body = b'\n'.join([ - b'GET', - b'1400003600', - url, - ]) - - -class TestTempURLBytesPathAndNonUtf8Key(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') - key = b'k\xffy' - expected_url = url + (b'?temp_url_sig=temp_url_signature' - b'&temp_url_expires=1400003600') - expected_body = b'\n'.join([ - b'GET', - b'1400003600', - url, - ]) - - -class TestReadableToIterable(unittest.TestCase): - - def test_iter(self): - chunk_size = 4 - write_data = tuple(x.encode() for x in ('a', 'b', 'c', 'd')) - actual_md5sum = md5() - - with tempfile.TemporaryFile() as f: - for x in write_data: - f.write(x * chunk_size) - actual_md5sum.update(x * chunk_size) - f.seek(0) - data = u.ReadableToIterable(f, chunk_size, True) - - for i, data_chunk in enumerate(data): - self.assertEqual(chunk_size, len(data_chunk)) - self.assertEqual(data_chunk, write_data[i] * chunk_size) - - self.assertEqual(actual_md5sum.hexdigest(), data.get_md5sum()) - - def test_md5_creation(self): - # Check creation with a real and noop md5 class - data = u.ReadableToIterable(None, None, md5=True) - self.assertEqual(md5().hexdigest(), data.get_md5sum()) - self.assertIs(type(md5()), type(data.md5sum)) - - data = u.ReadableToIterable(None, None, md5=False) - self.assertEqual('', data.get_md5sum()) - self.assertIs(u.NoopMD5, type(data.md5sum)) - - def test_unicode(self): - # Check no errors are raised if unicode data is feed in. - unicode_data = u'abc' - actual_md5sum = md5(unicode_data.encode()).hexdigest() - chunk_size = 2 - - with tempfile.TemporaryFile(mode='w+') as f: - f.write(unicode_data) - f.seek(0) - data = u.ReadableToIterable(f, chunk_size, True) - - x = next(data) - self.assertEqual(2, len(x)) - self.assertEqual(unicode_data[:2], x) - - x = next(data) - self.assertEqual(1, len(x)) - self.assertEqual(unicode_data[2:], x) - - self.assertEqual(actual_md5sum, data.get_md5sum()) - - -class TestLengthWrapper(unittest.TestCase): - - def test_stringio(self): - contents = six.StringIO(u'a' * 50 + u'b' * 50) - contents.seek(22) - data = u.LengthWrapper(contents, 42, True) - s = u'a' * 28 + u'b' * 14 - read_data = u''.join(iter(data.read, '')) - - self.assertEqual(42, len(data)) - self.assertEqual(42, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) - - data.reset() - self.assertEqual(md5().hexdigest(), data.get_md5sum()) - - read_data = u''.join(iter(data.read, '')) - self.assertEqual(42, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) - - def test_bytesio(self): - contents = six.BytesIO(b'a' * 50 + b'b' * 50) - contents.seek(22) - data = u.LengthWrapper(contents, 42, True) - s = b'a' * 28 + b'b' * 14 - read_data = b''.join(iter(data.read, '')) - - self.assertEqual(42, len(data)) - self.assertEqual(42, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) - - def test_tempfile(self): - with tempfile.NamedTemporaryFile(mode='wb') as f: - f.write(b'a' * 100) - f.flush() - contents = open(f.name, 'rb') - data = u.LengthWrapper(contents, 42, True) - s = b'a' * 42 - read_data = b''.join(iter(data.read, '')) - - self.assertEqual(42, len(data)) - self.assertEqual(42, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) - - def test_segmented_file(self): - with tempfile.NamedTemporaryFile(mode='wb') as f: - segment_length = 1024 - segments = ('a', 'b', 'c', 'd') - for c in segments: - f.write((c * segment_length).encode()) - f.flush() - for i, c in enumerate(segments): - contents = open(f.name, 'rb') - contents.seek(i * segment_length) - data = u.LengthWrapper(contents, segment_length, True) - read_data = b''.join(iter(data.read, '')) - s = (c * segment_length).encode() - - self.assertEqual(segment_length, len(data)) - self.assertEqual(segment_length, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) - - data.reset() - self.assertEqual(md5().hexdigest(), data.get_md5sum()) - read_data = b''.join(iter(data.read, '')) - self.assertEqual(segment_length, len(data)) - self.assertEqual(segment_length, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) - - -class TestGroupers(unittest.TestCase): - def test_n_at_a_time(self): - result = list(u.n_at_a_time(range(100), 9)) - self.assertEqual([9] * 11 + [1], list(map(len, result))) - - result = list(u.n_at_a_time(range(100), 10)) - self.assertEqual([10] * 10, list(map(len, result))) - - result = list(u.n_at_a_time(range(100), 11)) - self.assertEqual([11] * 9 + [1], list(map(len, result))) - - result = list(u.n_at_a_time(range(100), 12)) - self.assertEqual([12] * 8 + [4], list(map(len, result))) - - def test_n_groups(self): - result = list(u.n_groups(range(100), 9)) - self.assertEqual([12] * 8 + [4], list(map(len, result))) - - result = list(u.n_groups(range(100), 10)) - self.assertEqual([10] * 10, list(map(len, result))) - - result = list(u.n_groups(range(100), 11)) - self.assertEqual([10] * 10, list(map(len, result))) - - result = list(u.n_groups(range(100), 12)) - self.assertEqual([9] * 11 + [1], list(map(len, result))) - - -class TestApiResponeParser(unittest.TestCase): - - def test_utf8_default(self): - result = u.parse_api_response( - {}, u'{"test": "\u2603"}'.encode('utf8')) - self.assertEqual({'test': u'\u2603'}, result) - - result = u.parse_api_response( - {}, u'{"test": "\\u2603"}'.encode('utf8')) - self.assertEqual({'test': u'\u2603'}, result) - - def test_bad_json(self): - self.assertRaises(ValueError, u.parse_api_response, - {}, b'{"foo": "bar}') - - def test_bad_utf8(self): - self.assertRaises(UnicodeDecodeError, u.parse_api_response, - {}, b'{"foo": "b\xffr"}') - - def test_latin_1(self): - result = u.parse_api_response( - {'content-type': 'application/json; charset=iso8859-1'}, - b'{"t\xe9st": "\xff"}') - self.assertEqual({u't\xe9st': u'\xff'}, result) - - def test_gzipped_utf8(self): - buf = six.BytesIO() - gz = gzip.GzipFile(fileobj=buf, mode='w') - gz.write(u'{"test": "\u2603"}'.encode('utf8')) - gz.close() - result = u.parse_api_response( - {'content-encoding': 'gzip'}, - buf.getvalue()) - self.assertEqual({'test': u'\u2603'}, result) - - -class TestGetBody(unittest.TestCase): - - def test_not_gzipped(self): - result = u.parse_api_response( - {}, u'{"test": "\\u2603"}'.encode('utf8')) - self.assertEqual({'test': u'\u2603'}, result) - - def test_gzipped_body(self): - buf = six.BytesIO() - gz = gzip.GzipFile(fileobj=buf, mode='w') - gz.write(u'{"test": "\u2603"}'.encode('utf8')) - gz.close() - result = u.parse_api_response( - {'content-encoding': 'gzip'}, - buf.getvalue()) - self.assertEqual({'test': u'\u2603'}, result) - - -class JSONTracker(object): - def __init__(self, data): - self.data = data - self.calls = [] - - def __iter__(self): - for item in self.data: - self.calls.append(('read', item)) - yield item - - def write(self, s): - self.calls.append(('write', s)) - - -class TestJSONableIterable(unittest.TestCase): - def test_json_dump_iterencodes(self): - t = JSONTracker([1, 'fish', 2, 'fish']) - json.dump(u.JSONableIterable(t), t) - self.assertEqual(t.calls, [ - ('read', 1), - ('write', '[1'), - ('read', 'fish'), - ('write', ', "fish"'), - ('read', 2), - ('write', ', 2'), - ('read', 'fish'), - ('write', ', "fish"'), - ('write', ']'), - ]) - - def test_json_dump_empty_iter(self): - t = JSONTracker([]) - json.dump(u.JSONableIterable(t), t) - self.assertEqual(t.calls, [ - ('write', '[]'), - ]) diff -Nru python-swiftclient-3.8.1/tests/unit/utils.py python-swiftclient-3.9.0/tests/unit/utils.py --- python-swiftclient-3.8.1/tests/unit/utils.py 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/tests/unit/utils.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,582 +0,0 @@ -# Copyright (c) 2010-2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import functools -import sys -from requests import RequestException -from requests.structures import CaseInsensitiveDict -from time import sleep -import unittest -import mock -import six -import os -from six.moves import reload_module -from six.moves.urllib.parse import urlparse, ParseResult -from swiftclient import client as c -from swiftclient import shell as s -from swiftclient.utils import EMPTY_ETAG - - -def fake_get_auth_keystone(expected_os_options=None, exc=None, - storage_url='http://url/', token='token', - **kwargs): - def fake_get_auth_keystone(auth_url, - user, - key, - actual_os_options, **actual_kwargs): - if exc: - raise exc('test') - # TODO: some way to require auth_url, user and key? - if expected_os_options: - for key, value in actual_os_options.items(): - if value and value != expected_os_options.get(key): - return "", None - if 'required_kwargs' in kwargs: - for k, v in kwargs['required_kwargs'].items(): - if v != actual_kwargs.get(k): - return "", None - - if auth_url.startswith("https") and \ - auth_url.endswith("invalid-certificate") and \ - not actual_kwargs['insecure']: - from swiftclient import client as c - raise c.ClientException("invalid-certificate") - if auth_url.startswith("https") and \ - auth_url.endswith("self-signed-certificate") and \ - not actual_kwargs['insecure'] and \ - actual_kwargs['cacert'] is None: - from swiftclient import client as c - raise c.ClientException("unverified-certificate") - if auth_url.startswith("https") and \ - auth_url.endswith("client-certificate") and \ - not (actual_kwargs['cert'] and actual_kwargs['cert_key']): - from swiftclient import client as c - raise c.ClientException("noclient-certificate") - - return storage_url, token - return fake_get_auth_keystone - - -class StubResponse(object): - """ - Placeholder structure for use with fake_http_connect's code_iter to modify - response attributes (status, body, headers) on a per-request basis. - """ - - def __init__(self, status=200, body='', headers=None): - self.status = status - self.body = body - self.headers = headers or {} - - def __repr__(self): - return '%s(%r, %r, %r)' % (self.__class__.__name__, self.status, - self.body, self.headers) - - -def fake_http_connect(*code_iter, **kwargs): - """ - Generate a callable which yields a series of stubbed responses. Because - swiftclient will reuse an HTTP connection across pipelined requests it is - not always the case that this fake is used strictly for mocking an HTTP - connection, but rather each HTTP response (i.e. each call to requests - get_response). - """ - - class FakeConn(object): - - def __init__(self, status, etag=None, body='', timestamp='1', - headers=None): - self.status_code = self.status = status - self.reason = 'Fake' - self.scheme = 'http' - self.host = '1.2.3.4' - self.port = '1234' - self.sent = 0 - self.received = 0 - self.etag = etag - self.content = self.body = body - self.timestamp = timestamp - self.headers = headers or {} - self.request = None - - def getresponse(self): - if kwargs.get('raise_exc'): - raise Exception('test') - return self - - def getheaders(self): - if self.headers: - return self.headers.items() - headers = {'content-length': str(len(self.body)), - 'content-type': 'x-application/test', - 'x-timestamp': self.timestamp, - 'last-modified': self.timestamp, - 'x-object-meta-test': 'testing', - 'etag': - self.etag or '"%s"' % EMPTY_ETAG, - 'x-works': 'yes', - 'x-account-container-count': '12345'} - if not self.timestamp: - del headers['x-timestamp'] - try: - if next(container_ts_iter) is False: - headers['x-container-timestamp'] = '1' - except StopIteration: - pass - if 'slow' in kwargs: - headers['content-length'] = '4' - if 'headers' in kwargs: - headers.update(kwargs['headers']) - if 'auth_v1' in kwargs: - headers.update( - {'x-storage-url': 'storageURL', - 'x-auth-token': 'someauthtoken'}) - return headers.items() - - def read(self, amt=None): - if 'slow' in kwargs: - if self.sent < 4: - self.sent += 1 - sleep(0.1) - return ' ' - rv = self.body[:amt] - if amt is not None: - self.body = self.body[amt:] - else: - self.body = '' - return rv - - def send(self, amt=None): - if 'slow' in kwargs: - if self.received < 4: - self.received += 1 - sleep(0.1) - - def getheader(self, name, default=None): - return dict(self.getheaders()).get(name.lower(), default) - - def close(self): - pass - - timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) - etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) - x = kwargs.get('missing_container', [False] * len(code_iter)) - if not isinstance(x, (tuple, list)): - x = [x] * len(code_iter) - container_ts_iter = iter(x) - code_iter = iter(code_iter) - - def connect(*args, **ckwargs): - if 'give_content_type' in kwargs: - if len(args) >= 7 and 'Content-Type' in args[6]: - kwargs['give_content_type'](args[6]['Content-Type']) - else: - kwargs['give_content_type']('') - if 'give_connect' in kwargs: - kwargs['give_connect'](*args, **ckwargs) - status = next(code_iter) - if isinstance(status, StubResponse): - fake_conn = FakeConn(status.status, body=status.body, - headers=status.headers) - else: - etag = next(etag_iter) - timestamp = next(timestamps_iter) - fake_conn = FakeConn(status, etag, body=kwargs.get('body', ''), - timestamp=timestamp) - if fake_conn.status <= 0: - raise RequestException() - return fake_conn - - connect.code_iter = code_iter - return connect - - -class MockHttpTest(unittest.TestCase): - - def setUp(self): - super(MockHttpTest, self).setUp() - self.fake_connect = None - self.request_log = [] - - # Capture output, since the test-runner stdout/stderr monkey-patching - # won't cover the references to sys.stdout/sys.stderr in - # swiftclient.multithreading - self.capture_output = CaptureOutput() - if 'SWIFTCLIENT_DEBUG' not in os.environ: - self.capture_output.__enter__() - self.addCleanup(self.capture_output.__exit__) - - # since we're going to steal all stderr output globally; we should - # give the developer an escape hatch or risk scorn - def blowup_but_with_the_helpful(*args, **kwargs): - raise Exception( - "You tried to enter a debugger while stderr is " - "patched, you need to set SWIFTCLIENT_DEBUG=1 " - "and try again") - import pdb - pdb.set_trace = blowup_but_with_the_helpful - - def fake_http_connection(*args, **kwargs): - self.validateMockedRequestsConsumed() - self.request_log = [] - self.fake_connect = fake_http_connect(*args, **kwargs) - _orig_http_connection = c.http_connection - query_string = kwargs.get('query_string') - storage_url = kwargs.get('storage_url') - auth_token = kwargs.get('auth_token') - exc = kwargs.get('exc') - on_request = kwargs.get('on_request') - - def wrapper(url, proxy=None, cacert=None, insecure=False, - cert=None, cert_key=None, - ssl_compression=True, timeout=None): - if storage_url: - self.assertEqual(storage_url, url) - - parsed, _conn = _orig_http_connection(url, proxy=proxy) - - class RequestsWrapper(object): - def close(self): - pass - conn = RequestsWrapper() - - def request(method, path, *args, **kwargs): - try: - conn.resp = self.fake_connect() - except StopIteration: - self.fail('Unexpected %s request for %s' % ( - method, path)) - self.request_log.append((parsed, method, path, args, - kwargs, conn.resp)) - conn.host = conn.resp.host - conn.resp.request = RequestsWrapper() - conn.resp.request.url = '%s://%s%s' % ( - conn.resp.scheme, conn.resp.host, path) - conn.resp.has_been_read = False - _orig_read = conn.resp.read - - def read(*args, **kwargs): - conn.resp.has_been_read = True - return _orig_read(*args, **kwargs) - conn.resp.read = read - if on_request: - status = on_request(method, path, *args, **kwargs) - conn.resp.status = status - if auth_token: - headers = args[1] - self.assertEqual(auth_token, - headers.get('X-Auth-Token')) - if query_string: - self.assertTrue(path.endswith('?' + query_string)) - if path.endswith('invalid_cert') and not insecure: - from swiftclient import client as c - raise c.ClientException("invalid_certificate") - if exc: - raise exc - return conn.resp - - def putrequest(path, data=None, headers=None, **kwargs): - request('PUT', path, data, headers, **kwargs) - - conn.request = request - conn.putrequest = putrequest - - def getresponse(): - return conn.resp - conn.getresponse = getresponse - - return parsed, conn - return wrapper - self.fake_http_connection = fake_http_connection - - def iter_request_log(self): - for parsed, method, path, args, kwargs, resp in self.request_log: - parts = parsed._asdict() - parts['path'] = path - full_path = ParseResult(**parts).geturl() - args = list(args) - log = dict(zip(('body', 'headers'), args)) - log.update({ - 'method': method, - 'full_path': full_path, - 'parsed_path': urlparse(full_path), - 'path': path, - 'headers': CaseInsensitiveDict(log.get('headers')), - 'resp': resp, - 'status': resp.status, - }) - yield log - - orig_assertEqual = unittest.TestCase.assertEqual - - def assert_request_equal(self, expected, real_request): - method, path = expected[:2] - if urlparse(path).scheme: - match_path = real_request['full_path'] - else: - match_path = real_request['path'] - self.assertEqual((method, path), (real_request['method'], - match_path)) - if len(expected) > 2: - body = expected[2] - real_request['expected'] = body - err_msg = 'Body mismatch for %(method)s %(path)s, ' \ - 'expected %(expected)r, and got %(body)r' % real_request - self.orig_assertEqual(body, real_request['body'], err_msg) - - if len(expected) > 3: - headers = CaseInsensitiveDict(expected[3]) - for key, value in headers.items(): - real_request['key'] = key - real_request['expected_value'] = value - real_request['value'] = real_request['headers'].get(key) - err_msg = ( - 'Header mismatch on %(key)r, ' - 'expected %(expected_value)r and got %(value)r ' - 'for %(method)s %(path)s %(headers)r' % real_request) - self.orig_assertEqual(value, real_request['value'], - err_msg) - real_request['extra_headers'] = dict( - (key, value) for key, value in real_request['headers'].items() - if key not in headers) - if real_request['extra_headers']: - self.fail('Received unexpected headers for %(method)s ' - '%(path)s, got %(extra_headers)r' % real_request) - - def assertRequests(self, expected_requests): - """ - Make sure some requests were made like you expected, provide a list of - expected requests, typically in the form of [(method, path), ...] - or [(method, path, body, headers), ...] - """ - real_requests = self.iter_request_log() - for expected in expected_requests: - real_request = next(real_requests) - self.assert_request_equal(expected, real_request) - try: - real_request = next(real_requests) - except StopIteration: - pass - else: - self.fail('At least one extra request received: %r' % - real_request) - - def assert_request(self, expected_request): - """ - Make sure a request was made as expected. Provide the - expected request in the form of [(method, path), ...] - """ - real_requests = self.iter_request_log() - for real_request in real_requests: - try: - self.assert_request_equal(expected_request, real_request) - break - except AssertionError: - pass - else: - raise AssertionError( - "Expected request %s not found in actual requests %s" - % (expected_request, self.request_log) - ) - - def validateMockedRequestsConsumed(self): - if not self.fake_connect: - return - unused_responses = list(self.fake_connect.code_iter) - if unused_responses: - self.fail('Unused responses %r' % (unused_responses,)) - - def tearDown(self): - self.validateMockedRequestsConsumed() - super(MockHttpTest, self).tearDown() - # TODO: this nuke from orbit clean up seems to be encouraging - # un-hygienic mocking on the swiftclient.client module; which may lead - # to some unfortunate test order dependency bugs by way of the broken - # window theory if any other modules are similarly patched - reload_module(c) - - -class CaptureStreamPrinter(object): - """ - CaptureStreamPrinter is used for testing unicode writing for PY3. Anything - written here is encoded as utf-8 and written to the parent CaptureStream - """ - def __init__(self, captured_stream): - self._captured_stream = captured_stream - - def write(self, data): - # No encoding, just convert the raw bytes into a str for testing - # The below call also validates that we have a byte string. - self._captured_stream.write( - data if isinstance(data, six.binary_type) else data.encode('utf8')) - - -class CaptureStream(object): - - def __init__(self, stream): - self.stream = stream - self._buffer = six.BytesIO() - self._capture = CaptureStreamPrinter(self._buffer) - self.streams = [self._capture] - - @property - def buffer(self): - if six.PY3: - return self._buffer - else: - raise AttributeError( - 'Output stream has no attribute "buffer" in Python2') - - def flush(self): - pass - - def write(self, *args, **kwargs): - for stream in self.streams: - stream.write(*args, **kwargs) - - def writelines(self, *args, **kwargs): - for stream in self.streams: - stream.writelines(*args, **kwargs) - - def getvalue(self): - return self._buffer.getvalue() - - def clear(self): - self._buffer.truncate(0) - self._buffer.seek(0) - - -class CaptureOutput(object): - - def __init__(self, suppress_systemexit=False): - self._out = CaptureStream(sys.stdout) - self._err = CaptureStream(sys.stderr) - self.patchers = [] - - WrappedOutputManager = functools.partial(s.OutputManager, - print_stream=self._out, - error_stream=self._err) - - if suppress_systemexit: - self.patchers += [ - mock.patch('swiftclient.shell.OutputManager.get_error_count', - return_value=0) - ] - - self.patchers += [ - mock.patch('swiftclient.shell.OutputManager', - WrappedOutputManager), - mock.patch('sys.stdout', self._out), - mock.patch('sys.stderr', self._err), - ] - - def __enter__(self): - for patcher in self.patchers: - patcher.start() - return self - - def __exit__(self, *args, **kwargs): - for patcher in self.patchers: - patcher.stop() - - @property - def out(self): - return self._out.getvalue().decode('utf8') - - @property - def err(self): - return self._err.getvalue().decode('utf8') - - def clear(self): - self._out.clear() - self._err.clear() - - # act like the string captured by stdout - - def __str__(self): - return self.out - - def __len__(self): - return len(self.out) - - def __eq__(self, other): - return self.out == other - - def __ne__(self, other): - return not self.__eq__(other) - - def __getattr__(self, name): - return getattr(self.out, name) - - -class FakeKeystone(object): - ''' - Fake keystone client module. Returns given endpoint url and auth token. - ''' - def __init__(self, endpoint, token): - self.calls = [] - self.auth_version = None - self.endpoint = endpoint - self.token = token - - class _Client(object): - def __init__(self, endpoint, auth_token, **kwargs): - self.auth_token = auth_token - self.endpoint = endpoint - self.service_catalog = self.ServiceCatalog(endpoint) - - class ServiceCatalog(object): - def __init__(self, endpoint): - self.calls = [] - self.endpoint_url = endpoint - - def url_for(self, **kwargs): - self.calls.append(kwargs) - return self.endpoint_url - - def Client(self, **kwargs): - self.calls.append(kwargs) - self.client = self._Client( - endpoint=self.endpoint, auth_token=self.token, **kwargs) - return self.client - - class Unauthorized(Exception): - pass - - class AuthorizationFailure(Exception): - pass - - class EndpointNotFound(Exception): - pass - - -class FakeStream(object): - def __init__(self, size): - self.bytes_read = 0 - self.size = size - - def read(self, size=-1): - if self.bytes_read == self.size: - return b'' - - if size == -1 or size + self.bytes_read > self.size: - remaining = self.size - self.bytes_read - self.bytes_read = self.size - return b'A' * remaining - - self.bytes_read += size - return b'A' * size - - def __len__(self): - return self.size diff -Nru python-swiftclient-3.8.1/tox.ini python-swiftclient-3.9.0/tox.ini --- python-swiftclient-3.8.1/tox.ini 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/tox.ini 2020-02-13 17:31:20.000000000 +0000 @@ -24,7 +24,7 @@ [testenv:pep8] basepython = python3 commands = - python -m flake8 swiftclient tests + python -m flake8 swiftclient test [testenv:venv] basepython = python3 @@ -44,7 +44,7 @@ [testenv:func] basepython = python3 setenv = - OS_TEST_PATH=tests.functional + OS_TEST_PATH=test.functional PYTHON=coverage run --source swiftclient --parallel-mode whitelist_externals = coverage diff -Nru python-swiftclient-3.8.1/.unittests python-swiftclient-3.9.0/.unittests --- python-swiftclient-3.8.1/.unittests 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/.unittests 2020-02-13 17:31:20.000000000 +0000 @@ -1,7 +1,7 @@ #!/bin/bash set -e -python setup.py testr --coverage --testr-args="tests.unit" +python setup.py testr --coverage --testr-args="test.unit" RET=$? coverage report -m rm -f .coverage diff -Nru python-swiftclient-3.8.1/.zuul.yaml python-swiftclient-3.9.0/.zuul.yaml --- python-swiftclient-3.8.1/.zuul.yaml 2019-09-13 21:39:55.000000000 +0000 +++ python-swiftclient-3.9.0/.zuul.yaml 2020-02-13 17:31:20.000000000 +0000 @@ -39,7 +39,7 @@ - openstack-lower-constraints-jobs - openstack-pypy-jobs-nonvoting - openstack-python-jobs - - openstack-python3-train-jobs + - openstack-python3-ussuri-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: @@ -47,11 +47,15 @@ - swiftclient-swift-functional - swiftclient-functional - swiftclient-functional-py2 + - openstack-tox-py38: + voting: true gate: jobs: - swiftclient-swift-functional - swiftclient-functional - swiftclient-functional-py2 + - openstack-tox-py38: + voting: true post: jobs: - openstack-tox-cover