diff -Nru swift-2.25.1/bin/swift-container-info swift-2.25.2/bin/swift-container-info --- swift-2.25.1/bin/swift-container-info 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/bin/swift-container-info 2021-09-01 08:44:03.000000000 +0000 @@ -42,6 +42,10 @@ parser.add_option( '--drop-prefixes', default=False, action="store_true", help="When outputting metadata, drop the per-section common prefixes") + parser.add_option( + '-v', '--verbose', default=False, action="store_true", + help="Show all shard ranges. By default, only the number of shard " + "ranges is displayed if there are many shards.") options, args = parser.parse_args() diff -Nru swift-2.25.1/bin/swift-container-sharder swift-2.25.2/bin/swift-container-sharder --- swift-2.25.1/bin/swift-container-sharder 2020-10-09 18:48:13.000000000 +0000 +++ swift-2.25.2/bin/swift-container-sharder 2021-09-01 08:44:03.000000000 +0000 @@ -29,5 +29,9 @@ help='Shard containers only in given partitions. ' 'Comma-separated list. ' 'Only has effect if --once is used.') + parser.add_option('--no-auto-shard', action='store_false', + dest='auto_shard', default=None, + help='Disable auto-sharding. Overrides the auto_shard ' + 'value in the config file.') conf_file, options = parse_options(parser=parser, once=True) run_daemon(ContainerSharder, conf_file, **options) diff -Nru swift-2.25.1/debian/changelog swift-2.25.2/debian/changelog --- swift-2.25.1/debian/changelog 2020-10-19 18:33:46.000000000 +0000 +++ swift-2.25.2/debian/changelog 2021-09-17 10:05:09.000000000 +0000 @@ -1,3 +1,9 @@ +swift (2.25.2-0ubuntu1) focal; urgency=medium + + * New stable point release for OpenStack Ussuri (LP: #1943712). + + -- Chris MacNaughton Fri, 17 Sep 2021 10:05:09 +0000 + swift (2.25.1-0ubuntu1) focal; urgency=medium [ Chris MacNaughton ] diff -Nru swift-2.25.1/etc/memcache.conf-sample swift-2.25.2/etc/memcache.conf-sample --- swift-2.25.1/etc/memcache.conf-sample 2020-10-09 18:48:13.000000000 +0000 +++ swift-2.25.2/etc/memcache.conf-sample 2021-09-01 08:44:03.000000000 +0000 @@ -26,3 +26,25 @@ # tries = 3 # Timeout for read and writes # io_timeout = 2.0 +# +# (Optional) Global toggle for TLS usage when comunicating with +# the caching servers. +# tls_enabled = false +# +# (Optional) Path to a file of concatenated CA certificates in PEM +# format necessary to establish the caching server's authenticity. +# If tls_enabled is False, this option is ignored. +# tls_cafile = +# +# (Optional) Path to a single file in PEM format containing the +# client's certificate as well as any number of CA certificates +# needed to establish the certificate's authenticity. This file +# is only required when client side authentication is necessary. +# If tls_enabled is False, this option is ignored. +# tls_certfile = +# +# (Optional) Path to a single file containing the client's private +# key in. Otherwhise the private key will be taken from the file +# specified in tls_certfile. If tls_enabled is False, this option +# is ignored. +# tls_keyfile = diff -Nru swift-2.25.1/etc/proxy-server.conf-sample swift-2.25.2/etc/proxy-server.conf-sample --- swift-2.25.1/etc/proxy-server.conf-sample 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/etc/proxy-server.conf-sample 2021-09-01 08:44:03.000000000 +0000 @@ -671,6 +671,10 @@ # Sets the maximum number of connections to each memcached server per worker # memcache_max_connections = 2 # +# (Optional) Global toggle for TLS usage when comunicating with +# the caching servers. +# tls_enabled = +# # More options documented in memcache.conf-sample [filter:ratelimit] diff -Nru swift-2.25.1/PKG-INFO swift-2.25.2/PKG-INFO --- swift-2.25.1/PKG-INFO 2020-10-09 18:49:44.000000000 +0000 +++ swift-2.25.2/PKG-INFO 2021-09-01 08:44:41.733701500 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: swift -Version: 2.25.1 +Version: 2.25.2 Summary: OpenStack Object Storage Home-page: https://docs.openstack.org/swift/latest/ Author: OpenStack diff -Nru swift-2.25.1/swift/cli/info.py swift-2.25.2/swift/cli/info.py --- swift-2.25.1/swift/cli/info.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/swift/cli/info.py 2021-09-01 08:44:03.000000000 +0000 @@ -16,6 +16,7 @@ import os import sqlite3 from hashlib import md5 +from collections import defaultdict from six.moves import urllib @@ -193,7 +194,8 @@ 'real value is set in the config file on each storage node.') -def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False): +def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False, + verbose=False): """ print out data base info/metadata based on its type @@ -307,20 +309,31 @@ print(' Type: %s' % shard_type) print(' State: %s' % info['db_state']) if info.get('shard_ranges'): - print('Shard Ranges (%d):' % len(info['shard_ranges'])) + num_shards = len(info['shard_ranges']) + print('Shard Ranges (%d):' % num_shards) + count_by_state = defaultdict(int) for srange in info['shard_ranges']: - srange = dict(srange, state_text=srange.state_text) - print(' Name: %(name)s' % srange) - print(' lower: %(lower)r, upper: %(upper)r' % srange) - print(' Object Count: %(object_count)d, Bytes Used: ' - '%(bytes_used)d, State: %(state_text)s (%(state)d)' - % srange) - print(' Created at: %s (%s)' - % (Timestamp(srange['timestamp']).isoformat, - srange['timestamp'])) - print(' Meta Timestamp: %s (%s)' - % (Timestamp(srange['meta_timestamp']).isoformat, - srange['meta_timestamp'])) + count_by_state[(srange.state, srange.state_text)] += 1 + print(' States:') + for key_state, count in sorted(count_by_state.items()): + key, state = key_state + print(' %9s: %s' % (state, count)) + if verbose: + for srange in info['shard_ranges']: + srange = dict(srange, state_text=srange.state_text) + print(' Name: %(name)s' % srange) + print(' lower: %(lower)r, upper: %(upper)r' % srange) + print(' Object Count: %(object_count)d, Bytes Used: ' + '%(bytes_used)d, State: %(state_text)s (%(state)d)' + % srange) + print(' Created at: %s (%s)' + % (Timestamp(srange['timestamp']).isoformat, + srange['timestamp'])) + print(' Meta Timestamp: %s (%s)' + % (Timestamp(srange['meta_timestamp']).isoformat, + srange['meta_timestamp'])) + else: + print('(Use -v/--verbose to show more Shard Ranges details)') def print_obj_metadata(metadata, drop_prefixes=False): @@ -418,7 +431,7 @@ def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False, - drop_prefixes=False): + drop_prefixes=False, verbose=False): if db_type not in ('account', 'container'): print("Unrecognized DB type: internal error") raise InfoSystemExit() @@ -450,7 +463,8 @@ sranges = broker.get_shard_ranges() if sranges: info['shard_ranges'] = sranges - print_db_info_metadata(db_type, info, broker.metadata, drop_prefixes) + print_db_info_metadata( + db_type, info, broker.metadata, drop_prefixes, verbose) try: ring = Ring(swift_dir, ring_name=db_type) except Exception: diff -Nru swift-2.25.1/swift/cli/manage_shard_ranges.py swift-2.25.2/swift/cli/manage_shard_ranges.py --- swift-2.25.1/swift/cli/manage_shard_ranges.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/swift/cli/manage_shard_ranges.py 2021-09-01 08:44:03.000000000 +0000 @@ -343,7 +343,9 @@ # Crank up the timeout in an effort to *make sure* this succeeds with broker.updated_timeout(max(timeout, args.replace_timeout)): - delete_shard_ranges(broker, args) + delete_status = delete_shard_ranges(broker, args) + if delete_status != 0: + return delete_status broker.merge_shard_ranges(shard_ranges) print('Injected %d shard ranges.' % len(shard_ranges)) diff -Nru swift-2.25.1/swift/common/memcached.py swift-2.25.2/swift/common/memcached.py --- swift-2.25.1/swift/common/memcached.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/swift/common/memcached.py 2021-09-01 08:44:03.000000000 +0000 @@ -128,11 +128,12 @@ :func:`swift.common.utils.parse_socket_string` for details. """ - def __init__(self, server, size, connect_timeout): + def __init__(self, server, size, connect_timeout, tls_context=None): Pool.__init__(self, max_size=size) self.host, self.port = utils.parse_socket_string( server, DEFAULT_MEMCACHED_PORT) self._connect_timeout = connect_timeout + self._tls_context = tls_context def create(self): addrs = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, @@ -142,6 +143,9 @@ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) with Timeout(self._connect_timeout): sock.connect(sockaddr) + if self._tls_context: + sock = self._tls_context.wrap_socket(sock, + server_hostname=self.host) return (sock.makefile('rwb'), sock) def get(self): @@ -160,7 +164,7 @@ def __init__(self, servers, connect_timeout=CONN_TIMEOUT, io_timeout=IO_TIMEOUT, pool_timeout=POOL_TIMEOUT, tries=TRY_COUNT, allow_pickle=False, allow_unpickle=False, - max_conns=2): + max_conns=2, tls_context=None): self._ring = {} self._errors = dict(((serv, []) for serv in servers)) self._error_limited = dict(((serv, 0) for serv in servers)) @@ -169,10 +173,10 @@ self._ring[md5hash('%s-%s' % (server, i))] = server self._tries = tries if tries <= len(servers) else len(servers) self._sorted = sorted(self._ring) - self._client_cache = dict(((server, - MemcacheConnPool(server, max_conns, - connect_timeout)) - for server in servers)) + self._client_cache = dict(( + (server, MemcacheConnPool(server, max_conns, connect_timeout, + tls_context=tls_context)) + for server in servers)) self._connect_timeout = connect_timeout self._io_timeout = io_timeout self._pool_timeout = pool_timeout diff -Nru swift-2.25.1/swift/common/middleware/memcache.py swift-2.25.2/swift/common/middleware/memcache.py --- swift-2.25.1/swift/common/middleware/memcache.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/swift/common/middleware/memcache.py 2021-09-01 08:44:03.000000000 +0000 @@ -15,10 +15,12 @@ import os +from eventlet.green import ssl from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError from swift.common.memcached import (MemcacheRing, CONN_TIMEOUT, POOL_TIMEOUT, IO_TIMEOUT, TRY_COUNT) +from swift.common.utils import config_true_value class MemcacheMiddleware(object): @@ -84,6 +86,17 @@ 'pool_timeout', POOL_TIMEOUT)) tries = int(memcache_options.get('tries', TRY_COUNT)) io_timeout = float(memcache_options.get('io_timeout', IO_TIMEOUT)) + if config_true_value(memcache_options.get('tls_enabled', 'false')): + tls_cafile = memcache_options.get('tls_cafile') + tls_certfile = memcache_options.get('tls_certfile') + tls_keyfile = memcache_options.get('tls_keyfile') + self.tls_context = ssl.create_default_context( + cafile=tls_cafile) + if tls_certfile: + self.tls_context.load_cert_chain(tls_certfile, + tls_keyfile) + else: + self.tls_context = None if not self.memcache_servers: self.memcache_servers = '127.0.0.1:11211' @@ -102,7 +115,8 @@ io_timeout=io_timeout, allow_pickle=(serialization_format == 0), allow_unpickle=(serialization_format <= 1), - max_conns=max_conns) + max_conns=max_conns, + tls_context=self.tls_context) def __call__(self, env, start_response): env['swift.cache'] = self.memcache diff -Nru swift-2.25.1/swift/common/middleware/symlink.py swift-2.25.2/swift/common/middleware/symlink.py --- swift-2.25.1/swift/common/middleware/symlink.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/swift/common/middleware/symlink.py 2021-09-01 08:44:03.000000000 +0000 @@ -421,7 +421,7 @@ resp = self._app_call(req.environ) response_header_dict = HeaderKeyDict(self._response_headers) symlink_sysmeta_to_usermeta(response_header_dict) - self._response_headers = response_header_dict.items() + self._response_headers = list(response_header_dict.items()) return resp def handle_get_head(self, req): diff -Nru swift-2.25.1/swift/common/middleware/tempurl.py swift-2.25.2/swift/common/middleware/tempurl.py --- swift-2.25.1/swift/common/middleware/tempurl.py 2020-10-09 18:48:13.000000000 +0000 +++ swift-2.25.2/swift/common/middleware/tempurl.py 2021-09-01 08:44:03.000000000 +0000 @@ -839,7 +839,7 @@ if h.startswith(p): del headers[h] break - return headers.items() + return list(headers.items()) def filter_factory(global_conf, **local_conf): diff -Nru swift-2.25.1/swift/common/utils.py swift-2.25.2/swift/common/utils.py --- swift-2.25.1/swift/common/utils.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/swift/common/utils.py 2021-09-01 08:44:03.000000000 +0000 @@ -48,6 +48,7 @@ from copy import deepcopy from optparse import OptionParser import traceback +import warnings from tempfile import gettempdir, mkstemp, NamedTemporaryFile import glob @@ -4837,6 +4838,8 @@ value. :param epoch: optional epoch timestamp which represents the time at which sharding was enabled for a container. + :param reported: optional indicator that this shard and its stats have + been reported to the root container. """ FOUND = 10 CREATED = 20 @@ -4845,13 +4848,15 @@ SHRINKING = 50 SHARDING = 60 SHARDED = 70 + SHRUNK = 80 STATES = {FOUND: 'found', CREATED: 'created', CLEAVED: 'cleaved', ACTIVE: 'active', SHRINKING: 'shrinking', SHARDING: 'sharding', - SHARDED: 'sharded'} + SHARDED: 'sharded', + SHRUNK: 'shrunk'} STATES_BY_NAME = dict((v, k) for k, v in STATES.items()) class OuterBound(object): @@ -4887,7 +4892,8 @@ def __init__(self, name, timestamp, lower=MIN, upper=MAX, object_count=0, bytes_used=0, meta_timestamp=None, - deleted=False, state=None, state_timestamp=None, epoch=None): + deleted=False, state=None, state_timestamp=None, epoch=None, + reported=False): self.account = self.container = self._timestamp = \ self._meta_timestamp = self._state_timestamp = self._epoch = None self._lower = ShardRange.MIN @@ -4906,6 +4912,14 @@ self.state = self.FOUND if state is None else state self.state_timestamp = state_timestamp self.epoch = epoch + self.reported = reported + + @classmethod + def sort_key(cls, sr): + # defines the sort order for shard ranges + # note if this ever changes to *not* sort by upper first then it breaks + # a key assumption for bisect, which is used by utils.find_shard_range + return sr.upper, sr.state, sr.lower, sr.name @classmethod def _encode(cls, value): @@ -5009,8 +5023,10 @@ @lower.setter def lower(self, value): - if value in (None, b'', u''): - value = ShardRange.MIN + with warnings.catch_warnings(): + warnings.simplefilter('ignore', UnicodeWarning) + if value in (None, b'', u''): + value = ShardRange.MIN try: value = self._encode_bound(value) except TypeError as err: @@ -5035,8 +5051,10 @@ @upper.setter def upper(self, value): - if value in (None, b'', u''): - value = ShardRange.MAX + with warnings.catch_warnings(): + warnings.simplefilter('ignore', UnicodeWarning) + if value in (None, b'', u''): + value = ShardRange.MAX try: value = self._encode_bound(value) except TypeError as err: @@ -5082,8 +5100,14 @@ cast to an int, or if meta_timestamp is neither None nor can be cast to a :class:`~swift.common.utils.Timestamp`. """ - self.object_count = int(object_count) - self.bytes_used = int(bytes_used) + if self.object_count != int(object_count): + self.object_count = int(object_count) + self.reported = False + + if self.bytes_used != int(bytes_used): + self.bytes_used = int(bytes_used) + self.reported = False + if meta_timestamp is None: self.meta_timestamp = Timestamp.now() else: @@ -5164,6 +5188,14 @@ def epoch(self, epoch): self._epoch = self._to_timestamp(epoch) + @property + def reported(self): + return self._reported + + @reported.setter + def reported(self, value): + self._reported = bool(value) + def update_state(self, state, state_timestamp=None): """ Set state to the given value and optionally update the state_timestamp @@ -5180,6 +5212,7 @@ self.state = state if state_timestamp is not None: self.state_timestamp = state_timestamp + self.reported = False return True @property @@ -5302,6 +5335,7 @@ yield 'state', self.state yield 'state_timestamp', self.state_timestamp.internal yield 'epoch', self.epoch.internal if self.epoch is not None else None + yield 'reported', 1 if self.reported else 0 def copy(self, timestamp=None, **kwargs): """ @@ -5333,7 +5367,8 @@ params['name'], params['timestamp'], params['lower'], params['upper'], params['object_count'], params['bytes_used'], params['meta_timestamp'], params['deleted'], params['state'], - params['state_timestamp'], params['epoch']) + params['state_timestamp'], params['epoch'], + params.get('reported', 0)) def find_shard_range(item, ranges): diff -Nru swift-2.25.1/swift/common/wsgi.py swift-2.25.2/swift/common/wsgi.py --- swift-2.25.1/swift/common/wsgi.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/swift/common/wsgi.py 2021-09-01 08:44:03.000000000 +0000 @@ -1452,7 +1452,8 @@ 'SERVER_PROTOCOL', 'swift.cache', 'swift.source', 'swift.trans_id', 'swift.authorize_override', 'swift.authorize', 'HTTP_X_USER_ID', 'HTTP_X_PROJECT_ID', - 'HTTP_REFERER', 'swift.infocache'): + 'HTTP_REFERER', 'swift.infocache', + 'swift.shard_listing_history'): if name in env: newenv[name] = env[name] if method: diff -Nru swift-2.25.1/swift/container/backend.py swift-2.25.2/swift/container/backend.py --- swift-2.25.1/swift/container/backend.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/swift/container/backend.py 2021-09-01 08:44:03.000000000 +0000 @@ -60,7 +60,7 @@ # tuples and vice-versa SHARD_RANGE_KEYS = ('name', 'timestamp', 'lower', 'upper', 'object_count', 'bytes_used', 'meta_timestamp', 'deleted', 'state', - 'state_timestamp', 'epoch') + 'state_timestamp', 'epoch', 'reported') POLICY_STAT_TABLE_CREATE = ''' CREATE TABLE policy_stat ( @@ -267,6 +267,7 @@ if existing['timestamp'] < shard_data['timestamp']: # note that currently we do not roll forward any meta or state from # an item that was created at older time, newer created time trumps + shard_data['reported'] = 0 # reset the latch return True elif existing['timestamp'] > shard_data['timestamp']: return False @@ -283,6 +284,18 @@ else: new_content = True + # We can latch the reported flag + if existing['reported'] and \ + existing['object_count'] == shard_data['object_count'] and \ + existing['bytes_used'] == shard_data['bytes_used'] and \ + existing['state'] == shard_data['state'] and \ + existing['epoch'] == shard_data['epoch']: + shard_data['reported'] = 1 + else: + shard_data.setdefault('reported', 0) + if shard_data['reported'] and not existing['reported']: + new_content = True + if (existing['state_timestamp'] == shard_data['state_timestamp'] and shard_data['state'] > existing['state']): new_content = True @@ -396,7 +409,8 @@ own_shard_range = self.get_own_shard_range() if own_shard_range.state in (ShardRange.SHARDING, ShardRange.SHRINKING, - ShardRange.SHARDED): + ShardRange.SHARDED, + ShardRange.SHRUNK): return bool(self.get_shard_ranges()) return False @@ -595,7 +609,8 @@ deleted INTEGER DEFAULT 0, state INTEGER, state_timestamp TEXT, - epoch TEXT + epoch TEXT, + reported INTEGER DEFAULT 0 ); """ % SHARD_RANGE_TABLE) @@ -1428,10 +1443,13 @@ # sqlite3.OperationalError: cannot start a transaction # within a transaction conn.rollback() - if ('no such table: %s' % SHARD_RANGE_TABLE) not in str(err): - raise - self.create_shard_range_table(conn) - return _really_merge_items(conn) + if 'no such column: reported' in str(err): + self._migrate_add_shard_range_reported(conn) + return _really_merge_items(conn) + if ('no such table: %s' % SHARD_RANGE_TABLE) in str(err): + self.create_shard_range_table(conn) + return _really_merge_items(conn) + raise def get_reconciler_sync(self): with self.get() as conn: @@ -1579,6 +1597,17 @@ CONTAINER_STAT_VIEW_SCRIPT + 'COMMIT;') + def _migrate_add_shard_range_reported(self, conn): + """ + Add the reported column to the 'shard_range' table. + """ + conn.executescript(''' + BEGIN; + ALTER TABLE %s + ADD COLUMN reported INTEGER DEFAULT 0; + COMMIT; + ''' % SHARD_RANGE_TABLE) + def _reclaim_other_stuff(self, conn, age_timestamp, sync_timestamp): super(ContainerBroker, self)._reclaim_other_stuff( conn, age_timestamp, sync_timestamp) @@ -1628,7 +1657,7 @@ elif states is not None: included_states.add(states) - def do_query(conn): + def do_query(conn, use_reported_column=True): condition = '' conditions = [] params = [] @@ -1646,21 +1675,27 @@ params.append(self.path) if conditions: condition = ' WHERE ' + ' AND '.join(conditions) + if use_reported_column: + columns = SHARD_RANGE_KEYS + else: + columns = SHARD_RANGE_KEYS[:-1] + ('0 as reported', ) sql = ''' SELECT %s FROM %s%s; - ''' % (', '.join(SHARD_RANGE_KEYS), SHARD_RANGE_TABLE, condition) + ''' % (', '.join(columns), SHARD_RANGE_TABLE, condition) data = conn.execute(sql, params) data.row_factory = None return [row for row in data] - try: - with self.maybe_get(connection) as conn: + with self.maybe_get(connection) as conn: + try: return do_query(conn) - except sqlite3.OperationalError as err: - if ('no such table: %s' % SHARD_RANGE_TABLE) not in str(err): + except sqlite3.OperationalError as err: + if ('no such table: %s' % SHARD_RANGE_TABLE) in str(err): + return [] + if 'no such column: reported' in str(err): + return do_query(conn, use_reported_column=False) raise - return [] @classmethod def resolve_shard_range_states(cls, states): @@ -1741,9 +1776,7 @@ include_deleted=include_deleted, states=states, include_own=include_own, exclude_others=exclude_others)] - # note if this ever changes to *not* sort by upper first then it breaks - # a key assumption for bisect, which is used by utils.find_shard_ranges - shard_ranges.sort(key=lambda sr: (sr.upper, sr.state, sr.lower)) + shard_ranges.sort(key=ShardRange.sort_key) if includes: shard_range = find_shard_range(includes, shard_ranges) return [shard_range] if shard_range else [] @@ -2027,6 +2060,21 @@ else: return {k: v[0] for k, v in info.items()} + def _get_root_meta(self): + """ + Get the (unquoted) root path, plus the header the info came from. + If no info available, returns ``(None, None)`` + """ + path = self.get_sharding_sysmeta('Quoted-Root') + if path: + return 'X-Container-Sysmeta-Shard-Quoted-Root', unquote(path) + + path = self.get_sharding_sysmeta('Root') + if path: + return 'X-Container-Sysmeta-Shard-Root', path + + return None, None + def _load_root_info(self): """ Load the root container name and account for the container represented @@ -2039,13 +2087,7 @@ ``container`` attributes respectively. """ - path = self.get_sharding_sysmeta('Quoted-Root') - hdr = 'X-Container-Sysmeta-Shard-Quoted-Root' - if path: - path = unquote(path) - else: - path = self.get_sharding_sysmeta('Root') - hdr = 'X-Container-Sysmeta-Shard-Root' + hdr, path = self._get_root_meta() if not path: # Ensure account/container get populated @@ -2084,9 +2126,25 @@ A root container is a container that is not a shard of another container. """ - self._populate_instance_cache() - return (self.root_account == self.account and - self.root_container == self.container) + _, path = self._get_root_meta() + if path is not None: + # We have metadata telling us where the root is; it's authoritative + return self.path == path + + # Else, we're either a root or a deleted shard. + + # Use internal method so we don't try to update stats. + own_shard_range = self._own_shard_range(no_default=True) + if not own_shard_range: + return True # Never been sharded + + if own_shard_range.deleted: + # When shard ranges shrink, they get marked deleted + return False + else: + # But even when a root collapses, empties, and gets deleted, its + # own_shard_range is left alive + return True def _get_next_shard_range_upper(self, shard_size, last_upper=None): """ diff -Nru swift-2.25.1/swift/container/server.py swift-2.25.2/swift/container/server.py --- swift-2.25.1/swift/container/server.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/swift/container/server.py 2021-09-01 08:44:03.000000000 +0000 @@ -155,6 +155,8 @@ conf['auto_create_account_prefix'] else: self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX + self.shards_account_prefix = ( + self.auto_create_account_prefix + 'shards_') if config_true_value(conf.get('allow_versions', 'f')): self.save_headers.append('x-versions-location') if 'allow_versions' in conf: @@ -375,14 +377,12 @@ # auto create accounts) obj_policy_index = self.get_and_validate_policy_index(req) or 0 broker = self._get_container_broker(drive, part, account, container) - if account.startswith(self.auto_create_account_prefix) and obj and \ - not os.path.exists(broker.db_file): - try: - broker.initialize(req_timestamp.internal, obj_policy_index) - except DatabaseAlreadyExists: - pass - if not os.path.exists(broker.db_file): + if obj: + self._maybe_autocreate(broker, req_timestamp, account, + obj_policy_index, req) + elif not os.path.exists(broker.db_file): return HTTPNotFound() + if obj: # delete object # redirect if a shard range exists for the object name redirect = self._redirect_to_shard(req, broker, obj) @@ -449,11 +449,25 @@ broker.update_status_changed_at(timestamp) return recreated + def _should_autocreate(self, account, req): + auto_create_header = req.headers.get('X-Backend-Auto-Create') + if auto_create_header: + # If the caller included an explicit X-Backend-Auto-Create header, + # assume they know the behavior they want + return config_true_value(auto_create_header) + if account.startswith(self.shards_account_prefix): + # we have to specical case this subset of the + # auto_create_account_prefix because we don't want the updater + # accidently auto-creating shards; only the sharder creates + # shards and it will explicitly tell the server to do so + return False + return account.startswith(self.auto_create_account_prefix) + def _maybe_autocreate(self, broker, req_timestamp, account, - policy_index): + policy_index, req): created = False - if account.startswith(self.auto_create_account_prefix) and \ - not os.path.exists(broker.db_file): + should_autocreate = self._should_autocreate(account, req) + if should_autocreate and not os.path.exists(broker.db_file): if policy_index is None: raise HTTPBadRequest( 'X-Backend-Storage-Policy-Index header is required') @@ -506,8 +520,8 @@ # obj put expects the policy_index header, default is for # legacy support during upgrade. obj_policy_index = requested_policy_index or 0 - self._maybe_autocreate(broker, req_timestamp, account, - obj_policy_index) + self._maybe_autocreate( + broker, req_timestamp, account, obj_policy_index, req) # redirect if a shard exists for this object name response = self._redirect_to_shard(req, broker, obj) if response: @@ -531,8 +545,8 @@ for sr in json.loads(req.body)] except (ValueError, KeyError, TypeError) as err: return HTTPBadRequest('Invalid body: %r' % err) - created = self._maybe_autocreate(broker, req_timestamp, account, - requested_policy_index) + created = self._maybe_autocreate( + broker, req_timestamp, account, requested_policy_index, req) self._update_metadata(req, broker, req_timestamp, 'PUT') if shard_ranges: # TODO: consider writing the shard ranges into the pending @@ -805,7 +819,7 @@ requested_policy_index = self.get_and_validate_policy_index(req) broker = self._get_container_broker(drive, part, account, container) self._maybe_autocreate(broker, req_timestamp, account, - requested_policy_index) + requested_policy_index, req) try: objs = json.load(req.environ['wsgi.input']) except ValueError as err: diff -Nru swift-2.25.1/swift/container/sharder.py swift-2.25.2/swift/container/sharder.py --- swift-2.25.1/swift/container/sharder.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/swift/container/sharder.py 2021-09-01 08:44:03.000000000 +0000 @@ -313,6 +313,12 @@ self.cleaving_done = False self.cleave_to_row = self.max_row + def range_done(self, new_cursor): + self.ranges_done += 1 + self.ranges_todo -= 1 + if new_cursor is not None: + self.cursor = new_cursor + def done(self): return all((self.misplaced_done, self.cleaving_done, self.max_row == self.cleave_to_row)) @@ -618,7 +624,8 @@ def _send_shard_ranges(self, account, container, shard_ranges, headers=None): - body = json.dumps([dict(sr) for sr in shard_ranges]).encode('ascii') + body = json.dumps([dict(sr, reported=0) + for sr in shard_ranges]).encode('ascii') part, nodes = self.ring.get_nodes(account, container) headers = headers or {} headers.update({'X-Backend-Record-Type': RECORD_TYPE_SHARD, @@ -688,7 +695,8 @@ own_shard_range = broker.get_own_shard_range() if own_shard_range.state in (ShardRange.SHARDING, ShardRange.SHARDED): - shard_ranges = broker.get_shard_ranges() + shard_ranges = [sr for sr in broker.get_shard_ranges() + if sr.state != ShardRange.SHRINKING] missing_ranges = find_missing_ranges(shard_ranges) if missing_ranges: warnings.append( @@ -697,6 +705,10 @@ for lower, upper in missing_ranges])) for state in ShardRange.STATES: + if state == ShardRange.SHRINKING: + # Shrinking is how we resolve overlaps; we've got to + # allow multiple shards in that state + continue shard_ranges = broker.get_shard_ranges(states=state) overlaps = find_overlapping_ranges(shard_ranges) for overlapping_ranges in overlaps: @@ -717,7 +729,6 @@ return True def _audit_shard_container(self, broker): - # Get the root view of the world. self._increment_stat('audit_shard', 'attempted') warnings = [] errors = [] @@ -727,8 +738,10 @@ own_shard_range = broker.get_own_shard_range(no_default=True) - shard_range = None + shard_ranges = own_shard_range_from_root = None if own_shard_range: + # Get the root view of the world, at least that part of the world + # that overlaps with this shard's namespace shard_ranges = self._fetch_shard_ranges( broker, newest=True, params={'marker': str_to_wsgi(own_shard_range.lower_str), @@ -736,17 +749,21 @@ include_deleted=True) if shard_ranges: for shard_range in shard_ranges: - if (shard_range.lower == own_shard_range.lower and - shard_range.upper == own_shard_range.upper and - shard_range.name == own_shard_range.name): + # look for this shard range in the list of shard ranges + # received from root; the root may have different lower and + # upper bounds for this shard (e.g. if this shard has been + # expanded in the root to accept a shrinking shard) so we + # only match on name. + if shard_range.name == own_shard_range.name: + own_shard_range_from_root = shard_range break else: # this is not necessarily an error - some replicas of the # root may not yet know about this shard container warnings.append('root has no matching shard range') - shard_range = None - else: + elif not own_shard_range.deleted: warnings.append('unable to get shard ranges from root') + # else, our shard range is deleted, so root may have reclaimed it else: errors.append('missing own shard range') @@ -762,18 +779,46 @@ self._increment_stat('audit_shard', 'failure', statsd=True) return False - if shard_range: - self.logger.debug('Updating shard from root %s', dict(shard_range)) - broker.merge_shard_ranges(shard_range) + if own_shard_range_from_root: + # iff we find our own shard range in the root response, merge it + # and reload own shard range (note: own_range_from_root may not + # necessarily be 'newer' than the own shard range we already have, + # but merging will get us to the 'newest' state) + self.logger.debug('Updating own shard range from root') + broker.merge_shard_ranges(own_shard_range_from_root) + orig_own_shard_range = own_shard_range own_shard_range = broker.get_own_shard_range() - delete_age = time.time() - self.reclaim_age - if (own_shard_range.state == ShardRange.SHARDED and - own_shard_range.deleted and - own_shard_range.timestamp < delete_age and - broker.empty()): - broker.delete_db(Timestamp.now().internal) - self.logger.debug('Deleted shard container %s (%s)', - broker.db_file, quote(broker.path)) + if (orig_own_shard_range != own_shard_range or + orig_own_shard_range.state != own_shard_range.state): + self.logger.debug( + 'Updated own shard range from %s to %s', + orig_own_shard_range, own_shard_range) + if own_shard_range.state in (ShardRange.SHRINKING, + ShardRange.SHRUNK): + # If the up-to-date state is shrinking, save off *all* shards + # returned because these may contain shards into which this + # shard is to shrink itself; shrinking is the only case when we + # want to learn about *other* shard ranges from the root. + # We need to include shrunk state too, because one replica of a + # shard may already have moved the own_shard_range state to + # shrunk while another replica may still be in the process of + # shrinking. + other_shard_ranges = [sr for sr in shard_ranges + if sr is not own_shard_range_from_root] + self.logger.debug('Updating %s other shard range(s) from root', + len(other_shard_ranges)) + broker.merge_shard_ranges(other_shard_ranges) + + delete_age = time.time() - self.reclaim_age + deletable_states = (ShardRange.SHARDED, ShardRange.SHRUNK) + if (own_shard_range.state in deletable_states and + own_shard_range.deleted and + own_shard_range.timestamp < delete_age and + broker.empty()): + broker.delete_db(Timestamp.now().internal) + self.logger.debug('Deleted shard container %s (%s)', + broker.db_file, quote(broker.path)) + self._increment_stat('audit_shard', 'success', statsd=True) return True @@ -1096,7 +1141,7 @@ own_shard_range = broker.get_own_shard_range() shard_ranges = broker.get_shard_ranges() if shard_ranges and shard_ranges[-1].upper >= own_shard_range.upper: - self.logger.debug('Scan already completed for %s', + self.logger.debug('Scan for shard ranges already completed for %s', quote(broker.path)) return 0 @@ -1148,7 +1193,8 @@ 'X-Backend-Storage-Policy-Index': broker.storage_policy_index, 'X-Container-Sysmeta-Shard-Quoted-Root': quote( broker.root_path), - 'X-Container-Sysmeta-Sharding': True} + 'X-Container-Sysmeta-Sharding': 'True', + 'X-Backend-Auto-Create': 'True'} # NB: we *used* to send along # 'X-Container-Sysmeta-Shard-Root': broker.root_path # but that isn't safe for container names with nulls or newlines @@ -1229,9 +1275,7 @@ # SR because there was nothing there. So cleanup and # remove the shard_broker from its hand off location. self.delete_db(shard_broker) - cleaving_context.cursor = shard_range.upper_str - cleaving_context.ranges_done += 1 - cleaving_context.ranges_todo -= 1 + cleaving_context.range_done(shard_range.upper_str) if shard_range.upper >= own_shard_range.upper: # cleaving complete cleaving_context.cleaving_done = True @@ -1264,7 +1308,7 @@ # will atomically update its namespace *and* delete the donor. # Don't do this when sharding a shard because the donor # namespace should not be deleted until all shards are cleaved. - if own_shard_range.update_state(ShardRange.SHARDED): + if own_shard_range.update_state(ShardRange.SHRUNK): own_shard_range.set_deleted() broker.merge_shard_ranges(own_shard_range) shard_broker.merge_shard_ranges(own_shard_range) @@ -1304,9 +1348,7 @@ self._min_stat('cleaved', 'min_time', elapsed) self._max_stat('cleaved', 'max_time', elapsed) broker.merge_shard_ranges(shard_range) - cleaving_context.cursor = shard_range.upper_str - cleaving_context.ranges_done += 1 - cleaving_context.ranges_todo -= 1 + cleaving_context.range_done(shard_range.upper_str) if shard_range.upper >= own_shard_range.upper: # cleaving complete cleaving_context.cleaving_done = True @@ -1361,8 +1403,15 @@ ranges_done = [] for shard_range in ranges_todo: - if shard_range.state == ShardRange.FOUND: - break + if shard_range.state == ShardRange.SHRINKING: + # Ignore shrinking shard ranges: we never want to cleave + # objects to a shrinking shard. Shrinking shard ranges are to + # be expected in a root; shrinking shard ranges (other than own + # shard range) are not normally expected in a shard but can + # occur if there is an overlapping shard range that has been + # discovered from the root. + cleaving_context.range_done(None) # don't move the cursor + continue elif shard_range.state in (ShardRange.CREATED, ShardRange.CLEAVED, ShardRange.ACTIVE): @@ -1377,8 +1426,7 @@ # else, no errors, but no rows found either. keep going, # and don't count it against our batch size else: - self.logger.warning('Unexpected shard range state for cleave', - shard_range.state) + self.logger.info('Stopped cleave at unready %s', shard_range) break if not ranges_done: @@ -1402,7 +1450,12 @@ for sr in modified_shard_ranges: sr.update_state(ShardRange.ACTIVE) own_shard_range = broker.get_own_shard_range() - own_shard_range.update_state(ShardRange.SHARDED) + if own_shard_range.state in (ShardRange.SHRINKING, + ShardRange.SHRUNK): + next_state = ShardRange.SHRUNK + else: + next_state = ShardRange.SHARDED + own_shard_range.update_state(next_state) own_shard_range.update_meta(0, 0) if (not broker.is_root_container() and not own_shard_range.deleted): @@ -1468,7 +1521,7 @@ def _update_root_container(self, broker): own_shard_range = broker.get_own_shard_range(no_default=True) - if not own_shard_range: + if not own_shard_range or own_shard_range.reported: return # persist the reported shard metadata @@ -1478,9 +1531,12 @@ include_own=True, include_deleted=True) # send everything - self._send_shard_ranges( - broker.root_account, broker.root_container, - shard_ranges) + if self._send_shard_ranges( + broker.root_account, broker.root_container, shard_ranges): + # on success, mark ourselves as reported so we don't keep + # hammering the root + own_shard_range.reported = True + broker.merge_shard_ranges(own_shard_range) def _process_broker(self, broker, node, part): broker.get_info() # make sure account/container are populated @@ -1510,7 +1566,8 @@ own_shard_range = broker.get_own_shard_range() if own_shard_range.state in (ShardRange.SHARDING, ShardRange.SHRINKING, - ShardRange.SHARDED): + ShardRange.SHARDED, + ShardRange.SHRUNK): if broker.get_shard_ranges(): # container has been given shard ranges rather than # found them e.g. via replication or a shrink event @@ -1585,6 +1642,7 @@ - if not a root container, reports shard range stats to the root container """ + self.logger.info('Container sharder cycle starting, auto-sharding %s', self.auto_shard) if isinstance(devices_to_shard, (list, tuple)): @@ -1651,8 +1709,14 @@ self._report_stats() + def _set_auto_shard_from_command_line(self, **kwargs): + auto_shard = kwargs.get('auto_shard', None) + if auto_shard is not None: + self.auto_shard = config_true_value(auto_shard) + def run_forever(self, *args, **kwargs): """Run the container sharder until stopped.""" + self._set_auto_shard_from_command_line(**kwargs) self.reported = time.time() time.sleep(random() * self.interval) while True: @@ -1675,6 +1739,7 @@ override_options = parse_override_options(once=True, **kwargs) devices_to_shard = override_options.devices or Everything() partitions_to_shard = override_options.partitions or Everything() + self._set_auto_shard_from_command_line(**kwargs) begin = self.reported = time.time() self._one_shard_cycle(devices_to_shard=devices_to_shard, partitions_to_shard=partitions_to_shard) diff -Nru swift-2.25.1/swift/proxy/controllers/container.py swift-2.25.2/swift/proxy/controllers/container.py --- swift-2.25.1/swift/proxy/controllers/container.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/swift/proxy/controllers/container.py 2021-09-01 08:44:03.000000000 +0000 @@ -148,7 +148,19 @@ return resp def _get_from_shards(self, req, resp): - # construct listing using shards described by the response body + # Construct listing using shards described by the response body. + # The history of containers that have returned shard ranges is + # maintained in the request environ so that loops can be avoided by + # forcing an object listing if the same container is visited again. + # This can happen in at least two scenarios: + # 1. a container has filled a gap in its shard ranges with a + # shard range pointing to itself + # 2. a root container returns a (stale) shard range pointing to a + # shard that has shrunk into the root, in which case the shrunken + # shard may return the root's shard range. + shard_listing_history = req.environ.setdefault( + 'swift.shard_listing_history', []) + shard_listing_history.append((self.account_name, self.container_name)) shard_ranges = [ShardRange.from_dict(data) for data in json.loads(resp.body)] self.app.logger.debug('GET listing from %s shards for: %s', @@ -195,8 +207,8 @@ else: params['end_marker'] = str_to_wsgi(shard_range.end_marker) - if (shard_range.account == self.account_name and - shard_range.container == self.container_name): + if ((shard_range.account, shard_range.container) in + shard_listing_history): # directed back to same container - force GET of objects headers = {'X-Backend-Record-Type': 'object'} else: diff -Nru swift-2.25.1/swift.egg-info/pbr.json swift-2.25.2/swift.egg-info/pbr.json --- swift-2.25.1/swift.egg-info/pbr.json 2020-10-09 18:49:43.000000000 +0000 +++ swift-2.25.2/swift.egg-info/pbr.json 2021-09-01 08:44:41.000000000 +0000 @@ -1 +1 @@ -{"git_version": "a465c464c", "is_release": true} \ No newline at end of file +{"git_version": "4b0d6a87e", "is_release": true} \ No newline at end of file diff -Nru swift-2.25.1/swift.egg-info/PKG-INFO swift-2.25.2/swift.egg-info/PKG-INFO --- swift-2.25.1/swift.egg-info/PKG-INFO 2020-10-09 18:49:43.000000000 +0000 +++ swift-2.25.2/swift.egg-info/PKG-INFO 2021-09-01 08:44:41.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: swift -Version: 2.25.1 +Version: 2.25.2 Summary: OpenStack Object Storage Home-page: https://docs.openstack.org/swift/latest/ Author: OpenStack diff -Nru swift-2.25.1/swift.egg-info/requires.txt swift-2.25.2/swift.egg-info/requires.txt --- swift-2.25.1/swift.egg-info/requires.txt 2020-10-09 18:49:43.000000000 +0000 +++ swift-2.25.2/swift.egg-info/requires.txt 2021-09-01 08:44:41.000000000 +0000 @@ -28,7 +28,7 @@ oslo.config!=4.3.0,!=4.4.0,>=4.0.0 [test] -bandit>=1.1.0 +bandit<=1.6.2,>=1.1.0 boto3>=1.9 boto>=2.32.1 botocore>=1.12 diff -Nru swift-2.25.1/test/probe/test_sharder.py swift-2.25.2/test/probe/test_sharder.py --- swift-2.25.1/test/probe/test_sharder.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/test/probe/test_sharder.py 2021-09-01 08:44:03.000000000 +0000 @@ -16,6 +16,7 @@ import json import os import shutil +import subprocess import uuid from nose import SkipTest @@ -27,7 +28,8 @@ from swift.common.memcached import MemcacheRing from swift.common.utils import ShardRange, parse_db_filename, get_db_files, \ quorum_size, config_true_value, Timestamp -from swift.container.backend import ContainerBroker, UNSHARDED, SHARDING +from swift.container.backend import ContainerBroker, UNSHARDED, SHARDING, \ + SHARDED from swift.container.sharder import CleavingContext from swiftclient import client, get_auth, ClientException @@ -145,25 +147,24 @@ wait_for_server_to_hangup(ipport) def put_objects(self, obj_names, contents=None): + conn = client.Connection(preauthurl=self.url, preauthtoken=self.token) results = [] for obj in obj_names: rdict = {} - client.put_object(self.url, token=self.token, - container=self.container_name, name=obj, - contents=contents, response_dict=rdict) + conn.put_object(self.container_name, obj, + contents=contents, response_dict=rdict) results.append((obj, rdict['headers'].get('x-object-version-id'))) return results def delete_objects(self, obj_names_and_versions): + conn = client.Connection(preauthurl=self.url, preauthtoken=self.token) for obj in obj_names_and_versions: if isinstance(obj, tuple): obj, version = obj - client.delete_object( - self.url, self.token, self.container_name, obj, - query_string='version-id=%s' % version) + conn.delete_object(self.container_name, obj, + query_string='version-id=%s' % version) else: - client.delete_object( - self.url, self.token, self.container_name, obj) + conn.delete_object(self.container_name, obj) def get_container_shard_ranges(self, account=None, container=None): account = account if account else self.account @@ -195,12 +196,16 @@ return (utils.storage_directory(datadir, part, container_hash), container_hash) - def get_broker(self, part, node, account=None, container=None): + def get_db_file(self, part, node, account=None, container=None): container_dir, container_hash = self.get_storage_dir( part, node, account=account, container=container) db_file = os.path.join(container_dir, container_hash + '.db') self.assertTrue(get_db_files(db_file)) # sanity check - return ContainerBroker(db_file) + return db_file + + def get_broker(self, part, node, account=None, container=None): + return ContainerBroker( + self.get_db_file(part, node, account, container)) def categorize_container_dir_content(self, account=None, container=None): account = account or self.brain.account @@ -1607,7 +1612,7 @@ broker = self.get_broker( part, node, donor.account, donor.container) own_sr = broker.get_own_shard_range() - self.assertEqual(ShardRange.SHARDED, own_sr.state) + self.assertEqual(ShardRange.SHRUNK, own_sr.state) self.assertTrue(own_sr.deleted) # delete all the second shard's object apart from 'alpha' @@ -2409,3 +2414,266 @@ self.container_name = cont_name.encode('utf8').ljust(name_length, b'x') if not six.PY2: self.container_name = self.container_name.decode('utf8') + + +class TestManagedContainerSharding(BaseTestContainerSharding): + '''Test sharding using swift-manage-shard-ranges''' + + def sharders_once(self, **kwargs): + # inhibit auto_sharding regardless of the config setting + additional_args = kwargs.get('additional_args', []) + if not isinstance(additional_args, list): + additional_args = [additional_args] + additional_args.append('--no-auto-shard') + kwargs['additional_args'] = additional_args + self.sharders.once(**kwargs) + + def test_manage_shard_ranges(self): + obj_names = self._make_object_names(4) + self.put_objects(obj_names) + + client.post_container(self.url, self.admin_token, self.container_name, + headers={'X-Container-Sharding': 'on'}) + + # run replicators first time to get sync points set + self.replicators.once() + + # sanity check: we don't have nearly enough objects for this to shard + # automatically + self.sharders_once(number=self.brain.node_numbers[0], + additional_args='--partitions=%s' % self.brain.part) + self.assert_container_state(self.brain.nodes[0], 'unsharded', 0) + + subprocess.check_output([ + 'swift-manage-shard-ranges', + self.get_db_file(self.brain.part, self.brain.nodes[0]), + 'find_and_replace', '2', '--enable'], stderr=subprocess.STDOUT) + self.assert_container_state(self.brain.nodes[0], 'unsharded', 2) + + # "Run container-replicator to replicate them to other nodes." + self.replicators.once() + # "Run container-sharder on all nodes to shard the container." + self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + + # Everybody's settled + self.assert_container_state(self.brain.nodes[0], 'sharded', 2) + self.assert_container_state(self.brain.nodes[1], 'sharded', 2) + self.assert_container_state(self.brain.nodes[2], 'sharded', 2) + self.assert_container_listing(obj_names) + + def test_manage_shard_ranges_used_poorly(self): + obj_names = self._make_object_names(8) + self.put_objects(obj_names) + + client.post_container(self.url, self.admin_token, self.container_name, + headers={'X-Container-Sharding': 'on'}) + + # run replicators first time to get sync points set + self.replicators.once() + + # find 4 shard ranges on nodes[0] - let's denote these ranges 0.0, 0.1, + # 0.2 and 0.3 that are installed with epoch_0 + subprocess.check_output([ + 'swift-manage-shard-ranges', + self.get_db_file(self.brain.part, self.brain.nodes[0]), + 'find_and_replace', '2', '--enable'], stderr=subprocess.STDOUT) + shard_ranges_0 = self.assert_container_state(self.brain.nodes[0], + 'unsharded', 4) + + # *Also* go find 3 shard ranges on *another node*, like a dumb-dumb - + # let's denote these ranges 1.0, 1.1 and 1.2 that are installed with + # epoch_1 + subprocess.check_output([ + 'swift-manage-shard-ranges', + self.get_db_file(self.brain.part, self.brain.nodes[1]), + 'find_and_replace', '3', '--enable'], stderr=subprocess.STDOUT) + shard_ranges_1 = self.assert_container_state(self.brain.nodes[1], + 'unsharded', 3) + + # Run sharder in specific order so that the replica with the older + # epoch_0 starts sharding first - this will prove problematic later! + # On first pass the first replica passes audit, creates shards and then + # syncs shard ranges with the other replicas. It proceeds to cleave + # shard 0.0, but after 0.0 cleaving stalls because it will now have + # shard range 1.0 in 'found' state from the other replica that it + # cannot yet cleave. + self.sharders_once(number=self.brain.node_numbers[0], + additional_args='--partitions=%s' % self.brain.part) + + # On first pass the second replica passes audit (it has its own found + # ranges and the first replicas created shard ranges but none in the + # same state overlap), creates its shards and then syncs shard ranges + # with the other replicas. All of the 7 shard ranges on this replica + # are now in created state so it proceeds to cleave the first two shard + # ranges, 0.1 and 1.0. + self.sharders_once(number=self.brain.node_numbers[1], + additional_args='--partitions=%s' % self.brain.part) + self.replicators.once() + + # Uh-oh + self.assert_container_state(self.brain.nodes[0], 'sharding', 7) + self.assert_container_state(self.brain.nodes[1], 'sharding', 7) + # There's a race: the third replica may be sharding, may be unsharded + + # Try it again a few times + self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + self.replicators.once() + self.sharders_once(additional_args='--partitions=%s' % self.brain.part) + + # It's not really fixing itself... the sharder audit will detect + # overlapping ranges which prevents cleaving proceeding; expect the + # shard ranges to be mostly still in created state, with one or two + # possibly cleaved during first pass before the sharding got stalled + shard_ranges = self.assert_container_state(self.brain.nodes[0], + 'sharding', 7) + for sr in shard_ranges: + self.assertIn(sr.state, (ShardRange.CREATED, ShardRange.CLEAVED)) + shard_ranges = self.assert_container_state(self.brain.nodes[1], + 'sharding', 7) + for sr in shard_ranges: + self.assertIn(sr.state, (ShardRange.CREATED, ShardRange.CLEAVED)) + + # But hey, at least listings still work! They're just going to get + # horribly out of date as more objects are added + self.assert_container_listing(obj_names) + + # Let's pretend that some actor in the system has determined that the + # second set of 3 shard ranges (1.*) are correct and the first set of 4 + # (0.*) are not desired, so shrink shard ranges 0.*. We've already + # checked they are in cleaved or created state so it's ok to move them + # to shrinking. + # TODO: replace this db manipulation if/when manage_shard_ranges can + # manage shrinking... + for sr in shard_ranges_0: + self.assertTrue(sr.update_state(ShardRange.SHRINKING)) + sr.epoch = sr.state_timestamp = Timestamp.now() + broker = self.get_broker(self.brain.part, self.brain.nodes[0]) + broker.merge_shard_ranges(shard_ranges_0) + + # make sure all root replicas now sync their shard ranges + self.replicators.once() + # At this point one of the first two replicas may have done some useful + # cleaving of 1.* shards, the other may have only cleaved 0.* shards, + # and the third replica may have cleaved no shards. We therefore need + # two more passes of the sharder to get to a predictable state where + # all replicas have cleaved all three 0.* shards. + self.sharders_once() + self.sharders_once() + + # now we expect all replicas to have just the three 1.* shards, with + # the 0.* shards all deleted + brokers = {} + orig_shard_ranges = sorted(shard_ranges_0 + shard_ranges_1, + key=ShardRange.sort_key) + for node in (0, 1, 2): + with annotate_failure('node %s' % node): + broker = self.get_broker(self.brain.part, + self.brain.nodes[node]) + brokers[node] = broker + shard_ranges = broker.get_shard_ranges() + self.assertEqual(shard_ranges_1, shard_ranges) + shard_ranges = broker.get_shard_ranges(include_deleted=True) + self.assertLengthEqual(shard_ranges, len(orig_shard_ranges)) + self.assertEqual(orig_shard_ranges, shard_ranges) + self.assertEqual(ShardRange.SHARDED, + broker._own_shard_range().state) + # Sadly, the first replica to start sharding us still reporting its db + # state to be 'unsharded' because, although it has sharded, it's shard + # db epoch (epoch_0) does not match its own shard range epoch + # (epoch_1), and that is because the second replica (with epoch_1) + # updated the own shard range and replicated it to all other replicas. + # If we had run the sharder on the second replica before the first + # replica, then by the time the first replica started sharding it would + # have learnt the newer epoch_1 and we wouldn't see this inconsistency. + self.assertEqual(UNSHARDED, brokers[0].get_db_state()) + self.assertEqual(SHARDED, brokers[1].get_db_state()) + self.assertEqual(SHARDED, brokers[2].get_db_state()) + epoch_1 = brokers[1].db_epoch + self.assertEqual(epoch_1, brokers[2].db_epoch) + self.assertLess(brokers[0].db_epoch, epoch_1) + # the root replica that thinks it is unsharded is problematic - it will + # not return shard ranges for listings, but has no objects, so it's + # luck of the draw whether we get a listing or not at this point :( + + # check the unwanted shards did shrink away... + for shard_range in shard_ranges_0: + with annotate_failure(shard_range): + found_for_shard = self.categorize_container_dir_content( + shard_range.account, shard_range.container) + self.assertLengthEqual(found_for_shard['shard_dbs'], 3) + actual = [] + for shard_db in found_for_shard['shard_dbs']: + broker = ContainerBroker(shard_db) + own_sr = broker.get_own_shard_range() + actual.append( + (broker.get_db_state(), own_sr.state, own_sr.deleted)) + self.assertEqual([(SHARDED, ShardRange.SHRUNK, True)] * 3, + actual) + + # Run the sharders again: the first replica that is still 'unsharded' + # because of the older epoch_0 in its db filename will now start to + # shard again with a newer epoch_1 db, and will start to re-cleave the + # 3 active shards, albeit with zero objects to cleave. + self.sharders_once() + for node in (0, 1, 2): + with annotate_failure('node %s' % node): + broker = self.get_broker(self.brain.part, + self.brain.nodes[node]) + brokers[node] = broker + shard_ranges = broker.get_shard_ranges() + self.assertEqual(shard_ranges_1, shard_ranges) + shard_ranges = broker.get_shard_ranges(include_deleted=True) + self.assertLengthEqual(shard_ranges, len(orig_shard_ranges)) + self.assertEqual(orig_shard_ranges, shard_ranges) + self.assertEqual(ShardRange.SHARDED, + broker._own_shard_range().state) + self.assertEqual(epoch_1, broker.db_epoch) + self.assertIn(brokers[0].get_db_state(), (SHARDING, SHARDED)) + self.assertEqual(SHARDED, brokers[1].get_db_state()) + self.assertEqual(SHARDED, brokers[2].get_db_state()) + + # This cycle of the sharders also guarantees that all shards have had + # their state updated to ACTIVE from the root; this was not necessarily + # true at end of the previous sharder pass because a shard audit (when + # the shard is updated from a root) may have happened before all roots + # have had their shard ranges transitioned to ACTIVE. + for shard_range in shard_ranges_1: + with annotate_failure(shard_range): + found_for_shard = self.categorize_container_dir_content( + shard_range.account, shard_range.container) + self.assertLengthEqual(found_for_shard['normal_dbs'], 3) + actual = [] + for shard_db in found_for_shard['normal_dbs']: + broker = ContainerBroker(shard_db) + own_sr = broker.get_own_shard_range() + actual.append( + (broker.get_db_state(), own_sr.state, own_sr.deleted)) + self.assertEqual([(UNSHARDED, ShardRange.ACTIVE, False)] * 3, + actual) + + # We may need one more pass of the sharder before all three shard + # ranges are cleaved (2 per pass) and all the root replicas are + # predictably in sharded state. Note: the accelerated cleaving of >2 + # zero-object shard ranges per cycle is defeated if a shard happens + # to exist on the same node as the root because the roots cleaving + # process doesn't think that it created the shard db and will therefore + # replicate it as per a normal cleave. + self.sharders_once() + for node in (0, 1, 2): + with annotate_failure('node %s' % node): + broker = self.get_broker(self.brain.part, + self.brain.nodes[node]) + brokers[node] = broker + shard_ranges = broker.get_shard_ranges() + self.assertEqual(shard_ranges_1, shard_ranges) + shard_ranges = broker.get_shard_ranges(include_deleted=True) + self.assertLengthEqual(shard_ranges, len(orig_shard_ranges)) + self.assertEqual(orig_shard_ranges, shard_ranges) + self.assertEqual(ShardRange.SHARDED, + broker._own_shard_range().state) + self.assertEqual(epoch_1, broker.db_epoch) + self.assertEqual(SHARDED, broker.get_db_state()) + + # Finally, with all root replicas in a consistent state, the listing + # will be be predictably correct + self.assert_container_listing(obj_names) diff -Nru swift-2.25.1/test/unit/cli/test_info.py swift-2.25.2/test/unit/cli/test_info.py --- swift-2.25.1/test/unit/cli/test_info.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/test/unit/cli/test_info.py 2021-09-01 08:44:03.000000000 +0000 @@ -141,8 +141,8 @@ No system metadata found in db file User Metadata: {'x-account-meta-mydata': 'swift'}''' - self.assertEqual(sorted(out.getvalue().strip().split('\n')), - sorted(exp_out.split('\n'))) + self.assertEqual(out.getvalue().strip().split('\n'), + exp_out.split('\n')) info = dict( account='acct', @@ -269,7 +269,7 @@ id='abadf100d0ddba11') out = StringIO() with mock.patch('sys.stdout', out): - print_db_info_metadata('container', info, {}) + print_db_info_metadata('container', info, {}, verbose=True) exp_out = '''Path: /acct/cont Account: acct Container: cont @@ -295,6 +295,10 @@ Type: root State: sharded Shard Ranges (3): + States: + found: 1 + created: 1 + cleaved: 1 Name: .sharded_a/shard_range_1 lower: '1a', upper: '1z' Object Count: 1, Bytes Used: 1, State: cleaved (30) @@ -311,8 +315,77 @@ Created at: 1970-01-01T00:00:03.000000 (0000000003.00000) Meta Timestamp: 1970-01-01T00:00:03.000000 (0000000003.00000)''' %\ POLICIES[0].name - self.assertEqual(sorted(out.getvalue().strip().split('\n')), - sorted(exp_out.strip().split('\n'))) + self.assertEqual(out.getvalue().strip().split('\n'), + exp_out.strip().split('\n')) + + def test_print_db_info_metadata_with_many_shard_ranges(self): + + shard_ranges = [utils.ShardRange( + name='.sharded_a/shard_range_%s' % i, + timestamp=utils.Timestamp(i), lower='%02da' % i, + upper='%02dz' % i, object_count=i, bytes_used=i, + meta_timestamp=utils.Timestamp(i)) for i in range(1, 20)] + shard_ranges[0].state = utils.ShardRange.CLEAVED + shard_ranges[1].state = utils.ShardRange.CREATED + + info = dict( + account='acct', + container='cont', + storage_policy_index=0, + created_at='0000000100.10000', + put_timestamp='0000000106.30000', + delete_timestamp='0000000107.90000', + status_changed_at='0000000108.30000', + object_count='20', + bytes_used='42', + reported_put_timestamp='0000010106.30000', + reported_delete_timestamp='0000010107.90000', + reported_object_count='20', + reported_bytes_used='42', + db_state=SHARDED, + is_root=True, + shard_ranges=shard_ranges, + is_deleted=False, + hash='abaddeadbeefcafe', + id='abadf100d0ddba11') + out = StringIO() + with mock.patch('sys.stdout', out): + print_db_info_metadata('container', info, {}) + exp_out = ''' +Path: /acct/cont + Account: acct + Container: cont + Deleted: False + Container Hash: d49d0ecbb53be1fcc49624f2f7c7ccae +Metadata: + Created at: 1970-01-01T00:01:40.100000 (0000000100.10000) + Put Timestamp: 1970-01-01T00:01:46.300000 (0000000106.30000) + Delete Timestamp: 1970-01-01T00:01:47.900000 (0000000107.90000) + Status Timestamp: 1970-01-01T00:01:48.300000 (0000000108.30000) + Object Count: 20 + Bytes Used: 42 + Storage Policy: %s (0) + Reported Put Timestamp: 1970-01-01T02:48:26.300000 (0000010106.30000) + Reported Delete Timestamp: 1970-01-01T02:48:27.900000 (0000010107.90000) + Reported Object Count: 20 + Reported Bytes Used: 42 + Chexor: abaddeadbeefcafe + UUID: abadf100d0ddba11 +No system metadata found in db file +No user metadata found in db file +Sharding Metadata: + Type: root + State: sharded +Shard Ranges (19): + States: + found: 17 + created: 1 + cleaved: 1 +(Use -v/--verbose to show more Shard Ranges details) +''' %\ + POLICIES[0].name + self.assertEqual(out.getvalue().strip().split('\n'), + exp_out.strip().split('\n')) def test_print_db_info_metadata_with_shard_ranges_bis(self): @@ -346,7 +419,7 @@ info['is_deleted'] = False out = StringIO() with mock.patch('sys.stdout', out): - print_db_info_metadata('container', info, {}) + print_db_info_metadata('container', info, {}, verbose=True) if six.PY2: s_a = '\\xe3\\x82\\xa2' s_ya = '\\xe3\\x83\\xa4' @@ -378,6 +451,10 @@ Type: root State: sharded Shard Ranges (3): + States: + found: 1 + created: 1 + cleaved: 1 Name: .sharded_a/shard_range_1 lower: '1%s', upper: '1%s' Object Count: 1, Bytes Used: 1, State: cleaved (30) diff -Nru swift-2.25.1/test/unit/cli/test_manage_shard_ranges.py swift-2.25.2/test/unit/cli/test_manage_shard_ranges.py --- swift-2.25.1/test/unit/cli/test_manage_shard_ranges.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/test/unit/cli/test_manage_shard_ranges.py 2021-09-01 08:44:03.000000000 +0000 @@ -189,6 +189,7 @@ ' "meta_timestamp": "%s",' % now.internal, ' "name": "a/c",', ' "object_count": 0,', + ' "reported": 0,', ' "state": "sharding",', ' "state_timestamp": "%s",' % now.internal, ' "timestamp": "%s",' % now.internal, @@ -230,6 +231,7 @@ ' "meta_timestamp": "%s",' % now.internal, ' "name": "a/c",', ' "object_count": 0,', + ' "reported": 0,', ' "state": "sharding",', ' "state_timestamp": "%s",' % now.internal, ' "timestamp": "%s",' % now.internal, @@ -358,6 +360,23 @@ self.assertEqual(['Loaded db broker for a/c.'], err.getvalue().splitlines()) self._assert_enabled(broker, now) + found_shard_ranges = broker.get_shard_ranges() self.assertEqual( [(data['lower'], data['upper']) for data in self.shard_data], - [(sr.lower_str, sr.upper_str) for sr in broker.get_shard_ranges()]) + [(sr.lower_str, sr.upper_str) for sr in found_shard_ranges]) + + # Do another find & replace but quit when prompted about existing + # shard ranges + out = StringIO() + err = StringIO() + to_patch = 'swift.cli.manage_shard_ranges.input' + with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err), \ + mock_timestamp_now() as now, \ + mock.patch(to_patch, return_value='q'): + main([broker.db_file, 'find_and_replace', '10']) + # Shard ranges haven't changed at all + self.assertEqual(found_shard_ranges, broker.get_shard_ranges()) + expected = ['This will delete existing 10 shard ranges.'] + self.assertEqual(expected, out.getvalue().splitlines()) + self.assertEqual(['Loaded db broker for a/c.'], + err.getvalue().splitlines()) diff -Nru swift-2.25.1/test/unit/common/middleware/test_memcache.py swift-2.25.2/test/unit/common/middleware/test_memcache.py --- swift-2.25.1/test/unit/common/middleware/test_memcache.py 2020-10-09 18:48:13.000000000 +0000 +++ swift-2.25.2/test/unit/common/middleware/test_memcache.py 2021-09-01 08:44:03.000000000 +0000 @@ -17,6 +17,7 @@ from textwrap import dedent import unittest +from eventlet.green import ssl import mock from six.moves.configparser import NoSectionError, NoOptionError @@ -160,6 +161,22 @@ self.assertEqual( app.memcache._client_cache['6.7.8.9:10'].max_size, 5) + def test_conf_inline_tls(self): + fake_context = mock.Mock() + with mock.patch.object(ssl, 'create_default_context', + return_value=fake_context): + with mock.patch.object(memcache, 'ConfigParser', + get_config_parser()): + memcache.MemcacheMiddleware( + FakeApp(), + {'tls_enabled': 'true', + 'tls_cafile': 'cafile', + 'tls_certfile': 'certfile', + 'tls_keyfile': 'keyfile'}) + ssl.create_default_context.assert_called_with(cafile='cafile') + fake_context.load_cert_chain.assert_called_with('certfile', + 'keyfile') + def test_conf_extra_no_section(self): with mock.patch.object(memcache, 'ConfigParser', get_config_parser(section='foobar')): @@ -323,6 +340,7 @@ pool_timeout = 0.5 tries = 4 io_timeout = 1.0 + tls_enabled = true """ config_path = os.path.join(tempdir, 'test.conf') with open(config_path, 'w') as f: @@ -336,6 +354,9 @@ # tries is limited to server count self.assertEqual(memcache_ring._tries, 4) self.assertEqual(memcache_ring._io_timeout, 1.0) + self.assertIsInstance( + list(memcache_ring._client_cache.values())[0]._tls_context, + ssl.SSLContext) @with_tempdir def test_real_memcache_config(self, tempdir): diff -Nru swift-2.25.1/test/unit/common/middleware/test_symlink.py swift-2.25.2/test/unit/common/middleware/test_symlink.py --- swift-2.25.1/test/unit/common/middleware/test_symlink.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/test/unit/common/middleware/test_symlink.py 2021-09-01 08:44:03.000000000 +0000 @@ -402,6 +402,7 @@ req = Request.blank('/v1/a/c/symlink?symlink=get', method='GET') status, headers, body = self.call_sym(req) self.assertEqual(status, '200 OK') + self.assertIsInstance(headers, list) self.assertIn(('X-Symlink-Target', 'c1/o'), headers) self.assertNotIn('X-Symlink-Target-Account', dict(headers)) diff -Nru swift-2.25.1/test/unit/common/middleware/test_tempurl.py swift-2.25.2/test/unit/common/middleware/test_tempurl.py --- swift-2.25.1/test/unit/common/middleware/test_tempurl.py 2020-10-09 18:48:13.000000000 +0000 +++ swift-2.25.2/test/unit/common/middleware/test_tempurl.py 2021-09-01 08:44:03.000000000 +0000 @@ -476,6 +476,42 @@ self.assertEqual(req.environ['swift.authorize_override'], True) self.assertEqual(req.environ['REMOTE_USER'], '.wsgi.tempurl') + def test_put_response_headers_in_list(self): + class Validator(object): + def __init__(self, app): + self.app = app + self.status = None + self.headers = None + self.exc_info = None + + def start_response(self, status, headers, exc_info=None): + self.status = status + self.headers = headers + self.exc_info = exc_info + + def __call__(self, env, start_response): + resp_iter = self.app(env, self.start_response) + start_response(self.status, self.headers, self.exc_info) + return resp_iter + + method = 'PUT' + expires = int(time() + 86400) + path = '/v1/a/c/o' + key = b'abc' + hmac_body = ('%s\n%i\n%s' % (method, expires, path)).encode('utf-8') + sig = hmac.new(key, hmac_body, hashlib.sha1).hexdigest() + req = self._make_request( + path, keys=[key], + environ={'REQUEST_METHOD': 'PUT', + 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( + sig, expires)}) + validator = Validator(self.tempurl) + resp = req.get_response(validator) + self.assertIsInstance(validator.headers, list) + self.assertEqual(resp.status_int, 404) + self.assertEqual(req.environ['swift.authorize_override'], True) + self.assertEqual(req.environ['REMOTE_USER'], '.wsgi.tempurl') + def test_get_not_allowed_by_put(self): method = 'PUT' expires = int(time() + 86400) diff -Nru swift-2.25.1/test/unit/common/test_memcached.py swift-2.25.2/test/unit/common/test_memcached.py --- swift-2.25.1/test/unit/common/test_memcached.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/test/unit/common/test_memcached.py 2021-09-01 08:44:03.000000000 +0000 @@ -188,6 +188,20 @@ self.addCleanup(patcher.stop) patcher.start() + def test_tls_context_kwarg(self): + with patch('swift.common.memcached.socket.socket'): + server = '%s:%s' % ('[::1]', 11211) + client = memcached.MemcacheRing([server]) + self.assertIsNone(client._client_cache[server]._tls_context) + + context = mock.Mock() + client = memcached.MemcacheRing([server], tls_context=context) + self.assertIs(client._client_cache[server]._tls_context, context) + + key = uuid4().hex.encode('ascii') + list(client._get_conns(key)) + context.wrap_socket.assert_called_once() + def test_get_conns(self): sock1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock1.bind(('127.0.0.1', 0)) diff -Nru swift-2.25.1/test/unit/common/test_utils.py swift-2.25.2/test/unit/common/test_utils.py --- swift-2.25.1/test/unit/common/test_utils.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/test/unit/common/test_utils.py 2021-09-01 08:44:03.000000000 +0000 @@ -44,6 +44,7 @@ import json import math import inspect +import warnings import six from six import StringIO @@ -7233,7 +7234,8 @@ upper='', object_count=0, bytes_used=0, meta_timestamp=ts_1.internal, deleted=0, state=utils.ShardRange.FOUND, - state_timestamp=ts_1.internal, epoch=None) + state_timestamp=ts_1.internal, epoch=None, + reported=0) assert_initialisation_ok(dict(empty_run, name='a/c', timestamp=ts_1), expect) assert_initialisation_ok(dict(name='a/c', timestamp=ts_1), expect) @@ -7242,11 +7244,13 @@ upper='u', object_count=2, bytes_used=10, meta_timestamp=ts_2, deleted=0, state=utils.ShardRange.CREATED, - state_timestamp=ts_3.internal, epoch=ts_4) + state_timestamp=ts_3.internal, epoch=ts_4, + reported=0) expect.update({'lower': 'l', 'upper': 'u', 'object_count': 2, 'bytes_used': 10, 'meta_timestamp': ts_2.internal, 'state': utils.ShardRange.CREATED, - 'state_timestamp': ts_3.internal, 'epoch': ts_4}) + 'state_timestamp': ts_3.internal, 'epoch': ts_4, + 'reported': 0}) assert_initialisation_ok(good_run.copy(), expect) # obj count and bytes used as int strings @@ -7264,6 +7268,11 @@ assert_initialisation_ok(good_deleted, dict(expect, deleted=1)) + good_reported = good_run.copy() + good_reported['reported'] = 1 + assert_initialisation_ok(good_reported, + dict(expect, reported=1)) + assert_initialisation_fails(dict(good_run, timestamp='water balloon')) assert_initialisation_fails( @@ -7302,7 +7311,7 @@ 'upper': upper, 'object_count': 10, 'bytes_used': 100, 'meta_timestamp': ts_2.internal, 'deleted': 0, 'state': utils.ShardRange.FOUND, 'state_timestamp': ts_3.internal, - 'epoch': ts_4} + 'epoch': ts_4, 'reported': 0} self.assertEqual(expected, sr_dict) self.assertIsInstance(sr_dict['lower'], six.string_types) self.assertIsInstance(sr_dict['upper'], six.string_types) @@ -7317,6 +7326,14 @@ for key in sr_dict: bad_dict = dict(sr_dict) bad_dict.pop(key) + if key == 'reported': + # This was added after the fact, and we need to be able to eat + # data from old servers + utils.ShardRange.from_dict(bad_dict) + utils.ShardRange(**bad_dict) + continue + + # The rest were present from the beginning with self.assertRaises(KeyError): utils.ShardRange.from_dict(bad_dict) # But __init__ still (generally) works! @@ -7595,8 +7612,10 @@ expected = u'\N{SNOWMAN}' if six.PY2: expected = expected.encode('utf-8') - do_test(u'\N{SNOWMAN}', expected) - do_test(u'\N{SNOWMAN}'.encode('utf-8'), expected) + with warnings.catch_warnings(record=True) as captured_warnings: + do_test(u'\N{SNOWMAN}', expected) + do_test(u'\N{SNOWMAN}'.encode('utf-8'), expected) + self.assertFalse(captured_warnings) sr = utils.ShardRange('a/c', utils.Timestamp.now(), 'b', 'y') sr.lower = '' @@ -7643,8 +7662,10 @@ expected = u'\N{SNOWMAN}' if six.PY2: expected = expected.encode('utf-8') - do_test(u'\N{SNOWMAN}', expected) - do_test(u'\N{SNOWMAN}'.encode('utf-8'), expected) + with warnings.catch_warnings(record=True) as captured_warnings: + do_test(u'\N{SNOWMAN}', expected) + do_test(u'\N{SNOWMAN}'.encode('utf-8'), expected) + self.assertFalse(captured_warnings) sr = utils.ShardRange('a/c', utils.Timestamp.now(), 'b', 'y') sr.upper = '' diff -Nru swift-2.25.1/test/unit/container/test_backend.py swift-2.25.2/test/unit/container/test_backend.py --- swift-2.25.1/test/unit/container/test_backend.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/test/unit/container/test_backend.py 2021-09-01 08:44:03.000000000 +0000 @@ -26,6 +26,7 @@ from collections import defaultdict from contextlib import contextmanager import sqlite3 +import string import pickle import json import itertools @@ -442,6 +443,7 @@ self.assertTrue(broker_to_test.empty()) self.assertTrue(broker.empty()) + self.assertFalse(broker.is_root_container()) check_object_counted(broker, broker) # own shard range is not considered for object count @@ -498,6 +500,20 @@ own_sr.update_meta(3, 4, meta_timestamp=next(self.ts)) broker.merge_shard_ranges([own_sr]) self.assertTrue(broker.empty()) + self.assertFalse(broker.is_deleted()) + self.assertFalse(broker.is_root_container()) + + # sharder won't call delete_db() unless own_shard_range is deleted + own_sr.deleted = True + own_sr.timestamp = next(self.ts) + broker.merge_shard_ranges([own_sr]) + broker.delete_db(next(self.ts).internal) + + # Get a fresh broker, with instance cache unset + broker = ContainerBroker(db_path, account='.shards_a', container='cc') + self.assertTrue(broker.empty()) + self.assertTrue(broker.is_deleted()) + self.assertFalse(broker.is_root_container()) def test_reclaim(self): broker = ContainerBroker(':memory:', account='test_account', @@ -735,10 +751,12 @@ self.assertEqual(info['put_timestamp'], start.internal) self.assertTrue(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], '0') - if self.__class__ in (TestContainerBrokerBeforeMetadata, - TestContainerBrokerBeforeXSync, - TestContainerBrokerBeforeSPI, - TestContainerBrokerBeforeShardRanges): + if self.__class__ in ( + TestContainerBrokerBeforeMetadata, + TestContainerBrokerBeforeXSync, + TestContainerBrokerBeforeSPI, + TestContainerBrokerBeforeShardRanges, + TestContainerBrokerBeforeShardRangeReportedColumn): self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], @@ -1025,6 +1043,8 @@ "SELECT object_count FROM shard_range").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT bytes_used FROM shard_range").fetchone()[0], 0) + self.assertEqual(conn.execute( + "SELECT reported FROM shard_range").fetchone()[0], 0) # Reput same event broker.merge_shard_ranges( @@ -1050,6 +1070,64 @@ "SELECT object_count FROM shard_range").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT bytes_used FROM shard_range").fetchone()[0], 0) + self.assertEqual(conn.execute( + "SELECT reported FROM shard_range").fetchone()[0], 0) + + # Mark it as reported + broker.merge_shard_ranges( + ShardRange('"a/{}"', timestamp, + 'low', 'up', meta_timestamp=meta_timestamp, + reported=True)) + with broker.get() as conn: + self.assertEqual(conn.execute( + "SELECT name FROM shard_range").fetchone()[0], + '"a/{}"') + self.assertEqual(conn.execute( + "SELECT timestamp FROM shard_range").fetchone()[0], + timestamp) + self.assertEqual(conn.execute( + "SELECT meta_timestamp FROM shard_range").fetchone()[0], + meta_timestamp) + self.assertEqual(conn.execute( + "SELECT lower FROM shard_range").fetchone()[0], 'low') + self.assertEqual(conn.execute( + "SELECT upper FROM shard_range").fetchone()[0], 'up') + self.assertEqual(conn.execute( + "SELECT deleted FROM shard_range").fetchone()[0], 0) + self.assertEqual(conn.execute( + "SELECT object_count FROM shard_range").fetchone()[0], 0) + self.assertEqual(conn.execute( + "SELECT bytes_used FROM shard_range").fetchone()[0], 0) + self.assertEqual(conn.execute( + "SELECT reported FROM shard_range").fetchone()[0], 1) + + # Reporting latches it + broker.merge_shard_ranges( + ShardRange('"a/{}"', timestamp, + 'low', 'up', meta_timestamp=meta_timestamp, + reported=False)) + with broker.get() as conn: + self.assertEqual(conn.execute( + "SELECT name FROM shard_range").fetchone()[0], + '"a/{}"') + self.assertEqual(conn.execute( + "SELECT timestamp FROM shard_range").fetchone()[0], + timestamp) + self.assertEqual(conn.execute( + "SELECT meta_timestamp FROM shard_range").fetchone()[0], + meta_timestamp) + self.assertEqual(conn.execute( + "SELECT lower FROM shard_range").fetchone()[0], 'low') + self.assertEqual(conn.execute( + "SELECT upper FROM shard_range").fetchone()[0], 'up') + self.assertEqual(conn.execute( + "SELECT deleted FROM shard_range").fetchone()[0], 0) + self.assertEqual(conn.execute( + "SELECT object_count FROM shard_range").fetchone()[0], 0) + self.assertEqual(conn.execute( + "SELECT bytes_used FROM shard_range").fetchone()[0], 0) + self.assertEqual(conn.execute( + "SELECT reported FROM shard_range").fetchone()[0], 1) # Put new event timestamp = next(self.ts).internal @@ -1077,11 +1155,14 @@ "SELECT object_count FROM shard_range").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT bytes_used FROM shard_range").fetchone()[0], 2) + self.assertEqual(conn.execute( + "SELECT reported FROM shard_range").fetchone()[0], 0) # Put old event broker.merge_shard_ranges( ShardRange('"a/{}"', old_put_timestamp, - 'lower', 'upper', 1, 2, meta_timestamp=meta_timestamp)) + 'lower', 'upper', 1, 2, meta_timestamp=meta_timestamp, + reported=True)) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM shard_range").fetchone()[0], @@ -1102,6 +1183,8 @@ "SELECT object_count FROM shard_range").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT bytes_used FROM shard_range").fetchone()[0], 2) + self.assertEqual(conn.execute( + "SELECT reported FROM shard_range").fetchone()[0], 0) # Put old delete event broker.merge_shard_ranges( @@ -1978,10 +2061,12 @@ self.assertEqual(info['hash'], '00000000000000000000000000000000') self.assertEqual(info['put_timestamp'], Timestamp(1).internal) self.assertEqual(info['delete_timestamp'], '0') - if self.__class__ in (TestContainerBrokerBeforeMetadata, - TestContainerBrokerBeforeXSync, - TestContainerBrokerBeforeSPI, - TestContainerBrokerBeforeShardRanges): + if self.__class__ in ( + TestContainerBrokerBeforeMetadata, + TestContainerBrokerBeforeXSync, + TestContainerBrokerBeforeSPI, + TestContainerBrokerBeforeShardRanges, + TestContainerBrokerBeforeShardRangeReportedColumn): self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], @@ -3275,10 +3360,12 @@ self.assertEqual(0, info['storage_policy_index']) # sanity check self.assertEqual(0, info['object_count']) self.assertEqual(0, info['bytes_used']) - if self.__class__ in (TestContainerBrokerBeforeMetadata, - TestContainerBrokerBeforeXSync, - TestContainerBrokerBeforeSPI, - TestContainerBrokerBeforeShardRanges): + if self.__class__ in ( + TestContainerBrokerBeforeMetadata, + TestContainerBrokerBeforeXSync, + TestContainerBrokerBeforeSPI, + TestContainerBrokerBeforeShardRanges, + TestContainerBrokerBeforeShardRangeReportedColumn): self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(timestamp.internal, info['status_changed_at']) @@ -3842,6 +3929,45 @@ self.assertFalse(actual) @with_tempdir + def test_overloap_shard_range_order(self, tempdir): + db_path = os.path.join(tempdir, 'container.db') + broker = ContainerBroker(db_path, account='a', container='c') + broker.initialize(next(self.ts).internal, 0) + + epoch0 = next(self.ts) + epoch1 = next(self.ts) + shard_ranges = [ + ShardRange('.shard_a/shard_%d-%d' % (e, s), epoch, l, u, + state=ShardRange.ACTIVE) + for s, (l, u) in enumerate(zip(string.ascii_letters[:7], + string.ascii_letters[1:])) + for e, epoch in enumerate((epoch0, epoch1)) + ] + + random.shuffle(shard_ranges) + for sr in shard_ranges: + broker.merge_shard_ranges([sr]) + + expected = [ + '.shard_a/shard_0-0', + '.shard_a/shard_1-0', + '.shard_a/shard_0-1', + '.shard_a/shard_1-1', + '.shard_a/shard_0-2', + '.shard_a/shard_1-2', + '.shard_a/shard_0-3', + '.shard_a/shard_1-3', + '.shard_a/shard_0-4', + '.shard_a/shard_1-4', + '.shard_a/shard_0-5', + '.shard_a/shard_1-5', + '.shard_a/shard_0-6', + '.shard_a/shard_1-6', + ] + self.assertEqual(expected, [ + sr.name for sr in broker.get_shard_ranges()]) + + @with_tempdir def test_get_shard_ranges_with_sharding_overlaps(self, tempdir): db_path = os.path.join(tempdir, 'container.db') broker = ContainerBroker(db_path, account='a', container='c') @@ -5315,6 +5441,75 @@ FROM shard_range''') +def pre_reported_create_shard_range_table(self, conn): + """ + Copied from ContainerBroker before the + reported column was added; used for testing with + TestContainerBrokerBeforeShardRangeReportedColumn. + + Create a shard_range table with no 'reported' column. + + :param conn: DB connection object + """ + conn.execute(""" + CREATE TABLE shard_range ( + ROWID INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + timestamp TEXT, + lower TEXT, + upper TEXT, + object_count INTEGER DEFAULT 0, + bytes_used INTEGER DEFAULT 0, + meta_timestamp TEXT, + deleted INTEGER DEFAULT 0, + state INTEGER, + state_timestamp TEXT, + epoch TEXT + ); + """) + + conn.execute(""" + CREATE TRIGGER shard_range_update BEFORE UPDATE ON shard_range + BEGIN + SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT'); + END; + """) + + +class TestContainerBrokerBeforeShardRangeReportedColumn( + ContainerBrokerMigrationMixin, TestContainerBroker): + """ + Tests for ContainerBroker against databases created + before the shard_ranges table was added. + """ + # *grumble grumble* This should include container_info/policy_stat :-/ + expected_db_tables = {'outgoing_sync', 'incoming_sync', 'object', + 'sqlite_sequence', 'container_stat', 'shard_range'} + + def setUp(self): + super(TestContainerBrokerBeforeShardRangeReportedColumn, + self).setUp() + ContainerBroker.create_shard_range_table = \ + pre_reported_create_shard_range_table + + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(Timestamp('1').internal, 0) + with self.assertRaises(sqlite3.DatabaseError) as raised, \ + broker.get() as conn: + conn.execute('''SELECT reported + FROM shard_range''') + self.assertIn('no such column: reported', str(raised.exception)) + + def tearDown(self): + super(TestContainerBrokerBeforeShardRangeReportedColumn, + self).tearDown() + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(Timestamp('1').internal, 0) + with broker.get() as conn: + conn.execute('''SELECT reported + FROM shard_range''') + + class TestUpdateNewItemFromExisting(unittest.TestCase): # TODO: add test scenarios that have swift_bytes in content_type t0 = '1234567890.00000' diff -Nru swift-2.25.1/test/unit/container/test_server.py swift-2.25.2/test/unit/container/test_server.py --- swift-2.25.1/test/unit/container/test_server.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/test/unit/container/test_server.py 2021-09-01 08:44:03.000000000 +0000 @@ -2380,15 +2380,17 @@ 'X-Container-Sysmeta-Test': 'set', 'X-Container-Meta-Test': 'persisted'} - # PUT shard range to non-existent container with non-autocreate prefix - req = Request.blank('/sda1/p/a/c', method='PUT', headers=headers, - body=json.dumps([dict(shard_range)])) + # PUT shard range to non-existent container without autocreate flag + req = Request.blank( + '/sda1/p/.shards_a/shard_c', method='PUT', headers=headers, + body=json.dumps([dict(shard_range)])) resp = req.get_response(self.controller) self.assertEqual(404, resp.status_int) - # PUT shard range to non-existent container with autocreate prefix, + # PUT shard range to non-existent container with autocreate flag, # missing storage policy headers['X-Timestamp'] = next(ts_iter).internal + headers['X-Backend-Auto-Create'] = 't' req = Request.blank( '/sda1/p/.shards_a/shard_c', method='PUT', headers=headers, body=json.dumps([dict(shard_range)])) @@ -2397,7 +2399,7 @@ self.assertIn(b'X-Backend-Storage-Policy-Index header is required', resp.body) - # PUT shard range to non-existent container with autocreate prefix + # PUT shard range to non-existent container with autocreate flag headers['X-Timestamp'] = next(ts_iter).internal policy_index = random.choice(POLICIES).idx headers['X-Backend-Storage-Policy-Index'] = str(policy_index) @@ -2407,7 +2409,7 @@ resp = req.get_response(self.controller) self.assertEqual(201, resp.status_int) - # repeat PUT of shard range to autocreated container - 204 response + # repeat PUT of shard range to autocreated container - 202 response headers['X-Timestamp'] = next(ts_iter).internal headers.pop('X-Backend-Storage-Policy-Index') # no longer required req = Request.blank( @@ -2416,7 +2418,7 @@ resp = req.get_response(self.controller) self.assertEqual(202, resp.status_int) - # regular PUT to autocreated container - 204 response + # regular PUT to autocreated container - 202 response headers['X-Timestamp'] = next(ts_iter).internal req = Request.blank( '/sda1/p/.shards_a/shard_c', method='PUT', @@ -4649,61 +4651,53 @@ "%d on param %s" % (resp.status_int, param)) def test_put_auto_create(self): - headers = {'x-timestamp': Timestamp(1).internal, - 'x-size': '0', - 'x-content-type': 'text/plain', - 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e'} - - req = Request.blank('/sda1/p/a/c/o', - environ={'REQUEST_METHOD': 'PUT'}, - headers=dict(headers)) - resp = req.get_response(self.controller) - self.assertEqual(resp.status_int, 404) - - req = Request.blank('/sda1/p/.a/c/o', - environ={'REQUEST_METHOD': 'PUT'}, - headers=dict(headers)) - resp = req.get_response(self.controller) - self.assertEqual(resp.status_int, 201) - - req = Request.blank('/sda1/p/a/.c/o', - environ={'REQUEST_METHOD': 'PUT'}, - headers=dict(headers)) - resp = req.get_response(self.controller) - self.assertEqual(resp.status_int, 404) + def do_test(expected_status, path, extra_headers=None, body=None): + headers = {'x-timestamp': Timestamp(1).internal, + 'x-size': '0', + 'x-content-type': 'text/plain', + 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e'} + if extra_headers: + headers.update(extra_headers) + req = Request.blank('/sda1/p/' + path, + environ={'REQUEST_METHOD': 'PUT'}, + headers=headers, body=body) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, expected_status) - req = Request.blank('/sda1/p/a/c/.o', - environ={'REQUEST_METHOD': 'PUT'}, - headers=dict(headers)) - resp = req.get_response(self.controller) - self.assertEqual(resp.status_int, 404) + do_test(404, 'a/c/o') + do_test(404, '.a/c/o', {'X-Backend-Auto-Create': 'no'}) + do_test(201, '.a/c/o') + do_test(404, 'a/.c/o') + do_test(404, 'a/c/.o') + do_test(201, 'a/c/o', {'X-Backend-Auto-Create': 'yes'}) + + do_test(404, '.shards_a/c/o') + create_shard_headers = { + 'X-Backend-Record-Type': 'shard', + 'X-Backend-Storage-Policy-Index': '0'} + do_test(404, '.shards_a/c', create_shard_headers, '[]') + create_shard_headers['X-Backend-Auto-Create'] = 't' + do_test(201, '.shards_a/c', create_shard_headers, '[]') def test_delete_auto_create(self): - headers = {'x-timestamp': Timestamp(1).internal} - - req = Request.blank('/sda1/p/a/c/o', - environ={'REQUEST_METHOD': 'DELETE'}, - headers=dict(headers)) - resp = req.get_response(self.controller) - self.assertEqual(resp.status_int, 404) - - req = Request.blank('/sda1/p/.a/c/o', - environ={'REQUEST_METHOD': 'DELETE'}, - headers=dict(headers)) - resp = req.get_response(self.controller) - self.assertEqual(resp.status_int, 204) - - req = Request.blank('/sda1/p/a/.c/o', - environ={'REQUEST_METHOD': 'DELETE'}, - headers=dict(headers)) - resp = req.get_response(self.controller) - self.assertEqual(resp.status_int, 404) + def do_test(expected_status, path, extra_headers=None): + headers = {'x-timestamp': Timestamp(1).internal} + if extra_headers: + headers.update(extra_headers) + req = Request.blank('/sda1/p/' + path, + environ={'REQUEST_METHOD': 'DELETE'}, + headers=headers) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, expected_status) - req = Request.blank('/sda1/p/a/.c/.o', - environ={'REQUEST_METHOD': 'DELETE'}, - headers=dict(headers)) - resp = req.get_response(self.controller) - self.assertEqual(resp.status_int, 404) + do_test(404, 'a/c/o') + do_test(404, '.a/c/o', {'X-Backend-Auto-Create': 'false'}) + do_test(204, '.a/c/o') + do_test(404, 'a/.c/o') + do_test(404, 'a/.c/.o') + do_test(404, '.shards_a/c/o') + do_test(204, 'a/c/o', {'X-Backend-Auto-Create': 'true'}) + do_test(204, '.shards_a/c/o', {'X-Backend-Auto-Create': 'true'}) def test_content_type_on_HEAD(self): Request.blank('/sda1/p/a/o', diff -Nru swift-2.25.1/test/unit/container/test_sharder.py swift-2.25.2/test/unit/container/test_sharder.py --- swift-2.25.1/test/unit/container/test_sharder.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/test/unit/container/test_sharder.py 2021-09-01 08:44:03.000000000 +0000 @@ -109,8 +109,11 @@ return broker def _make_shard_ranges(self, bounds, state=None, object_count=0): + if not isinstance(state, (tuple, list)): + state = [state] * len(bounds) + state_iter = iter(state) return [ShardRange('.shards_a/c_%s' % upper, Timestamp.now(), - lower, upper, state=state, + lower, upper, state=next(state_iter), object_count=object_count) for lower, upper in bounds] @@ -2219,6 +2222,71 @@ shard_broker.get_syncs()) self.assertEqual(objects[5:], shard_broker.get_objects()) + def test_cleave_skips_shrinking_and_stops_at_found(self): + broker = self._make_broker() + broker.enable_sharding(Timestamp.now()) + shard_bounds = (('', 'b'), + ('b', 'c'), + ('b', 'd'), + ('d', 'f'), + ('f', '')) + # make sure there is an object in every shard range so cleaving will + # occur in batches of 2 + objects = [ + ('a', self.ts_encoded(), 10, 'text/plain', 'etag_a', 0, 0), + ('b', self.ts_encoded(), 10, 'text/plain', 'etag_b', 0, 0), + ('c', self.ts_encoded(), 1, 'text/plain', 'etag_c', 0, 0), + ('d', self.ts_encoded(), 2, 'text/plain', 'etag_d', 0, 0), + ('e', self.ts_encoded(), 3, 'text/plain', 'etag_e', 0, 0), + ('f', self.ts_encoded(), 100, 'text/plain', 'etag_f', 0, 0), + ('x', self.ts_encoded(), 0, '', '', 1, 0), # deleted + ('z', self.ts_encoded(), 1000, 'text/plain', 'etag_z', 0, 0) + ] + for obj in objects: + broker.put_object(*obj) + shard_ranges = self._make_shard_ranges( + shard_bounds, state=[ShardRange.CREATED, + ShardRange.SHRINKING, + ShardRange.CREATED, + ShardRange.CREATED, + ShardRange.FOUND]) + broker.merge_shard_ranges(shard_ranges) + self.assertTrue(broker.set_sharding_state()) + + # run cleave - first batch is cleaved, shrinking range doesn't count + # towards batch size of 2 but does count towards ranges_done + with self._mock_sharder() as sharder: + self.assertFalse(sharder._cleave(broker)) + context = CleavingContext.load(broker) + self.assertTrue(context.misplaced_done) + self.assertFalse(context.cleaving_done) + self.assertEqual(shard_ranges[2].upper_str, context.cursor) + self.assertEqual(3, context.ranges_done) + self.assertEqual(2, context.ranges_todo) + + # run cleave - stops at shard range in FOUND state + with self._mock_sharder() as sharder: + self.assertFalse(sharder._cleave(broker)) + context = CleavingContext.load(broker) + self.assertTrue(context.misplaced_done) + self.assertFalse(context.cleaving_done) + self.assertEqual(shard_ranges[3].upper_str, context.cursor) + self.assertEqual(4, context.ranges_done) + self.assertEqual(1, context.ranges_todo) + + # run cleave - final shard range in CREATED state, cleaving proceeds + shard_ranges[4].update_state(ShardRange.CREATED, + state_timestamp=Timestamp.now()) + broker.merge_shard_ranges(shard_ranges[4:]) + with self._mock_sharder() as sharder: + self.assertTrue(sharder._cleave(broker)) + context = CleavingContext.load(broker) + self.assertTrue(context.misplaced_done) + self.assertTrue(context.cleaving_done) + self.assertEqual(shard_ranges[4].upper_str, context.cursor) + self.assertEqual(5, context.ranges_done) + self.assertEqual(0, context.ranges_todo) + def _check_complete_sharding(self, account, container, shard_bounds): broker = self._make_sharding_broker( account=account, container=container, shard_bounds=shard_bounds) @@ -4119,7 +4187,8 @@ for state in sorted(ShardRange.STATES): if state in (ShardRange.SHARDING, ShardRange.SHRINKING, - ShardRange.SHARDED): + ShardRange.SHARDED, + ShardRange.SHRUNK): epoch = None else: epoch = Timestamp.now() @@ -4189,6 +4258,7 @@ def capture_send(conn, data): bodies.append(data) + self.assertFalse(broker.get_own_shard_range().reported) # sanity with self._mock_sharder() as sharder: with mocked_http_conn(204, 204, 204, give_send=capture_send) as mock_conn: @@ -4198,6 +4268,7 @@ self.assertEqual('PUT', req['method']) self.assertEqual([expected_sent] * 3, [json.loads(b) for b in bodies]) + self.assertTrue(broker.get_own_shard_range().reported) def test_update_root_container_own_range(self): broker = self._make_broker() @@ -4230,6 +4301,32 @@ with annotate_failure(state): check_only_own_shard_range_sent(state) + def test_update_root_container_already_reported(self): + broker = self._make_broker() + + def check_already_reported_not_sent(state): + own_shard_range = broker.get_own_shard_range() + + own_shard_range.reported = True + self.assertTrue(own_shard_range.update_state( + state, state_timestamp=next(self.ts_iter))) + # Check that updating state clears the flag + self.assertFalse(own_shard_range.reported) + + # If we claim to have already updated... + own_shard_range.reported = True + broker.merge_shard_ranges([own_shard_range]) + + # ... then there's nothing to send + with self._mock_sharder() as sharder: + with mocked_http_conn() as mock_conn: + sharder._update_root_container(broker) + self.assertFalse(mock_conn.requests) + + for state in ShardRange.STATES: + with annotate_failure(state): + check_already_reported_not_sent(state) + def test_update_root_container_all_ranges(self): broker = self._make_broker() other_shard_ranges = self._make_shard_ranges((('', 'h'), ('h', ''))) @@ -4290,6 +4387,10 @@ expected_stats = {'attempted': 1, 'success': 0, 'failure': 1} shard_bounds = (('a', 'j'), ('k', 't'), ('s', 'z')) for state, state_text in ShardRange.STATES.items(): + if state in (ShardRange.SHRINKING, + ShardRange.SHARDED, + ShardRange.SHRUNK): + continue # tested separately below shard_ranges = self._make_shard_ranges(shard_bounds, state) broker.merge_shard_ranges(shard_ranges) with self._mock_sharder() as sharder: @@ -4303,62 +4404,124 @@ self._assert_stats(expected_stats, sharder, 'audit_root') mocked.assert_not_called() - def assert_missing_warning(line): - self.assertIn( - 'Audit failed for root %s' % broker.db_file, line) - self.assertIn('missing range(s): -a j-k z-', line) + shard_ranges = self._make_shard_ranges(shard_bounds, + ShardRange.SHRINKING) + broker.merge_shard_ranges(shard_ranges) + with self._mock_sharder() as sharder: + with mock.patch.object( + sharder, '_audit_shard_container') as mocked: + sharder._audit_container(broker) + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + self._assert_stats({'attempted': 1, 'success': 1, 'failure': 0}, + sharder, 'audit_root') + mocked.assert_not_called() - own_shard_range = broker.get_own_shard_range() - states = (ShardRange.SHARDING, ShardRange.SHARDED) - for state in states: - own_shard_range.update_state( - state, state_timestamp=next(self.ts_iter)) - broker.merge_shard_ranges([own_shard_range]) + for state in (ShardRange.SHRUNK, ShardRange.SHARDED): + shard_ranges = self._make_shard_ranges(shard_bounds, state) + for sr in shard_ranges: + sr.set_deleted(Timestamp.now()) + broker.merge_shard_ranges(shard_ranges) with self._mock_sharder() as sharder: with mock.patch.object( sharder, '_audit_shard_container') as mocked: sharder._audit_container(broker) - lines = sharder.logger.get_lines_for_level('warning') - assert_missing_warning(lines[0]) - assert_overlap_warning(lines[0], state_text) - self.assertFalse(lines[1:]) + self.assertFalse(sharder.logger.get_lines_for_level('warning')) self.assertFalse(sharder.logger.get_lines_for_level('error')) - self._assert_stats(expected_stats, sharder, 'audit_root') + self._assert_stats({'attempted': 1, 'success': 1, 'failure': 0}, + sharder, 'audit_root') mocked.assert_not_called() - def test_audit_old_style_shard_container(self): - broker = self._make_broker(account='.shards_a', container='shard_c') - broker.set_sharding_sysmeta('Root', 'a/c') + # Put the shards back to a "useful" state + shard_ranges = self._make_shard_ranges(shard_bounds, + ShardRange.ACTIVE) + broker.merge_shard_ranges(shard_ranges) + + def assert_missing_warning(line): + self.assertIn( + 'Audit failed for root %s' % broker.db_file, line) + self.assertIn('missing range(s): -a j-k z-', line) + + def check_missing(): + own_shard_range = broker.get_own_shard_range() + states = (ShardRange.SHARDING, ShardRange.SHARDED) + for state in states: + own_shard_range.update_state( + state, state_timestamp=next(self.ts_iter)) + broker.merge_shard_ranges([own_shard_range]) + with self._mock_sharder() as sharder: + with mock.patch.object( + sharder, '_audit_shard_container') as mocked: + sharder._audit_container(broker) + lines = sharder.logger.get_lines_for_level('warning') + assert_missing_warning(lines[0]) + assert_overlap_warning(lines[0], 'active') + self.assertFalse(lines[1:]) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + self._assert_stats(expected_stats, sharder, 'audit_root') + mocked.assert_not_called() + + check_missing() + + # fill the gaps with shrinking shards and check that these are still + # reported as 'missing' + missing_shard_bounds = (('', 'a'), ('j', 'k'), ('z', '')) + shrinking_shard_ranges = self._make_shard_ranges(missing_shard_bounds, + ShardRange.SHRINKING) + broker.merge_shard_ranges(shrinking_shard_ranges) + check_missing() + + def call_audit_container(self, broker, shard_ranges, exc=None): + with self._mock_sharder() as sharder: + with mock.patch.object(sharder, '_audit_root_container') \ + as mocked, mock.patch.object( + sharder, 'int_client') as mock_swift: + mock_response = mock.MagicMock() + mock_response.headers = {'x-backend-record-type': + 'shard'} + mock_response.body = json.dumps( + [dict(sr) for sr in shard_ranges]) + mock_swift.make_request.return_value = mock_response + mock_swift.make_request.side_effect = exc + mock_swift.make_path = (lambda a, c: + '/v1/%s/%s' % (a, c)) + sharder.reclaim_age = 0 + sharder._audit_container(broker) + mocked.assert_not_called() + return sharder, mock_swift + + def assert_no_audit_messages(self, sharder, mock_swift, + marker='k', end_marker='t'): + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + expected_stats = {'attempted': 1, 'success': 1, 'failure': 0} + self._assert_stats(expected_stats, sharder, 'audit_shard') + expected_headers = {'X-Backend-Record-Type': 'shard', + 'X-Newest': 'true', + 'X-Backend-Include-Deleted': 'True', + 'X-Backend-Override-Deleted': 'true'} + params = {'format': 'json', 'marker': marker, 'end_marker': end_marker} + mock_swift.make_request.assert_called_once_with( + 'GET', '/v1/a/c', expected_headers, acceptable_statuses=(2,), + params=params) + + def _do_test_audit_shard_container(self, *args): # include overlaps to verify correct match for updating own shard range + broker = self._make_broker(account='.shards_a', container='shard_c') + broker.set_sharding_sysmeta(*args) shard_bounds = ( - ('a', 'j'), ('k', 't'), ('k', 's'), ('l', 's'), ('s', 'z')) - shard_ranges = self._make_shard_ranges(shard_bounds, ShardRange.ACTIVE) + ('a', 'j'), ('k', 't'), ('k', 'u'), ('l', 'v'), ('s', 'z')) + shard_states = ( + ShardRange.ACTIVE, ShardRange.ACTIVE, ShardRange.ACTIVE, + ShardRange.FOUND, ShardRange.CREATED + ) + shard_ranges = self._make_shard_ranges(shard_bounds, shard_states) shard_ranges[1].name = broker.path expected_stats = {'attempted': 1, 'success': 0, 'failure': 1} - def call_audit_container(exc=None): - with self._mock_sharder() as sharder: - sharder.logger = debug_logger() - with mock.patch.object(sharder, '_audit_root_container') \ - as mocked, mock.patch.object( - sharder, 'int_client') as mock_swift: - mock_response = mock.MagicMock() - mock_response.headers = {'x-backend-record-type': - 'shard'} - mock_response.body = json.dumps( - [dict(sr) for sr in shard_ranges]) - mock_swift.make_request.return_value = mock_response - mock_swift.make_request.side_effect = exc - mock_swift.make_path = (lambda a, c: - '/v1/%s/%s' % (a, c)) - sharder.reclaim_age = 0 - sharder._audit_container(broker) - mocked.assert_not_called() - return sharder, mock_swift - # bad account name broker.account = 'bad_account' - sharder, mock_swift = call_audit_container() + sharder, mock_swift = self.call_audit_container(broker, shard_ranges) lines = sharder.logger.get_lines_for_level('warning') self._assert_stats(expected_stats, sharder, 'audit_shard') self.assertIn('Audit warnings for shard %s' % broker.db_file, lines[0]) @@ -4372,7 +4535,7 @@ # missing own shard range broker.get_info() - sharder, mock_swift = call_audit_container() + sharder, mock_swift = self.call_audit_container(broker, shard_ranges) lines = sharder.logger.get_lines_for_level('warning') self._assert_stats(expected_stats, sharder, 'audit_shard') self.assertIn('Audit failed for shard %s' % broker.db_file, lines[0]) @@ -4382,21 +4545,24 @@ self.assertFalse(sharder.logger.get_lines_for_level('error')) self.assertFalse(broker.is_deleted()) - # create own shard range, no match in root + # own shard range bounds don't match what's in root (e.g. this shard is + # expanding to be an acceptor) expected_stats = {'attempted': 1, 'success': 1, 'failure': 0} own_shard_range = broker.get_own_shard_range() # get the default own_shard_range.lower = 'j' own_shard_range.upper = 'k' + own_shard_range.name = broker.path broker.merge_shard_ranges([own_shard_range]) - sharder, mock_swift = call_audit_container() - lines = sharder.logger.get_lines_for_level('warning') - self.assertIn('Audit warnings for shard %s' % broker.db_file, lines[0]) - self.assertNotIn('account not in shards namespace', lines[0]) - self.assertNotIn('missing own shard range', lines[0]) - self.assertIn('root has no matching shard range', lines[0]) - self.assertNotIn('unable to get shard ranges from root', lines[0]) + # bump timestamp of root shard range to be newer than own + now = Timestamp.now() + self.assertTrue(shard_ranges[1].update_state(ShardRange.ACTIVE, + state_timestamp=now)) + shard_ranges[1].timestamp = now + sharder, mock_swift = self.call_audit_container(broker, shard_ranges) self._assert_stats(expected_stats, sharder, 'audit_shard') - self.assertFalse(lines[1:]) + self.assertEqual(['Updating own shard range from root', mock.ANY], + sharder.logger.get_lines_for_level('debug')) + self.assertFalse(sharder.logger.get_lines_for_level('warning')) self.assertFalse(sharder.logger.get_lines_for_level('error')) self.assertFalse(broker.is_deleted()) expected_headers = {'X-Backend-Record-Type': 'shard', @@ -4407,14 +4573,58 @@ mock_swift.make_request.assert_called_once_with( 'GET', '/v1/a/c', expected_headers, acceptable_statuses=(2,), params=params) + # own shard range bounds are updated from root version + own_shard_range = broker.get_own_shard_range() + self.assertEqual(ShardRange.ACTIVE, own_shard_range.state) + self.assertEqual(now, own_shard_range.state_timestamp) + self.assertEqual('k', own_shard_range.lower) + self.assertEqual('t', own_shard_range.upper) + # check other shard ranges from root are not merged (not shrinking) + self.assertEqual([own_shard_range], + broker.get_shard_ranges(include_own=True)) + + # move root shard range to shrinking state + now = Timestamp.now() + self.assertTrue(shard_ranges[1].update_state(ShardRange.SHRINKING, + state_timestamp=now)) + # bump own shard range state timestamp so it is newer than root + now = Timestamp.now() + own_shard_range = broker.get_own_shard_range() + own_shard_range.update_state(ShardRange.ACTIVE, state_timestamp=now) + broker.merge_shard_ranges([own_shard_range]) + + sharder, mock_swift = self.call_audit_container(broker, shard_ranges) + self._assert_stats(expected_stats, sharder, 'audit_shard') + self.assertEqual(['Updating own shard range from root'], + sharder.logger.get_lines_for_level('debug')) + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + self.assertFalse(broker.is_deleted()) + expected_headers = {'X-Backend-Record-Type': 'shard', + 'X-Newest': 'true', + 'X-Backend-Include-Deleted': 'True', + 'X-Backend-Override-Deleted': 'true'} + params = {'format': 'json', 'marker': 'k', 'end_marker': 't'} + mock_swift.make_request.assert_called_once_with( + 'GET', '/v1/a/c', expected_headers, acceptable_statuses=(2,), + params=params) + # check own shard range bounds + own_shard_range = broker.get_own_shard_range() + # own shard range state has not changed (root is older) + self.assertEqual(ShardRange.ACTIVE, own_shard_range.state) + self.assertEqual(now, own_shard_range.state_timestamp) + self.assertEqual('k', own_shard_range.lower) + self.assertEqual('t', own_shard_range.upper) - # create own shard range, failed response from root + # reset own shard range bounds, failed response from root expected_stats = {'attempted': 1, 'success': 1, 'failure': 0} own_shard_range = broker.get_own_shard_range() # get the default own_shard_range.lower = 'j' own_shard_range.upper = 'k' + own_shard_range.timestamp = Timestamp.now() broker.merge_shard_ranges([own_shard_range]) - sharder, mock_swift = call_audit_container( + sharder, mock_swift = self.call_audit_container( + broker, shard_ranges, exc=internal_client.UnexpectedResponse('bad', 'resp')) lines = sharder.logger.get_lines_for_level('warning') self.assertIn('Failed to get shard ranges', lines[0]) @@ -4427,178 +4637,180 @@ self.assertFalse(lines[2:]) self.assertFalse(sharder.logger.get_lines_for_level('error')) self.assertFalse(broker.is_deleted()) + params = {'format': 'json', 'marker': 'j', 'end_marker': 'k'} mock_swift.make_request.assert_called_once_with( 'GET', '/v1/a/c', expected_headers, acceptable_statuses=(2,), params=params) - def assert_ok(): - sharder, mock_swift = call_audit_container() - self.assertFalse(sharder.logger.get_lines_for_level('warning')) - self.assertFalse(sharder.logger.get_lines_for_level('error')) - self._assert_stats(expected_stats, sharder, 'audit_shard') - params = {'format': 'json', 'marker': 'k', 'end_marker': 't'} - mock_swift.make_request.assert_called_once_with( - 'GET', '/v1/a/c', expected_headers, acceptable_statuses=(2,), - params=params) - # make own shard range match one in root, but different state shard_ranges[1].timestamp = Timestamp.now() broker.merge_shard_ranges([shard_ranges[1]]) now = Timestamp.now() shard_ranges[1].update_state(ShardRange.SHARDING, state_timestamp=now) - assert_ok() + sharder, mock_swift = self.call_audit_container(broker, shard_ranges) + self.assert_no_audit_messages(sharder, mock_swift) self.assertFalse(broker.is_deleted()) # own shard range state is updated from root version own_shard_range = broker.get_own_shard_range() self.assertEqual(ShardRange.SHARDING, own_shard_range.state) self.assertEqual(now, own_shard_range.state_timestamp) + self.assertEqual(['Updating own shard range from root', mock.ANY], + sharder.logger.get_lines_for_level('debug')) own_shard_range.update_state(ShardRange.SHARDED, state_timestamp=Timestamp.now()) broker.merge_shard_ranges([own_shard_range]) - assert_ok() + sharder, mock_swift = self.call_audit_container(broker, shard_ranges) + self.assert_no_audit_messages(sharder, mock_swift) own_shard_range.deleted = 1 own_shard_range.timestamp = Timestamp.now() broker.merge_shard_ranges([own_shard_range]) - assert_ok() + sharder, mock_swift = self.call_audit_container(broker, shard_ranges) + self.assert_no_audit_messages(sharder, mock_swift) self.assertTrue(broker.is_deleted()) - def test_audit_shard_container(self): - broker = self._make_broker(account='.shards_a', container='shard_c') - broker.set_sharding_sysmeta('Quoted-Root', 'a/c') - # include overlaps to verify correct match for updating own shard range - shard_bounds = ( - ('a', 'j'), ('k', 't'), ('k', 's'), ('l', 's'), ('s', 'z')) - shard_ranges = self._make_shard_ranges(shard_bounds, ShardRange.ACTIVE) - shard_ranges[1].name = broker.path - expected_stats = {'attempted': 1, 'success': 0, 'failure': 1} - - def call_audit_container(exc=None): - with self._mock_sharder() as sharder: - with mock.patch.object(sharder, '_audit_root_container') \ - as mocked, mock.patch.object( - sharder, 'int_client') as mock_swift: - mock_response = mock.MagicMock() - mock_response.headers = {'x-backend-record-type': - 'shard'} - mock_response.body = json.dumps( - [dict(sr) for sr in shard_ranges]) - mock_swift.make_request.return_value = mock_response - mock_swift.make_request.side_effect = exc - mock_swift.make_path = (lambda a, c: - '/v1/%s/%s' % (a, c)) - sharder.reclaim_age = 0 - sharder._audit_container(broker) - mocked.assert_not_called() - return sharder, mock_swift - - # bad account name - broker.account = 'bad_account' - sharder, mock_swift = call_audit_container() - lines = sharder.logger.get_lines_for_level('warning') - self._assert_stats(expected_stats, sharder, 'audit_shard') - self.assertIn('Audit warnings for shard %s' % broker.db_file, lines[0]) - self.assertIn('account not in shards namespace', lines[0]) - self.assertNotIn('root has no matching shard range', lines[0]) - self.assertNotIn('unable to get shard ranges from root', lines[0]) - self.assertIn('Audit failed for shard %s' % broker.db_file, lines[1]) - self.assertIn('missing own shard range', lines[1]) - self.assertFalse(lines[2:]) - self.assertFalse(broker.is_deleted()) - - # missing own shard range - broker.get_info() - sharder, mock_swift = call_audit_container() - lines = sharder.logger.get_lines_for_level('warning') - self._assert_stats(expected_stats, sharder, 'audit_shard') - self.assertIn('Audit failed for shard %s' % broker.db_file, lines[0]) - self.assertIn('missing own shard range', lines[0]) - self.assertNotIn('unable to get shard ranges from root', lines[0]) - self.assertFalse(lines[1:]) - self.assertFalse(sharder.logger.get_lines_for_level('error')) - self.assertFalse(broker.is_deleted()) + def test_audit_old_style_shard_container(self): + self._do_test_audit_shard_container('Root', 'a/c') - # create own shard range, no match in root - expected_stats = {'attempted': 1, 'success': 1, 'failure': 0} - own_shard_range = broker.get_own_shard_range() # get the default - own_shard_range.lower = 'j' - own_shard_range.upper = 'k' - broker.merge_shard_ranges([own_shard_range]) - sharder, mock_swift = call_audit_container() - lines = sharder.logger.get_lines_for_level('warning') - self.assertIn('Audit warnings for shard %s' % broker.db_file, lines[0]) - self.assertNotIn('account not in shards namespace', lines[0]) - self.assertNotIn('missing own shard range', lines[0]) - self.assertIn('root has no matching shard range', lines[0]) - self.assertNotIn('unable to get shard ranges from root', lines[0]) - self._assert_stats(expected_stats, sharder, 'audit_shard') - self.assertFalse(lines[1:]) - self.assertFalse(sharder.logger.get_lines_for_level('error')) - self.assertFalse(broker.is_deleted()) - expected_headers = {'X-Backend-Record-Type': 'shard', - 'X-Newest': 'true', - 'X-Backend-Include-Deleted': 'True', - 'X-Backend-Override-Deleted': 'true'} - params = {'format': 'json', 'marker': 'j', 'end_marker': 'k'} - mock_swift.make_request.assert_called_once_with( - 'GET', '/v1/a/c', expected_headers, acceptable_statuses=(2,), - params=params) + def test_audit_shard_container(self): + self._do_test_audit_shard_container('Quoted-Root', 'a/c') - # create own shard range, failed response from root - expected_stats = {'attempted': 1, 'success': 1, 'failure': 0} - own_shard_range = broker.get_own_shard_range() # get the default - own_shard_range.lower = 'j' - own_shard_range.upper = 'k' - broker.merge_shard_ranges([own_shard_range]) - sharder, mock_swift = call_audit_container( - exc=internal_client.UnexpectedResponse('bad', 'resp')) - lines = sharder.logger.get_lines_for_level('warning') - self.assertIn('Failed to get shard ranges', lines[0]) - self.assertIn('Audit warnings for shard %s' % broker.db_file, lines[1]) - self.assertNotIn('account not in shards namespace', lines[1]) - self.assertNotIn('missing own shard range', lines[1]) - self.assertNotIn('root has no matching shard range', lines[1]) - self.assertIn('unable to get shard ranges from root', lines[1]) - self._assert_stats(expected_stats, sharder, 'audit_shard') - self.assertFalse(lines[2:]) - self.assertFalse(sharder.logger.get_lines_for_level('error')) - self.assertFalse(broker.is_deleted()) - mock_swift.make_request.assert_called_once_with( - 'GET', '/v1/a/c', expected_headers, acceptable_statuses=(2,), - params=params) + def _do_test_audit_shard_container_merge_other_ranges(self, *args): + # verify that shard only merges other ranges from root when it is + # shrinking or shrunk + shard_bounds = ( + ('a', 'p'), ('k', 't'), ('p', 'u')) + shard_states = ( + ShardRange.ACTIVE, ShardRange.ACTIVE, ShardRange.FOUND, + ) + shard_ranges = self._make_shard_ranges(shard_bounds, shard_states) - def assert_ok(): - sharder, mock_swift = call_audit_container() + def check_audit(own_state, root_state): + broker = self._make_broker( + account='.shards_a', + container='shard_c_%s' % root_ts.normal) + broker.set_sharding_sysmeta(*args) + shard_ranges[1].name = broker.path + + # make own shard range match shard_ranges[1] + own_sr = shard_ranges[1] + expected_stats = {'attempted': 1, 'success': 1, 'failure': 0} + self.assertTrue(own_sr.update_state(own_state, + state_timestamp=own_ts)) + own_sr.timestamp = own_ts + broker.merge_shard_ranges([shard_ranges[1]]) + + # bump state and timestamp of root shard_ranges[1] to be newer + self.assertTrue(shard_ranges[1].update_state( + root_state, state_timestamp=root_ts)) + shard_ranges[1].timestamp = root_ts + sharder, mock_swift = self.call_audit_container(broker, + shard_ranges) + self._assert_stats(expected_stats, sharder, 'audit_shard') + debug_lines = sharder.logger.get_lines_for_level('debug') + self.assertGreater(len(debug_lines), 0) + self.assertEqual('Updating own shard range from root', + debug_lines[0]) self.assertFalse(sharder.logger.get_lines_for_level('warning')) self.assertFalse(sharder.logger.get_lines_for_level('error')) - self._assert_stats(expected_stats, sharder, 'audit_shard') + self.assertFalse(broker.is_deleted()) + expected_headers = {'X-Backend-Record-Type': 'shard', + 'X-Newest': 'true', + 'X-Backend-Include-Deleted': 'True', + 'X-Backend-Override-Deleted': 'true'} params = {'format': 'json', 'marker': 'k', 'end_marker': 't'} mock_swift.make_request.assert_called_once_with( 'GET', '/v1/a/c', expected_headers, acceptable_statuses=(2,), params=params) + return broker, shard_ranges - # make own shard range match one in root, but different state - shard_ranges[1].timestamp = Timestamp.now() - broker.merge_shard_ranges([shard_ranges[1]]) - now = Timestamp.now() - shard_ranges[1].update_state(ShardRange.SHARDING, state_timestamp=now) - assert_ok() - self.assertFalse(broker.is_deleted()) - # own shard range state is updated from root version + # make root's copy of shard range newer than shard's local copy, so + # shard will always update its own shard range from root, and may merge + # other shard ranges + for own_state in ShardRange.STATES: + for root_state in ShardRange.STATES: + with annotate_failure('own_state=%s, root_state=%s' % + (own_state, root_state)): + own_ts = Timestamp.now() + root_ts = Timestamp(float(own_ts) + 1) + broker, shard_ranges = check_audit(own_state, root_state) + # own shard range is updated from newer root version + own_shard_range = broker.get_own_shard_range() + self.assertEqual(root_state, own_shard_range.state) + self.assertEqual(root_ts, own_shard_range.state_timestamp) + updated_ranges = broker.get_shard_ranges(include_own=True) + if root_state in (ShardRange.SHRINKING, ShardRange.SHRUNK): + # check other shard ranges from root are merged + self.assertEqual(shard_ranges, updated_ranges) + else: + # check other shard ranges from root are not merged + self.assertEqual(shard_ranges[1:2], updated_ranges) + + # make root's copy of shard range older than shard's local copy, so + # shard will never update its own shard range from root, but may merge + # other shard ranges + for own_state in ShardRange.STATES: + for root_state in ShardRange.STATES: + with annotate_failure('own_state=%s, root_state=%s' % + (own_state, root_state)): + root_ts = Timestamp.now() + own_ts = Timestamp(float(root_ts) + 1) + broker, shard_ranges = check_audit(own_state, root_state) + # own shard range is not updated from older root version + own_shard_range = broker.get_own_shard_range() + self.assertEqual(own_state, own_shard_range.state) + self.assertEqual(own_ts, own_shard_range.state_timestamp) + updated_ranges = broker.get_shard_ranges(include_own=True) + if own_state in (ShardRange.SHRINKING, ShardRange.SHRUNK): + # check other shard ranges from root are merged + self.assertEqual(shard_ranges, updated_ranges) + else: + # check other shard ranges from root are not merged + self.assertEqual(shard_ranges[1:2], updated_ranges) + + def test_audit_old_style_shard_container_merge_other_ranges(self): + self._do_test_audit_shard_container_merge_other_ranges('Root', 'a/c') + + def test_audit_shard_container_merge_other_ranges(self): + self._do_test_audit_shard_container_merge_other_ranges('Quoted-Root', + 'a/c') + + def test_audit_deleted_range_in_root_container(self): + broker = self._make_broker(account='.shards_a', container='shard_c') + broker.set_sharding_sysmeta('Quoted-Root', 'a/c') own_shard_range = broker.get_own_shard_range() - self.assertEqual(ShardRange.SHARDING, own_shard_range.state) - self.assertEqual(now, own_shard_range.state_timestamp) + own_shard_range.lower = 'k' + own_shard_range.upper = 't' + broker.merge_shard_ranges([own_shard_range]) - own_shard_range.update_state(ShardRange.SHARDED, + shard_bounds = ( + ('a', 'j'), ('k', 't'), ('k', 's'), ('l', 's'), ('s', 'z')) + shard_ranges = self._make_shard_ranges(shard_bounds, ShardRange.ACTIVE) + shard_ranges[1].name = broker.path + shard_ranges[1].update_state(ShardRange.SHARDED, state_timestamp=Timestamp.now()) - broker.merge_shard_ranges([own_shard_range]) - assert_ok() + shard_ranges[1].deleted = 1 + + sharder, mock_swift = self.call_audit_container(broker, shard_ranges) + self.assert_no_audit_messages(sharder, mock_swift) + self.assertTrue(broker.is_deleted()) + def test_audit_deleted_range_missing_from_root_container(self): + broker = self._make_broker(account='.shards_a', container='shard_c') + broker.set_sharding_sysmeta('Quoted-Root', 'a/c') + own_shard_range = broker.get_own_shard_range() + own_shard_range.lower = 'k' + own_shard_range.upper = 't' + own_shard_range.update_state(ShardRange.SHARDED, + state_timestamp=Timestamp.now()) own_shard_range.deleted = 1 - own_shard_range.timestamp = Timestamp.now() broker.merge_shard_ranges([own_shard_range]) - assert_ok() + + self.assertFalse(broker.is_deleted()) + + sharder, mock_swift = self.call_audit_container(broker, []) + self.assert_no_audit_messages(sharder, mock_swift) self.assertTrue(broker.is_deleted()) def test_find_and_enable_sharding_candidates(self): @@ -5313,8 +5525,10 @@ self.assertEqual(0, ctx.ranges_done) self.assertEqual(0, ctx.ranges_todo) ctx.reset() + check_context() # check idempotency ctx.reset() + check_context() def test_start(self): ctx = CleavingContext('test', 'curs', 12, 11, 2, True, True) @@ -5330,5 +5544,30 @@ self.assertEqual(0, ctx.ranges_done) self.assertEqual(0, ctx.ranges_todo) ctx.start() + check_context() # check idempotency ctx.start() + check_context() + + def test_range_done(self): + ctx = CleavingContext('test', '', 12, 11, 2, True, True) + self.assertEqual(0, ctx.ranges_done) + self.assertEqual(0, ctx.ranges_todo) + self.assertEqual('', ctx.cursor) + + ctx.ranges_todo = 5 + ctx.range_done('b') + self.assertEqual(1, ctx.ranges_done) + self.assertEqual(4, ctx.ranges_todo) + self.assertEqual('b', ctx.cursor) + + ctx.range_done(None) + self.assertEqual(2, ctx.ranges_done) + self.assertEqual(3, ctx.ranges_todo) + self.assertEqual('b', ctx.cursor) + + ctx.ranges_todo = 9 + ctx.range_done('c') + self.assertEqual(3, ctx.ranges_done) + self.assertEqual(8, ctx.ranges_todo) + self.assertEqual('c', ctx.cursor) diff -Nru swift-2.25.1/test/unit/proxy/controllers/test_container.py swift-2.25.2/test/unit/proxy/controllers/test_container.py --- swift-2.25.1/test/unit/proxy/controllers/test_container.py 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/test/unit/proxy/controllers/test_container.py 2021-09-01 08:44:03.000000000 +0000 @@ -501,7 +501,7 @@ self.assertEqual(dict(exp_params, format='json'), got_params) for k, v in exp_headers.items(): self.assertIn(k, req['headers']) - self.assertEqual(v, req['headers'][k]) + self.assertEqual(v, req['headers'][k], k) self.assertNotIn('X-Backend-Override-Delete', req['headers']) return resp @@ -938,6 +938,166 @@ query_string='?delimiter=/') self.check_response(resp, root_resp_hdrs) + def test_GET_sharded_container_shard_redirects_to_root(self): + # check that if the root redirects listing to a shard, but the shard + # returns the root shard (e.g. it was the final shard to shrink into + # the root) objects are requested from the root, rather than a loop. + + # single shard spanning entire namespace + shard_sr = ShardRange('.shards_a/c_xyz', Timestamp.now(), '', '') + all_objects = self._make_shard_objects(shard_sr) + size_all_objects = sum([obj['bytes'] for obj in all_objects]) + num_all_objects = len(all_objects) + limit = CONTAINER_LISTING_LIMIT + + # when shrinking the final shard will return the root shard range into + # which it is shrinking + shard_resp_hdrs = { + 'X-Backend-Sharding-State': 'sharded', + 'X-Container-Object-Count': 0, + 'X-Container-Bytes-Used': 0, + 'X-Backend-Storage-Policy-Index': 0, + 'X-Backend-Record-Type': 'shard' + } + + # root still thinks it has a shard + root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', + 'X-Backend-Timestamp': '99', + 'X-Container-Object-Count': num_all_objects, + 'X-Container-Bytes-Used': size_all_objects, + 'X-Backend-Storage-Policy-Index': 0} + root_shard_resp_hdrs = dict(root_resp_hdrs) + root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + + root_sr = ShardRange('a/c', Timestamp.now(), '', '') + mock_responses = [ + # status, body, headers + (200, [dict(shard_sr)], root_shard_resp_hdrs), # from root + (200, [dict(root_sr)], shard_resp_hdrs), # from shard + (200, all_objects, root_resp_hdrs), # from root + ] + expected_requests = [ + # path, headers, params + # first request to root should specify auto record type + ('a/c', {'X-Backend-Record-Type': 'auto'}, + dict(states='listing')), + # request to shard should specify auto record type + (wsgi_quote(str_to_wsgi(shard_sr.name)), + {'X-Backend-Record-Type': 'auto'}, + dict(marker='', end_marker='', limit=str(limit), + states='listing')), # 200 + # second request to root should specify object record type + ('a/c', {'X-Backend-Record-Type': 'object'}, + dict(marker='', end_marker='', limit=str(limit))), # 200 + ] + + expected_objects = all_objects + resp = self._check_GET_shard_listing( + mock_responses, expected_objects, expected_requests) + self.check_response(resp, root_resp_hdrs, + expected_objects=expected_objects) + self.assertEqual( + [('a', 'c'), ('.shards_a', 'c_xyz')], + resp.request.environ.get('swift.shard_listing_history')) + + def test_GET_sharded_container_shard_redirects_between_shards(self): + # check that if one shard redirects listing to another shard that + # somehow redirects listing back to the first shard, then we will break + # out of the loop (this isn't an expected scenario, but could perhaps + # happen if multiple conflicting shard-shrinking decisions are made) + shard_bounds = ('', 'a', 'b', '') + shard_ranges = [ + ShardRange('.shards_a/c_%s' % upper, Timestamp.now(), lower, upper) + for lower, upper in zip(shard_bounds[:-1], shard_bounds[1:])] + self.assertEqual([ + '.shards_a/c_a', + '.shards_a/c_b', + '.shards_a/c_', + ], [sr.name for sr in shard_ranges]) + sr_dicts = [dict(sr) for sr in shard_ranges] + sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] + all_objects = [] + for objects in sr_objs: + all_objects.extend(objects) + size_all_objects = sum([obj['bytes'] for obj in all_objects]) + num_all_objects = len(all_objects) + + root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', + 'X-Backend-Timestamp': '99', + 'X-Container-Object-Count': num_all_objects, + 'X-Container-Bytes-Used': size_all_objects, + 'X-Backend-Storage-Policy-Index': 0, + 'X-Backend-Record-Type': 'shard', + } + shard_resp_hdrs = {'X-Backend-Sharding-State': 'unsharded', + 'X-Container-Object-Count': 2, + 'X-Container-Bytes-Used': 4, + 'X-Backend-Storage-Policy-Index': 0} + shrinking_resp_hdrs = { + 'X-Backend-Sharding-State': 'sharded', + 'X-Backend-Record-Type': 'shard', + } + limit = CONTAINER_LISTING_LIMIT + + mock_responses = [ + # status, body, headers + (200, sr_dicts, root_resp_hdrs), # from root + (200, sr_objs[0], shard_resp_hdrs), # objects from 1st shard + (200, [sr_dicts[2]], shrinking_resp_hdrs), # 2nd points to 3rd + (200, [sr_dicts[1]], shrinking_resp_hdrs), # 3rd points to 2nd + (200, sr_objs[1], shard_resp_hdrs), # objects from 2nd + (200, sr_objs[2], shard_resp_hdrs), # objects from 3rd + ] + expected_requests = [ + # each list item is tuple (path, headers, params) + # request to root + # context GET(a/c) + ('a/c', {'X-Backend-Record-Type': 'auto'}, + dict(states='listing')), + # request to 1st shard as per shard list from root; + # context GET(a/c); + # end_marker dictated by 1st shard range upper bound + ('.shards_a/c_a', {'X-Backend-Record-Type': 'auto'}, + dict(marker='', end_marker='a\x00', states='listing', + limit=str(limit))), # 200 + # request to 2nd shard as per shard list from root; + # context GET(a/c); + # end_marker dictated by 2nd shard range upper bound + ('.shards_a/c_b', {'X-Backend-Record-Type': 'auto'}, + dict(marker='a', end_marker='b\x00', states='listing', + limit=str(limit - len(sr_objs[0])))), + # request to 3rd shard as per shard list from *2nd shard*; + # new context GET(a/c)->GET(.shards_a/c_b); + # end_marker still dictated by 2nd shard range upper bound + ('.shards_a/c_', {'X-Backend-Record-Type': 'auto'}, + dict(marker='a', end_marker='b\x00', states='listing', + limit=str( + limit - len(sr_objs[0])))), + # request to 2nd shard as per shard list from *3rd shard*; this one + # should specify record type object; + # new context GET(a/c)->GET(.shards_a/c_b)->GET(.shards_a/c_); + # end_marker still dictated by 2nd shard range upper bound + ('.shards_a/c_b', {'X-Backend-Record-Type': 'object'}, + dict(marker='a', end_marker='b\x00', + limit=str( + limit - len(sr_objs[0])))), + # request to 3rd shard *as per shard list from root*; this one + # should specify record type object; + # context GET(a/c); + # end_marker dictated by 3rd shard range upper bound + ('.shards_a/c_', {'X-Backend-Record-Type': 'object'}, + dict(marker='b', end_marker='', + limit=str( + limit - len(sr_objs[0]) - len(sr_objs[1])))), # 200 + ] + resp = self._check_GET_shard_listing( + mock_responses, all_objects, expected_requests) + self.check_response(resp, root_resp_hdrs, + expected_objects=all_objects) + self.assertEqual( + [('a', 'c'), ('.shards_a', 'c_b'), ('.shards_a', 'c_')], + resp.request.environ.get('swift.shard_listing_history')) + def test_GET_sharded_container_overlapping_shards(self): # verify ordered listing even if unexpected overlapping shard ranges shard_bounds = (('', 'ham', ShardRange.CLEAVED), diff -Nru swift-2.25.1/test-requirements.txt swift-2.25.2/test-requirements.txt --- swift-2.25.1/test-requirements.txt 2020-10-09 18:48:13.000000000 +0000 +++ swift-2.25.2/test-requirements.txt 2021-09-01 08:44:03.000000000 +0000 @@ -21,6 +21,6 @@ keystonemiddleware>=4.17.0 # Apache-2.0 # Security checks -bandit>=1.1.0 # Apache-2.0 +bandit>=1.1.0,<=1.6.2 # Apache-2.0 docutils>=0.11 # OSI-Approved Open Source, Public Domain diff -Nru swift-2.25.1/tools/playbooks/ceph-s3tests/post.yaml swift-2.25.2/tools/playbooks/ceph-s3tests/post.yaml --- swift-2.25.1/tools/playbooks/ceph-s3tests/post.yaml 2020-10-09 18:48:13.000000000 +0000 +++ swift-2.25.2/tools/playbooks/ceph-s3tests/post.yaml 2021-09-01 08:44:03.000000000 +0000 @@ -1,6 +1,10 @@ - hosts: all become: true tasks: + - name: Check for s3-tests outputs + stat: + path: '{{ ansible_env.HOME }}/s3compat/output' + register: s3_tests_output - name: Copy s3-tests outputs from worker nodes to executor node synchronize: src: '{{ ansible_env.HOME }}/s3compat/output' @@ -8,3 +12,4 @@ mode: pull copy_links: true verify_host: true + when: s3_tests_output.stat.exists == true diff -Nru swift-2.25.1/tools/playbooks/ceph-s3tests/run.yaml swift-2.25.2/tools/playbooks/ceph-s3tests/run.yaml --- swift-2.25.1/tools/playbooks/ceph-s3tests/run.yaml 2020-10-09 18:48:13.000000000 +0000 +++ swift-2.25.2/tools/playbooks/ceph-s3tests/run.yaml 2021-09-01 08:44:03.000000000 +0000 @@ -25,7 +25,7 @@ - name: Clone s3compat repository git: - repo: "https://github.com/swiftstack/s3compat.git" + repo: "https://github.com/tipabu/s3compat.git" dest: "{{ ansible_env.HOME }}/s3compat" - name: Install s3compat requirements diff -Nru swift-2.25.1/tools/playbooks/common/install_dependencies.yaml swift-2.25.2/tools/playbooks/common/install_dependencies.yaml --- swift-2.25.1/tools/playbooks/common/install_dependencies.yaml 2020-10-09 18:48:13.000000000 +0000 +++ swift-2.25.2/tools/playbooks/common/install_dependencies.yaml 2021-09-01 08:44:03.000000000 +0000 @@ -17,9 +17,11 @@ roles: - ensure-pip tasks: - - name: upgrade pip + - name: upgrade pip, but not too far pip: - name: pip + # 20.* works on both py2 and py3, and the pip for centos7 in EPEL + # isn't smart enough to prevent us upgrading to 21+ + name: pip<21 extra_args: --upgrade - name: installing dependencies diff -Nru swift-2.25.1/tools/playbooks/probetests/run.yaml swift-2.25.2/tools/playbooks/probetests/run.yaml --- swift-2.25.1/tools/playbooks/probetests/run.yaml 2020-10-09 18:48:13.000000000 +0000 +++ swift-2.25.2/tools/playbooks/probetests/run.yaml 2021-09-01 08:44:03.000000000 +0000 @@ -21,6 +21,6 @@ shell: cmd: | source ~/.bashrc - nosetests test/probe/ + nosetests test/probe/ --with-id || nosetests --failed executable: /bin/bash chdir: '{{ zuul.project.src_dir }}' diff -Nru swift-2.25.1/tox.ini swift-2.25.2/tox.ini --- swift-2.25.1/tox.ini 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/tox.ini 2021-09-01 08:44:03.000000000 +0000 @@ -14,11 +14,13 @@ -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/ussuri} -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = find {envdir} ( -type f -o -type l ) -name "*.py[co]" -delete - find {envdir} -type d -name "__pycache__" -delete - nosetests {posargs:test/unit} -whitelist_externals = find - rm +commands = + find {envdir} ( -type f -o -type l ) -name "*.py[co]" -delete + find {envdir} -type d -name "__pycache__" -delete + bash -ec "nosetests {posargs:test/unit} --with-id || nosetests --failed" +allowlist_externals = + bash + find passenv = SWIFT_* *_proxy [testenv:cover] @@ -102,7 +104,9 @@ [testenv:docs] basepython = python3 -deps = -r{toxinidir}/doc/requirements.txt +deps = + -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/ussuri} + -r{toxinidir}/doc/requirements.txt commands = sphinx-build -W -b html doc/source doc/build/html [testenv:api-ref] @@ -110,6 +114,8 @@ # the API Ref to docs.openstack.org. basepython = python3 deps = -r{toxinidir}/doc/requirements.txt +allowlist_externals = + rm commands = rm -rf api-ref/build sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html @@ -151,7 +157,9 @@ [testenv:releasenotes] basepython = python3 -deps = -r{toxinidir}/doc/requirements.txt +deps = + -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/ussuri} + -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [testenv:lower-constraints] @@ -167,7 +175,7 @@ [testenv:pdf-docs] basepython = python3 deps = {[testenv:docs]deps} -whitelist_externals = +allowlist_externals = make commands = sphinx-build -W -b latex doc/source doc/build/pdf diff -Nru swift-2.25.1/.zuul.yaml swift-2.25.2/.zuul.yaml --- swift-2.25.1/.zuul.yaml 2020-10-09 18:48:14.000000000 +0000 +++ swift-2.25.2/.zuul.yaml 2021-09-01 08:44:03.000000000 +0000 @@ -235,7 +235,7 @@ - opendev.org/openstack/requirements - opendev.org/openstack/swift - opendev.org/openstack/keystone - timeout: 2700 + timeout: 3600 vars: tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/openstack/requirements/upper-constraints.txt' # This tox env get run twice; once for Keystone and once for tempauth @@ -292,7 +292,7 @@ nodeset: centos-7 description: | Setup a SAIO dev environment and run ceph-s3tests - timeout: 2400 + timeout: 3600 pre-run: - tools/playbooks/common/install_dependencies.yaml - tools/playbooks/saio_single_node_setup/setup_saio.yaml @@ -309,7 +309,7 @@ nodeset: centos-7 description: | Setup a SAIO dev environment and run Swift's probe tests - timeout: 3600 + timeout: 7200 pre-run: - tools/playbooks/common/install_dependencies.yaml - tools/playbooks/saio_single_node_setup/setup_saio.yaml