diff -Nru cinder-16.3.0/AUTHORS cinder-16.4.0/AUTHORS --- cinder-16.3.0/AUTHORS 2021-03-11 10:27:08.000000000 +0000 +++ cinder-16.4.0/AUTHORS 2021-07-29 10:16:45.000000000 +0000 @@ -163,6 +163,7 @@ Chris Yeoh Christian Berendt Christian Berendt +Christian Rohmann Christoph Kassen Christopher J Schaefer Christopher MacGown @@ -363,6 +364,7 @@ Ivan Kolodyazhny Ivan Kolodyazhny Ivan Pchelintsev +Ivan Pchelintsev Ivaylo Mitev Ivy Zhang Jack Lu diff -Nru cinder-16.3.0/ChangeLog cinder-16.4.0/ChangeLog --- cinder-16.3.0/ChangeLog 2021-03-11 10:27:07.000000000 +0000 +++ cinder-16.4.0/ChangeLog 2021-07-29 10:16:44.000000000 +0000 @@ -1,6 +1,22 @@ CHANGES ======= +16.4.0 +------ + +* Fix typo in Dell EMC Unity driver documentation +* Open local image files with "rb" mode +* Fix sporadic cleanup unit test failure +* Label temporary files created by image\_utils +* Add ports filtering support to Dell EMC XtremIO driver +* Tests: Don't assert notifier not called +* Backup manager: Synchronously call remove\_export +* Add support for RBD fast-diff feature for backups stored in Ceph +* Drop lower-constraints job +* API validation: Add cinder\_host type to support ipv6 in manage +* Pure: Add default value to pure\_host\_personality +* [Pure] Fix failing consistency group tempest tests + 16.3.0 ------ diff -Nru cinder-16.3.0/cinder/api/schemas/volume_manage.py cinder-16.4.0/cinder/api/schemas/volume_manage.py --- cinder-16.3.0/cinder/api/schemas/volume_manage.py 2021-03-11 10:26:20.000000000 +0000 +++ cinder-16.4.0/cinder/api/schemas/volume_manage.py 2021-07-29 10:16:07.000000000 +0000 @@ -33,7 +33,7 @@ "bootable": parameter_types.boolean, "volume_type": parameter_types.name_allow_zero_min_length, "name": parameter_types.name_allow_zero_min_length, - "host": parameter_types.hostname, + "host": parameter_types.cinder_host, "ref": {'type': ['object', 'string']}, "metadata": parameter_types.metadata_allows_null, }, diff -Nru cinder-16.3.0/cinder/api/validation/parameter_types.py cinder-16.4.0/cinder/api/validation/parameter_types.py --- cinder-16.3.0/cinder/api/validation/parameter_types.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/api/validation/parameter_types.py 2021-07-29 10:16:08.000000000 +0000 @@ -232,6 +232,18 @@ 'pattern': '^[a-zA-Z0-9-._#@:/+]*$' } +cinder_host = { + # A string that represents a cinder host. + # Examples: + # hostname + # hostname.domain + # hostname.domain@backend + # hostname.domain@backend#pool + # hostname.domain@backend#[dead:beef::cafe]:/complex_ipv6_pool_w_share + 'type': ['string', 'null'], 'minLength': 1, 'maxLength': 255, + 'pattern': r'^[a-zA-Z0-9-._#@:/+\[\]]*$' # hostname plus brackets +} + resource_type = {'type': ['string', 'null'], 'minLength': 0, 'maxLength': 40} diff -Nru cinder-16.3.0/cinder/backup/drivers/ceph.py cinder-16.4.0/cinder/backup/drivers/ceph.py --- cinder-16.3.0/cinder/backup/drivers/ceph.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/backup/drivers/ceph.py 2021-07-29 10:16:08.000000000 +0000 @@ -244,6 +244,11 @@ """Determine if journaling is supported by our version of librbd.""" return hasattr(self.rbd, 'RBD_FEATURE_JOURNALING') + @property + def _supports_fast_diff(self): + """Determine if fast-diff is supported by our version of librbd.""" + return hasattr(self.rbd, 'RBD_FEATURE_FAST_DIFF') + def _get_rbd_support(self): """Determine RBD features supported by our version of librbd.""" old_format = True @@ -255,24 +260,22 @@ old_format = False features |= self.rbd.RBD_FEATURE_STRIPINGV2 - # journaling requires exclusive_lock; check both together if CONF.backup_ceph_image_journals: - if self._supports_exclusive_lock and self._supports_journaling: - old_format = False - features |= (self.rbd.RBD_FEATURE_EXCLUSIVE_LOCK | - self.rbd.RBD_FEATURE_JOURNALING) - else: - # FIXME (tasker): when the backup manager supports loading the - # driver during its initialization, this exception should be - # moved to the driver's initialization so that it can stop - # the service from starting when the underyling RBD does not - # support the requested features. - LOG.error("RBD journaling not supported - unable to " - "support per image mirroring in backup pool") - raise exception.BackupInvalidCephArgs( - _("Image Journaling set but RBD backend does " - "not support journaling") - ) + LOG.debug("RBD journaling supported by backend and requested " + "via config. Enabling it together with " + "exclusive-lock") + old_format = False + features |= (self.rbd.RBD_FEATURE_EXCLUSIVE_LOCK | + self.rbd.RBD_FEATURE_JOURNALING) + + # NOTE(christian_rohmann): Check for fast-diff support and enable it + if self._supports_fast_diff: + LOG.debug("RBD also supports fast-diff, enabling it " + "together with exclusive-lock and object-map") + old_format = False + features |= (self.rbd.RBD_FEATURE_EXCLUSIVE_LOCK | + self.rbd.RBD_FEATURE_OBJECT_MAP | + self.rbd.RBD_FEATURE_FAST_DIFF) return (old_format, features) @@ -294,6 +297,16 @@ with rbd_driver.RADOSClient(self, self._ceph_backup_pool): pass + # NOTE(christian_rohmann): Check features required for journaling + if CONF.backup_ceph_image_journals: + if not self._supports_exclusive_lock and self._supports_journaling: + LOG.error("RBD journaling not supported - unable to " + "support per image mirroring in backup pool") + raise exception.BackupInvalidCephArgs( + _("Image Journaling set but RBD backend does " + "not support journaling") + ) + def _connect_to_rados(self, pool=None): """Establish connection to the backup Ceph cluster.""" client = eventlet.tpool.Proxy(self.rados.Rados( diff -Nru cinder-16.3.0/cinder/backup/manager.py cinder-16.4.0/cinder/backup/manager.py --- cinder-16.3.0/cinder/backup/manager.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/backup/manager.py 2021-07-29 10:16:08.000000000 +0000 @@ -1072,7 +1072,7 @@ if not is_snapshot: rpcapi.terminate_connection(ctxt, device, properties, force=force) - rpcapi.remove_export(ctxt, device) + rpcapi.remove_export(ctxt, device, sync=True) else: rpcapi.terminate_connection_snapshot(ctxt, device, properties, force=force) diff -Nru cinder-16.3.0/cinder/image/glance.py cinder-16.4.0/cinder/image/glance.py --- cinder-16.3.0/cinder/image/glance.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/image/glance.py 2021-07-29 10:16:08.000000000 +0000 @@ -366,7 +366,7 @@ # a system call to cp could have significant performance # advantages, however we do not have the path to files at # this point in the abstraction. - with open(parsed_url.path, "r") as f: + with open(parsed_url.path, "rb") as f: shutil.copyfileobj(f, data) return diff -Nru cinder-16.3.0/cinder/image/image_utils.py cinder-16.4.0/cinder/image/image_utils.py --- cinder-16.3.0/cinder/image/image_utils.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/image/image_utils.py 2021-07-29 10:16:08.000000000 +0000 @@ -590,7 +590,7 @@ # large and cause disk full errors which would confuse users. # Unfortunately it seems that you can't pipe to 'qemu-img convert' because # it seeks. Maybe we can think of something for a future version. - with temporary_file() as tmp: + with temporary_file(prefix='image_download_%s_' % image_id) as tmp: has_meta = False if not image_meta else True try: format_raw = True if image_meta['disk_format'] == 'raw' else False @@ -708,7 +708,7 @@ base_image_ref=base_image_ref) return - with temporary_file() as tmp: + with temporary_file(prefix='vol_upload_') as tmp: LOG.debug("%s was %s, converting to %s", image_id, volume_format, image_meta['disk_format']) @@ -942,7 +942,8 @@ @contextlib.contextmanager def fetch(cls, image_service, context, image_id, suffix=''): tmp_images = cls.for_image_service(image_service).temporary_images - with temporary_file(suffix=suffix) as tmp: + with temporary_file(prefix='image_fetch_%s_' % image_id, + suffix=suffix) as tmp: fetch_verify_image(context, image_service, image_id, tmp) user = context.user_id if not tmp_images.get(user): diff -Nru cinder-16.3.0/cinder/tests/unit/api/contrib/test_volume_manage.py cinder-16.4.0/cinder/tests/unit/api/contrib/test_volume_manage.py --- cinder-16.3.0/cinder/tests/unit/api/contrib/test_volume_manage.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/tests/unit/api/contrib/test_volume_manage.py 2021-07-29 10:16:08.000000000 +0000 @@ -205,7 +205,8 @@ @ddt.data({'host': 'host_ok'}, {'host': 'user@host#backend:/vol_path'}, - {'host': 'host@backend#parts+of+pool'}) + {'host': 'host@backend#parts+of+pool'}, + {'host': 'host@backend#[dead:beef::cafe]:/vol01'}) @ddt.unpack @mock.patch('cinder.volume.api.API.manage_existing', wraps=api_manage) def test_manage_volume_ok(self, mock_api_manage, host): diff -Nru cinder-16.3.0/cinder/tests/unit/backup/drivers/test_backup_ceph.py cinder-16.4.0/cinder/tests/unit/backup/drivers/test_backup_ceph.py --- cinder-16.3.0/cinder/tests/unit/backup/drivers/test_backup_ceph.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/tests/unit/backup/drivers/test_backup_ceph.py 2021-07-29 10:16:08.000000000 +0000 @@ -238,11 +238,15 @@ del self.service.rbd.RBD_FEATURE_STRIPINGV2 del self.service.rbd.RBD_FEATURE_EXCLUSIVE_LOCK del self.service.rbd.RBD_FEATURE_JOURNALING + del self.service.rbd.RBD_FEATURE_OBJECT_MAP + del self.service.rbd.RBD_FEATURE_FAST_DIFF self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_LAYERING')) self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_STRIPINGV2')) self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_EXCLUSIVE_LOCK')) self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_JOURNALING')) + self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_OBJECT_MAP')) + self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_FAST_DIFF')) oldformat, features = self.service._get_rbd_support() self.assertTrue(oldformat) @@ -282,6 +286,17 @@ self.assertFalse(oldformat) self.assertEqual(1 | 2 | 4 | 64, features) + # + # test that FAST_DIFF is enabled if supported by RBD + # this also enables OBJECT_MAP as required by Ceph + # + self.service.rbd.RBD_FEATURE_OBJECT_MAP = 8 + self.service.rbd.RBD_FEATURE_FAST_DIFF = 16 + + oldformat, features = self.service._get_rbd_support() + self.assertFalse(oldformat) + self.assertEqual(1 | 2 | 4 | 8 | 16 | 64, features) + @common_mocks def test_get_backup_snap_name(self): snap_name = 'backup.%s.snap.3824923.1412' % (uuid.uuid4()) diff -Nru cinder-16.3.0/cinder/tests/unit/image/test_glance.py cinder-16.4.0/cinder/tests/unit/image/test_glance.py --- cinder-16.3.0/cinder/tests/unit/image/test_glance.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/tests/unit/image/test_glance.py 2021-07-29 10:16:08.000000000 +0000 @@ -773,6 +773,7 @@ self.flags(allowed_direct_url_schemes=['file']) self.service.download(self.context, image_id, writer) mock_copyfileobj.assert_called_once_with(mock.ANY, writer) + mock_open.assert_called_once_with('/tmp/test', 'rb') @mock.patch('six.moves.builtins.open') @mock.patch('shutil.copyfileobj') diff -Nru cinder-16.3.0/cinder/tests/unit/test_cleanable_manager.py cinder-16.4.0/cinder/tests/unit/test_cleanable_manager.py --- cinder-16.3.0/cinder/tests/unit/test_cleanable_manager.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/tests/unit/test_cleanable_manager.py 2021-07-29 10:16:08.000000000 +0000 @@ -110,8 +110,15 @@ def test_do_cleanup_not_cleaning_already_claimed_by_us(self): """Basic cleanup that doesn't touch other thread's claimed works.""" - original_time = timeutils.utcnow() - other_thread_claimed_time = timeutils.utcnow() + now = timeutils.utcnow() + delta = timeutils.datetime.timedelta(seconds=1) + original_time = now - delta + # Creating the worker in the future, and then changing the in-memory + # value of worker2.updated_at to an earlier time, we effectively + # simulate that the worker entry was created in the past and that it + # has been just updated between worker_get_all and trying + # to claim a work for cleanup + other_thread_claimed_time = now + delta vol = utils.create_volume(self.context, status='creating') worker1 = db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, @@ -124,14 +131,14 @@ service_id=self.service.id, updated_at=other_thread_claimed_time) worker2 = db.worker_get(self.context, id=worker2.id) - - # Simulate that the change to vol2 worker happened between - # worker_get_all and trying to claim a work for cleanup + # This with the mock below simulates worker2 was created in the past + # and updated right between worker_get_all and worker_claim_for_cleanup worker2.updated_at = original_time clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) - with mock.patch('cinder.db.worker_get_all') as get_all_mock: + with mock.patch('cinder.manager.timeutils.utcnow', return_value=now),\ + mock.patch('cinder.db.worker_get_all') as get_all_mock: get_all_mock.return_value = [worker1, worker2] mngr.do_cleanup(self.context, clean_req) diff -Nru cinder-16.3.0/cinder/tests/unit/test_image_utils.py cinder-16.4.0/cinder/tests/unit/test_image_utils.py --- cinder-16.3.0/cinder/tests/unit/test_image_utils.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/tests/unit/test_image_utils.py 2021-07-29 10:16:08.000000000 +0000 @@ -1142,7 +1142,8 @@ volume_format, blocksize) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=True), mock.call(tmp, run_as_root=True)]) @@ -1194,7 +1195,8 @@ run_as_root=run_as_root) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1250,7 +1252,8 @@ run_as_root=run_as_root) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1302,7 +1305,8 @@ run_as_root=run_as_root) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1406,7 +1410,8 @@ self.assertIsNone(output) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_called_once_with(tmp, force_share=False, run_as_root=run_as_root) @@ -1453,7 +1458,8 @@ run_as_root=run_as_root) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_called_once_with(tmp, force_share=False, run_as_root=run_as_root) @@ -1498,7 +1504,8 @@ run_as_root=run_as_root) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_called_once_with(tmp, force_share=False, run_as_root=run_as_root) @@ -1549,7 +1556,8 @@ run_as_root=run_as_root) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1597,7 +1605,8 @@ run_as_root=run_as_root) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1645,7 +1654,8 @@ run_as_root=run_as_root) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1694,7 +1704,8 @@ run_as_root=run_as_root) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1852,7 +1863,8 @@ volume_format, blocksize) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=True), mock.call(tmp, run_as_root=True)]) diff -Nru cinder-16.3.0/cinder/tests/unit/volume/drivers/dell_emc/test_xtremio.py cinder-16.4.0/cinder/tests/unit/volume/drivers/dell_emc/test_xtremio.py --- cinder-16.3.0/cinder/tests/unit/volume/drivers/dell_emc/test_xtremio.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/tests/unit/volume/drivers/dell_emc/test_xtremio.py 2021-07-29 10:16:08.000000000 +0000 @@ -62,6 +62,14 @@ "name": "10.205.68.5/16", "index": 1, }, + '10.205.68.6/16': + {"port-address": + "iqn.2008-05.com.xtremio:002e67939c34", + "ip-port": 3260, + "ip-addr": "10.205.68.6/16", + "name": "10.205.68.6/16", + "index": 1, + }, }, 'targets': {'X1-SC2-target1': {'index': 1, "name": "X1-SC2-fc1", "port-address": @@ -347,7 +355,8 @@ driver_ssl_cert_path='/test/path/root_ca.crt', xtremio_array_busy_retry_count=5, xtremio_array_busy_retry_interval=5, - xtremio_clean_unused_ig=False) + xtremio_clean_unused_ig=False, + xtremio_ports=[]) def safe_get(key): return getattr(self.config, key) @@ -650,10 +659,26 @@ portals = xms_data['iscsi-portals'].copy() xms_data['iscsi-portals'].clear() lunmap = {'lun': 4} - self.assertRaises(exception.VolumeDriverException, + self.assertRaises(exception.VolumeBackendAPIException, self.driver._get_iscsi_properties, lunmap) xms_data['iscsi-portals'] = portals + def test_no_allowed_portals(self, req): + req.side_effect = xms_request + lunmap = {'lun': 4} + self.driver.allowed_ports = ['1.2.3.4'] + self.assertRaises(exception.VolumeBackendAPIException, + self.driver._get_iscsi_properties, lunmap) + + def test_filtered_portals(self, req): + req.side_effect = xms_request + lunmap = {'lun': 4} + self.driver.allowed_ports = ['10.205.68.6'] + connection_properties = self.driver._get_iscsi_properties(lunmap) + self.assertEqual(1, len(connection_properties['target_portals'])) + self.assertIn('10.205.68.6:3260', + connection_properties['target_portals']) + def test_initialize_connection(self, req): req.side_effect = xms_request self.driver.create_volume(self.data.test_volume) @@ -1365,6 +1390,27 @@ configuration=self.config) # ##### Connection FC##### + def test_no_targets_configured(self, req): + req.side_effect = xms_request + targets = xms_data['targets'].copy() + xms_data['targets'].clear() + self.assertRaises(exception.VolumeBackendAPIException, + self.driver.get_targets) + xms_data['targets'] = targets + + def test_no_allowed_targets(self, req): + req.side_effect = xms_request + self.driver.allowed_ports = ['58:cc:f0:98:49:22:07:02'] + self.assertRaises(exception.VolumeBackendAPIException, + self.driver.get_targets) + + def test_filtered_targets(self, req): + req.side_effect = xms_request + self.driver.allowed_ports = ['21:00:00:24:ff:57:b2:36'] + targets = self.driver.get_targets() + self.assertEqual(1, len(targets)) + self.assertIn('21000024ff57b236', targets) + def test_initialize_connection(self, req): req.side_effect = xms_request diff -Nru cinder-16.3.0/cinder/tests/unit/volume/drivers/test_pure.py cinder-16.4.0/cinder/tests/unit/volume/drivers/test_pure.py --- cinder-16.3.0/cinder/tests/unit/volume/drivers/test_pure.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/tests/unit/volume/drivers/test_pure.py 2021-07-29 10:16:08.000000000 +0000 @@ -1589,34 +1589,6 @@ self.assertEqual((None, None), result) mock_create_cg.assert_called_with(mock_context, mock_group) self.assertTrue(self.array.create_pgroup_snapshot.called) - self.assertEqual(num_volumes, self.array.copy_volume.call_count) - self.assertEqual(num_volumes, self.array.set_pgroup.call_count) - self.assertTrue(self.array.destroy_pgroup.called) - - @mock.patch(BASE_DRIVER_OBJ + ".create_consistencygroup") - def test_create_consistencygroup_from_cg_with_error(self, mock_create_cg): - num_volumes = 5 - mock_context = mock.MagicMock() - mock_group = mock.MagicMock() - mock_source_cg = mock.MagicMock() - mock_volumes = [mock.MagicMock() for i in range(num_volumes)] - mock_source_vols = [mock.MagicMock() for i in range(num_volumes)] - - self.array.copy_volume.side_effect = FakePureStorageHTTPError() - - self.assertRaises( - FakePureStorageHTTPError, - self.driver.create_consistencygroup_from_src, - mock_context, - mock_group, - mock_volumes, - source_cg=mock_source_cg, - source_vols=mock_source_vols - ) - mock_create_cg.assert_called_with(mock_context, mock_group) - self.assertTrue(self.array.create_pgroup_snapshot.called) - # Make sure that the temp snapshot is cleaned up even when copying - # the volume fails! self.assertTrue(self.array.destroy_pgroup.called) @mock.patch(BASE_DRIVER_OBJ + ".delete_volume", autospec=True) diff -Nru cinder-16.3.0/cinder/tests/unit/volume/test_volume.py cinder-16.4.0/cinder/tests/unit/volume/test_volume.py --- cinder-16.3.0/cinder/tests/unit/volume/test_volume.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/tests/unit/volume/test_volume.py 2021-07-29 10:16:08.000000000 +0000 @@ -253,7 +253,7 @@ volume_id = volume['id'] self.assertIsNone(volume['encryption_key_id']) - mock_notify.assert_not_called() + self.assertRaises(exception.DriverNotInitialized, self.volume.create_volume, self.context, volume) @@ -336,7 +336,6 @@ **self.volume_params) self.assertIsNone(volume['encryption_key_id']) - mock_notify.assert_not_called() self.assertRaises(exception.DriverNotInitialized, self.volume.delete_volume, self.context, volume) @@ -356,8 +355,6 @@ **self.volume_params) volume_id = volume['id'] - mock_notify.assert_not_called() - self.assertIsNone(volume['encryption_key_id']) self.volume.create_volume(self.context, volume) diff -Nru cinder-16.3.0/cinder/volume/drivers/dell_emc/xtremio.py cinder-16.4.0/cinder/volume/drivers/dell_emc/xtremio.py --- cinder-16.3.0/cinder/volume/drivers/dell_emc/xtremio.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/volume/drivers/dell_emc/xtremio.py 2021-07-29 10:16:08.000000000 +0000 @@ -31,6 +31,7 @@ 1.0.9 - performance improvements, support force detach, support for X2 1.0.10 - option to clean unused IGs 1.0.11 - add support for multiattach + 1.0.12 - add support for ports filtering """ import json @@ -82,7 +83,13 @@ 'the IG be, we default to False (not deleting IGs ' 'without connected volumes); setting this parameter ' 'to True will remove any IG after terminating its ' - 'connection to the last volume.')] + 'connection to the last volume.'), + cfg.ListOpt('xtremio_ports', + default=[], + help='Allowed ports. Comma separated list of XtremIO ' + 'iSCSI IPs or FC WWNs (ex. 58:cc:f0:98:49:22:07:02) ' + 'to be used. If option is not set all ports are allowed.') +] CONF.register_opts(XTREMIO_OPTS, group=configuration.SHARED_CONF_GROUP) @@ -420,7 +427,7 @@ class XtremIOVolumeDriver(san.SanDriver): """Executes commands relating to Volumes.""" - VERSION = '1.0.11' + VERSION = '1.0.12' # ThirdPartySystems wiki CI_WIKI_NAME = "EMC_XIO_CI" @@ -444,6 +451,10 @@ self.clean_ig = (self.configuration.safe_get('xtremio_clean_unused_ig') or False) self._stats = {} + self.allowed_ports = [ + port.strip().lower() for port in + self.configuration.safe_get('xtremio_ports') + ] self.client = XtremIOClient3(self.configuration, self.cluster_id) @classmethod @@ -1063,6 +1074,19 @@ raise (exception.VolumeBackendAPIException (data=_("Failed to create IG, %s") % name)) + def _port_is_allowed(self, port): + """Check if port is in allowed ports list. + + If allowed ports are empty then all ports are allowed. + + :param port: iSCSI IP/FC WWN to check + :return: is port allowed + """ + + if not self.allowed_ports: + return True + return port.lower() in self.allowed_ports + @interface.volumedriver class XtremIOISCSIDriver(XtremIOVolumeDriver, driver.ISCSIDriver): @@ -1180,26 +1204,33 @@ multiple values. The main portal information is also returned in :target_iqn, :target_portal, :target_lun for backward compatibility. """ - portals = self.client.get_iscsi_portals() - if not portals: - msg = _("XtremIO not configured correctly, no iscsi portals found") - LOG.error(msg) - raise exception.VolumeDriverException(message=msg) - portal = RANDOM.choice(portals) + iscsi_portals = self.client.get_iscsi_portals() + allowed_portals = [] + for iscsi_portal in iscsi_portals: + iscsi_portal['ip-addr'] = iscsi_portal['ip-addr'].split('/')[0] + if self._port_is_allowed(iscsi_portal['ip-addr']): + allowed_portals.append(iscsi_portal) + if not allowed_portals: + msg = _("There are no accessible iSCSI targets on the " + "system.") + raise exception.VolumeBackendAPIException(data=msg) + portal = RANDOM.choice(allowed_portals) portal_addr = ('%(ip)s:%(port)d' % - {'ip': portal['ip-addr'].split('/')[0], + {'ip': portal['ip-addr'], 'port': portal['ip-port']}) - tg_portals = ['%(ip)s:%(port)d' % {'ip': p['ip-addr'].split('/')[0], + tg_portals = ['%(ip)s:%(port)d' % {'ip': p['ip-addr'], 'port': p['ip-port']} - for p in portals] + for p in allowed_portals] properties = {'target_discovered': False, 'target_iqn': portal['port-address'], 'target_lun': lunmap['lun'], 'target_portal': portal_addr, - 'target_iqns': [p['port-address'] for p in portals], + 'target_iqns': [ + p['port-address'] for p in allowed_portals + ], 'target_portals': tg_portals, - 'target_luns': [lunmap['lun']] * len(portals)} + 'target_luns': [lunmap['lun']] * len(allowed_portals)} return properties def _get_initiator_names(self, connector): @@ -1222,8 +1253,17 @@ if not self._targets: try: targets = self.client.get_fc_up_ports() - self._targets = [target['port-address'].replace(':', '') - for target in targets] + allowed_targets = [] + for target in targets: + if self._port_is_allowed(target['port-address']): + allowed_targets.append( + target['port-address'].replace(':', '') + ) + if not allowed_targets: + msg = _("There are no accessible Fibre Channel targets " + "on the system.") + raise exception.VolumeBackendAPIException(data=msg) + self._targets = allowed_targets except exception.NotFound: raise (exception.VolumeBackendAPIException (data=_("Failed to get targets"))) diff -Nru cinder-16.3.0/cinder/volume/drivers/pure.py cinder-16.4.0/cinder/volume/drivers/pure.py --- cinder-16.3.0/cinder/volume/drivers/pure.py 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/cinder/volume/drivers/pure.py 2021-07-29 10:16:08.000000000 +0000 @@ -59,8 +59,9 @@ "this calculated value will override the " "max_over_subscription_ratio config option."), cfg.StrOpt("pure_host_personality", + default=None, choices=['aix', 'esxi', 'hitachi-vsp', 'hpux', - 'oracle-vm-server', 'solaris', 'vms'], + 'oracle-vm-server', 'solaris', 'vms', None], help="Determines how the Purity system tunes the protocol used " "between the array and the initiator."), # These are used as default settings. In future these can be overridden @@ -888,6 +889,7 @@ 'source_group': source_group.id}) current_array = self._get_current_array() current_array.create_pgroup_snapshot(pgroup_name, suffix=tmp_suffix) + volumes, _ = self.update_provider_info(volumes, None) try: for source_vol, cloned_vol in zip(source_vols, volumes): source_snap_name = self._get_pgroup_vol_snap_name( diff -Nru cinder-16.3.0/cinder/volume/rpcapi.py cinder-16.4.0/cinder/volume/rpcapi.py --- cinder-16.3.0/cinder/volume/rpcapi.py 2021-03-11 10:26:20.000000000 +0000 +++ cinder-16.4.0/cinder/volume/rpcapi.py 2021-07-29 10:16:08.000000000 +0000 @@ -231,9 +231,12 @@ return cctxt.call(ctxt, 'terminate_connection', volume_id=volume['id'], connector=connector, force=force) - def remove_export(self, ctxt, volume): + def remove_export(self, ctxt, volume, sync=False): cctxt = self._get_cctxt(volume.service_topic_queue) - cctxt.cast(ctxt, 'remove_export', volume_id=volume['id']) + if sync: + cctxt.call(ctxt, 'remove_export', volume_id=volume.id) + else: + cctxt.cast(ctxt, 'remove_export', volume_id=volume.id) def publish_service_capabilities(self, ctxt): cctxt = self._get_cctxt(fanout=True) diff -Nru cinder-16.3.0/cinder.egg-info/pbr.json cinder-16.4.0/cinder.egg-info/pbr.json --- cinder-16.3.0/cinder.egg-info/pbr.json 2021-03-11 10:27:08.000000000 +0000 +++ cinder-16.4.0/cinder.egg-info/pbr.json 2021-07-29 10:16:45.000000000 +0000 @@ -1 +1 @@ -{"git_version": "17d3ac3f2", "is_release": true} \ No newline at end of file +{"git_version": "62da20185", "is_release": true} \ No newline at end of file diff -Nru cinder-16.3.0/cinder.egg-info/PKG-INFO cinder-16.4.0/cinder.egg-info/PKG-INFO --- cinder-16.3.0/cinder.egg-info/PKG-INFO 2021-03-11 10:27:08.000000000 +0000 +++ cinder-16.4.0/cinder.egg-info/PKG-INFO 2021-07-29 10:16:45.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: cinder -Version: 16.3.0 +Version: 16.4.0 Summary: OpenStack Block Storage Home-page: https://docs.openstack.org/cinder/latest/ Author: OpenStack diff -Nru cinder-16.3.0/cinder.egg-info/SOURCES.txt cinder-16.4.0/cinder.egg-info/SOURCES.txt --- cinder-16.3.0/cinder.egg-info/SOURCES.txt 2021-03-11 10:27:08.000000000 +0000 +++ cinder-16.4.0/cinder.egg-info/SOURCES.txt 2021-07-29 10:16:45.000000000 +0000 @@ -2070,10 +2070,14 @@ releasenotes/notes/bug-1893107-ussuri-03765453337a7909.yaml releasenotes/notes/bug-1895510-REST-API-issue-to-get-bundle-198a3b89255759bb.yaml releasenotes/notes/bug-1898918-b24a93d7d5aff238.yaml +releasenotes/notes/bug-1900979-xtremio-ports-filtering-e68f90d47f17a7d9.yaml releasenotes/notes/bug-1901241-361b1b361bfa5152.yaml releasenotes/notes/bug-1904440-clone-rekey-fd57a2b5f6224e0f.yaml +releasenotes/notes/bug-1904892-ipv6-nfs-manage-391118115dfaaf54.yaml +releasenotes/notes/bug-1907964-9277e5ddec2abeda.yaml releasenotes/notes/bug-1908315-020fea3e244d49bb.yaml releasenotes/notes/bug-1913449-4796b366ae7e871b.yaml +releasenotes/notes/bug-1920237-backup-remove-export-race-941e2ab1f056e54c.yaml releasenotes/notes/bug-fix-1866871-f9d61defc00f4007.yaml releasenotes/notes/bug-fix-1867163-27afa39ac77b9e15.yaml releasenotes/notes/bug-invalid-content-type-1715094-8yu8i9w425ua08f3.yaml @@ -2427,6 +2431,7 @@ releasenotes/notes/pure-storage-multiattach-support-994da363e181d627.yaml releasenotes/notes/pure-v2.1-replication-0246223caaa8a9b5.yaml releasenotes/notes/pure-verify-https-requests-464320c97ba77a1f.yaml +releasenotes/notes/pure_tempest_cg_fix-913d405f7487de00.yaml releasenotes/notes/qb-backup-5b1f2161d160648a.yaml releasenotes/notes/qb-overlay-from-snap-cache-dc102acb4820e368.yaml releasenotes/notes/qb-switch-nas-sec-opts-635c6ef1205e4f3f.yaml diff -Nru cinder-16.3.0/debian/changelog cinder-16.4.0/debian/changelog --- cinder-16.3.0/debian/changelog 2021-04-12 12:13:29.000000000 +0000 +++ cinder-16.4.0/debian/changelog 2021-08-25 12:03:15.000000000 +0000 @@ -1,3 +1,9 @@ +cinder (2:16.4.0-0ubuntu1) focal; urgency=medium + + * New stable point release for OpenStack Ussuri (LP: #1941048). + + -- Chris MacNaughton Wed, 25 Aug 2021 12:03:15 +0000 + cinder (2:16.3.0-0ubuntu1) focal; urgency=medium * New stable point release for OpenStack Ussuri (LP: #1923036). diff -Nru cinder-16.3.0/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst cinder-16.4.0/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst --- cinder-16.3.0/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst 2021-03-11 10:26:20.000000000 +0000 +++ cinder-16.4.0/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst 2021-07-29 10:16:08.000000000 +0000 @@ -48,7 +48,7 @@ Driver configuration ~~~~~~~~~~~~~~~~~~~~ -.. note:: The following instructions should all be performed on Black Storage +.. note:: The following instructions should all be performed on Block Storage nodes. #. Install `storops` from pypi: diff -Nru cinder-16.3.0/doc/source/configuration/block-storage/drivers/dell-emc-xtremio-driver.rst cinder-16.4.0/doc/source/configuration/block-storage/drivers/dell-emc-xtremio-driver.rst --- cinder-16.3.0/doc/source/configuration/block-storage/drivers/dell-emc-xtremio-driver.rst 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/doc/source/configuration/block-storage/drivers/dell-emc-xtremio-driver.rst 2021-07-29 10:16:07.000000000 +0000 @@ -233,6 +233,16 @@ password) are generated automatically by the Block Storage driver. Therefore, there is no need to configure the initial CHAP credentials manually in XMS. +Configuring ports filtering +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The XtremIO Block Storage driver supports ports filtering to define a list +of iSCSI IP-addresses or FC WWNs which will be used to attach volumes. +If option is not set all ports are allowed. + +.. code-block:: ini + + xtremio_ports = iSCSI IPs or FC WWNs + .. _emc_extremio_configuration_example: Configuration example @@ -250,6 +260,7 @@ volume_driver = cinder.volume.drivers.dell_emc.xtremio.XtremIOFibreChannelDriver san_ip = XMS_IP xtremio_cluster_name = Cluster01 + xtremio_ports = 21:00:00:24:ff:57:b2:36,21:00:00:24:ff:57:b2:55 san_login = XMS_USER san_password = XMS_PASSWD volume_backend_name = XtremIOAFA diff -Nru cinder-16.3.0/PKG-INFO cinder-16.4.0/PKG-INFO --- cinder-16.3.0/PKG-INFO 2021-03-11 10:27:09.040432700 +0000 +++ cinder-16.4.0/PKG-INFO 2021-07-29 10:16:46.736173200 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: cinder -Version: 16.3.0 +Version: 16.4.0 Summary: OpenStack Block Storage Home-page: https://docs.openstack.org/cinder/latest/ Author: OpenStack diff -Nru cinder-16.3.0/releasenotes/notes/bug-1900979-xtremio-ports-filtering-e68f90d47f17a7d9.yaml cinder-16.4.0/releasenotes/notes/bug-1900979-xtremio-ports-filtering-e68f90d47f17a7d9.yaml --- cinder-16.3.0/releasenotes/notes/bug-1900979-xtremio-ports-filtering-e68f90d47f17a7d9.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-16.4.0/releasenotes/notes/bug-1900979-xtremio-ports-filtering-e68f90d47f17a7d9.yaml 2021-07-29 10:16:08.000000000 +0000 @@ -0,0 +1,5 @@ +--- +fixes: + - | + `Bug #1915800 `_: + Add support for ports filtering in XtremIO driver. diff -Nru cinder-16.3.0/releasenotes/notes/bug-1904892-ipv6-nfs-manage-391118115dfaaf54.yaml cinder-16.4.0/releasenotes/notes/bug-1904892-ipv6-nfs-manage-391118115dfaaf54.yaml --- cinder-16.3.0/releasenotes/notes/bug-1904892-ipv6-nfs-manage-391118115dfaaf54.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-16.4.0/releasenotes/notes/bug-1904892-ipv6-nfs-manage-391118115dfaaf54.yaml 2021-07-29 10:16:08.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + `Bug #1904892 `_: + Fix cinder manage operations for NFS backends using IPv6 addresses + in the NFS server address. These were previously rejected by the + Cinder API. diff -Nru cinder-16.3.0/releasenotes/notes/bug-1907964-9277e5ddec2abeda.yaml cinder-16.4.0/releasenotes/notes/bug-1907964-9277e5ddec2abeda.yaml --- cinder-16.3.0/releasenotes/notes/bug-1907964-9277e5ddec2abeda.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-16.4.0/releasenotes/notes/bug-1907964-9277e5ddec2abeda.yaml 2021-07-29 10:16:08.000000000 +0000 @@ -0,0 +1,11 @@ +--- +fixes: + - | + RBD driver `bug #1907964 + `_: Add support + for fast-diff on backup images stored in Ceph. + Provided fast-diff is supported by the backend it will automatically be + enabled and used. + With fast-diff enabled, the generation of diffs between images and + snapshots as well as determining the actual data usage of a snapshot + is speed up significantly. \ No newline at end of file diff -Nru cinder-16.3.0/releasenotes/notes/bug-1920237-backup-remove-export-race-941e2ab1f056e54c.yaml cinder-16.4.0/releasenotes/notes/bug-1920237-backup-remove-export-race-941e2ab1f056e54c.yaml --- cinder-16.3.0/releasenotes/notes/bug-1920237-backup-remove-export-race-941e2ab1f056e54c.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-16.4.0/releasenotes/notes/bug-1920237-backup-remove-export-race-941e2ab1f056e54c.yaml 2021-07-29 10:16:08.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + `Bug #1920237 `_: The + backup manager calls volume remove_export() but does not wait for it to + complete when detaching a volume after backup. This caused problems + when a subsequent operation started on that volume before it had fully + detached. diff -Nru cinder-16.3.0/releasenotes/notes/pure_tempest_cg_fix-913d405f7487de00.yaml cinder-16.4.0/releasenotes/notes/pure_tempest_cg_fix-913d405f7487de00.yaml --- cinder-16.3.0/releasenotes/notes/pure_tempest_cg_fix-913d405f7487de00.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-16.4.0/releasenotes/notes/pure_tempest_cg_fix-913d405f7487de00.yaml 2021-07-29 10:16:08.000000000 +0000 @@ -0,0 +1,5 @@ +--- +fixes: + - | + Pure Storage FlashArray driver fix to ensure cinder_tempest_plugin consistency + group tests pass. diff -Nru cinder-16.3.0/.zuul.yaml cinder-16.4.0/.zuul.yaml --- cinder-16.3.0/.zuul.yaml 2021-03-11 10:26:21.000000000 +0000 +++ cinder-16.4.0/.zuul.yaml 2021-07-29 10:16:08.000000000 +0000 @@ -1,6 +1,5 @@ - project: templates: - - openstack-lower-constraints-jobs - openstack-python3-ussuri-jobs - publish-openstack-docs-pti - periodic-stable-jobs