diff -Nru ceilometer-9.0.1/AUTHORS ceilometer-9.0.4/AUTHORS --- ceilometer-9.0.1/AUTHORS 2017-09-12 16:32:41.000000000 +0000 +++ ceilometer-9.0.4/AUTHORS 2018-01-09 16:15:00.000000000 +0000 @@ -7,11 +7,13 @@ Ala Rezmerita Alessandro Pilotti Alex Holden +Alexander Chadin Alexei Kornienko Alfredo Moralejo Amy Fong Ana Malagon Ananya Chatterjee +Andrea Frittoli Andreas Jaeger Andreas Jaeger Andrew Hutchings @@ -64,6 +66,7 @@ Darren Hague Davanum Srinivas David Peraza +David Rabel Dazhao Debo~ Dutta Deepthi V V @@ -121,6 +124,7 @@ Ionuț Arțăriși Jake Liu James E. Blair +James E. Blair Jason Myers Jason Zhang Jay Lau @@ -140,6 +144,7 @@ John Herndon Jonte Watford JordanP +Joseph Davis Joseph Richard Joshua Harlow JuPing @@ -197,6 +202,7 @@ Morgan Fainberg Nadya Privalova Nadya Shakhat +Nam Nguyen Hoai Nejc Saje Ngo Quoc Cuong Nguyen Phuong An @@ -217,6 +223,7 @@ Petr Kovar Petr Kuběna Phil Neal +Pierre Riteau Piyush Masrani Pradeep Kilambi Pradeep Kilambi @@ -242,6 +249,7 @@ Russell Bryant Ryan Petrello Ryota MIBU +SU, HAO-CHEN Saba Ahmed Sam Morrison Samta @@ -302,6 +310,7 @@ XiaBing Yao Xiang Li XieYingYun +Yaguang Tang Yaguang Tang Yanyan Hu Yarko Tymciurak @@ -320,6 +329,7 @@ ZhiQiang Fan Zhongyue Luo Zi Lian Ji +Zuul aggaatul alextricity25 ananya23d diff -Nru ceilometer-9.0.1/ceilometer/agent/manager.py ceilometer-9.0.4/ceilometer/agent/manager.py --- ceilometer-9.0.1/ceilometer/agent/manager.py 2017-09-12 16:29:53.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/agent/manager.py 2018-01-09 16:10:49.000000000 +0000 @@ -108,6 +108,7 @@ static_resources_group = self.agent_manager.construct_group_id( utils.hash_of_set(self._resources)) return [v for v in self._resources if + not self.agent_manager.partition_coordinator or self.agent_manager.hashrings[ static_resources_group].belongs_to_self( six.text_type(v))] + source_discovery @@ -279,13 +280,13 @@ self.discoveries = list(itertools.chain(*list(discoveries))) self.polling_periodics = None + self.hashrings = None + self.partition_coordinator = None if self.conf.coordination.backend_url: # XXX uuid4().bytes ought to work, but it requires ascii for now coordination_id = str(uuid.uuid4()).encode('ascii') self.partition_coordinator = coordination.get_coordinator( self.conf.coordination.backend_url, coordination_id) - else: - self.partition_coordinator = None # Compose coordination group prefix. # We'll use namespaces as the basement for this partitioning. diff -Nru ceilometer-9.0.1/ceilometer/compute/discovery.py ceilometer-9.0.4/ceilometer/compute/discovery.py --- ceilometer-9.0.1/ceilometer/compute/discovery.py 2017-09-12 16:29:56.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/compute/discovery.py 2018-01-09 16:10:49.000000000 +0000 @@ -145,12 +145,21 @@ def discover_libvirt_polling(self, manager, param=None): instances = [] for domain in self.connection.listAllDomains(): + try: + xml_string = domain.metadata( + libvirt.VIR_DOMAIN_METADATA_ELEMENT, + "http://openstack.org/xmlns/libvirt/nova/1.0") + except libvirt.libvirtError as e: + if libvirt_utils.is_disconnection_exception(e): + # Re-raise the exception so it's handled and retries + raise + LOG.error( + "Fail to get domain uuid %s metadata, libvirtError: %s", + domain.UUIDString(), e.message) + continue + full_xml = etree.fromstring(domain.XMLDesc()) os_type_xml = full_xml.find("./os/type") - - xml_string = domain.metadata( - libvirt.VIR_DOMAIN_METADATA_ELEMENT, - "http://openstack.org/xmlns/libvirt/nova/1.0") metadata_xml = etree.fromstring(xml_string) # TODO(sileht): We don't have the flavor ID here So the Gnocchi diff -Nru ceilometer-9.0.1/ceilometer/compute/pollsters/instance_stats.py ceilometer-9.0.4/ceilometer/compute/pollsters/instance_stats.py --- ceilometer-9.0.1/ceilometer/compute/pollsters/instance_stats.py 2017-09-12 16:29:56.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/compute/pollsters/instance_stats.py 2018-01-09 16:10:41.000000000 +0000 @@ -55,12 +55,14 @@ sample_name = 'memory.swap.in' sample_unit = 'MB' sample_stats_key = 'memory_swap_in' + sample_type = sample.TYPE_CUMULATIVE class MemorySwapOutPollster(InstanceStatsPollster): sample_name = 'memory.swap.out' sample_unit = 'MB' sample_stats_key = 'memory_swap_out' + sample_type = sample.TYPE_CUMULATIVE class PerfCPUCyclesPollster(InstanceStatsPollster): diff -Nru ceilometer-9.0.1/ceilometer/compute/virt/libvirt/inspector.py ceilometer-9.0.4/ceilometer/compute/virt/libvirt/inspector.py --- ceilometer-9.0.1/ceilometer/compute/virt/libvirt/inspector.py 2017-09-12 16:29:56.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/compute/virt/libvirt/inspector.py 2018-01-09 16:10:49.000000000 +0000 @@ -119,7 +119,8 @@ for device in filter( bool, [target.get("dev") - for target in tree.findall('devices/disk/target')]): + for target in tree.findall('devices/disk/target') + if target.getparent().find('source') is not None]): block_stats = domain.blockStats(device) yield virt_inspector.DiskStats(device=device, read_requests=block_stats[0], @@ -133,27 +134,19 @@ domain = self._get_domain_not_shut_off_or_raise(instance) tree = etree.fromstring(domain.XMLDesc(0)) for disk in tree.findall('devices/disk'): - disk_type = disk.get('type') - if disk_type: - if disk_type == 'network': - LOG.warning( - 'Inspection disk usage of network disk ' - '%(instance_uuid)s unsupported by libvirt' % { - 'instance_uuid': instance.id}) - continue - # NOTE(lhx): "cdrom" device associated to the configdrive - # no longer has a "source" element. Releated bug: - # https://bugs.launchpad.net/ceilometer/+bug/1622718 - if disk.find('source') is None: - continue - target = disk.find('target') - device = target.get('dev') - if device: - block_info = domain.blockInfo(device) - yield virt_inspector.DiskInfo(device=device, - capacity=block_info[0], - allocation=block_info[1], - physical=block_info[2]) + # NOTE(lhx): "cdrom" device associated to the configdrive + # no longer has a "source" element. Releated bug: + # https://bugs.launchpad.net/ceilometer/+bug/1622718 + if disk.find('source') is None: + continue + target = disk.find('target') + device = target.get('dev') + if device: + block_info = domain.blockInfo(device) + yield virt_inspector.DiskInfo(device=device, + capacity=block_info[0], + allocation=block_info[1], + physical=block_info[2]) @libvirt_utils.raise_nodata_if_unsupported @libvirt_utils.retry_on_disconnect diff -Nru ceilometer-9.0.1/ceilometer/hardware/pollsters/generic.py ceilometer-9.0.4/ceilometer/hardware/pollsters/generic.py --- ceilometer-9.0.1/ceilometer/hardware/pollsters/generic.py 2017-09-12 16:29:53.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/hardware/pollsters/generic.py 2018-01-09 16:10:49.000000000 +0000 @@ -161,11 +161,15 @@ parsed_url, i_cache[identifier])) except Exception as err: - LOG.exception('inspector call failed for %(ident)s ' - 'host %(host)s: %(err)s', - dict(ident=identifier, - host=parsed_url.hostname, - err=err)) + msg = ('inspector call failed for %(ident)s ' + 'host %(host)s: %(err)s' % + dict(ident=identifier, + host=parsed_url.hostname, + err=err)) + if "timeout" in str(err): + LOG.warning(msg) + else: + LOG.exception(msg) return itertools.chain(*sample_iters) def generate_samples(self, host_url, data): diff -Nru ceilometer-9.0.1/ceilometer/messaging.py ceilometer-9.0.4/ceilometer/messaging.py --- ceilometer-9.0.1/ceilometer/messaging.py 2017-09-12 16:29:53.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/messaging.py 2018-01-09 16:10:41.000000000 +0000 @@ -13,7 +13,9 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_config import cfg import oslo_messaging +from oslo_messaging._drivers import impl_rabbit from oslo_messaging.notify import notifier from oslo_messaging import serializer as oslo_serializer @@ -23,6 +25,19 @@ def setup(): oslo_messaging.set_transport_defaults('ceilometer') + # NOTE(sileht): When batch is not enabled, oslo.messaging read all messages + # in the queue and can consume a lot of memory, that works for rpc because + # you never have a lot of message, but sucks for notification. The + # default is not changeable on oslo.messaging side. And we can't expose + # this option to set set_transport_defaults because it a driver option. + # 100 allow to prefetch a lot of messages but limit memory to 1G per + # workers in worst case (~ 1M Nova notification) + # And even driver options are located in private module, this is not going + # to break soon. + cfg.set_defaults( + impl_rabbit.rabbit_opts, + rabbit_qos_prefetch_count=100, + ) def get_transport(conf, url=None, optional=False, cache=True): diff -Nru ceilometer-9.0.1/ceilometer/notification.py ceilometer-9.0.4/ceilometer/notification.py --- ceilometer-9.0.1/ceilometer/notification.py 2017-09-12 16:29:53.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/notification.py 2018-01-09 16:10:49.000000000 +0000 @@ -26,7 +26,6 @@ from oslo_config import cfg from oslo_log import log import oslo_messaging -import six from stevedore import extension from tooz import coordination @@ -128,6 +127,9 @@ str(uuid.uuid4()).encode('ascii')) self.partition_coordinator = coordination.get_coordinator( self.conf.coordination.backend_url, coordination_id) + self.partition_set = list(range( + self.conf.notification.pipeline_processing_queues)) + self.group_state = None else: self.partition_coordinator = None @@ -190,7 +192,7 @@ self.transport = messaging.get_transport(self.conf) if self.conf.notification.workload_partitioning: - self.partition_coordinator.start() + self.partition_coordinator.start(start_heart=True) else: # FIXME(sileht): endpoint uses the notification_topics option # and it should not because this is an oslo_messaging option @@ -214,16 +216,15 @@ run_immediately=True) def run_watchers(): self.partition_coordinator.run_watchers() + if self.group_state != self.hashring.ring.nodes: + self.group_state = self.hashring.ring.nodes.copy() + self._refresh_agent() self.periodic = periodics.PeriodicWorker.create( [], executor_factory=lambda: futures.ThreadPoolExecutor(max_workers=10)) self.periodic.add(run_watchers) - utils.spawn_thread(self.periodic.start) - # configure pipelines after all coordination is configured. - with self.coord_lock: - self._configure_pipeline_listener() def _configure_main_queue_listeners(self, pipe_manager, event_pipe_manager): @@ -266,7 +267,7 @@ ) self.listeners.append(listener) - def _refresh_agent(self, event): + def _refresh_agent(self): with self.coord_lock: if self.shutdown: # NOTE(sileht): We are going to shutdown we everything will be @@ -278,13 +279,8 @@ ev_pipes = self.event_pipeline_manager.pipelines pipelines = self.pipeline_manager.pipelines + ev_pipes transport = messaging.get_transport(self.conf) - partitioned = six.moves.range( - self.conf.notification.pipeline_processing_queues - ) - - if self.partition_coordinator: - partitioned = list(filter( - self.hashring.belongs_to_self, partitioned)) + partitioned = list(filter( + self.hashring.belongs_to_self, self.partition_set)) endpoints = [] targets = [] diff -Nru ceilometer-9.0.1/ceilometer/publisher/messaging.py ceilometer-9.0.4/ceilometer/publisher/messaging.py --- ceilometer-9.0.1/ceilometer/publisher/messaging.py 2017-09-12 16:29:53.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/publisher/messaging.py 2018-01-09 16:10:49.000000000 +0000 @@ -18,6 +18,7 @@ import abc import itertools import operator +import threading from oslo_config import cfg from oslo_log import log @@ -84,6 +85,7 @@ 'max_queue_length', [1024])[-1]) self.max_retry = 0 + self.queue_lock = threading.Lock() self.local_queue = [] if self.policy in ['default', 'queue', 'drop']: @@ -123,17 +125,16 @@ self.flush() def flush(self): - # NOTE(sileht): - # this is why the self.local_queue is emptied before processing the - # queue and the remaining messages in the queue are added to - # self.local_queue after in case of another call having already added - # something in the self.local_queue - queue = self.local_queue - self.local_queue = [] - self.local_queue = (self._process_queue(queue, self.policy) + - self.local_queue) - if self.policy == 'queue': - self._check_queue_length() + with self.queue_lock: + queue = self.local_queue + self.local_queue = [] + + queue = self._process_queue(queue, self.policy) + + with self.queue_lock: + self.local_queue = (queue + self.local_queue) + if self.policy == 'queue': + self._check_queue_length() def _check_queue_length(self): queue_length = len(self.local_queue) diff -Nru ceilometer-9.0.1/ceilometer/tests/functional/test_notification.py ceilometer-9.0.4/ceilometer/tests/functional/test_notification.py --- ceilometer-9.0.1/ceilometer/tests/functional/test_notification.py 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/tests/functional/test_notification.py 2018-01-09 16:10:50.000000000 +0000 @@ -82,6 +82,20 @@ } +class BaseNotificationTest(tests_base.BaseTestCase): + def run_service(self, srv): + srv.run() + self.addCleanup(srv.terminate) + if srv.conf.notification.workload_partitioning: + start = time.time() + while time.time() - start < 10: + if srv.group_state: # ensure pipeline is set if HA + break + time.sleep(0.1) + else: + self.fail('Did not start pipeline queues') + + class _FakeNotificationPlugin(plugin_base.NotificationBase): event_types = ['fake.event'] @@ -94,15 +108,13 @@ return [] -class TestNotification(tests_base.BaseTestCase): +class TestNotification(BaseNotificationTest): def setUp(self): super(TestNotification, self).setUp() self.CONF = service.prepare_service([], []) self.CONF.set_override("connection", "log://", group='database') self.CONF.set_override("backend_url", "zake://", group="coordination") - self.CONF.set_override("workload_partitioning", True, - group='notification') self.setup_messaging(self.CONF) self.srv = notification.NotificationService(0, self.CONF) @@ -117,16 +129,13 @@ ] ) - @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock()) - @mock.patch('ceilometer.pipeline.setup_event_pipeline', mock.MagicMock()) @mock.patch('ceilometer.event.endpoint.EventsNotificationEndpoint') def _do_process_notification_manager_start(self, fake_event_endpoint_class): with mock.patch.object(self.srv, '_get_notifications_manager') as get_nm: get_nm.side_effect = self.fake_get_notifications_manager - self.srv.run() - self.addCleanup(self.srv.terminate) + self.run_service(self.srv) self.fake_event_endpoint = fake_event_endpoint_class.return_value def test_start_multiple_listeners(self): @@ -147,15 +156,12 @@ self.assertEqual(2, len(self.srv.listeners[0].dispatcher.endpoints)) - @mock.patch('ceilometer.pipeline.setup_event_pipeline', mock.MagicMock()) def test_process_notification_with_events(self): self._do_process_notification_manager_start() self.assertEqual(2, len(self.srv.listeners[0].dispatcher.endpoints)) self.assertEqual(self.fake_event_endpoint, self.srv.listeners[0].dispatcher.endpoints[0]) - @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock()) - @mock.patch('ceilometer.pipeline.setup_event_pipeline', mock.MagicMock()) @mock.patch('oslo_messaging.get_batch_notification_listener') def test_unique_consumers(self, mock_listener): @@ -168,14 +174,13 @@ with mock.patch.object(self.srv, '_get_notifications_manager') as get_nm: get_nm.side_effect = fake_get_notifications_manager_dup_targets - self.srv.run() - self.addCleanup(self.srv.terminate) - self.assertEqual(2, len(mock_listener.call_args_list)) - args, kwargs = mock_listener.call_args + self.run_service(self.srv) + # 1 target, 1 listener + self.assertEqual(1, len(mock_listener.call_args_list[0][0][1])) self.assertEqual(1, len(self.srv.listeners)) -class BaseRealNotification(tests_base.BaseTestCase): +class BaseRealNotification(BaseNotificationTest): def setup_pipeline(self, counter_names): pipeline = yaml.dump({ 'sources': [{ @@ -237,9 +242,7 @@ self.publisher = test_publisher.TestPublisher(self.CONF, "") def _check_notification_service(self): - self.srv.run() - self.addCleanup(self.srv.terminate) - + self.run_service(self.srv) notifier = messaging.get_notifier(self.transport, "compute.vagrant-precise") notifier.info({}, 'compute.instance.create.end', @@ -270,8 +273,7 @@ @mock.patch('ceilometer.publisher.test.TestPublisher') def test_notification_service_error_topic(self, fake_publisher_cls): fake_publisher_cls.return_value = self.publisher - self.srv.run() - self.addCleanup(self.srv.terminate) + self.run_service(self.srv) notifier = messaging.get_notifier(self.transport, 'compute.vagrant-precise') notifier.error({}, 'compute.instance.error', @@ -308,7 +310,7 @@ override_pool_size=self.CONF.max_parallel_requests) m_listener.reset_mock() self.CONF.set_override('batch_size', 2, group='notification') - self.srv._refresh_agent(None) + self.srv._refresh_agent() m_listener.assert_called_with(override_pool_size=1) @mock.patch('oslo_messaging.get_batch_notification_listener') @@ -318,16 +320,13 @@ mock.MagicMock(), # pipeline listener mock.MagicMock(), # refresh pipeline listener ] - - self.srv.run() - self.addCleanup(self.srv.terminate) - + self.run_service(self.srv) listener = self.srv.pipeline_listener - self.srv._configure_pipeline_listener() + self.srv._refresh_agent() self.assertIsNot(listener, self.srv.pipeline_listener) @mock.patch('oslo_messaging.get_batch_notification_listener') - def test_retain_common_targets_on_refresh(self, mock_listener): + def test_hashring_targets(self, mock_listener): maybe = {"maybe": 0} def _once_over_five(item): @@ -338,27 +337,19 @@ hashring.belongs_to_self = _once_over_five self.srv.partition_coordinator = pc = mock.MagicMock() pc.join_partitioned_group.return_value = hashring - self.srv.run() - self.addCleanup(self.srv.terminate) - listened_before = [target.topic for target in - mock_listener.call_args[0][1]] - self.assertEqual(4, len(listened_before)) - self.srv._refresh_agent(None) - listened_after = [target.topic for target in - mock_listener.call_args[0][1]] - self.assertEqual(4, len(listened_after)) - common = set(listened_before) & set(listened_after) + self.run_service(self.srv) + topics = [target.topic for target in mock_listener.call_args[0][1]] + self.assertEqual(4, len(topics)) self.assertEqual( {'ceilometer-pipe-test_pipeline:test_sink-4', 'ceilometer-pipe-event:test_event:test_sink-4', 'ceilometer-pipe-event:test_event:test_sink-9', 'ceilometer-pipe-test_pipeline:test_sink-9'}, - common) + set(topics)) @mock.patch('oslo_messaging.get_batch_notification_listener') def test_notify_to_relevant_endpoint(self, mock_listener): - self.srv.run() - self.addCleanup(self.srv.terminate) + self.run_service(self.srv) targets = mock_listener.call_args[0][1] self.assertIsNotEmpty(targets) @@ -384,8 +375,7 @@ @mock.patch('oslo_messaging.Notifier.sample') def test_broadcast_to_relevant_pipes_only(self, mock_notifier): - self.srv.run() - self.addCleanup(self.srv.terminate) + self.run_service(self.srv) for endpoint in self.srv.listeners[0].dispatcher.endpoints: if (hasattr(endpoint, 'filter_rule') and not endpoint.filter_rule.match(None, None, 'nonmatching.end', @@ -421,7 +411,7 @@ mock_notifier.call_args_list[2][1]['event_type']) -class TestRealNotificationMultipleAgents(tests_base.BaseTestCase): +class TestRealNotificationMultipleAgents(BaseNotificationTest): def setup_pipeline(self, transformers): pipeline = yaml.dump({ 'sources': [{ @@ -471,6 +461,7 @@ group='notification') self.CONF.set_override('pipeline_processing_queues', 2, group='notification') + self.CONF.set_override('check_watchers', 1, group='coordination') self.publisher = test_publisher.TestPublisher(self.CONF, "") self.publisher2 = test_publisher.TestPublisher(self.CONF, "") @@ -485,11 +476,11 @@ self.srv = notification.NotificationService(0, self.CONF) self.srv.partition_coordinator = pc = mock.MagicMock() - hashring = mock.MagicMock() - hashring.belongs_to_self = _sometimes_srv - pc.join_partitioned_group.return_value = hashring - self.srv.run() - self.addCleanup(self.srv.terminate) + hashring_srv1 = mock.MagicMock() + hashring_srv1.belongs_to_self = _sometimes_srv + hashring_srv1.ring.nodes = {'id1': mock.Mock()} + pc.join_partitioned_group.return_value = hashring_srv1 + self.run_service(self.srv) def _sometimes_srv2(item): maybe["srv2"] += 1 @@ -499,9 +490,10 @@ self.srv2.partition_coordinator = pc = mock.MagicMock() hashring = mock.MagicMock() hashring.belongs_to_self = _sometimes_srv2 + hashring.ring.nodes = {'id1': mock.Mock(), 'id2': mock.Mock()} + self.srv.hashring.ring.nodes = hashring.ring.nodes.copy() pc.join_partitioned_group.return_value = hashring - self.srv2.run() - self.addCleanup(self.srv2.terminate) + self.run_service(self.srv2) notifier = messaging.get_notifier(self.transport, "compute.vagrant-precise") @@ -516,7 +508,8 @@ start = time.time() while time.time() - start < 10: if (len(self.publisher.samples + self.publisher2.samples) >= - self.expected_samples): + self.expected_samples and + len(self.srv.group_state) == 2): break time.sleep(0.1) @@ -526,6 +519,7 @@ s.resource_id for s in self.publisher.samples))) self.assertEqual(1, len(set( s.resource_id for s in self.publisher2.samples))) + self.assertEqual(2, len(self.srv.group_state)) @mock.patch('ceilometer.publisher.test.TestPublisher') def test_multiple_agents_no_transform(self, fake_publisher_cls): diff -Nru ceilometer-9.0.1/ceilometer/tests/tempest/service/client.py ceilometer-9.0.4/ceilometer/tests/tempest/service/client.py --- ceilometer-9.0.1/ceilometer/tests/tempest/service/client.py 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/tests/tempest/service/client.py 2018-01-09 16:10:50.000000000 +0000 @@ -16,16 +16,9 @@ from oslo_serialization import jsonutils as json from six.moves.urllib import parse as urllib +from tempest import clients from tempest import config from tempest.lib.common import rest_client -from tempest.lib.services.compute import flavors_client as flavor_cli -from tempest.lib.services.compute import floating_ips_client as floatingip_cli -from tempest.lib.services.compute import networks_client as network_cli -from tempest.lib.services.compute import servers_client as server_cli -from tempest.lib.services.image.v2 import images_client as img_cli_v2 -from tempest import manager -from tempest.services.object_storage import container_client as container_cli -from tempest.services.object_storage import object_client as obj_cli CONF = config.CONF @@ -89,43 +82,9 @@ return rest_client.ResponseBody(resp, body) -class Manager(manager.Manager): +class Manager(clients.Manager): - load_clients = [ - 'servers_client', - 'compute_networks_client', - 'compute_floating_ips_client', - 'flavors_client', - 'image_client_v2', - 'telemetry_client', - 'container_client', - 'object_client', - ] - - default_params = { - 'disable_ssl_certificate_validation': - CONF.identity.disable_ssl_certificate_validation, - 'ca_certs': CONF.identity.ca_certificates_file, - 'trace_requests': CONF.debug.trace_requests - } - - compute_params = { - 'service': CONF.compute.catalog_type, - 'region': CONF.compute.region or CONF.identity.region, - 'endpoint_type': CONF.compute.endpoint_type, - 'build_interval': CONF.compute.build_interval, - 'build_timeout': CONF.compute.build_timeout, - } - compute_params.update(default_params) - - image_params = { - 'service': CONF.image.catalog_type, - 'region': CONF.image.region or CONF.identity.region, - 'endpoint_type': CONF.image.endpoint_type, - 'build_interval': CONF.image.build_interval, - 'build_timeout': CONF.image.build_timeout, - } - image_params.update(default_params) + default_params = config.service_client_config() telemetry_params = { 'service': CONF.telemetry.catalog_type, @@ -134,53 +93,18 @@ } telemetry_params.update(default_params) - object_storage_params = { - 'service': CONF.object_storage.catalog_type, - 'region': CONF.object_storage.region or CONF.identity.region, - 'endpoint_type': CONF.object_storage.endpoint_type - } - object_storage_params.update(default_params) - - def __init__(self, credentials=None, service=None): - super(Manager, self).__init__(credentials) - for client in self.load_clients: - getattr(self, 'set_%s' % client)() - - def set_servers_client(self): - self.servers_client = server_cli.ServersClient( - self.auth_provider, - **self.compute_params) - - def set_compute_networks_client(self): - self.compute_networks_client = network_cli.NetworksClient( - self.auth_provider, - **self.compute_params) - - def set_compute_floating_ips_client(self): - self.compute_floating_ips_client = floatingip_cli.FloatingIPsClient( - self.auth_provider, - **self.compute_params) - - def set_flavors_client(self): - self.flavors_client = flavor_cli.FlavorsClient( - self.auth_provider, - **self.compute_params) - - def set_image_client_v2(self): - self.image_client_v2 = img_cli_v2.ImagesClient( - self.auth_provider, - **self.image_params) + def __init__(self, credentials): + # TODO(andreaf) Overriding Manager is a workaround. The "proper" way + # would it to expose the ceilometer service client via the plugin + # interface, use tempest.lib.clients and tempest master. + # Then ceilometer service client would be loaded and configured + # automatically into ServiceClients. + # In any case we're about to declare clients.Manager a stable + # interface for plugins and we won't change it, so this code won't + # break. + super(Manager, self).__init__(credentials=credentials) + self.set_telemetry_client() def set_telemetry_client(self): self.telemetry_client = TelemetryClient(self.auth_provider, **self.telemetry_params) - - def set_container_client(self): - self.container_client = container_cli.ContainerClient( - self.auth_provider, - **self.object_storage_params) - - def set_object_client(self): - self.object_client = obj_cli.ObjectClient( - self.auth_provider, - **self.object_storage_params) diff -Nru ceilometer-9.0.1/ceilometer/tests/unit/compute/test_discovery.py ceilometer-9.0.4/ceilometer/tests/unit/compute/test_discovery.py --- ceilometer-9.0.1/ceilometer/tests/unit/compute/test_discovery.py 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/tests/unit/compute/test_discovery.py 2018-01-09 16:10:41.000000000 +0000 @@ -15,6 +15,12 @@ import fixtures import iso8601 import mock +import testtools + +try: + import libvirt +except ImportError: + libvirt = None from ceilometer.compute import discovery from ceilometer.compute.pollsters import util @@ -58,6 +64,21 @@ """ +LIBVIRT_MANUAL_INSTANCE_DESC_XML = """ + + Manual-instance-00000001 + 5e637d0d-8c0e-441a-a11a-a9dc95aed84e + + hvm + /opt/instances/5e637d0d-8c0e-441a-a11a-a9dc95aed84e/kernel + /opt/instances/5e637d0d-8c0e-441a-a11a-a9dc95aed84e/ramdisk + root=/dev/vda console=tty0 console=ttyS0 + + + + +""" + class FakeDomain(object): def state(self): @@ -81,6 +102,33 @@ return [FakeDomain()] +class FakeManualInstanceDomain(object): + def state(self): + return [1, 2] + + def name(self): + return "Manual-instance-00000001" + + def UUIDString(self): + return "5e637d0d-8c0e-441a-a11a-a9dc95aed84e" + + def XMLDesc(self): + return LIBVIRT_MANUAL_INSTANCE_DESC_XML + + def metadata(self, flags, url): + # Note(xiexianbin): vm not create by nova-compute don't have metadata + # elements like: '' + # When invoke get metadata method, raise libvirtError. + raise libvirt.libvirtError( + "metadata not found: Requested metadata element is not present") + + +class FakeManualInstanceConn(object): + def listAllDomains(self): + return [FakeManualInstanceDomain()] + + class TestDiscovery(base.BaseTestCase): def setUp(self): @@ -236,3 +284,16 @@ mock.call('test', None)] self.assertEqual(expected_calls, self.client.instance_get_all_by_host.call_args_list) + + @testtools.skipUnless(libvirt, "libvirt not available") + @mock.patch.object(utils, "libvirt") + @mock.patch.object(discovery, "libvirt") + def test_discovery_with_libvirt_error(self, libvirt, libvirt2): + self.CONF.set_override("instance_discovery_method", + "libvirt_metadata", + group="compute") + libvirt.VIR_DOMAIN_METADATA_ELEMENT = 2 + libvirt2.openReadOnly.return_value = FakeManualInstanceConn() + dsc = discovery.InstanceDiscovery(self.CONF) + resources = dsc.discover(mock.MagicMock()) + self.assertEqual(0, len(resources)) diff -Nru ceilometer-9.0.1/ceilometer/tests/unit/compute/virt/libvirt/test_inspector.py ceilometer-9.0.4/ceilometer/tests/unit/compute/virt/libvirt/test_inspector.py --- ceilometer-9.0.1/ceilometer/tests/unit/compute/virt/libvirt/test_inspector.py 2017-09-12 16:29:56.000000000 +0000 +++ ceilometer-9.0.4/ceilometer/tests/unit/compute/virt/libvirt/test_inspector.py 2018-01-09 16:10:50.000000000 +0000 @@ -367,7 +367,7 @@ with mock.patch('ceilometer.compute.virt.libvirt.utils.' 'refresh_libvirt_connection', return_value=conn): disks = list(self.inspector.inspect_disk_info(self.instance, None)) - self.assertEqual(0, len(disks)) + self.assertEqual(1, len(disks)) def test_inspect_disk_info_without_source_element(self): dom_xml = """ @@ -397,6 +397,44 @@ disks = list(self.inspector.inspect_disk_info(self.instance, None)) self.assertEqual(0, len(disks)) + def test_inspect_disks_without_source_element(self): + dom_xml = """ + + + + + + + + +
+ + + + """ + blockStatsFlags = {'wr_total_times': 91752302267, + 'rd_operations': 6756, + 'flush_total_times': 1310427331, + 'rd_total_times': 29142253616, + 'rd_bytes': 171460096, + 'flush_operations': 746, + 'wr_operations': 1437, + 'wr_bytes': 13574656} + domain = mock.Mock() + domain.XMLDesc.return_value = dom_xml + domain.info.return_value = (0, 0, 0, 2, 999999) + domain.blockStats.return_value = (1, 2, 3, 4, -1) + domain.blockStatsFlags.return_value = blockStatsFlags + conn = mock.Mock() + conn.lookupByUUIDString.return_value = domain + + with mock.patch('ceilometer.compute.virt.libvirt.utils.' + 'refresh_libvirt_connection', return_value=conn): + disks = list(self.inspector.inspect_disks(self.instance, None)) + + self.assertEqual(0, len(disks)) + def test_inspect_memory_usage_with_domain_shutoff(self): domain = mock.Mock() domain.info.return_value = (5, 0, 51200, 2, 999999) diff -Nru ceilometer-9.0.1/ceilometer.egg-info/pbr.json ceilometer-9.0.4/ceilometer.egg-info/pbr.json --- ceilometer-9.0.1/ceilometer.egg-info/pbr.json 2017-09-12 16:32:41.000000000 +0000 +++ ceilometer-9.0.4/ceilometer.egg-info/pbr.json 2018-01-09 16:15:00.000000000 +0000 @@ -1 +1 @@ -{"git_version": "11be2c8", "is_release": true} \ No newline at end of file +{"git_version": "f168939", "is_release": true} \ No newline at end of file diff -Nru ceilometer-9.0.1/ceilometer.egg-info/PKG-INFO ceilometer-9.0.4/ceilometer.egg-info/PKG-INFO --- ceilometer-9.0.1/ceilometer.egg-info/PKG-INFO 2017-09-12 16:32:41.000000000 +0000 +++ ceilometer-9.0.4/ceilometer.egg-info/PKG-INFO 2018-01-09 16:15:00.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: ceilometer -Version: 9.0.1 +Version: 9.0.4 Summary: OpenStack Telemetry Home-page: https://docs.openstack.org/ceilometer/latest/ Author: OpenStack diff -Nru ceilometer-9.0.1/ceilometer.egg-info/requires.txt ceilometer-9.0.4/ceilometer.egg-info/requires.txt --- ceilometer-9.0.1/ceilometer.egg-info/requires.txt 2017-09-12 16:32:41.000000000 +0000 +++ ceilometer-9.0.4/ceilometer.egg-info/requires.txt 2018-01-09 16:15:00.000000000 +0000 @@ -6,7 +6,7 @@ jsonpath-rw-ext>=0.1.9 jsonschema!=2.5.0,<3.0.0,>=2.0.0 kafka-python>=1.3.2 -keystonemiddleware!=4.1.0,>=4.0.0 +keystonemiddleware!=4.1.0,!=4.19.0,>=4.0.0 lxml>=2.3 monotonic msgpack-python>=0.4.0 diff -Nru ceilometer-9.0.1/ceilometer.egg-info/SOURCES.txt ceilometer-9.0.4/ceilometer.egg-info/SOURCES.txt --- ceilometer-9.0.1/ceilometer.egg-info/SOURCES.txt 2017-09-12 16:32:43.000000000 +0000 +++ ceilometer-9.0.4/ceilometer.egg-info/SOURCES.txt 2018-01-09 16:15:01.000000000 +0000 @@ -1,6 +1,7 @@ .coveragerc .mailmap .testr.conf +.zuul.yaml AUTHORS CONTRIBUTING.rst ChangeLog @@ -528,23 +529,15 @@ doc/source/contributor/architecture.rst doc/source/contributor/ceilo-arch.png doc/source/contributor/ceilo-gnocchi-arch.png -doc/source/contributor/configuration.rst -doc/source/contributor/contributing.rst +doc/source/contributor/devstack.rst doc/source/contributor/events.rst doc/source/contributor/gmr.rst doc/source/contributor/index.rst doc/source/contributor/measurements.rst -doc/source/contributor/new_meters.rst doc/source/contributor/new_resource_types.rst doc/source/contributor/overview.rst doc/source/contributor/plugins.rst doc/source/contributor/testing.rst -doc/source/contributor/install/custom.rst -doc/source/contributor/install/dbreco.rst -doc/source/contributor/install/development.rst -doc/source/contributor/install/index.rst -doc/source/contributor/install/manual.rst -doc/source/contributor/install/upgrade.rst doc/source/install/conf.py doc/source/install/get_started.rst doc/source/install/index.rst @@ -559,6 +552,7 @@ doc/source/install/install-compute-ubuntu.rst doc/source/install/install-compute.rst doc/source/install/install-controller.rst +doc/source/install/install-gnocchi.inc doc/source/install/next-steps.rst doc/source/install/verify.rst doc/source/install/cinder/install-cinder-config-common.inc @@ -593,6 +587,22 @@ etc/ceilometer/examples/loadbalancer_v2_meter_definitions.yaml etc/ceilometer/examples/osprofiler_event_definitions.yaml etc/ceilometer/rootwrap.d/ipmi.filters +playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/post.yaml +playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/run.yaml +playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/post.yaml +playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/run.yaml +playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/post.yaml +playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/run.yaml +playbooks/legacy/ceilometer-tox-py27-mongodb/post.yaml +playbooks/legacy/ceilometer-tox-py27-mongodb/run.yaml +playbooks/legacy/ceilometer-tox-py27-mysql/post.yaml +playbooks/legacy/ceilometer-tox-py27-mysql/run.yaml +playbooks/legacy/ceilometer-tox-py27-postgresql/post.yaml +playbooks/legacy/ceilometer-tox-py27-postgresql/run.yaml +playbooks/legacy/grenade-dsvm-ceilometer/post.yaml +playbooks/legacy/grenade-dsvm-ceilometer/run.yaml +playbooks/legacy/telemetry-dsvm-integration-ceilometer/post.yaml +playbooks/legacy/telemetry-dsvm-integration-ceilometer/run.yaml rally-jobs/README.rst rally-jobs/ceilometer.yaml rally-jobs/extra/README.rst diff -Nru ceilometer-9.0.1/ChangeLog ceilometer-9.0.4/ChangeLog --- ceilometer-9.0.1/ChangeLog 2017-09-12 16:32:41.000000000 +0000 +++ ceilometer-9.0.4/ChangeLog 2018-01-09 16:14:59.000000000 +0000 @@ -1,6 +1,55 @@ CHANGES ======= +9.0.4 +----- + +* cleanup data collection admin guide +* cleanup admin-guide architecture +* reorder admin and contributor topics +* remove configuration from contributor guide +* remove install section from contributor guide +* partial clean up of contributor install info +* clean up non-install parts of contributor docs +* [doc] frequency of polling should be controlled via the polling configuration +* Fix incorrect yaml code block in pipeline definition +* Add missing command of adding admin role to gnocchi +* static resources not picked up +* Exclude keystonemiddleware 4.19.0, force gnocchi install in devstack +* snmp: warn if snmp call timeout +* Change oslo.messaging prefetch default + +9.0.3 +----- + +* Add cpu\_l3\_cache to polling yaml +* always declare partitioning variables +* fix ceilometer-compute invoke libvirt exception error +* zuul: run TripleO jobs with new zuulv3 layout + +9.0.2 +----- + +* stop double refreshing on start +* update install docs +* refresh agent if group membership changes +* Revert "devstack: Set an optimal processing queue" +* Modify memory swap metric type +* Fix bug for ceilometer polling generates an exception +* Remove direct publisher since it is deprecated +* Fix Format in contributor/plugins.rst +* set ceilometer\_backend +* Zuul: add file extension to playbook path +* Fix a typo in the Installation Guide +* remove branch matching +* Remove the wrap for skip inspect rbd disk info +* devstack: install ceilometer extra +* Migrate to Zuul v3 +* Use tempest.clients.Manager as base clients class +* Move object storage container\_client to match tempest +* messaging-publisher: fix threadsafe of flush() +* Initialize hashrings as AgentManager object + 9.0.1 ----- diff -Nru ceilometer-9.0.1/debian/changelog ceilometer-9.0.4/debian/changelog --- ceilometer-9.0.1/debian/changelog 2017-09-27 13:44:01.000000000 +0000 +++ ceilometer-9.0.4/debian/changelog 2018-02-02 20:15:01.000000000 +0000 @@ -1,3 +1,18 @@ +ceilometer (1:9.0.4-0ubuntu1) artful; urgency=medium + + * New upstream version. + * d/p/drop-kafka.patch: Rebased. + * New stable point release for OpenStack Pike (LP: #1747065). + + -- Corey Bryant Fri, 02 Feb 2018 15:15:01 -0500 + +ceilometer (1:9.0.2-0ubuntu1) artful; urgency=medium + + * d/gbp.conf: Create stable/pike branch. + * New stable point release for OpenStack Pike (LP: #1734990). + + -- Corey Bryant Tue, 28 Nov 2017 14:52:29 -0500 + ceilometer (1:9.0.1-0ubuntu1) artful; urgency=medium * New stable point release for OpenStack Pike (LP: #1719728). diff -Nru ceilometer-9.0.1/debian/gbp.conf ceilometer-9.0.4/debian/gbp.conf --- ceilometer-9.0.1/debian/gbp.conf 2017-09-27 13:44:01.000000000 +0000 +++ ceilometer-9.0.4/debian/gbp.conf 2018-02-02 20:15:01.000000000 +0000 @@ -1,5 +1,5 @@ [DEFAULT] -debian-branch = master +debian-branch = stable/pike upstream-tag = %(version)s pristine-tar = True diff -Nru ceilometer-9.0.1/debian/patches/drop-kafka.patch ceilometer-9.0.4/debian/patches/drop-kafka.patch --- ceilometer-9.0.1/debian/patches/drop-kafka.patch 2017-09-27 13:44:01.000000000 +0000 +++ ceilometer-9.0.4/debian/patches/drop-kafka.patch 2018-02-02 20:15:01.000000000 +0000 @@ -10,6 +10,6 @@ jsonpath-rw-ext>=0.1.9 # Apache-2.0 jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT -kafka-python>=1.3.2 # Apache-2.0 - keystonemiddleware!=4.1.0,>=4.0.0 # Apache-2.0 + keystonemiddleware!=4.19.0,!=4.1.0,>=4.0.0 # Apache-2.0 lxml>=2.3 # BSD monotonic diff -Nru ceilometer-9.0.1/devstack/plugin.sh ceilometer-9.0.4/devstack/plugin.sh --- ceilometer-9.0.1/devstack/plugin.sh 2017-09-12 16:29:56.000000000 +0000 +++ ceilometer-9.0.4/devstack/plugin.sh 2018-01-09 16:10:50.000000000 +0000 @@ -199,11 +199,13 @@ function install_gnocchi { echo_summary "Installing Gnocchi" - if [ $GNOCCHI_GIT_PATH ]; then - pip_install -e $GNOCCHI_GIT_PATH[redis,${DATABASE_TYPE},keystone] uwsgi + if python3_enabled; then + PY_VERS=${PYTHON3_VERSION} else - pip_install gnocchi[redis,${DATABASE_TYPE},keystone] uwsgi + PY_VERS=${PYTHON2_VERSION} fi + # workaround for upper-constraints. + sudo -H python${PY_VERS} -m pip install -U gnocchi[redis,${DATABASE_TYPE},keystone] recreate_database gnocchi sudo install -d -o $STACK_USER -m 755 $GNOCCHI_CONF_DIR @@ -338,7 +340,6 @@ iniset $CEILOMETER_CONF coordination backend_url $CEILOMETER_COORDINATION_URL iniset $CEILOMETER_CONF notification workload_partitioning True iniset $CEILOMETER_CONF notification workers $API_WORKERS - iniset $CEILOMETER_CONF notification pipeline_processing_queues $API_WORKERS fi if [[ -n "$CEILOMETER_CACHE_BACKEND" ]]; then @@ -443,7 +444,14 @@ fi install_ceilometerclient - setup_develop $CEILOMETER_DIR + + case $CEILOMETER_BACKEND in + mongodb) extra=mongo;; + gnocchi) extra=gnocchi;; + mysql) extra=mysql;; + postgresql) extra=postgresql;; + esac + setup_develop $CEILOMETER_DIR $extra sudo install -d -o $STACK_USER -m 755 $CEILOMETER_CONF_DIR } diff -Nru ceilometer-9.0.1/doc/source/admin/index.rst ceilometer-9.0.4/doc/source/admin/index.rst --- ceilometer-9.0.1/doc/source/admin/index.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/admin/index.rst 2018-01-09 16:10:50.000000000 +0000 @@ -4,14 +4,35 @@ Administrator Guide ===================== + +Overview +======== .. toctree:: :maxdepth: 2 telemetry-system-architecture - telemetry-measurements - telemetry-troubleshooting-guide - telemetry-data-pipelines + +Configuration +============= +.. toctree:: + :maxdepth: 2 + telemetry-data-collection + telemetry-data-pipelines telemetry-data-retrieval telemetry-best-practices + +Data Types +========== +.. toctree:: + :maxdepth: 2 + + telemetry-measurements telemetry-events + +Management +========== +.. toctree:: + :maxdepth: 2 + + telemetry-troubleshooting-guide diff -Nru ceilometer-9.0.1/doc/source/admin/telemetry-data-collection.rst ceilometer-9.0.4/doc/source/admin/telemetry-data-collection.rst --- ceilometer-9.0.1/doc/source/admin/telemetry-data-collection.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/admin/telemetry-data-collection.rst 2018-01-09 16:10:50.000000000 +0000 @@ -12,9 +12,6 @@ supported databases, which are listed in :ref:`telemetry-supported-databases`. -Samples capture a numerical measurement of a resource. The Telemetry service -leverages multiple methods to collect data samples. - The available data collection mechanisms are: Notifications @@ -37,7 +34,7 @@ Notifications -~~~~~~~~~~~~~ +============= All OpenStack services send notifications about the executed operations or system state. Several notifications carry information that can be @@ -63,141 +60,15 @@ operation. Not all these notifications are consumed by the Telemetry service, as the intention is only to capture the billable events and notifications that can be used for monitoring or profiling purposes. The -notification agent filters by the event type. Each notification -message contains the event type. The following table contains the event -types by each OpenStack service that Telemetry transforms into samples. - -.. list-table:: - :widths: 10 15 30 - :header-rows: 1 - - * - OpenStack service - - Event types - - Note - * - OpenStack Compute - - scheduler.run\_instance.scheduled - - scheduler.select\_\ - destinations - - compute.instance.\* - - For a more detailed list of Compute notifications please - check the `System Usage Data wiki page `__. - * - Bare metal service - - hardware.ipmi.\* - - - * - OpenStack Image - - image.update - - image.upload - - image.delete - - image.send - - - The required configuration for Image service can be found in the - `Configure the Image service for Telemetry `__ - section in the Installation Tutorials and Guides. - * - OpenStack Networking - - floatingip.create.end - - floatingip.update.\* - - floatingip.exists - - network.create.end - - network.update.\* - - network.exists - - port.create.end - - port.update.\* - - port.exists - - router.create.end - - router.update.\* - - router.exists - - subnet.create.end - - subnet.update.\* - - subnet.exists - - l3.meter - - - * - Orchestration service - - orchestration.stack\ - .create.end - - orchestration.stack\ - .update.end - - orchestration.stack\ - .delete.end - - orchestration.stack\ - .resume.end - - orchestration.stack\ - .suspend.end - - - * - OpenStack Block Storage - - volume.exists - - volume.create.\* - - volume.delete.\* - - volume.update.\* - - volume.resize.\* - - volume.attach.\* - - volume.detach.\* - - snapshot.exists - - snapshot.create.\* - - snapshot.delete.\* - - snapshot.update.\* - - volume.backup.create.\ - \* - - volume.backup.delete.\ - \* - - volume.backup.restore.\ - \* - - The required configuration for Block Storage service can be found in the - `Add the Block Storage service agent for Telemetry - `__ - section in the Installation Tutorials and Guides. +notifications handled are contained under the `ceilometer.sample.endpoint` +namespace. .. note:: Some services require additional configuration to emit the - notifications using the correct control exchange on the message - queue and so forth. These configuration needs are referred in the - above table for each OpenStack service that needs it. - -Specific notifications from the Compute service are important for -administrators and users. Configuring ``nova_notifications`` in the -``nova.conf`` file allows administrators to respond to events -rapidly. For more information on configuring notifications for the -compute service, see `Telemetry services -`__ in the -Installation Tutorials and Guides. + notifications. Please see the :ref:`install_controller` for more details. + +.. _meter_definitions: Meter definitions ----------------- @@ -302,8 +173,10 @@ project_id: $.payload.tenant_id resource_id: $.payload.instance_id +.. _Polling-Configuration: + Polling -~~~~~~~ +======= The Telemetry service is intended to store a complex picture of the infrastructure. This goal requires additional information than what is @@ -318,6 +191,54 @@ Telemetry uses an agent based architecture to fulfill the requirements against the data collection. +Configuration +------------- + +Polling rules are defined by the `polling.yaml` file. It defines the pollsters +to enable and the interval they should be polled. + +Each source configuration encapsulates meter name matching which matches +against the entry point of pollster. It also includes: polling +interval determination, optional resource enumeration or discovery. + +All samples generated by polling are placed on the queue to be handled by +the pipeline configuration loaded in the notification agent. + +The polling definition may look like the following:: + + --- + sources: + - name: 'source name' + interval: 'how often the samples should be generated' + meters: + - 'meter filter' + resources: + - 'list of resource URLs' + discovery: + - 'list of discoverers' + +The *interval* parameter in the sources section defines the cadence of sample +generation in seconds. + +Polling plugins are invoked according to each source's section whose *meters* +parameter matches the plugin's meter name. Its matching logic functions the +same as pipeline filtering. + +The optional *resources* section of a polling source allows a list of +static resource URLs to be configured. An amalgamated list of all +statically defined resources are passed to individual pollsters for polling. + +The optional *discovery* section of a polling source contains the list of +discoverers. These discoverers can be used to dynamically discover the +resources to be polled by the pollsters. + +If both *resources* and *discovery* are set, the final resources passed to the +pollsters will be the combination of the dynamic resources returned by the +discoverers and the static resources defined in the *resources* section. + +Agents +------ + There are three types of agents supporting the polling mechanism, the ``compute agent``, the ``central agent``, and the ``IPMI agent``. Under the hood, all the types of polling agents are the same @@ -359,7 +280,7 @@ used. Compute agent -------------- +~~~~~~~~~~~~~ This agent is responsible for collecting resource usage data of VM instances on individual compute nodes within an OpenStack deployment. @@ -369,15 +290,9 @@ information locally. A Compute agent instance has to be installed on each and every compute -node, installation instructions can be found in the `Install the Compute -agent for Telemetry -`__ +node, installation instructions can be found in the :ref:`install_compute` section in the Installation Tutorials and Guides. -The compute agent does not need direct database connection. The samples -collected by this agent are sent via AMQP to the notification agent to be -processed. - The list of supported hypervisors can be found in :ref:`telemetry-supported-hypervisors`. The Compute agent uses the API of the hypervisor installed on the compute hosts. Therefore, the supported meters may @@ -388,41 +303,27 @@ The support column provides the information about which meter is available for each hypervisor supported by the Telemetry service. -.. note:: - - Telemetry supports Libvirt, which hides the hypervisor under it. - Central agent -------------- +~~~~~~~~~~~~~ This agent is responsible for polling public REST APIs to retrieve additional information on OpenStack resources not already surfaced via notifications, and also for polling hardware resources over SNMP. -The following services can be polled with this agent: +Some of the services polled with this agent are: - OpenStack Networking - - OpenStack Object Storage - - OpenStack Block Storage - - Hardware resources via SNMP -- Energy consumption meters via `Kwapi `__ - framework (deprecated in Newton) - -To install and configure this service use the `Add the Telemetry service -`__ +To install and configure this service use the :ref:`install_rdo` section in the Installation Tutorials and Guides. -Just like the compute agent, this component also does not need a direct -database connection. The samples are sent via AMQP to the notification agent. - .. _telemetry-ipmi-agent: IPMI agent ----------- +~~~~~~~~~~ This agent is responsible for collecting IPMI sensor data and Intel Node Manager data on individual compute nodes within an OpenStack deployment. @@ -438,9 +339,6 @@ is suggested that you install the IPMI agent only on an IPMI capable node for performance reasons. -Just like the central agent, this component also does not need direct -database access. The samples are sent via AMQP to the notification agent. - The list of collected meters can be found in :ref:`telemetry-bare-metal-service`. diff -Nru ceilometer-9.0.1/doc/source/admin/telemetry-data-pipelines.rst ceilometer-9.0.4/doc/source/admin/telemetry-data-pipelines.rst --- ceilometer-9.0.1/doc/source/admin/telemetry-data-pipelines.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/admin/telemetry-data-pipelines.rst 2018-01-09 16:10:50.000000000 +0000 @@ -49,7 +49,7 @@ - name: 'source name' meters: - 'meter filter' - sinks + sinks: - 'sink name' sinks: - name: 'sink name' @@ -132,7 +132,7 @@ - name: 'source name' events: - 'event filter' - sinks + sinks: - 'sink name' sinks: - name: 'sink name' @@ -522,8 +522,6 @@ - panko:// - udp://10.0.0.2:1234 - notifier://?policy=drop&max_queue_length=512&topic=custom_target - - direct://?dispatcher=http - Deprecated publishers --------------------- @@ -616,3 +614,32 @@ - DB2 NoSQL does not have native TTL nor ``ceilometer-expirer`` support. +Pipeline Partitioning +~~~~~~~~~~~~~~~~~~~~~ + +.. note:: + + Partitioning is only required if pipelines contain transformations. It has + secondary benefit of supporting batching in certain publishers. + +On large workloads, multiple notification agents can be deployed to handle the +flood of incoming messages from monitored services. If transformations are +enabled in the pipeline, the notification agents must be coordinated to ensure +related messages are routed to the same agent. To enable coordination, set the +``workload_partitioning`` value in ``notification`` section. + +To distribute messages across agents, ``pipeline_processing_queues`` option +should be set. This value defines how many pipeline queues to create which will +then be distributed to the active notification agents. It is recommended that +the number of processing queues, at the very least, match the number of agents. + +Increasing the number of processing queues will improve the distribution of +messages across the agents. It will also help batching which minimises the +requests to Gnocchi storage backend. It will also increase the load the on +message queue as it uses the queue to shard data. + +.. warning:: + + Decreasing the number of processing queues may result in lost data as any + previously created queues may no longer be assigned to active agents. It + is only recommended that you **increase** processing queues. diff -Nru ceilometer-9.0.1/doc/source/admin/telemetry-system-architecture.rst ceilometer-9.0.4/doc/source/admin/telemetry-system-architecture.rst --- ceilometer-9.0.1/doc/source/admin/telemetry-system-architecture.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/admin/telemetry-system-architecture.rst 2018-01-09 16:10:50.000000000 +0000 @@ -5,10 +5,10 @@ =================== The Telemetry service uses an agent-based architecture. Several modules -combine their responsibilities to collect data, store samples in a -database, or provide an API service for handling incoming requests. +combine their responsibilities to collect, normalize, and redirect data +to be used for use cases such as metering, monitoring, and alerting. -The Telemetry service is built from the following agents and services: +The Telemetry service is built from the following agents: ceilometer-api (deprecated in Ocata) Presents aggregated metering data to consumers (such as billing @@ -18,12 +18,14 @@ ceilometer-polling Polls for different kinds of meter data by using the polling plug-ins (pollsters) registered in different namespaces. It provides a - single polling interface across different namespaces. The ``compute`` - namespace polls the local hypervisor to acquire performance data of local - instances. The ``central`` namespace polls the public RESTful APIs of other - OpenStack services such as Compute service and Image service. The ``ipmi`` - namespace polls the local node with IPMI support, in order to acquire IPMI - sensor data and Intel Node Manager datahost-level information. + single polling interface across different namespaces. + +.. note:: + + The ``ceilometer-polling`` service provides polling support on any + namespace but many distributions continue to provide namespace-scoped + agents: ``ceilometer-agent-central``, ``ceilometer-agent-compute``, + and ``ceilometer-agent-ipmi``. ceilometer-agent-notification Consumes AMQP messages from other OpenStack services, normalizes messages, @@ -48,7 +50,7 @@ ``ipmi`` namespaces, all the other services are placed on one or more controller nodes. -The Telemetry architecture highly depends on the AMQP service both for +The Telemetry architecture depends on the AMQP service both for consuming notifications coming from OpenStack services and internal communication. @@ -84,8 +86,6 @@ - `PostgreSQL `__ -- `HBase `__ - .. _telemetry-supported-hypervisors: @@ -98,67 +98,23 @@ The following is a list of supported hypervisors. -- The following hypervisors are supported via `libvirt `__ - - * `Kernel-based Virtual Machine (KVM) `__ - - * `Quick Emulator (QEMU) `__ - - * `Linux Containers (LXC) `__ - - * `User-mode Linux (UML) `__ - - .. note:: - - For details about hypervisor support in libvirt please check the - `Libvirt API support matrix `__. - +- `Libvirt supported hypervisors `__ such as KVM and QEMU - `Hyper-V `__ - - `XEN `__ - - `VMware vSphere `__ +.. note:: -Supported networking services -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Telemetry is able to retrieve information from OpenStack Networking and -external networking services: + For details about hypervisor support in libvirt please see the + `Libvirt API support matrix `__. -- OpenStack Networking: - - Basic network meters - - - Firewall-as-a-Service (FWaaS) meters - - - Load-Balancer-as-a-Service (LBaaS) meters +Supported networking services +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - VPN-as-a-Service (VPNaaS) meters +Telemetry is able to retrieve information from external networking services: - SDN controller meters: - `OpenDaylight `__ - - `OpenContrail `__ - - -.. _telemetry-users-roles-projects: - -Users, roles, and projects -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This service of OpenStack uses OpenStack Identity for authenticating and -authorizing users. The required configuration options are listed in the -`Telemetry section -`__ in the -OpenStack Configuration Reference. Alternatively, gnocchi can be configured -without authentication to minimize overhead. - -The system uses two roles:``admin`` and ``non-admin``. The authorization -happens before processing each API request. The amount of returned data -depends on the role the requestor owns. - -The creation of alarm definitions also highly depends on the role of the -user, who initiated the action. Further details about :ref:`telemetry-alarms` -handling can be found in this guide. diff -Nru ceilometer-9.0.1/doc/source/contributor/architecture.rst ceilometer-9.0.4/doc/source/contributor/architecture.rst --- ceilometer-9.0.1/doc/source/contributor/architecture.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/architecture.rst 2018-01-09 16:10:50.000000000 +0000 @@ -60,15 +60,11 @@ The Ceilometer project created 2 methods to collect data: 1. :term:`Notification agent` which takes messages generated on the - notification bus and transforms them into Ceilometer samples or events. This - is **the preferred method** of data collection. If you are working on some - OpenStack related project and are using the Oslo library, you are kindly - invited to come and talk to one of the project members to learn how you - could quickly add instrumentation for your project. -2. :term:`Polling agents`, which is the less preferred method, will poll - some API or other tool to collect information at a regular interval. - The polling approach is less preferred due to the load it can impose - on the API services. + notification bus and transforms them into Ceilometer samples or events. +2. :term:`Polling agent`, will poll some API or other tool to collect + information at a regular interval. The polling approach may impose + significant on the API services so should only be used on optimised + endpoints. The first method is supported by the ceilometer-notification agent, which monitors the message queues for notifications. Polling agents can be configured @@ -134,8 +130,8 @@ combination of ``ceilometer.poll.compute``, ``ceilometer.poll.central``, and ``ceilometer.poll.ipmi`` namespaces -The frequency of polling is controlled via the pipeline configuration. See -:ref:`Pipeline-Configuration` for details. The agent framework then passes the +The frequency of polling is controlled via the polling configuration. See +:ref:`Polling-Configuration` for details. The agent framework then passes the generated samples to the notification agent for processing. @@ -174,6 +170,11 @@ derive even more data. Ceilometer offers various transformers which can be used to manipulate data in the pipeline. +.. note:: + + The equivalent functionality can be handled more stably by storage + drivers such as Gnocchi. + Publishing the data ------------------- @@ -191,10 +192,7 @@ which can be consumed by an external system; 3. udp, which publishes samples using UDP packets; 4. http, which targets a REST interface; -5. kafka, which publishes data to a Kafka message queue to be consumed by any system - that supports Kafka. -6. file, which publishes samples to a file with specified name and location; -7. database, which stores samples to the legacy ceilometer database system. +5. file, which publishes samples to a file with specified name and location; Storing/Accessing the data diff -Nru ceilometer-9.0.1/doc/source/contributor/configuration.rst ceilometer-9.0.4/doc/source/contributor/configuration.rst --- ceilometer-9.0.1/doc/source/contributor/configuration.rst 2017-09-12 16:29:56.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/configuration.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,215 +0,0 @@ -.. - Copyright 2012 New Dream Network, LLC (DreamHost) - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -=============== - Configuration -=============== - -Polling -~~~~~~~ - -Polling rules are defined by the `polling.yaml` file. It defines the pollsters -to enable and the interval they should be polled. See :doc:`plugins` for -details on how to write and plug in your plugins. - -Each source configuration encapsulates meter name matching which matches -against the entry point of pollster. It also includes: polling -interval determination, optional resource enumeration or discovery. - -All samples generated by polling are placed on the queue to be handled by -the pipeline configuration loaded in the notification agent. - -The polling definition may look like the following:: - - --- - sources: - - name: 'source name' - interval: 'how often the samples should be generated' - meters: - - 'meter filter' - resources: - - 'list of resource URLs' - discovery: - - 'list of discoverers' - -The *interval* parameter in the sources section should be defined in seconds. -It determines the cadence of sample generation. - -A given polling plugin is invoked according to each source section -whose *meters* parameter matches the plugin's meter name. That is, -the matching source sections are combined by union, not intersection, -of the prescribed time series. It functions the same as Pipelines_ -filtering - -The optional *resources* section of a pipeline source allows a list of -static resource URLs to be configured. An amalgamated list of all -statically configured resources for a set of pipeline sources with a -common interval is passed to individual pollsters matching those pipelines. - -The optional *discovery* section of a pipeline source contains the list of -discoverers. These discoverers can be used to dynamically discover the -resources to be polled by the pollsters defined in this pipeline. The name -of the discoverers should be the same as the related names of plugins in -:file:`setup.cfg`. - -If *resources* or *discovery* section is not set, the default value would -be an empty list. If both *resources* and *discovery* are set, the final -resources passed to the pollsters will be the combination of the dynamic -resources returned by the discoverers and the static resources defined -in the *resources* section. If there are some duplications between the -resources returned by the discoverers and those defined in the *resources* -section, the duplication will be removed before passing those resources -to the pollsters. - -There are three ways a pollster can get a list of resources to poll, as the -following in descending order of precedence: - - 1. From the per-pipeline configured discovery and/or static resources. - 2. From the per-pollster default discovery. - 3. From the per-agent default discovery. - - -.. _Pipeline-Configuration: - -Pipelines -~~~~~~~~~ - -Pipelines describe a coupling between sources of samples and the -corresponding sinks for transformation and publication of the samples. - -A source is a set of samples that should be passed to specified sinks. These -samples may come from polling agents or service notifications. - -A sink on the other hand is a consumer of samples, providing logic for -the transformation and publication of samples emitted from related sources. -Each sink configuration is concerned `only` with the transformation rules -and publication conduits for samples. - -In effect, a sink describes a chain of handlers. The chain starts with -zero or more transformers and ends with one or more publishers. The first -transformer in the chain is passed samples from the corresponding source, -takes some action such as deriving rate of change, performing unit conversion, -or aggregating, before passing the modified sample to the next step. - -The chains end with one or more publishers. This component makes it possible -to persist the data into storage through the message bus or to send it to one -or more external consumers. One chain can contain multiple publishers, see the -:ref:`multi-publisher` section. - - -Pipeline configuration ----------------------- - -Pipeline configuration by default, is stored in a separate configuration file, -called :file:`pipeline.yaml`, next to the :file:`ceilometer.conf` file. The -pipeline configuration file can be set in the *pipeline_cfg_file* parameter in -:file:`ceilometer.conf`. Multiple chains can be defined in one configuration -file. - -The chain definition looks like the following:: - - --- - sources: - - name: 'source name' - sinks - - 'sink name' - sinks: - - name: 'sink name' - transformers: 'definition of transformers' - publishers: - - 'list of publishers' - -The *name* parameter of a source is unrelated to anything else; -nothing references a source by name, and a source's name does not have -to match anything. - -There are several ways to define the list of meters for a pipeline source. The -list of valid meters can be found in the :ref:`measurements` section. There is -a possibility to define all the meters, or just included or excluded meters, -with which a source should operate: - -* To include all meters, use the ``*`` wildcard symbol. -* To define the list of meters, use either of the following: - - * To define the list of included meters, use the ``meter_name`` syntax - * To define the list of excluded meters, use the ``!meter_name`` syntax - * For meters, which identify a complex Sample field, use the wildcard - symbol to select all, e.g. for ``disk.read.bytes``, use ``disk.*`` - -The above definition methods can be used in the following combinations: - -* Only the wildcard symbol -* The list of included meters -* The list of excluded meters -* Wildcard symbol with the list of excluded meters - -.. note:: - At least one of the above variations should be included in the meters - section. Included and excluded meters cannot co-exist in the same - pipeline. Wildcard and included meters cannot co-exist in the same - pipeline definition section. - -The *transformers* section of a pipeline sink provides the possibility to add a -list of transformer definitions. The names of the transformers should be the -same as the names of the related extensions in :file:`setup.cfg`. For a more -detailed description, please see the `transformers`_ section of the -Administrator Guide of Ceilometer. - -.. _transformers: https://docs.openstack.org/admin-guide/telemetry-data-collection.html#transformers - -The *publishers* section contains the list of publishers, where the samples -data should be sent after the possible transformations. The names of the -publishers should be the same as the related names of the plugins in -:file:`setup.cfg`. - -The default configuration can be found in `pipeline.yaml`_. For more details about -how to configure publishers, see :ref:`publisher-configuration`. - -.. _pipeline.yaml: https://git.openstack.org/cgit/openstack/ceilometer/tree/ceilometer/pipeline/data/pipeline.yaml - -Pipeline Processing -------------------- - -On large workloads, multiple notification agents can be deployed to handle the -flood of incoming messages from monitored services. If transformations are -enabled in the pipeline, the notification agents must be coordinated to ensure -related messages are routed to the same agent. To enable coordination, set the -``workload_partitioning`` value in ``notification`` section. - -To distribute messages across agents, ``pipeline_processing_queues`` option -should be set. This value defines how many pipeline queues to create which will -then be distributed to the active notification agents. It is recommended that -the number of processing queues, at the very least, match the number of agents. - -Increasing the number of processing queues will improve the distribution of -messages across the agents. It will also help batching which minimises the -requests to Gnocchi storage backend. - -.. warning:: - - Decreasing the number of processing queues may result in lost data as any - previously created queues may no longer be assigned to active agents. It - is only recommended that you **increase** processing queues. - - -.. _pipeline-publishers: - -Publishers -~~~~~~~~~~ - -For more information about publishers see the `publishers`_ section of the -Administrator Guide of Ceilometer. - -.. _publishers: https://docs.openstack.org/admin-guide/telemetry-data-pipelines.html#publishers diff -Nru ceilometer-9.0.1/doc/source/contributor/contributing.rst ceilometer-9.0.4/doc/source/contributor/contributing.rst --- ceilometer-9.0.1/doc/source/contributor/contributing.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/contributing.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,33 +0,0 @@ -.. - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _contributing: - -============================ - Contributing to Ceilometer -============================ - -Ceilometer follows the same workflow as other OpenStack projects. To start -contributing to Ceilometer, please follow the workflow found here_. - -.. _here: https://wiki.openstack.org/wiki/Gerrit_Workflow - - -Project Hosting Details -======================= - -:Bug tracker: https://launchpad.net/ceilometer -:Mailing list: http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-dev (prefix subjects with ``[Ceilometer]`` for faster responses) -:Wiki: https://wiki.openstack.org/wiki/Ceilometer -:Code Hosting: https://git.openstack.org/cgit/openstack/ceilometer/ -:Code Review: https://review.openstack.org/#/q/status:open+project:openstack/ceilometer,n,z diff -Nru ceilometer-9.0.1/doc/source/contributor/devstack.rst ceilometer-9.0.4/doc/source/contributor/devstack.rst --- ceilometer-9.0.1/doc/source/contributor/devstack.rst 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/devstack.rst 2018-01-09 16:10:49.000000000 +0000 @@ -0,0 +1,37 @@ +=============================== + Installing development sandbox +=============================== + +In a development environment created by devstack_, Ceilometer can be tested +alongside other OpenStack services. + +Configuring devstack +==================== + +1. Download devstack_. + +2. Create a ``local.conf`` file as input to devstack. + +3. The ceilometer services are not enabled by default, so they must be + enabled in ``local.conf`` but adding the following:: + + # Enable the Ceilometer devstack plugin + enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer.git + + By default, all ceilometer services except for ceilometer-ipmi agent will + be enabled + +4. Enable Gnocchi storage support by including the following in ``local.conf``:: + + CEILOMETER_BACKEND=gnocchi + + Optionally, services which extend Ceilometer can be enabled:: + + enable_plugin aodh https://git.openstack.org/openstack/aodh + enable_plugin panko https://git.openstack.org/openstack/panko + + These plugins should be added before ceilometer. + +5. ``./stack.sh`` + +.. _devstack: https://docs.openstack.org/devstack/latest/ diff -Nru ceilometer-9.0.1/doc/source/contributor/index.rst ceilometer-9.0.4/doc/source/contributor/index.rst --- ceilometer-9.0.1/doc/source/contributor/index.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/index.rst 2018-01-09 16:10:49.000000000 +0000 @@ -1,26 +1,53 @@ -Contribution Guidelines -======================= +================= +Contributor Guide +================= -In the Contributions Guide, you will find documented policies for -developing with Ceiloemter. This includes the processes we use for +In the Contributor Guide, you will find documented policies for +developing with Ceilometer. This includes the processes we use for bugs, contributor onboarding, core reviewer memberships, and other procedural items. -Developer reference -------------------- +Ceilometer follows the same workflow as other OpenStack projects. To start +contributing to Ceilometer, please follow the workflow found here_. +.. _here: https://wiki.openstack.org/wiki/Gerrit_Workflow + +:Bug tracker: https://launchpad.net/ceilometer +:Mailing list: http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-dev (prefix subjects with ``[Ceilometer]`` for faster responses) +:Wiki: https://wiki.openstack.org/wiki/Ceilometer +:Code Hosting: https://git.openstack.org/cgit/openstack/ceilometer/ +:Code Review: https://review.openstack.org/#/q/status:open+project:openstack/ceilometer,n,z + +Overview +======== .. toctree:: :maxdepth: 2 overview architecture + +Data Types +========== +.. toctree:: + :maxdepth: 2 + measurements events - contributing - install/index - configuration - plugins - new_meters - new_resource_types + +Getting Started +=============== +.. toctree:: + :maxdepth: 2 + + devstack testing gmr + + +Development +=========== +.. toctree:: + :maxdepth: 2 + + plugins + new_resource_types diff -Nru ceilometer-9.0.1/doc/source/contributor/install/custom.rst ceilometer-9.0.4/doc/source/contributor/install/custom.rst --- ceilometer-9.0.1/doc/source/contributor/install/custom.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/install/custom.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,153 +0,0 @@ -.. - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _customizing_deployment: - -=================================== - Customizing Ceilometer Deployment -=================================== - -Notifications queues -==================== - -.. index:: - double: customizing deployment; notifications queues; multiple topics - -By default, Ceilometer consumes notifications on the messaging bus sent to -**topics** by using a queue/pool name that is identical to the -topic name. You shouldn't have different applications consuming messages from -this queue. If you want to also consume the topic notifications with a system -other than Ceilometer, you should configure a separate queue that listens for -the same messages. - -Ceilometer allows multiple topics to be configured so that the polling agent can -send the same messages of notifications to other queues. Notification agents -also use **topics** to configure which queue to listen for. If -you use multiple topics, you should configure notification agent and polling -agent separately, otherwise Ceilometer collects duplicate samples. - -By default, the ceilometer.conf file is as follows:: - - [oslo_messaging_notifications] - topics = notifications - -To use multiple topics, you should give ceilometer-agent-notification and -ceilometer-polling services different ceilometer.conf files. The Ceilometer -configuration file ceilometer.conf is normally locate in the /etc/ceilometer -directory. Make changes according to your requirements which may look like -the following:: - -For notification agent using ceilometer-notification.conf, settings like:: - - [oslo_messaging_notifications] - topics = notifications,xxx - -For polling agent using ceilometer-polling.conf, settings like:: - - [oslo_messaging_notifications] - topics = notifications,foo - -.. note:: - - notification_topics in ceilometer-notification.conf should only have one same - topic in ceilometer-polling.conf - -Doing this, it's easy to listen/receive data from multiple internal and external services. - -.. _publisher-configuration: - -Using multiple publishers -========================= - -.. index:: - double: customizing deployment; multiple publishers - -Ceilometer allows multiple publishers to be configured in pipeline so that -data can be easily sent to multiple internal and external systems. Ceilometer -allows to set two types of pipelines. One is ``pipeline.yaml`` which is for -meters, another is ``event_pipeline.yaml`` which is for events. - -By default, Ceilometer only saves event and meter data into Gnocchi_. If you -want Ceilometer to send data to other systems, instead of or in addition to -the default storage services, multiple publishers can be enabled by modifying -the Ceilometer pipeline. - -Ceilometer ships multiple publishers currently. They are ``database``, -``notifier``, ``file``, ``http`` and ``gnocchi`` publishers. - -.. _Gnocchi: http://gnocchi.xyz - -To configure one or multiple publishers for Ceilometer, find the Ceilometer -configuration file ``pipeline.yaml`` and/or ``event_pipeline.yaml`` which is -normally located at /etc/ceilometer directory and make changes accordingly. -Your configuration file can be in a different directory. - -To use multiple publishers, add multiple publisher lines in ``pipeline.yaml`` and/or -``event_pipeline.yaml`` file like the following:: - - --- - sources: - - name: source_name - events: - - "*" - sinks: - - sink_name - sinks: - - name: sink_name - transformers: - publishers: - - database:// - - gnocchi:// - - file:// - -For the Gnocchi publisher, the following configuration settings should be added -into /etc/ceilometer/ceilometer.conf:: - - [dispatcher_gnocchi] - archive_policy = low - -The value specified for ``archive_policy`` should correspond to the name of an -``archive_policy`` configured within Gnocchi. - -For the Gnocchi publisher backed by Swift storage, the following additional -configuration settings should be added:: - - [dispatcher_gnocchi] - filter_project = gnocchi_swift - filter_service_activity = True - -Custom pipeline -=============== - -The paths of all pipeline files including ``pipeline.yaml`` and ``event_pipeline.yaml`` -are located to ceilometer/pipeline/data by default. And it's possible to set the -path through ``pipeline_cfg_file`` being assigned to another one in ``ceilometer.conf``. - -Ceilometer allow users to customize pipeline files. Before that, copy the following -yaml files:: - - $ cp ceilometer/pipeline/data/*.yaml /etc/ceilometer - -Then you can add configurations according to the former section. - -Efficient polling -================= - -- There is an optional config called ``shuffle_time_before_polling_task`` - in ceilometer.conf. Enable this by setting an integer greater than zero to - shuffle polling time for agents. This will add some random jitter to the time - of sending requests to Nova or other components to avoid large number of - requests in a short time period. -- There is an option to stream samples to minimise latency (at the - expense of load) by setting ``batch_polled_samples`` to ``False`` in - ``ceilometer.conf``. diff -Nru ceilometer-9.0.1/doc/source/contributor/install/dbreco.rst ceilometer-9.0.4/doc/source/contributor/install/dbreco.rst --- ceilometer-9.0.1/doc/source/contributor/install/dbreco.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/install/dbreco.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,57 +0,0 @@ -.. - Copyright 2013 Nicolas Barcet for eNovance - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _choosing_db_backend: - -============================ - Choosing a database backend -============================ - -Ceilometer is a data collection service. It normalizes data across OpenStack -and can be configured to persist the normalized data to multiple services. -Gnocchi_ is designed to store time-series **measurement data**. Panko_ is -intended to capture **event data**. Lastly, Aodh_ provides **alarming** -functionality. - -Moving from Ceilometer to Gnocchi -================================= - -Gnocchi represents a fundamental change in how data is represented and stored. -Installation and configuration can be found in :ref:`installing_manually`. -Differences between APIs can be found here_. - -There currently exists no migration tool between the services. To transition -to Gnocchi, multiple publishers can be enabled in the Collector to capture -data in both the native Ceilometer database and Gnocchi. This will allow you -to test Gnocchi and transition to it fully when comfortable. Edit the -``pipeline.yaml`` and ``event_pipeline.yaml`` to include multiple publishers:: - - --- - sources: - - name: event_source - events: - - "*" - sinks: - - event_sink - sinks: - - name: event_sink - publishers: - - gnocchi:// - - database:// - -.. _Gnocchi: http://gnocchi.xyz -.. _Aodh: https://docs.openstack.org/aodh/latest/ -.. _Panko: https://docs.openstack.org/panko/latest/ -.. _here: https://www.slideshare.net/GordonChung/ceilometer-to-gnocchi diff -Nru ceilometer-9.0.1/doc/source/contributor/install/development.rst ceilometer-9.0.4/doc/source/contributor/install/development.rst --- ceilometer-9.0.1/doc/source/contributor/install/development.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/install/development.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,57 +0,0 @@ -.. - Copyright 2012 Nicolas Barcet for Canonical - 2013 New Dream Network, LLC (DreamHost) - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -=============================== - Installing development sandbox -=============================== - -Ceilometer has several daemons. The basic are: :term:`polling agent` running -either on the Nova compute node(s) or :term:`polling agent` running on the -central management node(s), :term:`notification agent` running on the cloud's -management node(s). - -In a development environment created by devstack_, these services are -typically running on the same server. - - -Configuring devstack -==================== - -.. index:: - double: installing; devstack - -1. Download devstack_. - -2. Create a ``local.conf`` file as input to devstack. - -3. Ceilometer makes extensive use of the messaging bus, but has not - yet been tested with ZeroMQ. We recommend using Rabbit for - now. By default, RabbitMQ will be used by devstack. - -4. The ceilometer services are not enabled by default, so they must be - enabled in ``local.conf`` before running ``stack.sh``. - - This example ``local.conf`` file shows all of the settings required for - ceilometer:: - - [[local|localrc]] - # Enable the Ceilometer devstack plugin - enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer.git - - By default, all ceilometer services except for ceilometer-ipmi agent will - be enabled - -.. _devstack: http://www.devstack.org/ diff -Nru ceilometer-9.0.1/doc/source/contributor/install/index.rst ceilometer-9.0.4/doc/source/contributor/install/index.rst --- ceilometer-9.0.1/doc/source/contributor/install/index.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/install/index.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,29 +0,0 @@ -.. - Copyright 2013 New Dream Network, LLC (DreamHost) - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _install: - -======================= - Installing Ceilometer -======================= - -.. toctree:: - :maxdepth: 2 - - dbreco - development - manual - custom - upgrade diff -Nru ceilometer-9.0.1/doc/source/contributor/install/manual.rst ceilometer-9.0.4/doc/source/contributor/install/manual.rst --- ceilometer-9.0.1/doc/source/contributor/install/manual.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/install/manual.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,261 +0,0 @@ -.. - Copyright 2012 Nicolas Barcet for Canonical - 2013 New Dream Network, LLC (DreamHost) - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _installing_manually: - -===================== - Installing Manually -===================== - -.. note:: - - Ceilometer collector service is deprecated. Configure dispatchers under publisher - in pipeline to push data instead. For more details about how to configure - publishers in the :ref:`publisher-configuration`. - -Storage Backend Installation -============================ - - -Gnocchi -------- - -#. Follow `Gnocchi installation`_ instructions - -#. Edit `/etc/ceilometer/ceilometer.conf` for the collector service: - - * With Keystone authentication enabled:: - - [dispatcher_gnocchi] - filter_service_activity = False # Enable if using swift backend - filter_project = # if using swift backend - - [service_credentials] - auth_url = :5000 - region_name = RegionOne - password = password - username = ceilometer - project_name = service - project_domain_id = default - user_domain_id = default - auth_type = password - - * In somes cases, it is possible to disable keystone authentication for - Gnocchi to remove the overhead of token creation/verification when request - authentication doesn't matter. This will increase the performance of - Gnocchi:: - - [dispatcher_gnocchi] - filter_service_activity = False # Enable if using swift backend - filter_project = # if using swift backend - auth_section=service_credentials_gnocchi - - [service_credentials_gnocchi] - auth_type=gnocchi-noauth - roles = admin - user_id = - project_id = - endpoint = - -#. Copy gnocchi_resources.yaml to config directory (e.g./etc/ceilometer) - -#. Initialize Gnocchi database by creating ceilometer resources:: - - ceilometer-upgrade --skip-metering-database - -#. To minimize data requests, caching and batch processing should be enabled: - - 1. Enable resource caching (oslo.cache_ should be installed):: - - [cache] - backend_argument = redis_expiration_time:600 - backend_argument = db:0 - backend_argument = distributed_lock:True - backend_argument = url:redis://localhost:6379 - backend = dogpile.cache.redis - - 2. Enable batch processing:: - - [notification] - batch_size = 100 - batch_timeout = 5 - -#. Start notification service - -.. _oslo.cache: https://docs.openstack.org/oslo.cache/latest/configuration/index.html -.. _`Gnocchi installation`: http://gnocchi.xyz/install.html - - -Installing the notification agent -================================= - -.. index:: - double: installing; agent-notification - -1. Clone the ceilometer git repository to the management server:: - - $ cd /opt/stack - $ git clone https://git.openstack.org/openstack/ceilometer.git - -2. As a user with ``root`` permissions or ``sudo`` privileges, run the - ceilometer installer:: - - $ cd ceilometer - $ sudo python setup.py install - -3. Generate configuration file:: - - $ tox -egenconfig - -4. Copy the sample configuration files from the source tree - to their final location:: - - $ mkdir -p /etc/ceilometer - $ cp etc/ceilometer/ceilometer.conf /etc/ceilometer - $ cp ceilometer/pipeline/data/*.yaml /etc/ceilometer - -5. Edit ``/etc/ceilometer/ceilometer.conf`` - - 1. Configure messaging:: - - [oslo_messaging_notifications] - topics = notifications - - [oslo_messaging_rabbit] - rabbit_userid = stackrabbit - rabbit_password = openstack1 - rabbit_hosts = 10.0.2.15 - - 2. Set the ``telemetry_secret`` value. - - Set the ``telemetry_secret`` value to a large, random, value. Use - the same value in all ceilometer configuration files, on all - nodes, so that messages passing between the nodes can be - validated. This value can be left empty to disable message signing. - - .. note:: - - Disabling signing will improve message handling performance - - Refer to :doc:`/configuration` for details about any other options - you might want to modify before starting the service. - -6. Edit ``/etc/ceilometer/ceilometer.conf``: - - Change publisher endpoints to expected targets. By default, it pushes to a - `metering.sample` topic on the oslo.messaging queue. Available publishers - are listed in :ref:`pipeline-publishers` section. - -5. Start the notification daemon:: - - $ ceilometer-agent-notification - - .. note:: - - The default development configuration of the notification logs to - stderr, so you may want to run this step using a screen session - or other tool for maintaining a long-running program in the - background. - - -Installing the Polling Agent -============================ - -.. index:: - double: installing; agent - -.. note:: - - The polling agent needs to be able to talk to Keystone and any of - the services being polled for updates. It also needs to run on your compute - nodes to poll instances. - -1. Clone the ceilometer git repository to the server:: - - $ cd /opt/stack - $ git clone https://git.openstack.org/openstack/ceilometer.git - -2. As a user with ``root`` permissions or ``sudo`` privileges, run the - ceilometer installer:: - - $ cd ceilometer - $ sudo python setup.py install - -3. Generate configuration file:: - - $ tox -egenconfig - -4. Copy the sample configuration files from the source tree - to their final location:: - - $ mkdir -p /etc/ceilometer - $ cp etc/ceilometer/ceilometer.conf /etc/ceilometer/ceilometer.conf - $ cp ceilometer/pipeline/data/*.yaml /etc/ceilometer - -5. Configure messaging by editing ``/etc/ceilometer/ceilometer.conf``:: - - [oslo_messaging_rabbit] - rabbit_userid = stackrabbit - rabbit_password = openstack1 - rabbit_hosts = 10.0.2.15 - -6. In order to retrieve object store statistics, ceilometer needs - access to swift with ``ResellerAdmin`` role. You should give this - role to your ``os_username`` user for tenant ``os_tenant_name``:: - - $ openstack role create ResellerAdmin - +-----------+----------------------------------+ - | Field | Value | - +-----------+----------------------------------+ - | domain_id | None | - | id | f5153dae801244e8bb4948f0a6fb73b7 | - | name | ResellerAdmin | - +-----------+----------------------------------+ - - $ openstack role add f5153dae801244e8bb4948f0a6fb73b7 \ - --project $SERVICE_TENANT \ - --user $CEILOMETER_USER - -7. Start the agent:: - - $ ceilometer-polling - -8. By default, the polling agent polls the `compute` and `central` namespaces. - You can specify which namespace to poll in the `ceilometer.conf` - configuration file or on the command line:: - - $ ceilometer-polling --polling-namespaces central,ipmi - - -Installing the API Server -========================= - -.. index:: - double: installing; API - -.. note:: - - The Ceilometer's API service is no longer supported. Data storage should be - handled by a separate service such as Gnocchi. - - -Enabling Service Notifications -============================== - -See the `install guide`_ for instructions on how to enable meters for specific -OpenStack services. - -.. _`install guide`: https://docs.openstack.org/project-install-guide/telemetry/draft/install-controller.html diff -Nru ceilometer-9.0.1/doc/source/contributor/install/upgrade.rst ceilometer-9.0.4/doc/source/contributor/install/upgrade.rst --- ceilometer-9.0.1/doc/source/contributor/install/upgrade.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/install/upgrade.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,114 +0,0 @@ -.. - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _upgrade: - -========== - Upgrading -========== - -Ceilometer's services support both full upgrades as well as partial -(rolling) upgrades. The required steps for each process are described below. - - -Full upgrades -============= - -The following describes how to upgrade your entire Ceilometer environment in -one pass. - -.. _full upgrade path: - -1. Upgrade the database (if applicable) - - Run ceilometer-upgrade to upgrade the storage backend if using one of - Ceilometer's databases (see :ref:`choosing_db_backend`). The database does - not need to be taken offline. Ideally this should be done during a period of - low activity. Best practices should still be followed (ie. back up your - data). If not using a Ceilometer database, you should consult the - documentation of that storage beforehand. - -2. Upgrade the collector service(s) - - Shutdown all collector services. The new collector, that knows how to - interpret the new payload, can then be started. It will disregard any - historical attributes and can continue to process older data from the - agents. You may restart as many new collectors as required. - -3. Upgrade the notification agent(s) - - The notification agent can then be taken offline and upgraded with the - same conditions as the collector service. - -4. Upgrade the polling agent(s) - - In this path, you'll want to take down agents on all hosts before starting. - After starting the first agent, you should verify that data is again being - polled. Additional agents can be added to support coordination if enabled. - -.. note:: - - The API service can be taken offline and upgraded at any point in the - process (if applicable). - - -Partial upgrades -================ - -The following describes how to upgrade parts of your Ceilometer environment -gradually. The ultimate goal is to have all services upgraded to the new -version in time. - -1. Upgrade the database (if applicable) - - Upgrading the database here is the same as the `full upgrade path`_. - -2. Upgrade the collector service(s) - - The new collector services can be started alongside the old collectors. - Collectors old and new will disregard any new or historical attributes. - -3. Upgrade the notification agent(s) - - The new notification agent can be started alongside the old agent if no - workload_partioning is enabled OR if it has the same pipeline configuration. - If the pipeline configuration is changed, the old agents must be loaded with - the same pipeline configuration first to ensure the notification agents all - work against same pipeline sets. - -4. Upgrade the polling agent(s) - - The new polling agent can be started alongside the old agent only if no new - pollsters were added. If not, new polling agents must start only in its - own partitioning group and poll only the new pollsters. After all old agents - are upgraded, the polling agents can be changed to poll both new pollsters - AND the old ones. - -5. Upgrade the API service(s) - - API management is handled by WSGI so there is only ever one version of API - service running - -.. note:: - - Upgrade ordering does not matter in partial upgrade path. The only - requirement is that the database be upgraded first. It is advisable to - upgrade following the same ordering as currently described: database, - collector, notification agent, polling agent, api. - - -Developer notes -=============== - -When updating data models in the database or IPC, we need to adhere to a single -mantra: 'always add, never delete or modify.' diff -Nru ceilometer-9.0.1/doc/source/contributor/measurements.rst ceilometer-9.0.4/doc/source/contributor/measurements.rst --- ceilometer-9.0.1/doc/source/contributor/measurements.rst 2017-09-12 16:29:56.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/measurements.rst 2018-01-09 16:10:50.000000000 +0000 @@ -27,9 +27,82 @@ .. _Measurements page: https://docs.openstack.org/admin-guide/telemetry-measurements.html -Adding new meters -================= +New measurements +================ -If you would like to add new meters please check the -:ref:`add_new_meters` page under in the Contributing +Ceilometer is designed to collect measurements from OpenStack services and +from other external components. If you would like to add new meters to the +currently existing ones, you need to follow the guidelines given in this section. + +.. _meter_types: + +Types +----- + +Three type of meters are defined in Ceilometer: + +.. index:: + double: meter; cumulative + double: meter; gauge + double: meter; delta + +========== ============================================================================== +Type Definition +========== ============================================================================== +Cumulative Increasing over time (instance hours) +Gauge Discrete items (floating IPs, image uploads) and fluctuating values (disk I/O) +Delta Changing over time (bandwidth) +========== ============================================================================== + +When you're about to add a new meter choose one type from the above list, which +is applicable. + +Units +----- + +1. Whenever a volume is to be measured, SI approved units and their + approved symbols or abbreviations should be used. Information units + should be expressed in bits ('b') or bytes ('B'). +2. For a given meter, the units should NEVER, EVER be changed. +3. When the measurement does not represent a volume, the unit + description should always describe WHAT is measured (ie: apples, + disk, routers, floating IPs, etc.). +4. When creating a new meter, if another meter exists measuring + something similar, the same units and precision should be used. +5. Meters and samples should always document their units in Ceilometer (API + and Documentation) and new sampling code should not be merged without the + appropriate documentation. + +============ ======== ============== ======================= +Dimension Unit Abbreviations Note +============ ======== ============== ======================= +None N/A Dimension-less variable +Volume byte B +Time seconds s +============ ======== ============== ======================= + +Naming convention +----------------- + +If you plan on adding meters, please follow the convention below: + +1. Always use '.' as separator and go from least to most discriminant word. + For example, do not use ephemeral_disk_size but disk.ephemeral.size + +2. When a part of the name is a variable, it should always be at the end and start with a ':'. + For example, do not use .image but image:, where type is your variable name. + +3. If you have any hesitation, come and ask in #openstack-telemetry + +Meter definitions +----------------- +Meters definitions by default, are stored in separate configuration +file, called :file:`ceilometer/data/meters.d/meters.yaml`. This is essentially +a replacement for prior approach of writing notification handlers to consume +specific topics. + +A detailed description of how to use meter definition is illustrated in +the `admin_guide`_. + +.. _admin_guide: https://docs.openstack.org/ceilometer/latest/admin/telemetry-data-collection.html#meter-definitions diff -Nru ceilometer-9.0.1/doc/source/contributor/new_meters.rst ceilometer-9.0.4/doc/source/contributor/new_meters.rst --- ceilometer-9.0.1/doc/source/contributor/new_meters.rst 2017-09-12 16:29:56.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/new_meters.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,115 +0,0 @@ -.. - Copyright 2012 New Dream Network (DreamHost) - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _add_new_meters: - -================== - New measurements -================== - -Ceilometer is designed to collect measurements from OpenStack services and -from other external components. If you would like to add new meters to the -currently existing ones, you need to follow the guidelines given in this -section. - -.. _meter_types: - -Types -===== - -Three type of meters are defined in Ceilometer: - -.. index:: - double: meter; cumulative - double: meter; gauge - double: meter; delta - -========== ============================================================================== -Type Definition -========== ============================================================================== -Cumulative Increasing over time (instance hours) -Gauge Discrete items (floating IPs, image uploads) and fluctuating values (disk I/O) -Delta Changing over time (bandwidth) -========== ============================================================================== - -When you're about to add a new meter choose one type from the above list, which -is applicable. - - -Units -===== - -1. Whenever a volume is to be measured, SI approved units and their - approved symbols or abbreviations should be used. Information units - should be expressed in bits ('b') or bytes ('B'). -2. For a given meter, the units should NEVER, EVER be changed. -3. When the measurement does not represent a volume, the unit - description should always describe WHAT is measured (ie: apples, - disk, routers, floating IPs, etc.). -4. When creating a new meter, if another meter exists measuring - something similar, the same units and precision should be used. -5. Meters and samples should always document their units in Ceilometer (API - and Documentation) and new sampling code should not be merged without the - appropriate documentation. - -============ ======== ============== ======================= -Dimension Unit Abbreviations Note -============ ======== ============== ======================= -None N/A Dimension-less variable -Volume byte B -Time seconds s -============ ======== ============== ======================= - - -Meters -====== - -Naming convention ------------------ - -If you plan on adding meters, please follow the convention below: - -1. Always use '.' as separator and go from least to most discriminant word. - For example, do not use ephemeral_disk_size but disk.ephemeral.size - -2. When a part of the name is a variable, it should always be at the end and start with a ':'. - For example, do not use .image but image:, where type is your variable name. - -3. If you have any hesitation, come and ask in #openstack-telemetry - -Meter definitions ------------------ -Meters definitions by default, are stored in separate configuration -file, called :file:`ceilometer/data/meters.d/meters.yaml`. This is essentially -a replacement for prior approach of writing notification handlers to consume -specific topics. - -A detailed description of how to use meter definition is illustrated in -the `admin_guide`_. - -.. _admin_guide: https://docs.openstack.org/admin-guide/telemetry-data-collection.html#meter-definitions - -Non-metric meters and events ----------------------------- - -Ceilometer supports collecting notifications as events. It is highly -recommended to use events for capturing if something happened in the system -or not as opposed to defining meters of which volume will be constantly '1'. -Events enable better representation and querying of metadata rather than -statistical aggregations required for Samples. When the event support is -turned on for Ceilometer, event type meters are collected into the event -database too, which can lead to the duplication of a huge amount of data. - -In order to learn more about events see the :ref:`events` section. diff -Nru ceilometer-9.0.1/doc/source/contributor/plugins.rst ceilometer-9.0.4/doc/source/contributor/plugins.rst --- ceilometer-9.0.1/doc/source/contributor/plugins.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/plugins.rst 2018-01-09 16:10:50.000000000 +0000 @@ -63,7 +63,7 @@ which is defined in the ``ceilometer/telemetry/notifications`` folder, Though in most cases, this is not needed. A meter definition can be directly added to :file:`ceilometer/data/meters.d/meters.yaml` to match the event type. For -more information, see the :ref:`add_new_meters` page. +more information, see the :ref:`meter_definitions` page. We are using these two existing plugins as examples as the first one provides an example of how to interact when you need to retrieve information from an @@ -126,13 +126,13 @@ :class:`ceilometer.agent.plugin_base.NotificationBase` meta class. Notifications must implement: - ``event_types`` - A sequence of strings defining the event types to be given to the plugin +``event_types`` + A sequence of strings defining the event types to be given to the plugin - ``process_notification(self, message)`` - Receives an event message from the list provided to ``event_types`` and - returns a sequence of ``Sample`` objects as defined in the - :file:`ceilometer/sample.py` file. +``process_notification(self, message)`` + Receives an event message from the list provided to ``event_types`` and + returns a sequence of ``Sample`` objects as defined in the + :file:`ceilometer/sample.py` file. In the ``InstanceNotifications`` plugin, it listens to three events: diff -Nru ceilometer-9.0.1/doc/source/contributor/testing.rst ceilometer-9.0.4/doc/source/contributor/testing.rst --- ceilometer-9.0.1/doc/source/contributor/testing.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/contributor/testing.rst 2018-01-09 16:10:41.000000000 +0000 @@ -24,59 +24,28 @@ $ sudo pip install tox -2. On Ubuntu install ``mongodb`` and ``libmysqlclient-dev`` packages:: - - $ sudo apt-get install mongodb - $ sudo apt-get install libmysqlclient-dev - - For Fedora20 there is no ``libmysqlclient-dev`` package, so you’ll need - to install ``mariadb-devel.x86-64`` (or ``mariadb-devel.i386``) instead:: - - $ sudo yum install mongodb - $ sudo yum install mariadb-devel.x86_64 - -3. Install the test dependencies:: - - $ sudo pip install -r /opt/stack/ceilometer/test-requirements.txt - -4. Run the unit and code-style tests:: +2. Run the unit and code-style tests:: $ cd /opt/stack/ceilometer $ tox -e py27,pep8 - As tox is a wrapper around testr, it also accepts the same flags as testr. - See the `testr documentation`_ for details about these additional flags. +As tox is a wrapper around testr, it also accepts the same flags as testr. +See the `testr documentation`_ for details about these additional flags. .. _testr documentation: https://testrepository.readthedocs.org/en/latest/MANUAL.html - Use a double hyphen to pass options to testr. For example, to run only tests under tests/api/v2:: - - $ tox -e py27 -- api.v2 - - To debug tests (ie. break into pdb debugger), you can use ''debug'' tox - environment. Here's an example, passing the name of a test since you'll - normally only want to run the test that hits your breakpoint:: - - $ tox -e debug ceilometer.tests.test_bin - - For reference, the ``debug`` tox environment implements the instructions - here: https://wiki.openstack.org/wiki/Testr#Debugging_.28pdb.29_Tests - -5. There is a growing suite of tests which use a tool called `gabbi`_ to - test and validate the behavior of the Ceilometer API. These tests are run - when using the usual ``py27`` tox target but if desired they can be run by - themselves:: - - $ tox -e gabbi +Use a double hyphen to pass options to testr. For example, to run only tests +under tests/unit/image:: - The YAML files used to drive the gabbi tests can be found in - ``ceilometer/tests/gabbi/gabbits``. If you are adding to or adjusting the - API you should consider adding tests here. + $ tox -e py27 -- image -.. _gabbi: https://gabbi.readthedocs.org/ +To debug tests (ie. break into pdb debugger), you can use ''debug'' tox +environment. Here's an example, passing the name of a test since you'll +normally only want to run the test that hits your breakpoint:: -.. seealso:: + $ tox -e debug ceilometer.tests.unit.test_bin - * tox_ +For reference, the ``debug`` tox environment implements the instructions +here: https://wiki.openstack.org/wiki/Testr#Debugging_.28pdb.29_Tests -.. _tox: http://tox.testrun.org/latest/ +.. _tox: https://tox.readthedocs.io/en/latest/ diff -Nru ceilometer-9.0.1/doc/source/install/index.rst ceilometer-9.0.4/doc/source/install/index.rst --- ceilometer-9.0.1/doc/source/install/index.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/install/index.rst 2018-01-09 16:10:50.000000000 +0000 @@ -17,7 +17,7 @@ Ocata ~~~~~ -To install Ceilometer, see the Ocata Telmetry Data Collection service +To install Ceilometer, see the Ocata Telemetry Data Collection service `install guide `__ Newton diff -Nru ceilometer-9.0.1/doc/source/install/install-base-config-common.inc ceilometer-9.0.4/doc/source/install/install-base-config-common.inc --- ceilometer-9.0.1/doc/source/install/install-base-config-common.inc 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/install/install-base-config-common.inc 2018-01-09 16:10:50.000000000 +0000 @@ -42,36 +42,7 @@ Replace ``CEILOMETER_PASS`` with the password you chose for the ``ceilometer`` user in the Identity service. -3. Edit the ``/etc/gnocchi/gnocchi.conf`` file and add Keystone options: - - * In the ``[api]`` section, configure gnocchi to use keystone: - - .. code-block:: ini - - [api] - auth_mode = keystone - - * In the ``[keystone_authtoken]`` section, configure keystone - authentication: - - .. code-block:: ini - - [keystone_authtoken] - ... - auth_type = password - auth_url = http://controller:5000/v3 - project_domain_name = Default - user_domain_name = Default - project_name = service - username = gnocchi - password = GNOCCHI_PASS - interface = internalURL - region_name = RegionOne - - Replace ``GNOCCHI_PASS`` with the password you chose for - the ``gnocchi`` user in the Identity service. - -4. Create Ceilometer resources in Gnocchi. Gnocchi should be running by this +3. Create Ceilometer resources in Gnocchi. Gnocchi should be running by this stage: .. code-block:: console diff -Nru ceilometer-9.0.1/doc/source/install/install-base-obs.rst ceilometer-9.0.4/doc/source/install/install-base-obs.rst --- ceilometer-9.0.1/doc/source/install/install-base-obs.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/install/install-base-obs.rst 2018-01-09 16:10:41.000000000 +0000 @@ -11,20 +11,51 @@ Before you install and configure the Telemetry service, you must configure a target to send metering data to. The recommended endpoint -is Gnocchi_. To enable Gnocchi, please see its install guide. +is Gnocchi_. .. _Gnocchi: http://gnocchi.xyz .. include:: install-base-prereq-common.inc +Install Gnocchi +--------------- + +#. Install the Gnocchi packages. Alternatively, Gnocchi can be install using + pip: + + .. code-block:: console + + # zypper install openstack-gnocchi-api openstack-gnocchi-metricd \ + python-gnocchiclient + + .. note:: + + Depending on your environment size, consider installing Gnocchi + separately as it makes extensive use of the cpu. + +.. include:: install-gnocchi.inc + +Finalize Gnocchi installation +----------------------------- + +#. Start the Gnocchi services and configure them to start when the + system boots: + + .. code-block:: console + + # systemctl enable openstack-gnocchi-api.service \ + openstack-gnocchi-metricd.service + # systemctl start openstack-gnocchi-api.service \ + openstack-gnocchi-metricd.service + Install and configure components -------------------------------- -1. Install the packages: +#. Install the packages: .. code-block:: console # zypper install openstack-ceilometer-agent-notification \ - openstack-ceilometer-agent-central python-ceilometerclient + openstack-ceilometer-agent-central .. include:: install-base-config-common.inc diff -Nru ceilometer-9.0.1/doc/source/install/install-base-prereq-common.inc ceilometer-9.0.4/doc/source/install/install-base-prereq-common.inc --- ceilometer-9.0.1/doc/source/install/install-base-prereq-common.inc 2017-09-12 16:29:56.000000000 +0000 +++ ceilometer-9.0.4/doc/source/install/install-base-prereq-common.inc 2018-01-09 16:10:50.000000000 +0000 @@ -83,6 +83,16 @@ | type | metric | +-------------+----------------------------------+ + * Add the ``admin`` role to the ``gnocchi`` user. + + .. code-block:: console + + $ openstack role add --project service --user gnocchi admin + + .. note:: + + This command provides no output. + * Create the Metric service API endpoints: .. code-block:: console diff -Nru ceilometer-9.0.1/doc/source/install/install-base-rdo.rst ceilometer-9.0.4/doc/source/install/install-base-rdo.rst --- ceilometer-9.0.1/doc/source/install/install-base-rdo.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/install/install-base-rdo.rst 2018-01-09 16:10:41.000000000 +0000 @@ -11,20 +11,51 @@ Before you install and configure the Telemetry service, you must configure a target to send metering data to. The recommended endpoint -is Gnocchi_. To enable Gnocchi, please see its install guide. +is Gnocchi_. .. _Gnocchi: http://gnocchi.xyz .. include:: install-base-prereq-common.inc +Install Gnocchi +--------------- + +#. Install the Gnocchi packages. Alternatively, Gnocchi can be install using + pip: + + .. code-block:: console + + # yum install openstack-gnocchi-api openstack-gnocchi-metricd \ + python-gnocchiclient + + .. note:: + + Depending on your environment size, consider installing Gnocchi + separately as it makes extensive use of the cpu. + +.. include:: install-gnocchi.inc + +Finalize Gnocchi installation +----------------------------- + +#. Start the Gnocchi services and configure them to start when the + system boots: + + .. code-block:: console + + # systemctl enable openstack-gnocchi-api.service \ + openstack-gnocchi-metricd.service + # systemctl start openstack-gnocchi-api.service \ + openstack-gnocchi-metricd.service + Install and configure components -------------------------------- -1. Install the packages: +#. Install the Ceilometer packages: .. code-block:: console # yum install openstack-ceilometer-notification \ - openstack-ceilometer-central python-ceilometerclient + openstack-ceilometer-central .. include:: install-base-config-common.inc diff -Nru ceilometer-9.0.1/doc/source/install/install-base-ubuntu.rst ceilometer-9.0.4/doc/source/install/install-base-ubuntu.rst --- ceilometer-9.0.1/doc/source/install/install-base-ubuntu.rst 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/doc/source/install/install-base-ubuntu.rst 2018-01-09 16:10:41.000000000 +0000 @@ -11,20 +11,47 @@ Before you install and configure the Telemetry service, you must configure a target to send metering data to. The recommended endpoint -is Gnocchi_. To enable Gnocchi, please see its install guide. +is Gnocchi_. .. _Gnocchi: http://gnocchi.xyz .. include:: install-base-prereq-common.inc +Install Gnocchi +--------------- + +#. Install the Gnocchi packages. Alternatively, Gnocchi can be install using + pip: + + .. code-block:: console + + # apt-get install gnocchi-api gnocchi-metricd python-gnocchiclient + + .. note:: + + Depending on your environment size, consider installing Gnocchi + separately as it makes extensive use of the cpu. + +.. include:: install-gnocchi.inc + +Finalize Gnocchi installation +----------------------------- + +#. Restart the Gnocchi services: + + .. code-block:: console + + # service gnocchi-api restart + # service gnocchi-metricd restart + Install and configure components -------------------------------- -#. Install the packages: +#. Install the ceilometer packages: .. code-block:: console # apt-get install ceilometer-agent-notification \ - ceilometer-agent-central python-ceilometerclient + ceilometer-agent-central .. include:: install-base-config-common.inc diff -Nru ceilometer-9.0.1/doc/source/install/install-gnocchi.inc ceilometer-9.0.4/doc/source/install/install-gnocchi.inc --- ceilometer-9.0.1/doc/source/install/install-gnocchi.inc 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/doc/source/install/install-gnocchi.inc 2018-01-09 16:10:41.000000000 +0000 @@ -0,0 +1,83 @@ +2. Create the database for Gnocchi's indexer: + + * Use the database access client to connect to the database + server as the ``root`` user: + + .. code-block:: console + + $ mysql -u root -p + + * Create the ``gnocchi`` database: + + .. code-block:: console + + CREATE DATABASE gnocchi; + + * Grant proper access to the ``gnocchi`` database: + + .. code-block:: console + + GRANT ALL PRIVILEGES ON gnocchi.* TO 'gnocchi'@'localhost' \ + IDENTIFIED BY 'GNOCCHI_DBPASS'; + GRANT ALL PRIVILEGES ON gnocchi.* TO 'gnocchi'@'%' \ + IDENTIFIED BY 'GNOCCHI_DBPASS'; + + Replace ``GNOCCHI_DBPASS`` with a suitable password. + + * Exit the database access client. + +3. Edit the ``/etc/gnocchi/gnocchi.conf`` file and add Keystone options: + + * In the ``[api]`` section, configure gnocchi to use keystone: + + .. code-block:: ini + + [api] + auth_mode = keystone + + * In the ``[keystone_authtoken]`` section, configure keystone + authentication: + + .. code-block:: ini + + [keystone_authtoken] + ... + auth_type = password + auth_url = http://controller:5000/v3 + project_domain_name = Default + user_domain_name = Default + project_name = service + username = gnocchi + password = GNOCCHI_PASS + interface = internalURL + region_name = RegionOne + + Replace ``GNOCCHI_PASS`` with the password you chose for + the ``gnocchi`` user in the Identity service. + + * In the ``[indexer]`` section, configure database access: + + .. code-block:: ini + + [indexer] + url = mysql+pymysql://gnocchi:GNOCCHI_DBPASS@controller/gnocchi + + Replace ``GNOCCHI_DBPASS`` with the password you chose for Gnocchi's + indexer database. + + * In the ``[storage]`` section, configure location to store metric data. + In this case, we will store it to the local file system. See Gnocchi + documenation for a list of more durable and performant drivers: + + .. code-block:: ini + + [storage] + coordination_url = redis://controller:6379 + file_basepath = /var/lib/gnocchi + driver = file + +4. Initialize Gnocchi: + + .. code-block:: console + + gnocchi-upgrade diff -Nru ceilometer-9.0.1/etc/ceilometer/polling.yaml ceilometer-9.0.4/etc/ceilometer/polling.yaml --- ceilometer-9.0.1/etc/ceilometer/polling.yaml 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/etc/ceilometer/polling.yaml 2018-01-09 16:10:41.000000000 +0000 @@ -4,6 +4,7 @@ interval: 300 meters: - cpu + - cpu_l3_cache - memory.usage - network.incoming.bytes - network.incoming.packets diff -Nru ceilometer-9.0.1/PKG-INFO ceilometer-9.0.4/PKG-INFO --- ceilometer-9.0.1/PKG-INFO 2017-09-12 16:32:44.000000000 +0000 +++ ceilometer-9.0.4/PKG-INFO 2018-01-09 16:15:01.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: ceilometer -Version: 9.0.1 +Version: 9.0.4 Summary: OpenStack Telemetry Home-page: https://docs.openstack.org/ceilometer/latest/ Author: OpenStack diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/post.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/post.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/post.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/post.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,80 @@ +- hosts: primary + tasks: + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*nose_results.html + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testr_results.html.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.testrepository/tmp* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testrepository.subunit.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}/tox' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.tox/*/log/* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/logs/** + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/run.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/run.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/run.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/run.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,52 @@ +- hosts: all + name: Autoconverted job legacy-ceilometer-dsvm-tempest-plugin-mongodb from old job + gate-ceilometer-dsvm-tempest-plugin-mongodb-ubuntu-xenial + tasks: + + - name: Ensure legacy workspace directory + file: + path: '{{ ansible_user_dir }}/workspace' + state: directory + + - shell: + cmd: | + set -e + set -x + cat > clonemap.yaml << EOF + clonemap: + - name: openstack-infra/devstack-gate + dest: devstack-gate + EOF + /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ + git://git.openstack.org \ + openstack-infra/devstack-gate + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + set -e + set -x + export PYTHONUNBUFFERED=true + export DEVSTACK_GATE_TEMPEST=1 + export DEVSTACK_GATE_TEMPEST_REGEX="^ceilometer\." + export DEVSTACK_GATE_TEMPEST_ALL_PLUGINS=1 + export DEVSTACK_LOCAL_CONFIG="enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"CEILOMETER_BACKEND=mongodb" + + export BRANCH_OVERRIDE=default + if [ "$BRANCH_OVERRIDE" != "default" ] ; then + export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE + fi + if [ "mongodb" = "postgresql" ] ; then + export DEVSTACK_GATE_POSTGRES=1 + fi + if [ "" == "-identity-v3-only" ] ; then + export DEVSTACK_LOCAL_CONFIG+=$'\n'"ENABLE_IDENTITY_V2=False" + fi + cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh + ./safe-devstack-vm-gate-wrap.sh + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/post.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/post.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/post.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/post.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,80 @@ +- hosts: primary + tasks: + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*nose_results.html + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testr_results.html.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.testrepository/tmp* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testrepository.subunit.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}/tox' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.tox/*/log/* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/logs/** + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/run.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/run.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/run.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/run.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,52 @@ +- hosts: all + name: Autoconverted job legacy-ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only + from old job gate-ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only-ubuntu-xenial + tasks: + + - name: Ensure legacy workspace directory + file: + path: '{{ ansible_user_dir }}/workspace' + state: directory + + - shell: + cmd: | + set -e + set -x + cat > clonemap.yaml << EOF + clonemap: + - name: openstack-infra/devstack-gate + dest: devstack-gate + EOF + /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ + git://git.openstack.org \ + openstack-infra/devstack-gate + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + set -e + set -x + export PYTHONUNBUFFERED=true + export DEVSTACK_GATE_TEMPEST=1 + export DEVSTACK_GATE_TEMPEST_REGEX="^ceilometer\." + export DEVSTACK_GATE_TEMPEST_ALL_PLUGINS=1 + export DEVSTACK_LOCAL_CONFIG="enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"CEILOMETER_BACKEND=mongodb" + + export BRANCH_OVERRIDE=default + if [ "$BRANCH_OVERRIDE" != "default" ] ; then + export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE + fi + if [ "mongodb" = "postgresql" ] ; then + export DEVSTACK_GATE_POSTGRES=1 + fi + if [ "-identity-v3-only" == "-identity-v3-only" ] ; then + export DEVSTACK_LOCAL_CONFIG+=$'\n'"ENABLE_IDENTITY_V2=False" + fi + cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh + ./safe-devstack-vm-gate-wrap.sh + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/post.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/post.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/post.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/post.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,80 @@ +- hosts: primary + tasks: + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*nose_results.html + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testr_results.html.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.testrepository/tmp* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testrepository.subunit.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}/tox' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.tox/*/log/* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/logs/** + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/run.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/run.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/run.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/run.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,53 @@ +- hosts: all + name: Autoconverted job legacy-ceilometer-dsvm-tempest-plugin-mysql from old job + gate-ceilometer-dsvm-tempest-plugin-mysql-ubuntu-xenial + tasks: + + - name: Ensure legacy workspace directory + file: + path: '{{ ansible_user_dir }}/workspace' + state: directory + + - shell: + cmd: | + set -e + set -x + cat > clonemap.yaml << EOF + clonemap: + - name: openstack-infra/devstack-gate + dest: devstack-gate + EOF + /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ + git://git.openstack.org \ + openstack-infra/devstack-gate + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + set -e + set -x + export PYTHONUNBUFFERED=true + export DEVSTACK_GATE_TEMPEST=1 + export DEVSTACK_GATE_TEMPEST_REGEX="^ceilometer\." + export DEVSTACK_GATE_TEMPEST_ALL_PLUGINS=1 + export DEVSTACK_GATE_CEILOMETER_BACKEND=mysql + export DEVSTACK_LOCAL_CONFIG="enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"CEILOMETER_BACKEND=mysql" + + export BRANCH_OVERRIDE=default + if [ "$BRANCH_OVERRIDE" != "default" ] ; then + export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE + fi + if [ "mysql" = "postgresql" ] ; then + export DEVSTACK_GATE_POSTGRES=1 + fi + if [ "" == "-identity-v3-only" ] ; then + export DEVSTACK_LOCAL_CONFIG+=$'\n'"ENABLE_IDENTITY_V2=False" + fi + cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh + ./safe-devstack-vm-gate-wrap.sh + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-mongodb/post.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-mongodb/post.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-mongodb/post.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-mongodb/post.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,67 @@ +- hosts: primary + tasks: + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*nose_results.html + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testr_results.html.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.testrepository/tmp* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testrepository.subunit.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}/tox' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.tox/*/log/* + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-mongodb/run.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-mongodb/run.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-mongodb/run.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-mongodb/run.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,86 @@ +- hosts: all + name: Autoconverted job legacy-ceilometer-tox-py27-mongodb from old job gate-ceilometer-tox-py27-mongodb-ubuntu-xenial + tasks: + + - name: Ensure legacy workspace directory + file: + path: '{{ ansible_user_dir }}/workspace' + state: directory + + - shell: + cmd: | + set -e + set -x + CLONEMAP=`mktemp` + REQS_DIR=`mktemp -d` + function cleanup { + mkdir -p $WORKSPACE + rm -rf $CLONEMAP $REQS_DIR + } + trap cleanup EXIT + cat > $CLONEMAP << EOF + clonemap: + - name: $ZUUL_PROJECT + dest: . + EOF + # zuul cloner works poorly if there are 2 names that are the + # same in here. + if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then + cat >> $CLONEMAP << EOF + - name: openstack/requirements + dest: $REQS_DIR + EOF + fi + /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \ + git://git.openstack.org $ZUUL_PROJECT openstack/requirements + # REQS_DIR is not set for openstack/requirements and there is also + # no need to copy in this case. + if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then + cp $REQS_DIR/upper-constraints.txt ./ + fi + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: /usr/local/jenkins/slave_scripts/install-distro-packages.sh + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + if [ -x tools/test-setup.sh ] ; then + tools/test-setup.sh + fi + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + set -x + sudo rm -f /etc/sudoers.d/zuul + # Prove that general sudo access is actually revoked + ! sudo -n true + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: /usr/local/jenkins/slave_scripts/run-tox.sh py27-mongodb + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + OUT=`git ls-files --other --exclude-standard --directory` + if [ -z "$OUT" ]; then + echo "No extra files created during test." + exit 0 + else + echo "The following un-ignored files were created during the test:" + echo "$OUT" + exit 0 # TODO: change to 1 to fail tests. + fi + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-mysql/post.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-mysql/post.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-mysql/post.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-mysql/post.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,67 @@ +- hosts: primary + tasks: + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*nose_results.html + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testr_results.html.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.testrepository/tmp* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testrepository.subunit.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}/tox' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.tox/*/log/* + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-mysql/run.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-mysql/run.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-mysql/run.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-mysql/run.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,86 @@ +- hosts: all + name: Autoconverted job legacy-ceilometer-tox-py27-mysql from old job gate-ceilometer-tox-py27-mysql-ubuntu-xenial + tasks: + + - name: Ensure legacy workspace directory + file: + path: '{{ ansible_user_dir }}/workspace' + state: directory + + - shell: + cmd: | + set -e + set -x + CLONEMAP=`mktemp` + REQS_DIR=`mktemp -d` + function cleanup { + mkdir -p $WORKSPACE + rm -rf $CLONEMAP $REQS_DIR + } + trap cleanup EXIT + cat > $CLONEMAP << EOF + clonemap: + - name: $ZUUL_PROJECT + dest: . + EOF + # zuul cloner works poorly if there are 2 names that are the + # same in here. + if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then + cat >> $CLONEMAP << EOF + - name: openstack/requirements + dest: $REQS_DIR + EOF + fi + /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \ + git://git.openstack.org $ZUUL_PROJECT openstack/requirements + # REQS_DIR is not set for openstack/requirements and there is also + # no need to copy in this case. + if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then + cp $REQS_DIR/upper-constraints.txt ./ + fi + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: /usr/local/jenkins/slave_scripts/install-distro-packages.sh + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + if [ -x tools/test-setup.sh ] ; then + tools/test-setup.sh + fi + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + set -x + sudo rm -f /etc/sudoers.d/zuul + # Prove that general sudo access is actually revoked + ! sudo -n true + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: /usr/local/jenkins/slave_scripts/run-tox.sh py27-mysql + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + OUT=`git ls-files --other --exclude-standard --directory` + if [ -z "$OUT" ]; then + echo "No extra files created during test." + exit 0 + else + echo "The following un-ignored files were created during the test:" + echo "$OUT" + exit 0 # TODO: change to 1 to fail tests. + fi + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-postgresql/post.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-postgresql/post.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-postgresql/post.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-postgresql/post.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,67 @@ +- hosts: primary + tasks: + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*nose_results.html + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testr_results.html.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.testrepository/tmp* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testrepository.subunit.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}/tox' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.tox/*/log/* + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff -Nru ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-postgresql/run.yaml ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-postgresql/run.yaml --- ceilometer-9.0.1/playbooks/legacy/ceilometer-tox-py27-postgresql/run.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/ceilometer-tox-py27-postgresql/run.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,86 @@ +- hosts: all + name: Autoconverted job legacy-ceilometer-tox-py27-postgresql from old job gate-ceilometer-tox-py27-postgresql-ubuntu-xenial + tasks: + + - name: Ensure legacy workspace directory + file: + path: '{{ ansible_user_dir }}/workspace' + state: directory + + - shell: + cmd: | + set -e + set -x + CLONEMAP=`mktemp` + REQS_DIR=`mktemp -d` + function cleanup { + mkdir -p $WORKSPACE + rm -rf $CLONEMAP $REQS_DIR + } + trap cleanup EXIT + cat > $CLONEMAP << EOF + clonemap: + - name: $ZUUL_PROJECT + dest: . + EOF + # zuul cloner works poorly if there are 2 names that are the + # same in here. + if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then + cat >> $CLONEMAP << EOF + - name: openstack/requirements + dest: $REQS_DIR + EOF + fi + /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \ + git://git.openstack.org $ZUUL_PROJECT openstack/requirements + # REQS_DIR is not set for openstack/requirements and there is also + # no need to copy in this case. + if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then + cp $REQS_DIR/upper-constraints.txt ./ + fi + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: /usr/local/jenkins/slave_scripts/install-distro-packages.sh + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + if [ -x tools/test-setup.sh ] ; then + tools/test-setup.sh + fi + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + set -x + sudo rm -f /etc/sudoers.d/zuul + # Prove that general sudo access is actually revoked + ! sudo -n true + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: /usr/local/jenkins/slave_scripts/run-tox.sh py27-postgresql + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + OUT=`git ls-files --other --exclude-standard --directory` + if [ -z "$OUT" ]; then + echo "No extra files created during test." + exit 0 + else + echo "The following un-ignored files were created during the test:" + echo "$OUT" + exit 0 # TODO: change to 1 to fail tests. + fi + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' diff -Nru ceilometer-9.0.1/playbooks/legacy/grenade-dsvm-ceilometer/post.yaml ceilometer-9.0.4/playbooks/legacy/grenade-dsvm-ceilometer/post.yaml --- ceilometer-9.0.1/playbooks/legacy/grenade-dsvm-ceilometer/post.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/grenade-dsvm-ceilometer/post.yaml 2018-01-09 16:10:41.000000000 +0000 @@ -0,0 +1,15 @@ +- hosts: primary + tasks: + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/logs/** + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff -Nru ceilometer-9.0.1/playbooks/legacy/grenade-dsvm-ceilometer/run.yaml ceilometer-9.0.4/playbooks/legacy/grenade-dsvm-ceilometer/run.yaml --- ceilometer-9.0.1/playbooks/legacy/grenade-dsvm-ceilometer/run.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/grenade-dsvm-ceilometer/run.yaml 2018-01-09 16:10:50.000000000 +0000 @@ -0,0 +1,45 @@ +- hosts: all + name: Autoconverted job legacy-grenade-dsvm-ceilometer from old job gate-grenade-dsvm-ceilometer-ubuntu-xenial + tasks: + + - name: Ensure legacy workspace directory + file: + path: '{{ ansible_user_dir }}/workspace' + state: directory + + - shell: + cmd: | + set -e + set -x + cat > clonemap.yaml << EOF + clonemap: + - name: openstack-infra/devstack-gate + dest: devstack-gate + EOF + /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ + git://git.openstack.org \ + openstack-infra/devstack-gate + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + set -e + set -x + export PROJECTS="openstack-dev/grenade $PROJECTS" + export PYTHONUNBUFFERED=true + export DEVSTACK_GATE_TEMPEST=1 + export DEVSTACK_GATE_GRENADE=pullup + export BRANCH_OVERRIDE=default + if [ "$BRANCH_OVERRIDE" != "default" ] ; then + export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE + fi + export GRENADE_PLUGINRC="enable_grenade_plugin ceilometer https://git.openstack.org/openstack/ceilometer" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"CEILOMETER_BACKEND=mysql" + + cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh + ./safe-devstack-vm-gate-wrap.sh + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' diff -Nru ceilometer-9.0.1/playbooks/legacy/telemetry-dsvm-integration-ceilometer/post.yaml ceilometer-9.0.4/playbooks/legacy/telemetry-dsvm-integration-ceilometer/post.yaml --- ceilometer-9.0.1/playbooks/legacy/telemetry-dsvm-integration-ceilometer/post.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/telemetry-dsvm-integration-ceilometer/post.yaml 2018-01-09 16:10:41.000000000 +0000 @@ -0,0 +1,80 @@ +- hosts: primary + tasks: + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*nose_results.html + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testr_results.html.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.testrepository/tmp* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=**/*testrepository.subunit.gz + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}/tox' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/.tox/*/log/* + - --include=*/ + - --exclude=* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/workspace/ on node + synchronize: + src: '{{ ansible_user_dir }}/workspace/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/logs/** + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff -Nru ceilometer-9.0.1/playbooks/legacy/telemetry-dsvm-integration-ceilometer/run.yaml ceilometer-9.0.4/playbooks/legacy/telemetry-dsvm-integration-ceilometer/run.yaml --- ceilometer-9.0.1/playbooks/legacy/telemetry-dsvm-integration-ceilometer/run.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/playbooks/legacy/telemetry-dsvm-integration-ceilometer/run.yaml 2018-01-09 16:10:41.000000000 +0000 @@ -0,0 +1,56 @@ +- hosts: all + name: Autoconverted job legacy-telemetry-dsvm-integration-ceilometer from old job + gate-telemetry-dsvm-integration-ceilometer-ubuntu-xenial + tasks: + + - name: Ensure legacy workspace directory + file: + path: '{{ ansible_user_dir }}/workspace' + state: directory + + - shell: + cmd: | + set -e + set -x + cat > clonemap.yaml << EOF + clonemap: + - name: openstack-infra/devstack-gate + dest: devstack-gate + EOF + /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ + git://git.openstack.org \ + openstack-infra/devstack-gate + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' + + - shell: + cmd: | + set -e + set -x + export PYTHONUNBUFFERED=true + export DEVSTACK_GATE_HEAT=1 + export DEVSTACK_GATE_NEUTRON=1 + export DEVSTACK_GATE_TEMPEST=1 + export DEVSTACK_GATE_EXERCISES=0 + export DEVSTACK_GATE_INSTALL_TESTONLY=1 + export DEVSTACK_GATE_TEMPEST_NOTESTS=1 + export PROJECTS="openstack/ceilometer openstack/aodh openstack/panko" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin panko git://git.openstack.org/openstack/panko" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin aodh git://git.openstack.org/openstack/aodh" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin heat git://git.openstack.org/openstack/heat" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"CEILOMETER_BACKEND=gnocchi" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"GNOCCHI_ARCHIVE_POLICY=high" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"CEILOMETER_PIPELINE_INTERVAL=15" + export DEVSTACK_PROJECT_FROM_GIT=$ZUUL_SHORT_PROJECT_NAME + function post_test_hook { + cd /opt/stack/new/ceilometer/ceilometer/tests/integration/hooks/ + ./post_test_hook.sh + } + export -f post_test_hook + cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh + ./safe-devstack-vm-gate-wrap.sh + executable: /bin/bash + chdir: '{{ ansible_user_dir }}/workspace' + environment: '{{ zuul | zuul_legacy_vars }}' diff -Nru ceilometer-9.0.1/requirements.txt ceilometer-9.0.4/requirements.txt --- ceilometer-9.0.1/requirements.txt 2017-09-12 16:29:54.000000000 +0000 +++ ceilometer-9.0.4/requirements.txt 2018-01-09 16:10:50.000000000 +0000 @@ -10,7 +10,7 @@ jsonpath-rw-ext>=0.1.9 # Apache-2.0 jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT kafka-python>=1.3.2 # Apache-2.0 -keystonemiddleware!=4.1.0,>=4.0.0 # Apache-2.0 +keystonemiddleware!=4.19.0,!=4.1.0,>=4.0.0 # Apache-2.0 lxml>=2.3 # BSD monotonic msgpack-python>=0.4.0 # Apache-2.0 diff -Nru ceilometer-9.0.1/.zuul.yaml ceilometer-9.0.4/.zuul.yaml --- ceilometer-9.0.1/.zuul.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ceilometer-9.0.4/.zuul.yaml 2018-01-09 16:10:49.000000000 +0000 @@ -0,0 +1,131 @@ +- job: + name: ceilometer-dsvm-tempest-plugin-mongodb + parent: legacy-dsvm-base + run: playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/run.yaml + post-run: playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb/post.yaml + timeout: 7800 + required-projects: + - openstack-infra/devstack-gate + - openstack/ceilometer + - openstack/tempest + +- job: + name: ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only + parent: legacy-dsvm-base + run: playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/run.yaml + post-run: playbooks/legacy/ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only/post.yaml + timeout: 7800 + required-projects: + - openstack-infra/devstack-gate + - openstack/ceilometer + - openstack/tempest + +- job: + name: ceilometer-dsvm-tempest-plugin-mysql + parent: legacy-dsvm-base + run: playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/run.yaml + post-run: playbooks/legacy/ceilometer-dsvm-tempest-plugin-mysql/post.yaml + timeout: 7800 + required-projects: + - openstack-infra/devstack-gate + - openstack/ceilometer + - openstack/tempest + +- job: + name: ceilometer-tox-py27-mongodb + parent: legacy-base + run: playbooks/legacy/ceilometer-tox-py27-mongodb/run.yaml + post-run: playbooks/legacy/ceilometer-tox-py27-mongodb/post.yaml + timeout: 2400 + required-projects: + - openstack/requirements + +- job: + name: ceilometer-tox-py27-mysql + parent: legacy-base + run: playbooks/legacy/ceilometer-tox-py27-mysql/run.yaml + post-run: playbooks/legacy/ceilometer-tox-py27-mysql/post.yaml + timeout: 2400 + required-projects: + - openstack/requirements + +- job: + name: ceilometer-tox-py27-postgresql + parent: legacy-base + run: playbooks/legacy/ceilometer-tox-py27-postgresql/run.yaml + post-run: playbooks/legacy/ceilometer-tox-py27-postgresql/post.yaml + timeout: 2400 + required-projects: + - openstack/requirements + +- job: + name: grenade-dsvm-ceilometer + parent: legacy-dsvm-base + run: playbooks/legacy/grenade-dsvm-ceilometer/run.yaml + post-run: playbooks/legacy/grenade-dsvm-ceilometer/post.yaml + timeout: 10800 + required-projects: + - openstack-dev/grenade + - openstack-infra/devstack-gate + - openstack/ceilometer + +- job: + name: telemetry-dsvm-integration-ceilometer + parent: legacy-dsvm-base + run: playbooks/legacy/telemetry-dsvm-integration-ceilometer/run.yaml + post-run: playbooks/legacy/telemetry-dsvm-integration-ceilometer/post.yaml + timeout: 7800 + required-projects: + - openstack-infra/devstack-gate + - openstack/aodh + - openstack/ceilometer + - openstack/panko + # following are required when DEVSTACK_GATE_HEAT, which this + # job turns on + - openstack/dib-utils + - openstack/diskimage-builder + +- project: + name: openstack/ceilometer + check: + jobs: + - ceilometer-dsvm-tempest-plugin-mongodb + - ceilometer-dsvm-tempest-plugin-mysql + - ceilometer-tox-py27-mongodb + - ceilometer-tox-py27-mysql + - ceilometer-tox-py27-postgresql + - grenade-dsvm-ceilometer: + irrelevant-files: + - ^(test-|)requirements.txt$ + - ^setup.cfg$ + - telemetry-dsvm-integration-ceilometer + - ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only + # TripleO jobs that deploy Telemetry. + # Note we don't use a project-template here, so it's easier + # to disable voting on one specific job if things go wrong. + # tripleo-ci-centos-7-scenario00(1|2)-multinode-oooq will only + # run on stable/pike while the -container will run in Queens + # and beyond. + # If you need any support to debug these jobs in case of + # failures, please reach us on #tripleo IRC channel. + - tripleo-ci-centos-7-scenario001-multinode-oooq: + voting: false + - tripleo-ci-centos-7-scenario001-multinode-oooq-container: + voting: false + - tripleo-ci-centos-7-scenario002-multinode-oooq: + voting: false + - tripleo-ci-centos-7-scenario002-multinode-oooq-container: + voting: false + gate: + jobs: + - ceilometer-dsvm-tempest-plugin-mongodb + - ceilometer-dsvm-tempest-plugin-mysql + - ceilometer-tox-py27-mongodb + - ceilometer-tox-py27-mysql + - ceilometer-tox-py27-postgresql + - grenade-dsvm-ceilometer: + irrelevant-files: + - ^(test-|)requirements.txt$ + - ^setup.cfg$ + - telemetry-dsvm-integration-ceilometer + - ceilometer-dsvm-tempest-plugin-mongodb-identity-v3-only