diff -Nru domtrix-sql-20221014.1/bin/dom-sql-stop domtrix-sql-20221222.1/bin/dom-sql-stop --- domtrix-sql-20221014.1/bin/dom-sql-stop 1970-01-01 00:00:00.000000000 +0000 +++ domtrix-sql-20221222.1/bin/dom-sql-stop 2022-12-22 11:49:46.000000000 +0000 @@ -0,0 +1,39 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Brightbox - Configure Database +# Copyright (C) 2022, Brightbox Systems +# Author: Neil Wilson +# +# dom-sql-stop +begin + require 'domtrix_queue' +rescue LoadError + require File.join(File.dirname(__FILE__), 'relative-env') + retry +end +require 'optparse' + +optparse = OptionParser.new do |opts| + opts.banner = "Usage: #{File.basename $PROGRAM_NAME} vol_queue_name" + opts.separator '' + opts.separator 'Options:' + opts.on('-h', '--help', 'Display this screen') do + abort opts.to_s + end +end + +optparse.parse! + +abort optparse.to_s unless ARGV.length == 1 + +VOLUME_QUEUE_NAME = ARGV[0] +COMMAND = 'stop' +queue = DomtrixQueue.new(VOLUME_QUEUE_NAME) + +queue.execute( + COMMAND, + {}, + "Sending '#{COMMAND}' to #{VOLUME_QUEUE_NAME}" +) +exit diff -Nru domtrix-sql-20221014.1/bin/domtrix-service-worker domtrix-sql-20221222.1/bin/domtrix-service-worker --- domtrix-sql-20221014.1/bin/domtrix-service-worker 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/bin/domtrix-service-worker 2022-12-22 11:49:46.000000000 +0000 @@ -17,21 +17,18 @@ require 'domtrix_worker' require 'domtrix_syslog' +command_hash = { + 'configure' => CloudsqlConfigureCommand, + 'extend' => CloudsqlExtendCommand, + 'stop' => CloudsqlStopCommand +} case QueueConfig.instance['service_type'] when 'mys' - command_hash = { - 'configure' => CloudsqlConfigureCommand, - 'extend' => CloudsqlExtendCommand, - 'snapshot' => MysSnapshotCommand, - 'restore' => MysRestoreCommand - } + command_hash['snapshot'] = MysSnapshotCommand + command_hash['restore'] = MysRestoreCommand when 'pg' - command_hash = { - 'configure' => CloudsqlConfigureCommand, - 'extend' => CloudsqlExtendCommand, - 'snapshot' => PgSnapshotCommand, - 'restore' => PgRestoreCommand - } + command_hash['snapshot'] = PgSnapshotCommand + command_hash['restore'] = PgRestoreCommand else abort 'Unsupported or missing service type' end diff -Nru domtrix-sql-20221014.1/debian/changelog domtrix-sql-20221222.1/debian/changelog --- domtrix-sql-20221014.1/debian/changelog 2022-10-14 14:03:34.000000000 +0000 +++ domtrix-sql-20221222.1/debian/changelog 2022-12-22 11:55:57.000000000 +0000 @@ -1,8 +1,20 @@ -domtrix-sql (20221014.1-1~bpo22.04.1~ppa1) jammy; urgency=medium +domtrix-sql (20221222.1-1~bpo22.04.1~ppa1) jammy; urgency=medium * No-change backport to jammy. - -- Neil Wilson Fri, 14 Oct 2022 14:03:34 +0000 + -- Neil Wilson Thu, 22 Dec 2022 11:55:57 +0000 + +domtrix-sql (20221222.1-1) jammy; urgency=medium + + * New Upstream Version + + -- Neil Wilson Thu, 22 Dec 2022 11:52:21 +0000 + +domtrix-sql (20221213.1-1) jammy; urgency=medium + + * New Upstream Version + + -- Neil Wilson Tue, 13 Dec 2022 15:36:02 +0000 domtrix-sql (20221014.1-1) jammy; urgency=medium diff -Nru domtrix-sql-20221014.1/domtrix.gemspec domtrix-sql-20221222.1/domtrix.gemspec --- domtrix-sql-20221014.1/domtrix.gemspec 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/domtrix.gemspec 2022-12-22 11:49:46.000000000 +0000 @@ -25,5 +25,5 @@ gem.add_dependency('ruby-libvirt', '~> 0.7.0') gem.add_dependency('rubytree', '~> 0.9.7') gem.add_dependency('stomp', '~> 1.4.10') - gem.add_dependency('uricp', '= 0.0.30') + gem.add_dependency('uricp', '= 0.0.36') end diff -Nru domtrix-sql-20221014.1/Gemfile.lock domtrix-sql-20221222.1/Gemfile.lock --- domtrix-sql-20221014.1/Gemfile.lock 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/Gemfile.lock 2022-12-22 11:49:46.000000000 +0000 @@ -9,7 +9,7 @@ ruby-libvirt (~> 0.7.0) rubytree (~> 0.9.7) stomp (~> 1.4.10) - uricp (= 0.0.30) + uricp (= 0.0.36) GEM remote: https://rubygems.org/ @@ -49,7 +49,7 @@ structured_warnings (~> 0.2) stomp (1.4.10) structured_warnings (0.4.0) - uricp (0.0.30) + uricp (0.0.36) childprocess (~> 1.0) filesize (= 0.0.2) methadone (~> 2.0.2) diff -Nru domtrix-sql-20221014.1/lib/domtrix_async/active_clone.rb domtrix-sql-20221222.1/lib/domtrix_async/active_clone.rb --- domtrix-sql-20221014.1/lib/domtrix_async/active_clone.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/lib/domtrix_async/active_clone.rb 2022-12-22 11:49:46.000000000 +0000 @@ -26,7 +26,7 @@ end def shell_command - "exec nice setsid uricp #{force_writes} #{token_details} #{format_details} #{compression_details} #{segment_size_details} #{cache_details} #{@from} #{@to} 2>&1" + "exec nice setsid uricp #{force_writes} #{token_details} #{format_details} #{compression_details} #{segment_size_details} #{cache_details} '#{@from}' '#{@to}' 2>&1" end private diff -Nru domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_configure_command.rb domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_configure_command.rb --- domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_configure_command.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_configure_command.rb 2022-12-22 11:49:46.000000000 +0000 @@ -11,6 +11,7 @@ include RootPrivileges include DomtrixConfig include CommandRunner + include CloudsqlFacts def required_elements_present? true @@ -36,14 +37,6 @@ @data[:upgrade_hour] end - def systemd_area - '/etc/systemd/system' - end - - def runname - 'cloudsql' - end - def database_class case config['service_type'] when 'mys' @@ -55,11 +48,6 @@ end end - def container_unit_file - @container_unit_file ||= - File.join(systemd_area, "container-#{runname}.service") - end - def timer_override_dirs %w[podman-auto-update.timer.d apt-daily-upgrade.timer.d] end @@ -105,6 +93,17 @@ ) end + def ensure_admin_permissions(container_name) + run( + database_class.ensure_admin_permissions( + container_name, + admin_username + ), + "setting permissions for #{admin_username}", + "failed to set permissions for #{admin_username}" + ) + end + def update_timer_overrides hour = upgrade_hour || 6 day = %w[Sun Mon Tue Wed Thu Fri Day][upgrade_weekday.to_i] @@ -184,6 +183,7 @@ return unless @created_container logger.info { "#{self.class.name}: Upgrading database" } execute_upgrade_database(runname) + ensure_admin_permissions(runname) end def data_action diff -Nru domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_extend_command.rb domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_extend_command.rb --- domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_extend_command.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_extend_command.rb 2022-12-22 11:49:46.000000000 +0000 @@ -11,7 +11,7 @@ include CommandRunner def data_action - logger.debug 'Running extend process' + logger.debug { 'Running extend process' } run_extend @state = 'completed' end diff -Nru domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_facts.rb domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_facts.rb --- domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_facts.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_facts.rb 2022-12-22 11:49:46.000000000 +0000 @@ -14,4 +14,20 @@ .find { |line| line =~ /^MemTotal:\W+([0-9]+) kB$/ } && Regexp.last_match[1] end + + def systemd_area + '/etc/systemd/system' + end + + def runname + 'cloudsql' + end + + def service_name + "container-#{runname}.service" + end + + def container_unit_file + File.join(systemd_area, service_name) + end end diff -Nru domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_mysql.rb domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_mysql.rb --- domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_mysql.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_mysql.rb 2022-12-22 11:49:46.000000000 +0000 @@ -79,6 +79,13 @@ '"' end + def ensure_admin_permissions(name, user) + "podman exec #{name} "\ + 'mysql --database=mysql -uroot --comments --execute="'\ + "GRANT SUPER ON *.* TO '#{user}' WITH GRANT OPTION;"\ + '"' + end + def upgrade_database_command(name) "podman exec #{name} "\ 'mysql_upgrade' diff -Nru domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_postgresql.rb domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_postgresql.rb --- domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_postgresql.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_postgresql.rb 2022-12-22 11:49:46.000000000 +0000 @@ -54,6 +54,13 @@ '" postgres postgres' end + def ensure_admin_permissions(name, user) + "podman exec #{name} "\ + 'psql --command="'\ + "ALTER USER #{user} SUPERUSER CREATEDB CREATEROLE;"\ + '" postgres postgres' + end + def checkpoint_command(name) "if podman container exists #{name}; "\ "then podman exec #{name} "\ diff -Nru domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_stop_command.rb domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_stop_command.rb --- domtrix-sql-20221014.1/lib/domtrix_cloudsql/cloudsql_stop_command.rb 1970-01-01 00:00:00.000000000 +0000 +++ domtrix-sql-20221222.1/lib/domtrix_cloudsql/cloudsql_stop_command.rb 2022-12-22 11:49:46.000000000 +0000 @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# Brightbox - Command processor classes +# Copyright (C) 2022, Neil Wilson, Brightbox Systems +# +# CloudSQL Stop Command +class CloudsqlStopCommand < DataCommand + private + + include RootPrivileges + include CommandRunner + + def data_action + if File.exist?(CloudsqlFacts.container_unit_file) + logger.debug { 'Running stop process' } + run_stop + remove_unit_file + else + logger.debug { 'Unit file missing - nothing to stop' } + end + @state = 'completed' + end + + def run_stop + run( + "systemctl disable --now #{CloudsqlFacts.service_name}", + 'Stopped database process', + 'failed to stop database process' + ) + end + + def remove_unit_file + File.unlink(CloudsqlFacts.container_unit_file) + end + + def required_elements_present? + true + end +end diff -Nru domtrix-sql-20221014.1/lib/domtrix_reg/register_command.rb domtrix-sql-20221222.1/lib/domtrix_reg/register_command.rb --- domtrix-sql-20221014.1/lib/domtrix_reg/register_command.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/lib/domtrix_reg/register_command.rb 2022-12-22 11:49:46.000000000 +0000 @@ -12,8 +12,28 @@ private def decorated_action - @register.store_image(action_proc(resource_id)) - @register.metadata_stamp + run_upload(@register) + end + + def run_upload(registration) + registration.store_image(action_proc(registration.resource_id)) + if registration.compressed? + retarget + else + registration.metadata_stamp + end + end + + def retarget + @register.uncompress + new_data = @data.clone + new_data[:incoming] = @register.uncompressed_uri + new_register = Registration.make_register(new_data) + new_register.statistics_frequency = @register.statistics_frequency + run_upload(new_register) + @register = new_register + ensure + new_register.remove_file_source end def required_elements_present? diff -Nru domtrix-sql-20221014.1/lib/domtrix_reg/swift_registration.rb domtrix-sql-20221222.1/lib/domtrix_reg/swift_registration.rb --- domtrix-sql-20221014.1/lib/domtrix_reg/swift_registration.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/lib/domtrix_reg/swift_registration.rb 2022-12-22 11:49:46.000000000 +0000 @@ -225,35 +225,93 @@ end end + # Not really a clean way of testing the files, so we just look for + # known double compressed suffixes on the URI + def compressed? + if incoming_uri.path.end_with?('.qcow2.xz') + logger.info { "#{logname}: Detected compressed qcow2 file" } + return true + end + false + end + + def uncompress + logger.info { "#{logname}: Uncompressing qcow2 source" } + File.rename(cache_file, compressed_source) + system("xz --decompress '#{compressed_source}'") + raise "Failed to decompress #{compressed_source}" unless $?.success? + end + + def uncompressed_uri + "file://#{compressed_source.delete_suffix('.xz')}" + end + + def remove_file_source + return unless incoming_uri && incoming_uri.scheme == 'file' + logger.info { "#{logname}: Removing source file #{incoming_uri.path}" } + File.unlink(incoming_uri.path) + end + private + def curl_command 'curl --fail --silent --show-error' end def incoming_virtual_size - @incoming_virtual_size ||= - if incoming_uri - magic = File.open(incoming_uri.path) do |f| - f.read(4).to_s - end - if qcow2?(magic) - read_qcow2_size - elsif lz4?(magic) - read_lz4_size - else - read_file_size - end - end + @incoming_virtual_size ||= lookup_virtual_size + end + + def cache_file + @cache_file ||= File.join( + target_cache, + 'cache', + File.basename(incoming_uri.path) + ) + end + + def compressed_source + @temp_cache_file ||= File.join( + target_cache, + 'temp', + File.basename(incoming_uri.path) + ) + end + + def lookup_virtual_size + return unless incoming_uri + case incoming_uri.scheme + when 'file' + read_size(incoming_uri.path) + when 'http', 'https' + result = read_size(cache_file) + File.unlink(cache_file) + result + end + end + + def read_size(source) + magic = File.open(source) { |f| f.read(6).to_s } + case + when qcow2?(magic) + read_qcow2_size(source) + when lz4?(magic) + read_lz4_size(source) + when xz?(magic) + read_xz_size(source) + else + read_file_size(source) + end end def qcow2?(magic) magic.unpack('a3C') == ['QFI', 0xfb] end - def read_qcow2_size + def read_qcow2_size(source) logger.info { "#{logname}: Qcow2 file detected" } - info = qemu_image_info(incoming_uri.path) + info = qemu_image_info(source) info && info['virtual size'] end @@ -261,20 +319,33 @@ magic.unpack('V') == [0x184D2204] end - def read_lz4_size + def xz?(magic) + magic.unpack('H12') == %w[fd377a585a00] + end + + def read_lz4_size(source) logger.info { "#{logname}: LZ4 file detected" } - lz4_size =~ /\bdecoded (\d+) bytes\b/ && Regexp.last_match(1) + lz4_size(source) =~ /\bdecoded (\d+) bytes\b/ && Regexp.last_match(1) end # tail the output so we only get the decoded bytes line, not the # progress indicator (#17172) - def lz4_size - `env LC_ALL=C LANG=C lz4 -f -t #{incoming_uri.path} 2>&1` + def lz4_size(source) + `env LC_ALL=C LANG=C lz4 -f -t #{source} 2>&1` end - def read_file_size + def read_xz_size(source) + logger.info { "#{logname}: XZ file detected" } + xz_size(source).split[6] + end + + def xz_size(source) + `env LC_ALL=C LANG=C xz --robot --list #{source} 2>&1` + end + + def read_file_size(source) logger.info { "#{logname}: raw file detected" } - File.size(incoming_uri.path) + File.size(source) end def authentication @@ -321,3 +392,12 @@ s.success? && @curl_error !~ /^curl: \(\d+\)/ end end + +unless String.method_defined?(:delete_suffix) + class String + def delete_suffix(suffix) + return self unless end_with?(suffix) + self[0..-suffix.length - 1] + end + end +end diff -Nru domtrix-sql-20221014.1/lib/domtrix_volumes/domtrix_container_image.rb domtrix-sql-20221222.1/lib/domtrix_volumes/domtrix_container_image.rb --- domtrix-sql-20221014.1/lib/domtrix_volumes/domtrix_container_image.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/lib/domtrix_volumes/domtrix_container_image.rb 2022-12-22 11:49:46.000000000 +0000 @@ -64,7 +64,7 @@ # Observer hook to extract tar file to container root def post_command_action - temp_name = uri.path + '.tar.xz' + temp_name = uri.path + DomtrixContainerIo.extension DomtrixContainerIo.rename(uri.path, temp_name) create DomtrixContainerIo.extract_tar(uri.path, temp_name) diff -Nru domtrix-sql-20221014.1/lib/domtrix_volumes/domtrix_container_io.rb domtrix-sql-20221222.1/lib/domtrix_volumes/domtrix_container_io.rb --- domtrix-sql-20221014.1/lib/domtrix_volumes/domtrix_container_io.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/lib/domtrix_volumes/domtrix_container_io.rb 2022-12-22 11:49:46.000000000 +0000 @@ -18,13 +18,17 @@ FileUtils.mkdir(path, mode: 0o755) end + def self.extension + '.tar' + end + def self.read_size(path) `du -s --block-size=1 #{path} 2>/dev/null`.split.first.to_i end def self.snapshot(source, snapshot_file) ContainerIoUtils.run( - "tar --directory=#{source} --create --file=#{snapshot_file} --sparse --xz .", + "tar --directory=#{source} --create --file=#{snapshot_file} --sparse .", "create tar snapshot of #{source}", "Failed to create snapshot of #{source}" ) diff -Nru domtrix-sql-20221014.1/spec/cloudsql_configure_command_spec.rb domtrix-sql-20221222.1/spec/cloudsql_configure_command_spec.rb --- domtrix-sql-20221014.1/spec/cloudsql_configure_command_spec.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/spec/cloudsql_configure_command_spec.rb 2022-12-22 11:49:46.000000000 +0000 @@ -94,17 +94,47 @@ .and_return(mysql_cloud_config) QueueConfig.instance.load_config Class.new.extend(DomtrixConfig).source = QueueConfig.instance - allow(FileUtils) - .to receive(:mkdir_p) - allow(File) - .to receive(:write) end context('with missing systemd file') do + let(:file_notifier) do + class_double(File).as_stubbed_const + end + + let(:file_utils_notifier) do + class_double(FileUtils).as_stubbed_const + end + before do - allow(File).to receive(:exist?) + allow(file_notifier).to receive(:exist?) .with('/etc/systemd/system/container-cloudsql.service') .and_return(false) + allow(file_notifier).to receive(:join) + .with('/etc/domtrix', 'config.yml') + .and_return(config_yaml_file) + allow(file_notifier).to receive(:expand_path) + .with(config_yaml_file) + .and_return(config_yaml_file) + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'container-cloudsql.service') + .and_return('/etc/systemd/system/container-cloudsql.service') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'apt-daily-upgrade.timer.d') + .and_return('/etc/systemd/system/apt-daily-upgrade.timer.d') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'podman-auto-update.timer.d') + .and_return('/etc/systemd/system/podman-auto-update.timer.d') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'podman-auto-update.timer.d', 'override.conf') + .and_return('/etc/systemd/system/podman-auto-update.timer.d/override.conf') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'apt-daily-upgrade.timer.d', 'override.conf') + .and_return('/etc/systemd/system/apt-daily-upgrade.timer.d/override.conf') + allow(file_notifier).to receive(:foreach) + .with('/proc/meminfo') + .and_return( + ['MemTotal: 992724 kB', 'MemFree: 163576 kB'] + ) end it 'runs_create action properly with admin password' do @@ -116,10 +146,8 @@ ) allow(correct) .to receive(:run) - allow(FileUtils) - .to receive(:mkdir_p) - allow(File) - .to receive(:write) + allow(file_utils_notifier).to receive(:mkdir_p) + allow(file_notifier).to receive(:write) correct.action expect(correct.state).to eq('completed') expect(correct) @@ -149,6 +177,8 @@ ) allow(initial) .to receive(:run) + allow(file_notifier).to receive(:write) + allow(file_utils_notifier).to receive(:mkdir_p) initial.action expect(initial.state).to eq('completed') expect(initial) @@ -181,6 +211,8 @@ ) allow(correct) .to receive(:run) + allow(file_utils_notifier).to receive(:mkdir_p) + allow(file_notifier).to receive(:write) correct.action expect(correct.state).to eq('completed') expect(correct) @@ -191,6 +223,8 @@ it 'does not update user if admin password missing' do allow(initial) .to receive(:run) + allow(file_utils_notifier).to receive(:mkdir_p) + allow(file_notifier).to receive(:write) initial.action expect(initial.state).to eq('completed') expect(initial) @@ -208,17 +242,36 @@ .and_return(postgresql_cloud_config) QueueConfig.instance.load_config Class.new.extend(DomtrixConfig).source = QueueConfig.instance - allow(FileUtils) - .to receive(:mkdir_p) - allow(File) - .to receive(:write) end context('with systemd container file') do + let(:file_notifier) do + class_double(File).as_stubbed_const + end + + let(:file_utils_notifier) do + class_double(FileUtils).as_stubbed_const + end + before do - allow(File).to receive(:exist?) + allow(file_notifier).to receive(:exist?) .with('/etc/systemd/system/container-cloudsql.service') .and_return(true) + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'container-cloudsql.service') + .and_return('/etc/systemd/system/container-cloudsql.service') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'apt-daily-upgrade.timer.d') + .and_return('/etc/systemd/system/apt-daily-upgrade.timer.d') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'podman-auto-update.timer.d') + .and_return('/etc/systemd/system/podman-auto-update.timer.d') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'podman-auto-update.timer.d', 'override.conf') + .and_return('/etc/systemd/system/podman-auto-update.timer.d/override.conf') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'apt-daily-upgrade.timer.d', 'override.conf') + .and_return('/etc/systemd/system/apt-daily-upgrade.timer.d/override.conf') end it 'does not update auto-timer when upgrade settings missing' do @@ -226,8 +279,6 @@ .to receive(:run) initial.action expect(initial.state).to eq('completed') - expect(FileUtils).not_to have_received(:mkdir_p) - expect(File).not_to have_received(:write) expect(initial) .to have_received(:run) .with( @@ -240,13 +291,15 @@ it 'does update auto-timer when upgrade settings are supplied' do allow(correct) .to receive(:run) + allow(file_notifier).to receive(:write) correct.action expect(correct.state).to eq('completed') - expect(FileUtils).not_to have_received(:mkdir_p) - expect(File).to have_received(:write).twice expect(correct) .to have_received(:run) .with('systemctl daemon-reload', anything) + expect(file_notifier) + .to have_received(:write) + .twice expect(correct) .to have_received(:run) .with( @@ -265,6 +318,7 @@ ) allow(correct) .to receive(:run) + allow(file_notifier).to receive(:write) correct.action expect(correct.state).to eq('completed') expect(correct) @@ -284,10 +338,38 @@ end context('with missing systemd file') do + let(:file_notifier) do + class_double(File).as_stubbed_const + end + + let(:file_utils_notifier) do + class_double(FileUtils).as_stubbed_const + end + before do - allow(File).to receive(:exist?) + allow(file_notifier).to receive(:exist?) .with('/etc/systemd/system/container-cloudsql.service') .and_return(false) + allow(file_notifier).to receive(:foreach) + .with('/proc/meminfo') + .and_return( + ['MemTotal: 992724 kB', 'MemFree: 163576 kB'] + ) + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'container-cloudsql.service') + .and_return('/etc/systemd/system/container-cloudsql.service') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'apt-daily-upgrade.timer.d') + .and_return('/etc/systemd/system/apt-daily-upgrade.timer.d') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'podman-auto-update.timer.d') + .and_return('/etc/systemd/system/podman-auto-update.timer.d') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'podman-auto-update.timer.d', 'override.conf') + .and_return('/etc/systemd/system/podman-auto-update.timer.d/override.conf') + allow(file_notifier).to receive(:join) + .with('/etc/systemd/system', 'apt-daily-upgrade.timer.d', 'override.conf') + .and_return('/etc/systemd/system/apt-daily-upgrade.timer.d/override.conf') end it 'runs_create_action properly with admin password' do @@ -299,6 +381,8 @@ ) allow(correct) .to receive(:run) + allow(file_utils_notifier).to receive(:mkdir_p) + allow(file_notifier).to receive(:write) correct.action expect(correct.state).to eq('completed') expect(correct) @@ -307,7 +391,7 @@ expect(correct) .to have_received(:run) .with(/generate systemd.*cloudsql/, anything, anything) - expect(FileUtils).to have_received(:mkdir_p).twice + expect(file_utils_notifier).to have_received(:mkdir_p).twice expect(correct) .to have_received(:run) .with('systemctl daemon-reload', anything) @@ -329,6 +413,8 @@ ) allow(initial) .to receive(:run) + allow(file_utils_notifier).to receive(:mkdir_p) + allow(file_notifier).to receive(:write) initial.action expect(initial.state).to eq('completed') expect(initial) @@ -340,7 +426,7 @@ expect(initial) .not_to have_received(:run) .with(/POSTGRES_PASSWORD=''/, anything, anything) - expect(FileUtils).to have_received(:mkdir_p).twice + expect(file_utils_notifier).to have_received(:mkdir_p).twice expect(initial) .to have_received(:run) .with('systemctl daemon-reload', anything) @@ -362,6 +448,8 @@ ) allow(correct) .to receive(:run) + allow(file_utils_notifier).to receive(:mkdir_p) + allow(file_notifier).to receive(:write) correct.action expect(correct.state).to eq('completed') expect(correct) @@ -372,11 +460,13 @@ it 'does not update user if admin password missing' do allow(initial) .to receive(:run) + allow(file_utils_notifier).to receive(:mkdir_p) + allow(file_notifier).to receive(:write) initial.action expect(initial.state).to eq('completed') expect(initial) .not_to have_received(:run) - .with(/ALTER USER/, anything, anything) + .with(/ALTER USER admin PASSWORD/, anything, anything) end end end diff -Nru domtrix-sql-20221014.1/spec/cloudsql_stop_command_spec.rb domtrix-sql-20221222.1/spec/cloudsql_stop_command_spec.rb --- domtrix-sql-20221014.1/spec/cloudsql_stop_command_spec.rb 1970-01-01 00:00:00.000000000 +0000 +++ domtrix-sql-20221222.1/spec/cloudsql_stop_command_spec.rb 2022-12-22 11:49:46.000000000 +0000 @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'domtrix_cloudsql' + +describe CloudsqlStopCommand do + command_string = "systemctl disable --now container-cloudsql.service" + container_file = '/etc/systemd/system/container-cloudsql.service' + let(:name) { 'dummy' } + let(:dummy) do + described_class.new(name) + end + let(:sub) do + result = dummy + allow(result).to receive(:correct_privileges?).and_return(true) + result + end + + let(:correct) do + result = described_class.new(name, {}) + allow(result).to receive(:correct_privileges?).and_return(true) + result + end + + it 'has the correct root privileges' do + expect(dummy.correct_privileges?).to eq(Process.euid == 0) + end + + it 'runs_data_action properly' do + file_double = class_double(File) + .as_stubbed_const(transfer_nested_constants: true) + allow(correct).to receive(:run).and_return(`exit 0`) + allow(file_double).to receive(:join) + .and_return(container_file) + allow(file_double).to receive(:unlink) + allow(file_double).to receive(:exist?) + .with(container_file) + .and_return(true) + correct.action + expect(correct.state).to eq('completed') + expect(correct) + .to have_received(:run) + .with( + command_string, + anything, + anything + ) + expect(file_double).to have_received(:unlink) + .with(container_file) + end + + it 'handles missing unit file correctly' do + file_double = class_double(File) + .as_stubbed_const(transfer_nested_constants: true) + allow(file_double).to receive(:join) + .and_return(container_file) + allow(file_double).to receive(:exist?) + .with(container_file) + .and_return(false) + correct.action + expect(correct.state).to eq('completed') + end +end diff -Nru domtrix-sql-20221014.1/spec/domtrix_luks_spec.rb domtrix-sql-20221222.1/spec/domtrix_luks_spec.rb --- domtrix-sql-20221014.1/spec/domtrix_luks_spec.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/spec/domtrix_luks_spec.rb 2022-12-22 11:49:46.000000000 +0000 @@ -119,7 +119,7 @@ .and_return(dangling_secret) expect(described_class).to receive(:cmd).and_return(secret_password) expect(secret).to receive(:value=).with(secret_password) - expect(dangling_secret) + allow(dangling_secret) .to receive(:uuid) .and_return(uuid2) expect(dangling_secret) @@ -137,7 +137,7 @@ end it 'removes secret if provided' do - expect(secret) + allow(secret) .to receive(:uuid) .and_return('044bc461-7bfc-425b-87c8-ae5849a8eb38') expect(secret) diff -Nru domtrix-sql-20221014.1/spec/register_command_spec.rb domtrix-sql-20221222.1/spec/register_command_spec.rb --- domtrix-sql-20221014.1/spec/register_command_spec.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/spec/register_command_spec.rb 2022-12-22 11:49:46.000000000 +0000 @@ -8,7 +8,11 @@ end let(:swift_registration) do - instance_double(SwiftRegistration) + instance_double(SwiftRegistration, "Compressed Registration") + end + + let(:swift_registration2) do + instance_double(SwiftRegistration, "Uncompressed Registration") end it 'fails with missing data' do @@ -26,6 +30,33 @@ expect(swift_registration).to receive(:statistics).and_return('statistics') expect(swift_registration).to receive(:metadata_stamp) expect(swift_registration).to receive(:store_image).with(instance_of(Proc)) + expect(swift_registration).to receive(:compressed?).and_return(false) + reg = described_class.new('dummy', data) + expect { |b| reg.action(&b) }.to yield_with_args('resource', 'statistics', nil) + expect(reg.state).to eq('completed') + end + + it 'registers double_decompress image and send statistics with correct data' do + data = { auth_token: 'fish', image: 'http://example.com/somewhere', incoming: 'http://example.com/here/fred.qcow.xz' } + data2 = data.clone + data2[:incoming] = data[:incoming].delete_suffix('.xz') + expect(Registration).to receive(:make_register).with(data).and_return(swift_registration) + expect(swift_registration).to receive(:statistics_frequency=).with(described_class::DEFAULT_FREQUENCY) + expect(swift_registration).to receive(:register_required_elements_present?).and_return(true) + expect(swift_registration).to receive(:resource_id).and_return('resource') + expect(swift_registration).to receive(:store_image).with(instance_of(Proc)) + expect(swift_registration).to receive(:compressed?).and_return(true) + expect(swift_registration).to receive(:uncompress) + expect(swift_registration).to receive(:uncompressed_uri).and_return(data2[:incoming]) + expect(swift_registration).to receive(:statistics_frequency).and_return(described_class::DEFAULT_FREQUENCY) + expect(Registration).to receive(:make_register).with(data2).and_return(swift_registration2) + expect(swift_registration2).to receive(:statistics_frequency=).with(described_class::DEFAULT_FREQUENCY) + expect(swift_registration2).to receive(:resource_id).twice.and_return('resource') + expect(swift_registration2).to receive(:statistics).and_return('statistics') + expect(swift_registration2).to receive(:metadata_stamp) + expect(swift_registration2).to receive(:store_image).with(instance_of(Proc)) + expect(swift_registration2).to receive(:compressed?).and_return(false) + expect(swift_registration2).to receive(:remove_file_source) reg = described_class.new('dummy', data) expect { |b| reg.action(&b) }.to yield_with_args('resource', 'statistics', nil) expect(reg.state).to eq('completed') diff -Nru domtrix-sql-20221014.1/spec/swift_registration_spec.rb domtrix-sql-20221222.1/spec/swift_registration_spec.rb --- domtrix-sql-20221014.1/spec/swift_registration_spec.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/spec/swift_registration_spec.rb 2022-12-22 11:49:46.000000000 +0000 @@ -26,7 +26,7 @@ let(:config) do config = { 'upload_segment_size' => 10_000, - 'snapshot_cache_dir' => '/opt/cache' + 'snapshot_cache_dir' => '/tmp/cache' } config.default_proc = proc do |_hash, key| raise("Key #{key.inspect} is not valid") @@ -46,6 +46,12 @@ StringIO.new(temp, 'r') end + let(:xzmagic) do + temp = String.new + StringIO.open(temp, 'w') { |s| s.write(%w[fd377a585a00].pack('H12')) } + StringIO.new(temp, 'r') + end + let(:orbit_host) do 'test-orbit.brightbox.com' end @@ -66,6 +72,15 @@ { auth_token: '12345', image: "http://#{orbit_host}/v1/#{account}/images/#{image}" } end + let(:fake_source) do + { incoming: "http://#{orbit_host}/v1/#{account}/downloads/freedos.qcow2" } + end + + let(:temp_source) do + { incoming: 'https://orbit.gb1s.brightbox.com/v1/acc-yw7i0/downloads/freedos.qcow2?temp_url_sig=c285f67776b824bf01880da9d142dc9b42ca9ec4cc77d2926fdb024bda754d5b&temp_url_expires=1666347924' } + end + + let(:segment_range) do (0..3) end @@ -84,6 +99,12 @@ temp end + let(:double_compressed_data) do + temp = data.dup + temp[:incoming] = 'http://example.com/somewhere/target_image.qcow2.xz' + temp + end + let(:config_instance) do check = described_class.new(data) check.source = config @@ -96,6 +117,12 @@ check end + let(:double_compressed_instance) do + check = described_class.new(double_compressed_data) + check.source = config + check + end + let(:command_failure) do instance_double(Process::Status, success?: false) end @@ -131,6 +158,13 @@ it 'has required elements with registration data' do expect(register_instance).to be_register_required_elements_present + expect(register_instance).not_to be_compressed + end + + it 'detects double compressed data properly' do + expect(double_compressed_instance).to be_compressed + expect(double_compressed_instance.uncompressed_uri.start_with?('file:///')).to be_truthy + expect(double_compressed_instance.uncompressed_uri.end_with?('.qcow2')).to be_truthy end it 'raises error with bad config data' do @@ -322,7 +356,7 @@ allow(clone_image).to receive(:launch).and_return(true) allow(ActiveClone).to receive(:new).and_return(clone_image) register_instance.store_image(stats_proc) - expect(clone_image.shell_command).to match("exec nice setsid uricp --auth-token '#{data[:auth_token]}' --target-format 'raw' --compress --segment-size '#{config['upload_segment_size']}B' --cache '#{config['snapshot_cache_dir']}' file:/*/tmp/input #{registration_data[:image]} 2>&1") + expect(clone_image.shell_command).to match("exec nice setsid uricp --auth-token '#{data[:auth_token]}' --target-format 'raw' --compress --segment-size '#{config['upload_segment_size']}B' --cache '#{config['snapshot_cache_dir']}' 'file:/*/tmp/input' '#{registration_data[:image]}' 2>&1") end it 'generates correct size statistics' do @@ -380,6 +414,19 @@ meta_instance.metadata_stamp end + it 'reads xz metadata properly' do + local = data + local[:incoming] = "file:///#{__FILE__}" + local_file_size = File.size(__FILE__) + expect(File).to receive(:open).with("/#{__FILE__}").and_yield(xzmagic) + meta_instance = described_class.new(local) + expect(meta_instance).to receive(:xz_size).and_return("name\tFedora-Cloud-Base-35-1.2.x86_64.raw.xz\nfile\t1\t214\t300688964\t#{local_file_size}\t0.056\tCRC64\t0\ntotals\t1\t214\t300688964\t#{local_file_size}\t0.056\tCRC64\t0\t1\n") + expect(uri_io).to receive(:fetch_headers).with(instance_of(DomtrixUri)).and_return(initial_headers) + expect(uri_io).to receive(:post_headers).with(instance_of(DomtrixUri), stamp_headers(local_file_size)) + # expect(meta_instance).to receive(:run_curl_command).with("curl --fail --silent --show-error -H X-Auth-Token:#{local[:auth_token]} -H 'X-Object-Meta-Virtual-Size:#{local_file_size}' -X POST #{local[:image]}").and_return(true) + meta_instance.metadata_stamp + end + it 'reads lz4 metadata properly' do local = data local[:incoming] = "file:///#{__FILE__}" @@ -421,6 +468,38 @@ meta_instance.metadata_stamp end + context "with cache" do + let(:target_cache) do + config['snapshot_cache_dir'] + end + let(:cache_area) do + [ + File.join(target_cache, 'cache'), + File.join(target_cache, 'temp') + ] + end + + before do + FileUtils.rm_rf(target_cache) + FileUtils.mkdir_p(cache_area) + end + + after do + FileUtils.rm_rf(target_cache) + end + + it 'obtains http metadata from cache file' do + local_file_size = 100_000 + local = data.merge(fake_source) + incoming_cache_file = File.join(target_cache, 'cache', File.basename(local[:incoming])) + system("fallocate -l #{local_file_size} #{incoming_cache_file}") + meta_instance = described_class.new(local) + expect(uri_io).to receive(:fetch_headers).with(instance_of(DomtrixUri)).and_return(initial_headers) + expect(uri_io).to receive(:post_headers).with(instance_of(DomtrixUri), stamp_headers(local_file_size)) + meta_instance.metadata_stamp + end + end + it 'does not try to stamp if metadata stamp data is missing' do expect(uri_io).not_to receive(:post_headers) config_instance.metadata_stamp diff -Nru domtrix-sql-20221014.1/spec/vol_clone_spec.rb domtrix-sql-20221222.1/spec/vol_clone_spec.rb --- domtrix-sql-20221014.1/spec/vol_clone_spec.rb 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/spec/vol_clone_spec.rb 2022-12-22 11:49:46.000000000 +0000 @@ -346,7 +346,7 @@ describe 'with container uris' do container_target = '/var/lib/machines/srv-testy' - container_target_tar = "#{container_target}.tar.xz" + container_target_tar = "#{container_target}.tar" before do DomtrixVolume.source = config @@ -357,9 +357,10 @@ sourceuri: orbit_uri, targeturi: "file://#{container_target}" }) allow(uri_io).to receive(:fetch_headers).with(instance_of(DomtrixUri)).and_return(stamped_headers) + allow(container_io).to receive(:extension).and_return('.tar') expect(container_io).to receive(:create).twice.with(container_target, meta_virtual_size).and_return(true) expect(container_io).to receive(:zap).with(container_target).and_return(true) - expect(container_io).to receive(:zap).with(["#{container_target}.tar.xz"]).and_return(true) + expect(container_io).to receive(:zap).with(["#{container_target}.tar"]).and_return(true) expect(container_io).to receive(:rename).with(container_target, container_target_tar).and_return(true) expect(container_io).to receive(:extract_tar).with(container_target, container_target_tar).and_return(true) container_vol.action @@ -374,9 +375,10 @@ sourceuri: orbit_uri, targeturi: "file://#{container_target}" }) allow(uri_io).to receive(:fetch_headers).with(instance_of(DomtrixUri)).and_return(stamped_headers) + allow(container_io).to receive(:extension).and_return('.tar') expect(container_io).to receive(:create).twice.with(container_target, meta_virtual_size).and_return(true) expect(container_io).to receive(:zap).with(container_target).and_return(true) - expect(container_io).to receive(:zap).with(["#{container_target}.tar.xz"]).and_return(true) + expect(container_io).to receive(:zap).with(["#{container_target}.tar"]).and_return(true) expect(container_io).to receive(:rename).with(container_target, container_target_tar).and_return(true) expect(container_io).to receive(:extract_tar).with(container_target, container_target_tar).and_raise('failed to extract tar file') diff -Nru domtrix-sql-20221014.1/sql/create-thin-partition domtrix-sql-20221222.1/sql/create-thin-partition --- domtrix-sql-20221014.1/sql/create-thin-partition 1970-01-01 00:00:00.000000000 +0000 +++ domtrix-sql-20221222.1/sql/create-thin-partition 2022-12-22 11:49:46.000000000 +0000 @@ -0,0 +1,50 @@ +#!/bin/sh + +set -e + +create_thin_pool() { + # Create thin LVM data partition if missing + thin_vol_name=data + thin_vol_fs_type=xfs + thin_vol_fs_type_options='-m bigtime=1' + thin_pool_name=lvol0 + thin_metadata_size=1G + volgroup_name=cloudsql + thin_vol=/dev/mapper/${volgroup_name}-${thin_pool_name} + target=/dev/${volgroup_name}/${thin_vol_name} + + if [ -b "${target}" ] + then + echo "Data Partition already exists" + else + echo "Generating Thin Data Partition" + echo "options dm_thin_pool no_space_timeout=0" > /etc/modprobe.d/dm_thin_pool.conf + vgcreate -f "${volgroup_name}" /dev/?da2 + lvcreate --thin \ + --errorwhenfull n \ + --extents 100%FREE \ + --thinpool "${thin_pool_name}" \ + --poolmetadatasize "${thin_metadata_size}" \ + "${volgroup_name}" + thin_vol_size=$(lvs --noheading --unit S "${thin_vol}"|grep -o '[0-9]*S') + lvcreate --thin \ + --name "${thin_vol_name}" \ + --virtualsize "${thin_vol_size}" \ + "${thin_vol}" + mkfs -t "${thin_vol_fs_type}" ${thin_vol_fs_type_options} -q \ + -L "containers" "${target}" + fi +} + +create_container_area() { + mkdir -p /var/lib/containers + cat >> /etc/fstab <<-FSTAB +LABEL=containers /var/lib/containers xfs defaults 0 2 +/.swapfile none swap discard,nofail 0 0 +FSTAB + mount /var/lib/containers +} + + +create_thin_pool +create_container_area diff -Nru domtrix-sql-20221014.1/sql/sql-cloud-config-script domtrix-sql-20221222.1/sql/sql-cloud-config-script --- domtrix-sql-20221014.1/sql/sql-cloud-config-script 2022-10-14 13:52:46.000000000 +0000 +++ domtrix-sql-20221222.1/sql/sql-cloud-config-script 2022-12-22 11:49:46.000000000 +0000 @@ -7,3 +7,7 @@ disable_ec2_metadata: true package_update: no locale: en_GB.UTF-8 +write_files: +- path: /etc/udev/rules.d/40-vm-hotadd-brightbox.rules + content: | + SUBSYSTEM=="cpu", ACTION=="add", DEVPATH=="/devices/system/cpu/cpu[0-9]*", TEST=="online", ATTR{online}!="1", ATTR{online}="1"