diff -Nru curtin-21.3/curtin/block/deps.py curtin-22.1/curtin/block/deps.py --- curtin-21.3/curtin/block/deps.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/curtin/block/deps.py 2022-06-27 00:46:09.000000000 +0000 @@ -96,8 +96,12 @@ if osfamily not in distro_mapping: raise ValueError('No block package mapping for distro: %s' % osfamily) - return {1: {'handler': storage_config_required_packages, - 'mapping': distro_mapping.get(osfamily)}} + cfg_map = { + 'handler': storage_config_required_packages, + 'mapping': distro_mapping.get(osfamily), + } + + return {1: cfg_map, 2: cfg_map} # vi: ts=4 expandtab syntax=python diff -Nru curtin-21.3/curtin/block/__init__.py curtin-22.1/curtin/block/__init__.py --- curtin-21.3/curtin/block/__init__.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/curtin/block/__init__.py 2022-06-27 00:46:09.000000000 +0000 @@ -993,19 +993,12 @@ else: raise ValueError("Blockdev and sysfs_path cannot both be None") - # queue property is only on parent devices, ie, we can't read - # /sys/class/block/vda/vda1/queue/* as queue is only on the - # parent device sysfs_prefix = sysfs_path (parent, partnum) = get_blockdev_for_partition(blockdev) if partnum: sysfs_prefix = sys_block_path(parent) partnum = int(partnum) - block_size = int(util.load_file(os.path.join( - sysfs_prefix, 'queue/logical_block_size'))) - unit = block_size - ptdata = [] for part_sysfs in get_sysfs_partitions(sysfs_prefix): data = {} @@ -1015,8 +1008,12 @@ continue data[sfile] = int(util.load_file(dfile)) if partnum is None or data['partition'] == partnum: - ptdata.append((path_to_kname(part_sysfs), data['partition'], - data['start'] * unit, data['size'] * unit,)) + ptdata.append(( + path_to_kname(part_sysfs), + data['partition'], + data['start'] * SECTOR_SIZE_BYTES, + data['size'] * SECTOR_SIZE_BYTES, + )) return ptdata @@ -1371,4 +1368,9 @@ return {} +def get_resize_fstypes(): + from curtin.commands.block_meta_v2 import resizers + return {fstype for fstype in resizers.keys()} + + # vi: ts=4 expandtab syntax=python diff -Nru curtin-21.3/curtin/block/schemas.py curtin-22.1/curtin/block/schemas.py --- curtin-21.3/curtin/block/schemas.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/curtin/block/schemas.py 2022-06-27 00:46:09.000000000 +0000 @@ -284,8 +284,13 @@ 'properties': { 'id': {'$ref': '#/definitions/id'}, 'multipath': {'type': 'string'}, + # Permit path to device as output. + # This value is ignored for input. + 'path': {'type': 'string', + 'pattern': _path_dev}, 'name': {'$ref': '#/definitions/name'}, 'offset': {'$ref': '#/definitions/size'}, # XXX: This is not used + 'resize': {'type': 'boolean'}, 'preserve': {'$ref': '#/definitions/preserve'}, 'size': {'$ref': '#/definitions/size'}, 'uuid': {'$ref': '#/definitions/uuid'}, # XXX: This is not used @@ -299,6 +304,11 @@ 'enum': ['bios_grub', 'boot', 'extended', 'home', 'linux', 'logical', 'lvm', 'mbr', 'prep', 'raid', 'swap', '']}, + 'partition_type': {'type': 'string', + 'oneOf': [ + {'pattern': r'^0x[0-9a-fA-F]{1,2}$'}, + {'$ref': '#/definitions/uuid'}, + ]}, 'grub_device': { 'type': ['boolean', 'integer'], 'minimum': 0, diff -Nru curtin-21.3/curtin/commands/apt_config.py curtin-22.1/curtin/commands/apt_config.py --- curtin-21.3/curtin/commands/apt_config.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/curtin/commands/apt_config.py 2022-06-27 00:46:09.000000000 +0000 @@ -28,6 +28,9 @@ APT_CONFIG_FN = "/etc/apt/apt.conf.d/94curtin-config" APT_PROXY_FN = "/etc/apt/apt.conf.d/90curtin-aptproxy" +# Files to store pinning information +APT_PREFERENCES_FN = "/etc/apt/preferences.d/90curtin.pref" + # Default keyserver to use DEFAULT_KEYSERVER = "keyserver.ubuntu.com" @@ -37,7 +40,7 @@ PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports", "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"} PRIMARY_ARCHES = ['amd64', 'i386'] -PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el'] +PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el', 'riscv64'] APT_SOURCES_PROPOSED = ( "deb $MIRROR $RELEASE-proposed main restricted universe multiverse") @@ -81,6 +84,11 @@ except (IOError, OSError): LOG.exception("Failed to apply proxy or apt config info:") + try: + apply_apt_preferences(cfg, target + APT_PREFERENCES_FN) + except (IOError, OSError): + LOG.exception("Failed to apply apt preferences.") + # Process 'apt_source -> sources {dict}' if 'sources' in cfg: params = mirrors @@ -571,7 +579,7 @@ def apply_apt_proxy_config(cfg, proxy_fname, config_fname): """apply_apt_proxy_config - Applies any apt*proxy config from if specified + Applies any apt*proxy from config if specified """ # Set up any apt proxy cfgs = (('proxy', 'Acquire::http::Proxy "%s";'), @@ -584,8 +592,14 @@ LOG.debug("write apt proxy info to %s", proxy_fname) util.write_file(proxy_fname, '\n'.join(proxies) + '\n') elif os.path.isfile(proxy_fname): - util.del_file(proxy_fname) - LOG.debug("no apt proxy configured, removed %s", proxy_fname) + # When $ curtin apt-config is called with no proxy set, it makes + # sense to remove the proxy file (if present). Having said that, + # this code is also called automatically at the curthooks stage with an + # empty configuration. Since the installation of external packages and + # execution of unattended-upgrades (which happen after executing the + # curthooks) need to use the proxy if specified, we must not let the + # curthooks remove the proxy file. + pass if cfg.get('conf', None): LOG.debug("write apt config info to %s", config_fname) @@ -595,6 +609,38 @@ LOG.debug("no apt config configured, removed %s", config_fname) +def preference_to_str(preference): + """ Return a textual representation of a given preference as specified in + apt_preferences(5). + """ + + return """\ +Package: {package} +Pin: {pin} +Pin-Priority: {pin_priority} +""".format(package=preference["package"], + pin=preference["pin"], + pin_priority=preference["pin-priority"]) + + +def apply_apt_preferences(cfg, pref_fname): + """ Apply apt preferences if any is provided. + """ + + prefs = cfg.get("preferences") + if not prefs: + # When $ curtin apt-config is called with no preferences set, it makes + # sense to remove the preferences file (if present). Having said that, + # this code is also called automatically at the curthooks stage with an + # empty configuration. Since the installation of packages (which + # happens after executing the curthooks) needs to honor the preferences + # set, we must not let the curthooks remove the preferences file. + return + prefs_as_strings = [preference_to_str(pref) for pref in prefs] + LOG.debug("write apt preferences info to %s.", pref_fname) + util.write_file(pref_fname, "\n".join(prefs_as_strings)) + + def apt_command(args): """ Main entry point for curtin apt-config standalone command This does not read the global config as handled by curthooks, but diff -Nru curtin-21.3/curtin/commands/block_meta.py curtin-22.1/curtin/commands/block_meta.py --- curtin-21.3/curtin/commands/block_meta.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/curtin/commands/block_meta.py 2022-06-27 00:46:09.000000000 +0000 @@ -552,16 +552,30 @@ def image_handler(info, storage_config, handlers): path = info['path'] - if os.path.exists(path): - os.unlink(path) + size = int(util.human2bytes(info['size'])) + sector_size = str(int(util.human2bytes(info.get('sector_size', 512)))) + if info.get('preserve', False): + actual_size = os.stat(path).st_size + if size != actual_size: + raise RuntimeError( + 'image at {} was size {} not {} as expected.'.format( + path, actual_size, size)) + else: + if os.path.exists(path): + os.unlink(path) + try: + with open(path, 'wb') as fp: + fp.truncate(size) + except BaseException: + if os.path.exists(path): + os.unlink(path) + raise try: - with open(path, 'wb') as fp: - fp.truncate(int(util.human2bytes(info['size']))) dev = util.subp([ - 'losetup', '--show', '--find', path], + 'losetup', '--show', '--sector-size', sector_size, '--find', path], capture=True)[0].strip() except BaseException: - if os.path.exists(path): + if os.path.exists(path) and not info.get('preserve'): os.unlink(path) raise info['dev'] = dev @@ -765,12 +779,17 @@ raise RuntimeError("Device %s does not exist" % devpath) -def verify_size(devpath, expected_size_bytes, part_info): +def get_part_size_bytes(devpath, part_info): (found_type, _code) = ptable_uuid_to_flag_entry(part_info.get('type')) if found_type == 'extended': found_size_bytes = int(part_info['size']) * 512 else: found_size_bytes = block.read_sys_block_size_bytes(devpath) + return found_size_bytes + + +def verify_size(devpath, expected_size_bytes, part_info): + found_size_bytes = get_part_size_bytes(devpath, part_info) msg = ( 'Verifying %s size, expecting %s bytes, found %s bytes' % ( devpath, expected_size_bytes, found_size_bytes)) @@ -807,7 +826,7 @@ def partition_verify_sfdisk(part_action, label, sfdisk_part_info): - devpath = sfdisk_part_info['node'] + devpath = os.path.realpath(sfdisk_part_info['node']) verify_size( devpath, int(util.human2bytes(part_action['size'])), sfdisk_part_info) expected_flag = part_action.get('flag') @@ -1110,6 +1129,12 @@ return lsblock[kname]['TYPE'] +def _get_volume_fstype(device_path): + lsblock = block._lsblock([device_path]) + kname = block.path_to_kname(device_path) + return lsblock[kname]['FSTYPE'] + + def get_volume_spec(device_path): """ Return the most reliable spec for a device per Ubuntu FSTAB wiki @@ -2004,6 +2029,17 @@ storage_config_dict = extract_storage_ordered_dict(cfg) + version = cfg['storage']['version'] + if version > 1: + from curtin.commands.block_meta_v2 import ( + disk_handler_v2, + partition_handler_v2, + ) + command_handlers.update({ + 'disk': disk_handler_v2, + 'partition': partition_handler_v2, + }) + storage_config_dict = zfsroot_update_storage_config(storage_config_dict) # set up reportstack diff -Nru curtin-21.3/curtin/commands/block_meta_v2.py curtin-22.1/curtin/commands/block_meta_v2.py --- curtin-21.3/curtin/commands/block_meta_v2.py 1970-01-01 00:00:00.000000000 +0000 +++ curtin-22.1/curtin/commands/block_meta_v2.py 2022-06-27 00:46:09.000000000 +0000 @@ -0,0 +1,422 @@ +# This file is part of curtin. See LICENSE file for copyright and license info. + +import os +from typing import Optional + +import attr + +from curtin import (block, util) +from curtin.commands.block_meta import ( + _get_volume_fstype, + disk_handler as disk_handler_v1, + get_path_to_storage_volume, + make_dname, + partition_handler as partition_handler_v1, + verify_ptable_flag, + verify_size, + ) +from curtin.log import LOG +from curtin.storage_config import ( + GPT_GUID_TO_CURTIN_MAP, + select_configs, + ) +from curtin.udev import udevadm_settle + + +@attr.s(auto_attribs=True) +class PartTableEntry: + number: int + start: int + size: int + type: str + uuid: Optional[str] + bootable: bool = False + + def render(self): + r = f'{self.number}: ' + for a in 'start', 'size', 'type', 'uuid': + v = getattr(self, a) + if v is not None: + r += f' {a}={v}' + if self.bootable: + r += ' bootable' + return r + + +ONE_MIB_BYTES = 1 << 20 + + +def align_up(size, block_size): + return (size + block_size - 1) & ~(block_size - 1) + + +def align_down(size, block_size): + return size & ~(block_size - 1) + + +def resize_ext(path, size): + util.subp(['e2fsck', '-p', '-f', path]) + size_k = size // 1024 + util.subp(['resize2fs', path, f'{size_k}k']) + + +def resize_ntfs(path, size): + util.subp(['ntfsresize', '-s', str(size), path]) + + +def perform_resize(kname, resize): + path = block.kname_to_path(kname) + fstype = resize['fstype'] + size = resize['size'] + direction = resize['direction'] + LOG.debug('Resizing %s of type %s %s to %s', + path, fstype, direction, size) + resizers[fstype](path, size) + + +resizers = { + 'ext2': resize_ext, + 'ext3': resize_ext, + 'ext4': resize_ext, + 'ntfs': resize_ntfs, +} + + +FLAG_TO_GUID = { + flag: guid for (guid, (flag, typecode)) in GPT_GUID_TO_CURTIN_MAP.items() + } +FLAG_TO_MBR_TYPE = { + flag: typecode[:2].upper() for (guid, (flag, typecode)) + in GPT_GUID_TO_CURTIN_MAP.items() + } +FLAG_TO_MBR_TYPE['extended'] = '05' + + +class SFDiskPartTable: + + label = None + + def __init__(self, sector_bytes): + self.entries = [] + self.label_id = None + self._sector_bytes = sector_bytes + if ONE_MIB_BYTES % sector_bytes != 0: + raise Exception( + f"sector_bytes {sector_bytes} does not divide 1MiB, cannot " + "continue!") + self.one_mib_sectors = ONE_MIB_BYTES // sector_bytes + + def bytes2sectors(self, amount): + return int(util.human2bytes(amount)) // self._sector_bytes + + def sectors2bytes(self, amount): + return amount * self._sector_bytes + + def render(self): + r = ['label: ' + self.label] + if self.label_id: + r.extend(['label-id: ' + self.label_id]) + r.extend(['']) + r.extend([e.render() for e in self.entries]) + return '\n'.join(r) + + def apply(self, device): + sfdisk_script = self.render() + LOG.debug("sfdisk input:\n---\n%s\n---\n", sfdisk_script) + util.subp( + ['sfdisk', '--no-tell-kernel', '--no-reread', device], + data=sfdisk_script.encode('ascii')) + util.subp(['partprobe', device]) + # sfdisk and partprobe (as invoked here) use ioctls to inform the + # kernel that the partition table has changed so it can add and remove + # device nodes for the partitions as needed. Unfortunately this is + # asynchronous: we can return before the nodes are present in /dev (or + # /sys for that matter). Calling "udevadm settle" is slightly + # incoherent as udev has nothing to do with creating these nodes, but + # at the same time, udev won't finish processing the events triggered + # by the sfdisk until after the nodes for the partitions have been + # updated by the kernel. + udevadm_settle() + + +class GPTPartTable(SFDiskPartTable): + + label = 'gpt' + + def add(self, action): + number = action.get('number', len(self.entries) + 1) + if 'offset' in action: + start = self.bytes2sectors(action['offset']) + else: + if self.entries: + prev = self.entries[-1] + start = align_up(prev.start + prev.size, self.one_mib_sectors) + else: + start = self.one_mib_sectors + size = self.bytes2sectors(action['size']) + uuid = action.get('uuid') + type = action.get('partition_type', + FLAG_TO_GUID.get(action.get('flag'))) + entry = PartTableEntry(number, start, size, type, uuid) + self.entries.append(entry) + return entry + + +class DOSPartTable(SFDiskPartTable): + + label = 'dos' + _extended = None + + def add(self, action): + flag = action.get('flag', None) + start = action.get('offset', None) + if start is not None: + start = self.bytes2sectors(start) + if flag == 'logical': + if self._extended is None: + raise Exception("logical partition without extended partition") + prev = None + for entry in reversed(self.entries): + if entry.number > 4: + prev = entry + break + # The number of an logical partition cannot be specified (so the + # 'number' from the action is completely ignored here) as the + # partitions are numbered by the order they are found in the linked + # list of logical partitions. sfdisk just cares that we put a + # number > 4 here, in fact we could "number" every logical + # partition as "5" but it's not hard to put the number that the + # partition will end up getting into the sfdisk input. + if prev is None: + number = 5 + if start is None: + start = align_up( + self._extended.start + self.one_mib_sectors, + self.one_mib_sectors) + else: + number = prev.number + 1 + if start is None: + start = align_up( + prev.start + prev.size + self.one_mib_sectors, + self.one_mib_sectors) + else: + number = action.get('number', len(self.entries) + 1) + if number > 4: + raise Exception( + "primary partition cannot have number %s" % (number,)) + if start is None: + prev = None + for entry in self.entries: + if entry.number <= 4: + prev = entry + if prev is None: + start = self.one_mib_sectors + else: + start = align_up( + prev.start + prev.size, + self.one_mib_sectors) + size = self.bytes2sectors(action['size']) + type = action.get('partition_type', FLAG_TO_MBR_TYPE.get(flag)) + if flag == 'boot': + bootable = True + else: + bootable = None + entry = PartTableEntry( + number, start, size, type, uuid=None, bootable=bootable) + if flag == 'extended': + self._extended = entry + self.entries.append(entry) + return entry + + +def _find_part_info(sfdisk_info, offset): + for part in sfdisk_info['partitions']: + if part['start'] == offset: + return part + else: + raise Exception( + "could not find existing partition by offset") + + +def _wipe_for_action(action): + # If a wipe action is specified, do that. + if 'wipe' in action: + return action['wipe'] + # Existing partitions are left alone by default. + if action.get('preserve', False): + return None + # New partitions are wiped by default apart from extended partitions, where + # it would destroy the EBR. + if action.get('flag') == 'extended': + return None + return 'superblock' + + +def _prepare_resize(storage_config, part_action, table, part_info): + if not part_action.get('preserve') or not part_action.get('resize'): + return None + + devpath = os.path.realpath(part_info['node']) + fstype = _get_volume_fstype(devpath) + if fstype == '': + return None + + volume = part_action['id'] + format_actions = select_configs(storage_config, type='format', + volume=volume) + if len(format_actions) > 1: + raise Exception(f'too many format actions for volume {volume}') + + if len(format_actions) == 1: + if not format_actions[0].get('preserve'): + return None + + target_fstype = format_actions[0]['fstype'] + msg = ( + 'Verifying %s format, expecting %s, found %s' % ( + devpath, fstype, target_fstype)) + LOG.debug(msg) + if fstype != target_fstype: + raise RuntimeError(msg) + + msg = 'Resize requested for format %s' % (fstype, ) + LOG.debug(msg) + if fstype not in resizers: + raise RuntimeError(msg + ' is unsupported') + + start = table.sectors2bytes(part_info['size']) + end = int(util.human2bytes(part_action['size'])) + if start > end: + direction = 'down' + elif start < end: + direction = 'up' + else: + return None + + return { + 'fstype': fstype, + 'size': end, + 'direction': direction, + } + + +def verify_offset(devpath, part_action, current_info, table): + if 'offset' not in part_action: + return + current_offset = table.sectors2bytes(current_info['start']) + action_offset = int(util.human2bytes(part_action['offset'])) + msg = ( + 'Verifying %s offset, expecting %s, found %s' % ( + devpath, current_offset, action_offset)) + LOG.debug(msg) + if current_offset != action_offset: + raise RuntimeError(msg) + + +def partition_verify_sfdisk_v2(part_action, label, sfdisk_part_info, + storage_config, table): + devpath = os.path.realpath(sfdisk_part_info['node']) + if not part_action.get('resize'): + verify_size(devpath, int(util.human2bytes(part_action['size'])), + sfdisk_part_info) + verify_offset(devpath, part_action, sfdisk_part_info, table) + expected_flag = part_action.get('flag') + if expected_flag: + verify_ptable_flag(devpath, expected_flag, label, sfdisk_part_info) + + +def disk_handler_v2(info, storage_config, handlers): + disk_handler_v1(info, storage_config, handlers) + + part_actions = [] + + for action in storage_config.values(): + if action['type'] == 'partition' and action['device'] == info['id']: + part_actions.append(action) + + table_cls = { + 'msdos': DOSPartTable, + 'gpt': GPTPartTable, + }.get(info.get('ptable')) + + if table_cls is None: + for action in part_actions: + partition_handler_v1(action, storage_config, handlers) + return + + disk = get_path_to_storage_volume(info.get('id'), storage_config) + (sector_size, _) = block.get_blockdev_sector_size(disk) + + table = table_cls(sector_size) + preserved_offsets = set() + wipes = {} + resizes = {} + + sfdisk_info = None + for action in part_actions: + entry = table.add(action) + if action.get('preserve', False): + if sfdisk_info is None: + # Lazily computing sfdisk_info is slightly more efficient but + # the real reason for doing this is that calling sfdisk_info on + # a disk with no partition table logs messages that makes the + # vmtest infrastructure unhappy. + sfdisk_info = block.sfdisk_info(disk) + part_info = _find_part_info(sfdisk_info, entry.start) + partition_verify_sfdisk_v2(action, sfdisk_info['label'], part_info, + storage_config, table) + resizes[entry.start] = _prepare_resize(storage_config, action, + table, part_info) + preserved_offsets.add(entry.start) + wipes[entry.start] = _wipe_for_action(action) + + # preserve disk label ids + if info.get('preserve') and sfdisk_info is not None: + table.label_id = sfdisk_info['id'] + + for kname, nr, offset, size in block.sysfs_partition_data(disk): + offset_sectors = table.bytes2sectors(offset) + resize = resizes.get(offset_sectors) + if resize and resize['direction'] == 'down': + perform_resize(kname, resize) + + for kname, nr, offset, size in block.sysfs_partition_data(disk): + offset_sectors = table.bytes2sectors(offset) + if offset_sectors not in preserved_offsets: + # Do a superblock wipe of any partitions that are being deleted. + block.wipe_volume(block.kname_to_path(kname), 'superblock') + elif wipes.get(offset_sectors) is not None: + # We do a quick wipe of where any new partitions will be, + # because if there is bcache or other metadata there, this + # can cause the partition to be used by a storage + # subsystem and preventing the exclusive open done by the + # wipe_volume call below. See + # https://bugs.launchpad.net/curtin/+bug/1718699 for all + # the gory details. + LOG.debug('Wiping 1M on %s at offset %s', disk, offset) + block.zero_file_at_offsets(disk, [offset], exclusive=False) + + table.apply(disk) + + for kname, number, offset, size in block.sysfs_partition_data(disk): + offset_sectors = table.bytes2sectors(offset) + wipe = wipes[offset_sectors] + if wipe is not None: + # Wipe the new partitions as needed. + block.wipe_volume(block.kname_to_path(kname), wipe) + resize = resizes.get(offset_sectors) + if resize and resize['direction'] == 'up': + perform_resize(kname, resize) + + # Make the names if needed + if 'name' in info: + for action in part_actions: + if action.get('flag') != 'extended': + make_dname(action['id'], storage_config) + + +def partition_handler_v2(info, storage_config, handlers): + pass + + +# vi: ts=4 expandtab syntax=python diff -Nru curtin-21.3/curtin/commands/install_grub.py curtin-22.1/curtin/commands/install_grub.py --- curtin-21.3/curtin/commands/install_grub.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/curtin/commands/install_grub.py 2022-06-27 00:46:09.000000000 +0000 @@ -62,6 +62,9 @@ elif target_arch == 'i386': grub_name = 'grub-efi-ia32' grub_target = 'i386-efi' + elif target_arch == 'riscv64': + grub_name = 'grub-efi-riscv64' + grub_target = 'riscv64-efi' else: raise ValueError('Unsupported UEFI arch: %s' % target_arch) else: diff -Nru curtin-21.3/curtin/distro.py curtin-22.1/curtin/distro.py --- curtin-21.3/curtin/distro.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/curtin/distro.py 2022-06-27 00:46:09.000000000 +0000 @@ -23,7 +23,8 @@ DistroInfo = namedtuple('DistroInfo', ('variant', 'family')) DISTRO_NAMES = ['arch', 'centos', 'debian', 'fedora', 'freebsd', 'gentoo', - 'opensuse', 'redhat', 'rhel', 'sles', 'suse', 'ubuntu'] + 'opensuse', 'redhat', 'rhel', 'sles', 'suse', 'ubuntu', + 'rocky'] # python2.7 lacks PEP 435, so we must make use an alternative for py2.7/3.x @@ -37,7 +38,7 @@ OS_FAMILIES = { DISTROS.debian: [DISTROS.debian, DISTROS.ubuntu], DISTROS.redhat: [DISTROS.centos, DISTROS.fedora, DISTROS.redhat, - DISTROS.rhel], + DISTROS.rhel, DISTROS.rocky], DISTROS.gentoo: [DISTROS.gentoo], DISTROS.freebsd: [DISTROS.freebsd], DISTROS.suse: [DISTROS.opensuse, DISTROS.sles, DISTROS.suse], @@ -382,6 +383,9 @@ osfamily=None): LOG.debug("Upgrading system in %s", target) + if not osfamily: + osfamily = get_osfamily(target=target) + distro_cfg = { DISTROS.debian: {'function': run_apt_command, 'subcommands': ('dist-upgrade', 'autoremove')}, diff -Nru curtin-21.3/curtin/__init__.py curtin-22.1/curtin/__init__.py --- curtin-21.3/curtin/__init__.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/curtin/__init__.py 2022-06-27 00:46:09.000000000 +0000 @@ -40,6 +40,6 @@ 'FSTAB_DEFAULT_FSCK_ON_BLK' ] -__version__ = "21.3" +__version__ = "22.1" # vi: ts=4 expandtab syntax=python diff -Nru curtin-21.3/curtin/storage_config.py curtin-22.1/curtin/storage_config.py --- curtin-21.3/curtin/storage_config.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/curtin/storage_config.py 2022-06-27 00:46:09.000000000 +0000 @@ -79,7 +79,7 @@ 'required': ['version', 'config'], 'definitions': schemas.definitions, 'properties': { - 'version': {'type': 'integer', 'enum': [1]}, + 'version': {'type': 'integer', 'enum': [1, 2]}, 'config': { 'type': 'array', 'items': { @@ -753,6 +753,8 @@ return entry if entry['type'] == 'partition': + if devname: + entry['path'] = devname attrs = blockdev_data['attrs'] if self.is_mpath_partition(blockdev_data): entry['number'] = int(blockdev_data['DM_PART']) @@ -798,6 +800,8 @@ entry['size'] *= 512 ptype = blockdev_data.get('ID_PART_ENTRY_TYPE') + if ptype is not None: + entry['partition_type'] = ptype flag_name, _flag_code = ptable_uuid_to_flag_entry(ptype) if ptable and ptable.get('label') == 'dos': @@ -1315,7 +1319,7 @@ ordered = (dasd + disk + part + format + lvols + lparts + raids + dmcrypts + mounts + bcache + zpool + zfs) - final_config = {'storage': {'version': 1, 'config': ordered}} + final_config = {'storage': {'version': 2, 'config': ordered}} try: LOG.info('Validating extracted storage config components') validate_config(final_config['storage']) @@ -1346,7 +1350,7 @@ LOG.debug("Merging storage config dependencies") merged_config = { - 'version': 1, + 'version': 2, 'config': merge_config_trees_to_list(ctrees) } LOG.debug("Merged storage config:\n%s", @@ -1355,4 +1359,12 @@ return {'storage': merged_config} +def select_configs(storage_config, **kwargs): + """ Given a set of key=value arguments, return a list of the configs that + match all specified key-value pairs. + """ + return [cfg for cfg in storage_config.values() + if all(cfg.get(k) == v for k, v in kwargs.items())] + + # vi: ts=4 expandtab syntax=python diff -Nru curtin-21.3/curtin/util.py curtin-22.1/curtin/util.py --- curtin-21.3/curtin/util.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/curtin/util.py 2022-06-27 00:46:09.000000000 +0000 @@ -501,6 +501,13 @@ os.chdir(curdir) +@contextmanager +def mount(src, target): + do_mount(src, target) + yield + do_umount(target) + + def do_mount(src, target, opts=None): # mount src at target with opts and return True # if already mounted, return False diff -Nru curtin-21.3/debian/changelog curtin-22.1/debian/changelog --- curtin-21.3/debian/changelog 2021-12-07 20:34:05.000000000 +0000 +++ curtin-22.1/debian/changelog 2022-06-27 22:24:54.000000000 +0000 @@ -1,4 +1,73 @@ -curtin (21.3-0ubuntu1~20.04.1) focal; urgency=medium +curtin (22.1-0ubuntu1~20.04.1) focal; urgency=medium + + * New upstream release. (LP: #1979687) + - deb: fix dependencies [Dan Bungert] + - tox: drop xenial-py3 from default env list [Dan Bungert] + - block/v2: preserve disk label id [Dan Bungert] + - block/v2: docs for partition_type [Dan Bungert] + - block/v2: unit tests for partition_type [Dan Bungert] + - block/v2: raw partition table codes for gpt [Dan Bungert] + - block/v2: allow setting raw partition_type value [Dan Bungert] + - Make sure curthooks do not discard supplied proxy settings + [Olivier Gayot] + - block/v2: resize-friendly ordering of wipe [Dan Bungert] + - block/v2: handle resize when no format action [Dan Bungert] + - block/v2: resize of ntfs [Dan Bungert] + - vmtests: remove out of date skip [Dan Bungert] + - block: provide get_resize_fstypes [Dan Bungert] + - Add support for resize of ext{2,3,4} [Dan Bungert] + - Add riscv64 to supported UEFI architectures [William Wilson] + - block_meta_v2: call make_dname when required [Michael Hudson-Doyle] + - examples: even more tweaks for v2 [Michael Hudson-Doyle] + - Add riscv64 support [Heinrich Schuchardt] + - examples: stop assuming curtin accounts for overhead of logical + partitions [Michael Hudson-Doyle] + - block_meta_v2: zero start of partitions before they are created + [Michael Hudson-Doyle] + - examples: enlarge / for some more vmtests [Michael Hudson-Doyle] + - skip BionicTestPartitionExistingRAID.test_correct_ptype + [Michael Hudson-Doyle] + - examples: boost size of / in multipath-reuse.yaml [Michael Hudson-Doyle] + - examples: add offsets to preserved partitions [Michael Hudson-Doyle] + - block_meta_v2: change how we invoke sfdisk again, restore partprobe call + [Michael Hudson-Doyle] + - block_meta_v2: fix partitioning a device with sector size != 512 + [Michael Hudson-Doyle] + - block_meta_v2: fix implicit offset calculation for dos partitions + [Michael Hudson-Doyle] + - block_meta_v2: do not use aliases for partition types + [Michael Hudson-Doyle] + - Remove CentOS 6 tests. [Michael Hudson-Doyle] + - vmtests: boost the size of / in a few tests [Michael Hudson-Doyle] + - examples: sleep after creating bcache device in preserve-bcache.yaml + [Michael Hudson-Doyle] + - vmtests: fix parted invocation in partition-existing-raid.yaml + [Michael Hudson-Doyle] + - vmtests: bump VM memory size to 2048 MiB for all tests + [Michael Hudson-Doyle] + - vmtests: drop assertion that clear-holders ran [Michael Hudson-Doyle] + - block_meta_v2: a few more fixes for v2 partitioning + [Michael Hudson-Doyle] + - block_meta: call realpath on partition node returned by sfdisk + [Michael Hudson-Doyle] + - Add rocky linux as a RHEL-like variant [Dimitri John Ledkov] + - Update pylint version in tox.ini [Michael Hudson-Doyle] + - block_meta: implement v2 partitioning [Michael Hudson-Doyle] + - Stop running CI against Python 2 [Michael Hudson-Doyle] + - Make sure curthooks do not discard APT preferences [Olivier Gayot] + - Remove leftover debug print statement [Olivier Gayot] + - Fix format of examples/apt-source.yaml [Olivier Gayot] + - Implement support for APT preferences in apt-config [Olivier Gayot] + - build-deb: changelog gen with dch [Dan Bungert] + - vmtests uefi: relax the uefi check [Dan Bungert] + - block: output partition device path [Dan Bungert] + - support version 2 curtin storage configs [Michael Hudson-Doyle] + - system-upgrade: lookup os family [Dan Bungert] + - add preserve: true support to the image action [Michael Hudson-Doyle] + + -- Dan Bungert Mon, 27 Jun 2022 16:24:54 -0600 + +curtin (21.3-0ubuntu1) jammy; urgency=medium * New upstream release. - version to 21.3 [Dan Bungert] (LP: #1953410) @@ -64,7 +133,7 @@ [Michael Hudson-Doyle] (LP: #1893818) - vmtest/centos: handle different paths to grub config [Ryan Harper] - -- Dan Bungert Tue, 07 Dec 2021 13:34:05 -0700 + -- Dan Bungert Mon, 06 Dec 2021 17:02:22 -0700 curtin (21.2-0ubuntu1) hirsute; urgency=medium diff -Nru curtin-21.3/debian/changelog.trunk curtin-22.1/debian/changelog.trunk --- curtin-21.3/debian/changelog.trunk 2021-12-07 00:02:22.000000000 +0000 +++ curtin-22.1/debian/changelog.trunk 1970-01-01 00:00:00.000000000 +0000 @@ -1,5 +0,0 @@ -curtin (UPSTREAM_VER-0ubuntu1) UNRELEASED; urgency=low - - * Initial release - - -- Scott Moser Mon, 29 Jul 2013 16:12:09 -0400 diff -Nru curtin-21.3/debian/control curtin-22.1/debian/control --- curtin-21.3/debian/control 2021-12-07 00:02:22.000000000 +0000 +++ curtin-22.1/debian/control 2022-06-27 22:24:54.000000000 +0000 @@ -7,6 +7,7 @@ dh-python, python3, python3-apt, + python3-attr, python3-coverage, python3-mock, python3-nose, @@ -14,6 +15,8 @@ python3-setuptools, python3-yaml Homepage: http://launchpad.net/curtin +Vcs-Git: https://git.launchpad.net/curtin +Vcs-Browser: https://git.launchpad.net/curtin X-Python3-Version: >= 3.2 Package: curtin @@ -51,6 +54,7 @@ Priority: extra Depends: curtin-common (= ${binary:Version}), python3-apt, + python3-attr, python3-oauthlib, python3-yaml, wget, diff -Nru curtin-21.3/doc/topics/apt_source.rst curtin-22.1/doc/topics/apt_source.rst --- curtin-21.3/doc/topics/apt_source.rst 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/doc/topics/apt_source.rst 2022-06-27 00:46:09.000000000 +0000 @@ -31,6 +31,8 @@ - add arbitrary apt.conf settings + - add arbitrary apt preferences + - provide debconf configurations - disabling suites (=pockets) diff -Nru curtin-21.3/doc/topics/storage.rst curtin-22.1/doc/topics/storage.rst --- curtin-21.3/doc/topics/storage.rst 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/doc/topics/storage.rst 2022-06-27 00:46:09.000000000 +0000 @@ -13,8 +13,7 @@ in curtin. Partitioning layout is read as a list of in-order modifications to make to achieve the desired configuration. The top level configuration key containing this configuration is ``storage``. This key should contain a -dictionary with at least a version number and the configuration list. The -current config specification is ``version: 1``. +dictionary with at least a version number and the configuration list. **Config Example**:: @@ -27,6 +26,20 @@ serial: QM00002 model: QEMU_HARDDISK +Config versions +--------------- + +The current version of curtin supports versions ``1`` and ``2``. These +only differ in the interpretation of ``partition`` actions at this +time. ``lvm_partition`` actions will be interpreted differently at +some point in the future. + +.. note:: + + Config version ``2`` is under active development and subject to change. + Users are advised to use version ``1`` unless features enabled by version + ``2`` are required. + Configuration Types ------------------- Each entry in the config list is a dictionary with several keys which vary @@ -322,13 +335,41 @@ to be told which disk to use and the size of the partition. Additional options are available. +Partition actions are interpreted differently according to the version of the +storage config. + + * For version 1 configs, the actions are handled one by one and each + partition is created (or assumed to exist, in the ``preserve: true`` case) + just after that described by the previous action. + + * For version 2 configs, the actions are bundled together to create a + complete description of the partition table, and the ``offset`` of each + action is respected if present. Any partitions that already exist but are + not referenced in the new config are (superblock-) wiped and deleted. + + * Because the numbering of logical partitions is not stable (i.e. if there + are two logical partitions numbered 5 and 6, and partition 5 is deleted, + what was partition 6 will become partition 5), curtin checks if a + partition is deleted or not by checking for the presence of a partition + action with a matching offset. + +If the disk is being completely repartitioned, the two schemes are effectively +the same. + **number**: ** -The partition number can be specified using ``number``. However, numbers must -be in order and some situations, such as extended/logical partitions on msdos -partition tables will require special numbering, so it maybe better to omit -the partition number. If the ``number`` key is not present, curtin will attempt -determine the right number to use. +The partition number can be specified using ``number``. + +For GPT partition tables, this will just be the slot in the partition table +that is used to describe this partition. + +For DOS partition tables, a primary or extended partition must have a number +less than or equal to 4. Logical partitions have numbers 5 or greater but are +numbered by the order they are found when parsing the partitions, so the +``number`` field is ignored for them. + +If the ``number`` key is not present, curtin will attempt determine the right +number to use. **size**: ** @@ -338,8 +379,15 @@ .. note:: - Curtin does not adjust size values. If you specific a size that exceeds the - capacity of a device then installation will fail. + Curtin does not adjust or inspect size values. If you specify a size that + exceeds the capacity of a device then installation will fail. + +**offset**: ** + +The offset at which to create the partition. Only respected in a version 2 +config. If the offset field is not present, the partition will be placed after +that described by the preceding (logical or primary, if appropriate) partition +action, or at the start of the disk (or extended partition, as appropriate). **device**: ** @@ -368,9 +416,7 @@ The *logical/extended* partition flags can be used to create logical partitions on a msdos table. An extended partition should be created containing all of the empty space on the drive, and logical partitions can be created within it. A -extended partition must already be present to create logical partitions. If the -``number`` flag is set for an extended partition it must be set to 4, and -each logical partition should be numbered starting from 5. +extended partition must already be present to create logical partitions. On msdos partition tables, the *boot* flag sets the boot parameter to that partition. On gpt partition tables, the boot flag sets the esp flag on the @@ -385,10 +431,33 @@ at the beginning of the disk and should be 1MB in size. It should not contain a filesystem or be mounted anywhere on the system. +**partition_type**: *msdos: byte value in 0xnn style; gpt: GUID* + +Only applicable to v2 storage configuration. If both ``partition_type`` and +``flag`` are set, ``partition_type`` dictates the acutal type. + +The ``partition_type`` field allows for setting arbitrary partition type values +that do not have a matching ``flag``, or cases that are not handled by the +``flag`` system. For example, since the *boot* flag results in both setting +the bootable state for a MSDOS partition table and setting it to type *0xEF*, +one can override this behavior and achieve a bootable partition of a different +type by using ``flag``: *boot* and using ``partition_type``. + **preserve**: *true, false* If the preserve flag is set to true, curtin will verify that the partition exists and that the ``size`` and ``flag`` match the configuration provided. +See also the ``resize`` flag, which adjusts this behavior. + +**resize**: *true, false* + +Only applicable to v2 storage configuration. +If the ``preserve`` flag is set to false, this value is not applicable. +If the ``preserve`` flag is set to true, curtin will adjust the size of the +partition to the new size. When adjusting smaller, the size of the contents +must permit that. When adjusting larger, there must already be a gap beyond +the partition in question. +Resize is supported on filesystems of types ext2, ext3, ext4, ntfs. **name**: ** diff -Nru curtin-21.3/examples/apt-source.yaml curtin-22.1/examples/apt-source.yaml --- curtin-21.3/examples/apt-source.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/apt-source.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -77,7 +77,7 @@ # arches is list of architectures the following config applies to # the special keyword "default" applies to any architecture not explicitly # listed. - - arches: [amd64, i386, default] + - arches: [amd64, i386, default] # uri is just defining the target as-is uri: http://us.archive.ubuntu.com/ubuntu # @@ -100,7 +100,7 @@ # security is optional, if not defined it is set to the same value as primary security: uri: http://security.ubuntu.com/ubuntu - [...] + # [...] # if no mirrors are specified at all, or all lookups fail it will use: # primary: http://archive.ubuntu.com/ubuntu @@ -152,6 +152,18 @@ # The following example is also the builtin default if nothing is specified add_apt_repo_match: '^[\w-]+:\w' + # 1.9 preferences + # + # Any apt preferences that will be made available to apt + # see the APT_PREFERENCES(5) man page for details about what can be specified + preferences: + - package: python3-* + pin: origin *ubuntu.com* + pin-priority: 200 + - package: python-* + pin: origin *ubuntu.com* + pin-priority: -1 + ############################################################################## # Section 2: source list entries diff -Nru curtin-21.3/examples/tests/basic_iscsi.yaml curtin-22.1/examples/tests/basic_iscsi.yaml --- curtin-21.3/examples/tests/basic_iscsi.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/basic_iscsi.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -12,7 +12,7 @@ - id: vdb1 type: partition number: 1 - size: 3GB + size: 4GB device: vdb flag: boot - id: vdb2 diff -Nru curtin-21.3/examples/tests/basic.yaml curtin-22.1/examples/tests/basic.yaml --- curtin-21.3/examples/tests/basic.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/basic.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -17,7 +17,7 @@ - id: sda1 type: partition number: 1 - size: 3GB + size: 4GB device: sda flag: boot - id: sda2 diff -Nru curtin-21.3/examples/tests/bcache-partitions.yaml curtin-22.1/examples/tests/bcache-partitions.yaml --- curtin-21.3/examples/tests/bcache-partitions.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/bcache-partitions.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -18,7 +18,6 @@ type: disk name: rotary1 serial: disk-c - ptable: gpt wipe: superblock - id: id_rotary0_part1 type: partition diff -Nru curtin-21.3/examples/tests/centos6_basic.yaml curtin-22.1/examples/tests/centos6_basic.yaml --- curtin-21.3/examples/tests/centos6_basic.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/centos6_basic.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,101 +0,0 @@ -showtrace: true -storage: - version: 1 - config: - - id: sda - type: disk - ptable: msdos - model: QEMU HARDDISK - serial: disk-a - name: main_disk_with_in/\&valid@#dname - wipe: superblock - grub_device: true - - id: sda1 - type: partition - number: 1 - size: 3GB - device: sda - flag: boot - - id: sda2 - type: partition - number: 2 - size: 1GB - device: sda - - id: sda3 - type: partition - number: 3 - size: 1GB - device: sda - name: swap - - id: sda1_root - type: format - fstype: ext3 - volume: sda1 - label: 'cloudimg-rootfs' - - id: sda2_home - type: format - fstype: ext4 - volume: sda2 - - id: sda3_swap - type: format - fstype: swap - volume: sda3 - - id: sda1_mount - type: mount - path: / - device: sda1_root - - id: sda2_mount - type: mount - path: /home - device: sda2_home - - id: sparedisk_id - type: disk - serial: disk-b - name: sparedisk - wipe: superblock - - id: sparedisk_fat_fmt_id - type: format - fstype: fat32 - volume: sparedisk_id - - id: btrfs_disk_id - type: disk - serial: disk-c - name: btrfs_volume - wipe: superblock - - id: btrfs_disk_fmt_id - type: format - fstype: btrfs - volume: btrfs_disk_id - - id: btrfs_disk_mnt_id - type: mount - path: /btrfs - options: 'defaults,noatime' - device: btrfs_disk_fmt_id - - id: pnum_disk - type: disk - serial: disk-d - name: pnum_disk - wipe: superblock - ptable: gpt - - id: pnum_disk_p1 - type: partition - number: 1 - size: 1GB - device: pnum_disk - - id: pnum_disk_p2 - type: partition - number: 2 - size: 8MB - device: pnum_disk - flag: prep - wipe: zero - name: prep - - id: pnum_disk_p3 - type: partition - number: 10 - size: 1GB - device: pnum_disk - - id: swap_mnt - type: mount - path: "none" - device: sda3_swap diff -Nru curtin-21.3/examples/tests/lvm_iscsi.yaml curtin-22.1/examples/tests/lvm_iscsi.yaml --- curtin-21.3/examples/tests/lvm_iscsi.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/lvm_iscsi.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -12,7 +12,7 @@ - id: vdb1 type: partition number: 1 - size: 3GB + size: 4GB device: vdb flag: boot - id: vdb2 @@ -44,7 +44,7 @@ wipe: superblock - id: sda_extended type: partition - size: 5G + size: 5.5G flag: extended device: sda - id: sda1 @@ -100,7 +100,7 @@ wipe: superblock - id: sdb_extended type: partition - size: 4G + size: 4.5G flag: extended device: sdb - id: sdb1 diff -Nru curtin-21.3/examples/tests/lvm.yaml curtin-22.1/examples/tests/lvm.yaml --- curtin-21.3/examples/tests/lvm.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/lvm.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -23,7 +23,7 @@ flag: boot - id: sda_extended type: partition - size: 5G + size: 5.5G flag: extended device: sda - id: sda2 diff -Nru curtin-21.3/examples/tests/mirrorboot.yaml curtin-22.1/examples/tests/mirrorboot.yaml --- curtin-21.3/examples/tests/mirrorboot.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/mirrorboot.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -17,7 +17,7 @@ flag: bios_grub - id: sda1 type: partition - size: 3GB + size: 3.5GB device: sda - id: sdb type: disk @@ -28,7 +28,7 @@ name: second_disk - id: sdb1 type: partition - size: 3GB + size: 3.5GB device: sdb - id: mddevice name: md0 diff -Nru curtin-21.3/examples/tests/multipath-lvm-part-wipe.yaml curtin-22.1/examples/tests/multipath-lvm-part-wipe.yaml --- curtin-21.3/examples/tests/multipath-lvm-part-wipe.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/multipath-lvm-part-wipe.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -113,7 +113,7 @@ - id: root_vg_lv1 type: lvm_partition name: lv1_root - size: 2.5G + size: 3.5G volgroup: root_vg - id: lv1_root_fs type: format diff -Nru curtin-21.3/examples/tests/multipath-reuse.yaml curtin-22.1/examples/tests/multipath-reuse.yaml --- curtin-21.3/examples/tests/multipath-reuse.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/multipath-reuse.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -6,8 +6,8 @@ - &setup | parted /dev/disk/by-id/dm-name-mpatha --script -- \ mklabel msdos \ - mkpart primary ext4 1GiB 4GiB \ - mkpart primary ext4 4GiB 5GiB \ + mkpart primary ext4 1GiB 5GiB \ + mkpart primary ext4 5GiB 6GiB \ set 1 boot on udevadm settle @@ -32,16 +32,18 @@ - id: sda1 type: partition number: 1 - size: 3GB + size: 4GB device: sda flag: boot preserve: true + offset: 1G - id: sda2 type: partition number: 2 size: 1GB device: sda preserve: true + offset: 5G - id: sda1_root type: format fstype: ext4 diff -Nru curtin-21.3/examples/tests/multipath.yaml curtin-22.1/examples/tests/multipath.yaml --- curtin-21.3/examples/tests/multipath.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/multipath.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -16,7 +16,7 @@ - id: sda1 type: partition number: 1 - size: 3GB + size: 4GB device: sda flag: boot wipe: superblock diff -Nru curtin-21.3/examples/tests/partition-existing-raid.yaml curtin-22.1/examples/tests/partition-existing-raid.yaml --- curtin-21.3/examples/tests/partition-existing-raid.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/partition-existing-raid.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -15,7 +15,7 @@ /dev/disk/by-id/virtio-disk-b-part1 /dev/disk/by-id/virtio-disk-c-part1 udevadm settle parted /dev/md1 --script -- \ - mklabel dos + mklabel msdos udevadm settle mdadm --stop /dev/md1 udevadm settle @@ -51,7 +51,7 @@ id: id_disk0_part2 device: id_disk0 number: 2 - size: 3G + size: 4G - type: partition id: id_disk0_part3 device: id_disk0 @@ -63,6 +63,7 @@ flag: boot number: 1 size: 8G + offset: 1G preserve: true - type: partition id: id_disk2_part1 @@ -70,6 +71,7 @@ flag: boot number: 1 size: 8G + offset: 1G preserve: true - type: raid id: raid-md1 diff -Nru curtin-21.3/examples/tests/preserve-bcache.yaml curtin-22.1/examples/tests/preserve-bcache.yaml --- curtin-21.3/examples/tests/preserve-bcache.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/preserve-bcache.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -10,6 +10,7 @@ udevadm settle make-bcache -C /dev/disk/by-id/virtio-disk-b \ -B /dev/disk/by-id/virtio-disk-a-part2 --writeback + sleep 1 udevadm settle mkfs.ext4 /dev/bcache0 mount /dev/bcache0 /mnt @@ -46,6 +47,7 @@ size: 1024M preserve: true wipe: superblock + offset: 1M - id: id_rotary0_part2 type: partition name: rotary0-part2 @@ -53,6 +55,7 @@ number: 2 size: 8G preserve: true + offset: 1026M - id: id_bcache0 type: bcache name: bcache0 diff -Nru curtin-21.3/examples/tests/preserve-lvm.yaml curtin-22.1/examples/tests/preserve-lvm.yaml --- curtin-21.3/examples/tests/preserve-lvm.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/preserve-lvm.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -47,6 +47,7 @@ device: main_disk flag: bios_grub preserve: true + offset: 1MB - id: main_disk_p2 type: partition number: 2 @@ -54,6 +55,7 @@ device: main_disk flag: boot preserve: true + offset: 3MB - id: root_vg type: lvm_volgroup name: root_vg diff -Nru curtin-21.3/examples/tests/preserve-partition-wipe-vg-simple.yaml curtin-22.1/examples/tests/preserve-partition-wipe-vg-simple.yaml --- curtin-21.3/examples/tests/preserve-partition-wipe-vg-simple.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/preserve-partition-wipe-vg-simple.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -39,6 +39,7 @@ number: 1 type: partition id: disk-sda-part-1 + offset: 2M - device: disk-sda size: 3G flag: linux @@ -47,6 +48,7 @@ wipe: zero type: partition id: disk-sda-part-2 + offset: 4G - fstype: ext4 volume: disk-sda-part-2 preserve: false diff -Nru curtin-21.3/examples/tests/preserve-partition-wipe-vg.yaml curtin-22.1/examples/tests/preserve-partition-wipe-vg.yaml --- curtin-21.3/examples/tests/preserve-partition-wipe-vg.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/preserve-partition-wipe-vg.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -51,6 +51,7 @@ wipe: zero type: partition id: disk-sda-part-1 + offset: 2M - device: disk-sda size: 3G flag: linux @@ -58,6 +59,7 @@ wipe: zero type: partition id: disk-sda-part-2 + offset: 1G - device: disk-sdb flag: linux size: 3G @@ -65,12 +67,14 @@ wipe: zero type: partition id: disk-sdb-part-1 + offset: 1G - device: disk-sdb flag: linux size: 3G preserve: true type: partition id: disk-sdb-part-2 + offset: 4G - fstype: ext4 volume: disk-sda-part-2 preserve: false diff -Nru curtin-21.3/examples/tests/preserve-raid.yaml curtin-22.1/examples/tests/preserve-raid.yaml --- curtin-21.3/examples/tests/preserve-raid.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/preserve-raid.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -52,7 +52,7 @@ id: id_disk0_part2 device: id_disk0 number: 2 - size: 3G + size: 4G - type: partition id: id_disk0_part3 device: id_disk0 @@ -65,6 +65,7 @@ number: 1 size: 8G preserve: true + offset: 1G - type: partition id: id_disk2_part1 device: id_disk2 @@ -72,6 +73,7 @@ number: 1 size: 8G preserve: true + offset: 1G - type: raid id: raid-md1 name: md1 diff -Nru curtin-21.3/examples/tests/preserve.yaml curtin-22.1/examples/tests/preserve.yaml --- curtin-21.3/examples/tests/preserve.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/preserve.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -6,8 +6,8 @@ mklabel gpt \ mkpart primary ext4 2MiB 514MiB \ set 1 esp on \ - mkpart primary ext4 1GiB 4GiB \ - mkpart primary ext4 4GiB 7GiB + mkpart primary ext4 1GiB 5GiB \ + mkpart primary ext4 6GiB 9GiB udevadm settle mkfs.ext4 /dev/disk/by-id/virtio-disk-a-part3 mount /dev/disk/by-id/virtio-disk-a-part3 /mnt @@ -32,18 +32,21 @@ number: 1 size: 512M preserve: true + offset: 2M - type: partition id: id_disk0_part2 device: id_disk0 number: 2 - size: 3G + size: 4G preserve: true + offset: 1G - type: partition id: id_disk0_part3 device: id_disk0 number: 3 size: 3G preserve: true + offset: 6G - type: format id: id_efi_format volume: id_disk0_part1 diff -Nru curtin-21.3/examples/tests/reuse-lvm-member-partition.yaml curtin-22.1/examples/tests/reuse-lvm-member-partition.yaml --- curtin-21.3/examples/tests/reuse-lvm-member-partition.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/reuse-lvm-member-partition.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -69,12 +69,14 @@ flag: boot number: 1 size: 1G + offset: 1G - type: partition id: id_disk0_part2 preserve: true device: id_disk0 number: 2 size: 7G + offset: 2G - type: format id: id_efi_format volume: id_disk0_part1 diff -Nru curtin-21.3/examples/tests/reuse-msdos-partitions.yaml curtin-22.1/examples/tests/reuse-msdos-partitions.yaml --- curtin-21.3/examples/tests/reuse-msdos-partitions.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/reuse-msdos-partitions.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -43,6 +43,7 @@ flag: boot preserve: true wipe: superblock + offset: 1M - id: sda2 type: partition number: 2 @@ -50,6 +51,7 @@ flag: extended device: sda preserve: true + offset: 3074M - id: sda5 type: partition number: 5 @@ -58,6 +60,7 @@ device: sda preserve: true wipe: superblock + offset: 3075M - id: sda6 type: partition number: 6 @@ -66,6 +69,7 @@ device: sda preserve: true wipe: superblock + offset: 5123M - id: sda1_root type: format fstype: ext4 diff -Nru curtin-21.3/examples/tests/reuse-raid-member-wipe-partition.yaml curtin-22.1/examples/tests/reuse-raid-member-wipe-partition.yaml --- curtin-21.3/examples/tests/reuse-raid-member-wipe-partition.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/reuse-raid-member-wipe-partition.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -49,6 +49,7 @@ flag: boot number: 1 size: 1G + offset: 1G - type: partition id: id_disk0_part2 preserve: true @@ -56,6 +57,7 @@ number: 2 size: 7G wipe: superblock + offset: 2G - type: format id: id_efi_format volume: id_disk0_part1 diff -Nru curtin-21.3/examples/tests/uefi_basic.yaml curtin-22.1/examples/tests/uefi_basic.yaml --- curtin-21.3/examples/tests/uefi_basic.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/uefi_basic.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -31,7 +31,7 @@ - device: id_disk0 id: id_disk0_part2 number: 2 - size: 3G + size: 4G type: partition wipe: superblock - fstype: fat32 diff -Nru curtin-21.3/examples/tests/uefi_reuse_esp.yaml curtin-22.1/examples/tests/uefi_reuse_esp.yaml --- curtin-21.3/examples/tests/uefi_reuse_esp.yaml 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/examples/tests/uefi_reuse_esp.yaml 2022-06-27 00:46:09.000000000 +0000 @@ -8,7 +8,7 @@ mklabel gpt \ mkpart primary fat32 1MiB 513MiB \ set 1 esp on \ - mkpart primary ext4 513MiB 3585MiB + mkpart primary ext4 513MiB 4609MiB udevadm settle mkfs.vfat -I -n EFI -F 32 /dev/disk/by-id/virtio-disk-a-part1 @@ -67,9 +67,10 @@ - device: id_disk0 id: id_disk0_part2 number: 2 - size: 3G + size: 4G type: partition preserve: true + offset: 513M - fstype: fat32 id: id_efi_format label: efi diff -Nru curtin-21.3/pylintrc curtin-22.1/pylintrc --- curtin-21.3/pylintrc 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/pylintrc 2022-06-27 00:46:09.000000000 +0000 @@ -7,7 +7,7 @@ # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members=redhat,centos,fedora,debian,suse,opensuse,sles,arch,ubuntu,rhel,freebsd,gentoo +generated-members=redhat,centos,fedora,debian,suse,opensuse,sles,arch,ubuntu,rhel,freebsd,gentoo,rocky # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime diff -Nru curtin-21.3/test-requirements.txt curtin-22.1/test-requirements.txt --- curtin-21.3/test-requirements.txt 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/test-requirements.txt 2022-06-27 00:46:09.000000000 +0000 @@ -3,3 +3,4 @@ nose pyflakes coverage +parameterized diff -Nru curtin-21.3/tests/integration/test_block_meta.py curtin-22.1/tests/integration/test_block_meta.py --- curtin-21.3/tests/integration/test_block_meta.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/integration/test_block_meta.py 2022-06-27 00:46:09.000000000 +0000 @@ -1,13 +1,21 @@ # This file is part of curtin. See LICENSE file for copyright and license info. -from collections import namedtuple +import dataclasses +from dataclasses import dataclass import contextlib +import json +import os +from parameterized import parameterized +import re import sys +from typing import Optional import yaml -import os from curtin import block, udev, util +from curtin.commands.block_meta import _get_volume_fstype +from curtin.commands.block_meta_v2 import ONE_MIB_BYTES + from tests.unittests.helpers import CiTestCase from tests.integration.webserv import ImageServer @@ -17,10 +25,11 @@ @contextlib.contextmanager -def loop_dev(image): - dev = util.subp( - ['losetup', '--show', '--find', '--partscan', image], - capture=True, decode='ignore')[0].strip() +def loop_dev(image, sector_size=512): + dev = util.subp([ + 'losetup', '--show', '--find', '--partscan', + '--sector-size', str(sector_size), image, + ], capture=True, decode='ignore')[0].strip() try: udev.udevadm_trigger([dev]) yield dev @@ -28,18 +37,114 @@ util.subp(['losetup', '--detach', dev]) -PartData = namedtuple("PartData", ('number', 'offset', 'size')) +@dataclass(order=True) +class PartData: + number: Optional[int] = None + offset: Optional[int] = None + size: Optional[int] = None + boot: Optional[bool] = None + partition_type: Optional[str] = None + + # test cases may initialize the values they care about + # test utilities shall initialize all fields + def assertFieldsAreNotNone(self): + for field in dataclasses.fields(self): + assert getattr(self, field.name) is not None + + def __eq__(self, other): + for field in dataclasses.fields(self): + myval = getattr(self, field.name) + otherval = getattr(other, field.name) + if myval is not None and otherval is not None \ + and myval != otherval: + return False + return True + + +def _get_ext_size(dev, part_action): + num = part_action['number'] + cmd = ['dumpe2fs', '-h', f'{dev}p{num}'] + out = util.subp(cmd, capture=True)[0] + for line in out.splitlines(): + if line.startswith('Block count'): + block_count = line.split(':')[1].strip() + if line.startswith('Block size'): + block_size = line.split(':')[1].strip() + return int(block_count) * int(block_size) + + +def _get_ntfs_size(dev, part_action): + num = part_action['number'] + cmd = ['ntfsresize', + '--no-action', + '--force', # needed post-resize, which otherwise demands a CHKDSK + '--info', f'{dev}p{num}'] + out = util.subp(cmd, capture=True)[0] + # Sample input: + # Current volume size: 41939456 bytes (42 MB) + volsize_matcher = re.compile(r'^Current volume size: ([0-9]+) bytes') + for line in out.splitlines(): + m = volsize_matcher.match(line) + if m: + return int(m.group(1)) + raise Exception('ntfs volume size not found') + + +_get_fs_sizers = { + 'ext2': _get_ext_size, + 'ext3': _get_ext_size, + 'ext4': _get_ext_size, + 'ntfs': _get_ntfs_size, +} + + +def _get_filesystem_size(dev, part_action, fstype='ext4'): + if fstype not in _get_fs_sizers.keys(): + raise Exception(f'_get_filesystem_size: no support for {fstype}') + return _get_fs_sizers[fstype](dev, part_action) + + +def _get_extended_partition_size(dev, num): + # sysfs reports extended partitions as having 1K size + # sfdisk seems to have a better idea + ptable_json = util.subp(['sfdisk', '-J', dev], capture=True)[0] + ptable = json.loads(ptable_json) + + nodename = f'{dev}p{num}' + partitions = ptable['partitiontable']['partitions'] + partition = [part for part in partitions if part['node'] == nodename][0] + return partition['size'] * 512 + + +def _get_disk_label_id(dev): + ptable_json = util.subp(['sfdisk', '-J', dev], capture=True)[0] + ptable = json.loads(ptable_json) + # string in lowercase hex + return ptable['partitiontable']['id'] def summarize_partitions(dev): - # We don't care about the kname - return sorted( - [PartData(*d[1:]) for d in block.sysfs_partition_data(dev)]) + parts = [] + ptable_json = util.subp(['sfdisk', '-J', dev], capture=True)[0] + ptable = json.loads(ptable_json) + partitions = ptable['partitiontable']['partitions'] + for d in block.sysfs_partition_data(dev): + nodename = f'/dev/{d[0]}' + partition = [part for part in partitions + if part['node'] == nodename][0] + ptype = partition['type'] + boot = partition.get('bootable', False) + # We don't care about the kname + pd = PartData(*d[1:], partition_type=ptype, boot=boot) + pd.assertFieldsAreNotNone() + parts.append(pd) + return sorted(parts) class StorageConfigBuilder: - def __init__(self): + def __init__(self, *, version): + self.version = version self.config = [] self.cur_image = None @@ -47,37 +152,81 @@ return { 'storage': { 'config': self.config, + 'version': self.version, }, } - def add_image(self, *, path, size, create=False, **kw): - action = { - 'type': 'image', - 'id': 'id' + str(len(self.config)), - 'path': path, - 'size': size, - } - action.update(**kw) - self.cur_image = action['id'] + def _add(self, *, type, **kw): + if type != 'image' and self.cur_image is None: + raise Exception("no current image") + action = {'id': 'id' + str(len(self.config))} + action.update(type=type, **kw) self.config.append(action) + return action + + def add_image(self, *, path, size, create=False, **kw): if create: with open(path, "wb") as f: f.write(b"\0" * int(util.human2bytes(size))) + action = self._add(type='image', path=path, size=size, **kw) + self.cur_image = action['id'] + return action def add_part(self, *, size, **kw): - if self.cur_image is None: - raise Exception("no current image") - action = { - 'type': 'partition', - 'id': 'id' + str(len(self.config)), - 'device': self.cur_image, - 'size': size, - } - action.update(**kw) - self.config.append(action) + fstype = kw.pop('fstype', None) + part = self._add(type='partition', device=self.cur_image, size=size, + **kw) + if fstype: + self.add_format(part=part, fstype=fstype) + return part + + def add_format(self, *, part, fstype='ext4', **kw): + return self._add(type='format', volume=part['id'], fstype=fstype, **kw) + + def set_preserve(self): + for action in self.config: + action['preserve'] = True class TestBlockMeta(IntegrationTestCase): + def setUp(self): + self.data = self.random_string() + + def assertPartitions(self, *args): + with loop_dev(self.img) as dev: + self.assertEqual([*args], summarize_partitions(dev)) + + @contextlib.contextmanager + def mount(self, dev, partition_cfg): + mnt_point = self.tmp_dir() + num = partition_cfg['number'] + with util.mount(f'{dev}p{num}', mnt_point): + yield mnt_point + + @contextlib.contextmanager + def open_file_on_part(self, dev, part_action, mode): + with self.mount(dev, part_action) as mnt_point: + with open(f'{mnt_point}/data.txt', mode) as fp: + yield fp + + def create_data(self, dev, part_action): + with self.open_file_on_part(dev, part_action, 'w') as fp: + fp.write(self.data) + + def check_data(self, dev, part_action): + with self.open_file_on_part(dev, part_action, 'r') as fp: + self.assertEqual(self.data, fp.read()) + + def check_fssize(self, dev, part_action, fstype, expected): + tolerance = 0 + if fstype == 'ntfs': + # Per ntfsresize manpage, the actual fs size is at least one sector + # less than requested. + # In these tests it has been consistently 7 sectors fewer. + tolerance = 512 * 10 + actual_fssize = _get_filesystem_size(dev, part_action, fstype) + diff = expected - actual_fssize + self.assertTrue(0 <= diff <= tolerance, f'difference of {diff}') def run_bm(self, config, *args, **kwargs): config_path = self.tmp_path('config.yaml') @@ -102,34 +251,88 @@ ] util.subp(cmd, env=cmd_env, **kwargs) - def _test_default_offsets(self, ptable): + def _test_default_offsets(self, ptable, version, sector_size=512): psize = 40 << 20 img = self.tmp_path('image.img') - config = StorageConfigBuilder() + config = StorageConfigBuilder(version=version) + config.add_image( + path=img, size='200M', ptable=ptable, sector_size=sector_size) + p1 = config.add_part(size=psize, number=1) + p2 = config.add_part(size=psize, number=2) + p3 = config.add_part(size=psize, number=3) + self.run_bm(config.render()) + + with loop_dev(img, sector_size) as dev: + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=psize), + PartData(number=2, offset=(1 << 20) + psize, size=psize), + PartData(number=3, offset=(1 << 20) + 2*psize, size=psize), + ]) + p1['offset'] = 1 << 20 + p2['offset'] = (1 << 20) + psize + p3['offset'] = (1 << 20) + 2*psize + config.set_preserve() + self.run_bm(config.render()) + + def test_default_offsets_gpt_v1(self): + self._test_default_offsets('gpt', 1) + + def test_default_offsets_msdos_v1(self): + self._test_default_offsets('msdos', 1) + + def test_default_offsets_gpt_v2(self): + self._test_default_offsets('gpt', 2) + + def test_default_offsets_msdos_v2(self): + self._test_default_offsets('msdos', 2) + + def test_default_offsets_gpt_v1_4k(self): + self._test_default_offsets('gpt', 1, 4096) + + def test_default_offsets_msdos_v1_4k(self): + self._test_default_offsets('msdos', 1, 4096) + + def test_default_offsets_gpt_v2_4k(self): + self._test_default_offsets('gpt', 2, 4096) + + def test_default_offsets_msdos_v2_4k(self): + self._test_default_offsets('msdos', 2, 4096) + + def _test_specified_offsets(self, ptable, version): + psize = 20 << 20 + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=version) config.add_image(path=img, size='100M', ptable=ptable) - config.add_part(size=psize, number=1) - config.add_part(size=psize, number=2) + config.add_part(size=psize, number=1, offset=psize) + config.add_part(size=psize, number=2, offset=psize * 3) self.run_bm(config.render()) with loop_dev(img) as dev: self.assertEqual( summarize_partitions(dev), [ - PartData( - number=1, offset=1 << 20, size=psize), - PartData( - number=2, offset=(1 << 20) + psize, size=psize), + PartData(number=1, offset=psize, size=psize), + PartData(number=2, offset=psize*3, size=psize), ]) + config.set_preserve() + self.run_bm(config.render()) + + def DONT_test_specified_offsets_gpt_v1(self): + self._test_specified_offsets('gpt', 1) - def test_default_offsets_gpt(self): - self._test_default_offsets('gpt') + def DONT_test_specified_offsets_msdos_v1(self): + self._test_specified_offsets('msdos', 1) - def test_default_offsets_msdos(self): - self._test_default_offsets('msdos') + def test_specified_offsets_gpt_v2(self): + self._test_specified_offsets('gpt', 2) - def _test_non_default_numbering(self, ptable): + def test_specified_offsets_msdos_v2(self): + self._test_specified_offsets('msdos', 2) + + def _test_non_default_numbering(self, ptable, version): psize = 40 << 20 img = self.tmp_path('image.img') - config = StorageConfigBuilder() + config = StorageConfigBuilder(version=version) config.add_image(path=img, size='100M', ptable=ptable) config.add_part(size=psize, number=1) config.add_part(size=psize, number=4) @@ -138,23 +341,30 @@ with loop_dev(img) as dev: self.assertEqual( summarize_partitions(dev), [ - PartData( - number=1, offset=1 << 20, size=psize), - PartData( - number=4, offset=(1 << 20) + psize, size=psize), + PartData(number=1, offset=1 << 20, size=psize), + PartData(number=4, offset=(1 << 20) + psize, size=psize), ]) - def test_non_default_numbering_gpt(self): - self._test_non_default_numbering('gpt') + def test_non_default_numbering_gpt_v1(self): + self._test_non_default_numbering('gpt', 1) + + def BROKEN_test_non_default_numbering_msdos_v1(self): + self._test_non_default_numbering('msdos', 2) - def BROKEN_test_non_default_numbering_msdos(self): - self._test_non_default_numbering('msdos') + def test_non_default_numbering_gpt_v2(self): + self._test_non_default_numbering('gpt', 2) - def test_logical(self): + def test_non_default_numbering_msdos_v2(self): + self._test_non_default_numbering('msdos', 2) + + def _test_logical(self, version): img = self.tmp_path('image.img') - config = StorageConfigBuilder() + config = StorageConfigBuilder(version=version) config.add_image(path=img, size='100M', ptable='msdos') - config.add_part(size='50M', number=1, flag='extended') + # curtin adds 1MiB to the size of the extend partition per contained + # logical partition, but only in v1 mode + size = '97M' if version == 1 else '99M' + config.add_part(size=size, number=1, flag='extended') config.add_part(size='10M', number=5, flag='logical') config.add_part(size='10M', number=6, flag='logical') self.run_bm(config.render()) @@ -163,19 +373,162 @@ self.assertEqual( summarize_partitions(dev), [ # extended partitions get a strange size in sysfs - PartData(number=1, offset=1 << 20, size=1 << 10), - PartData(number=5, offset=2 << 20, size=10 << 20), + PartData(number=1, offset=1 << 20, size=1 << 10), + PartData(number=5, offset=2 << 20, size=10 << 20), # part 5 takes us to 12 MiB offset, curtin leaves a 1 MiB # gap. PartData(number=6, offset=13 << 20, size=10 << 20), ]) + self.assertEqual(99 << 20, _get_extended_partition_size(dev, 1)) p1kname = block.partition_kname(block.path_to_kname(dev), 1) self.assertTrue(block.is_extended_partition('/dev/' + p1kname)) + def test_logical_v1(self): + self._test_logical(1) + + def test_logical_v2(self): + self._test_logical(2) + + def _test_replace_partition(self, ptable): + psize = 20 << 20 + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='100M', ptable=ptable) + config.add_part(size=psize, number=1) + config.add_part(size=psize, number=2) + self.run_bm(config.render()) + + with loop_dev(img) as dev: + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=psize), + PartData(number=2, offset=(1 << 20) + psize, size=psize), + ]) + + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='100M', ptable=ptable, preserve=True) + config.add_part(size=psize, number=1, offset=1 << 20, preserve=True) + config.add_part(size=psize*2, number=2) + self.run_bm(config.render()) + + with loop_dev(img) as dev: + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=psize), + PartData(number=2, offset=(1 << 20) + psize, size=2*psize), + ]) + + def test_replace_partition_gpt_v2(self): + self._test_replace_partition('gpt') + + def test_replace_partition_msdos_v2(self): + self._test_replace_partition('msdos') + + def test_delete_logical_partition(self): + # The test case that resulted in a lot of hair-pulling: + # deleting a logical partition renumbers any later partitions + # (so you cannot stably refer to partitions by number!) + psize = 20 << 20 + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='100M', ptable='msdos') + config.add_part(size='90M', number=1, flag='extended') + config.add_part(size=psize, number=5, flag='logical') + config.add_part(size=psize, number=6, flag='logical') + self.run_bm(config.render()) + + with loop_dev(img) as dev: + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=1 << 10), + PartData(number=5, offset=(2 << 20), size=psize), + PartData(number=6, offset=(3 << 20) + psize, size=psize), + ]) + self.assertEqual(90 << 20, _get_extended_partition_size(dev, 1)) + + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='100M', ptable='msdos', preserve=True) + config.add_part(size='90M', number=1, flag='extended', preserve=True) + config.add_part( + size=psize, number=5, flag='logical', offset=(3 << 20) + psize, + preserve=True) + self.run_bm(config.render()) + + with loop_dev(img) as dev: + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=1 << 10), + PartData(number=5, offset=(3 << 20) + psize, size=psize), + ]) + self.assertEqual(90 << 20, _get_extended_partition_size(dev, 1)) + + def _test_wiping(self, ptable): + # Test wiping behaviour. + # + # Paritions that should be (superblock, i.e. first and last + # megabyte) wiped: + # + # 1) New partitions + # 2) Partitions that are being removed, i.e. no longer present + # 3) Preserved partitions with an explicit wipe + # + # Partitions that should not be wiped: + # + # 4) Preserved partitions with no wipe field. + # + # We test this by creating some partitions with block-meta, + # writing content to them, then running block-meta again, with + # each partition matching one of the conditions above. + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='30M', ptable=ptable) + config.add_part(size='5M', number=1, offset='5M') + config.add_part(size='5M', number=2, offset='10M') + config.add_part(size='5M', number=3, offset='15M') + config.add_part(size='5M', number=4, offset='20M') + self.run_bm(config.render()) + + part_offset_sizes = {} + with loop_dev(img) as dev: + for kname, number, offset, size in block.sysfs_partition_data(dev): + content = bytes([number]) + with open(block.kname_to_path(kname), 'wb') as fp: + fp.write(content*size) + part_offset_sizes[number] = (offset, size) + + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='30M', ptable=ptable, preserve=True) + config.add_part(size='5M', number=1, offset='5M') + # Partition 2 is being deleted. + config.add_part( + size='5M', number=3, offset='15M', preserve=True, + wipe='superblock') + config.add_part(size='5M', number=4, offset='20M', preserve=True) + self.run_bm(config.render()) + + expected_content = {1: {0}, 2: {0}, 3: {0}, 4: {4}} + + with loop_dev(img) as dev: + with open(dev, 'rb') as fp: + for nr, (offset, size) in part_offset_sizes.items(): + expected = expected_content[nr] + fp.seek(offset) + first = set(fp.read(ONE_MIB_BYTES)) + fp.seek(offset + size - ONE_MIB_BYTES) + last = set(fp.read(ONE_MIB_BYTES)) + self.assertEqual(first, expected) + self.assertEqual(last, expected) + + def test_wiping_gpt(self): + self._test_wiping('gpt') + + def test_wiping_msdos(self): + self._test_wiping('msdos') + def test_raw_image(self): img = self.tmp_path('image.img') - config = StorageConfigBuilder() + config = StorageConfigBuilder(version=1) config.add_image(path=img, size='2G', ptable='gpt', create=True) curtin_cfg = config.render() @@ -206,3 +559,432 @@ ) finally: server.stop() + + def _do_test_resize(self, start, end, fstype): + start <<= 20 + end <<= 20 + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='200M', ptable='gpt') + p1 = config.add_part(size=start, offset=1 << 20, number=1, + fstype=fstype) + self.run_bm(config.render()) + with loop_dev(img) as dev: + self.assertEqual(fstype, _get_volume_fstype(f'{dev}p1')) + self.create_data(dev, p1) + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=start), + ]) + self.check_fssize(dev, p1, fstype, start) + + config.set_preserve() + p1['resize'] = True + p1['size'] = end + self.run_bm(config.render()) + with loop_dev(img) as dev: + self.check_data(dev, p1) + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=end), + ]) + self.check_fssize(dev, p1, fstype, end) + + def test_resize_up_ext2(self): + self._do_test_resize(40, 80, 'ext2') + + def test_resize_down_ext2(self): + self._do_test_resize(80, 40, 'ext2') + + def test_resize_up_ext3(self): + self._do_test_resize(40, 80, 'ext3') + + def test_resize_down_ext3(self): + self._do_test_resize(80, 40, 'ext3') + + def test_resize_up_ext4(self): + self._do_test_resize(40, 80, 'ext4') + + def test_resize_down_ext4(self): + self._do_test_resize(80, 40, 'ext4') + + def test_resize_up_ntfs(self): + self._do_test_resize(40, 80, 'ntfs') + + def test_resize_down_ntfs(self): + self._do_test_resize(80, 40, 'ntfs') + + def test_resize_logical(self): + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='100M', ptable='msdos') + config.add_part(size='50M', number=1, flag='extended', offset=1 << 20) + config.add_part(size='10M', number=5, flag='logical', offset=2 << 20) + p6 = config.add_part(size='10M', number=6, flag='logical', + offset=13 << 20, fstype='ext4') + self.run_bm(config.render()) + + with loop_dev(img) as dev: + self.create_data(dev, p6) + self.assertEqual( + summarize_partitions(dev), [ + # extended partitions get a strange size in sysfs + PartData(number=1, offset=1 << 20, size=1 << 10), + PartData(number=5, offset=2 << 20, size=10 << 20), + # part 5 takes us to 12 MiB offset, curtin leaves a 1 MiB + # gap. + PartData(number=6, offset=13 << 20, size=10 << 20), + ]) + self.assertEqual(50 << 20, _get_extended_partition_size(dev, 1)) + + config.set_preserve() + p6['resize'] = True + p6['size'] = '20M' + self.run_bm(config.render()) + + with loop_dev(img) as dev: + self.check_data(dev, p6) + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=1 << 10), + PartData(number=5, offset=2 << 20, size=10 << 20), + PartData(number=6, offset=13 << 20, size=20 << 20), + ]) + self.assertEqual(50 << 20, _get_extended_partition_size(dev, 1)) + + def test_resize_extended(self): + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='100M', ptable='msdos') + p1 = config.add_part(size='50M', number=1, flag='extended', + offset=1 << 20) + p5 = config.add_part(size='49M', number=5, flag='logical', + offset=2 << 20) + self.run_bm(config.render()) + + with loop_dev(img) as dev: + self.assertEqual( + summarize_partitions(dev), [ + # extended partitions get a strange size in sysfs + PartData(number=1, offset=1 << 20, size=1 << 10), + PartData(number=5, offset=2 << 20, size=49 << 20), + ]) + self.assertEqual(50 << 20, _get_extended_partition_size(dev, 1)) + + config.set_preserve() + p1['resize'] = True + p1['size'] = '99M' + p5['resize'] = True + p5['size'] = '98M' + self.run_bm(config.render()) + + with loop_dev(img) as dev: + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=1 << 10), + PartData(number=5, offset=2 << 20, size=98 << 20), + ]) + self.assertEqual(99 << 20, _get_extended_partition_size(dev, 1)) + + def test_split(self): + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='200M', ptable='gpt') + config.add_part(size=9 << 20, offset=1 << 20, number=1) + p2 = config.add_part(size='180M', offset=10 << 20, number=2, + fstype='ext4') + self.run_bm(config.render()) + with loop_dev(img) as dev: + self.create_data(dev, p2) + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=9 << 20), + PartData(number=2, offset=10 << 20, size=180 << 20), + ]) + self.assertEqual(180 << 20, _get_filesystem_size(dev, p2)) + + config.set_preserve() + p2['resize'] = True + p2['size'] = '80M' + p3 = config.add_part(size='100M', offset=90 << 20, number=3, + fstype='ext4') + self.run_bm(config.render()) + with loop_dev(img) as dev: + self.check_data(dev, p2) + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=9 << 20), + PartData(number=2, offset=10 << 20, size=80 << 20), + PartData(number=3, offset=90 << 20, size=100 << 20), + ]) + self.assertEqual(80 << 20, _get_filesystem_size(dev, p2)) + self.assertEqual(100 << 20, _get_filesystem_size(dev, p3)) + + def test_partition_unify(self): + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='200M', ptable='gpt') + config.add_part(size=9 << 20, offset=1 << 20, number=1) + p2 = config.add_part(size='40M', offset=10 << 20, number=2, + fstype='ext4') + p3 = config.add_part(size='60M', offset=50 << 20, number=3, + fstype='ext4') + self.run_bm(config.render()) + with loop_dev(img) as dev: + self.create_data(dev, p2) + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=9 << 20), + PartData(number=2, offset=10 << 20, size=40 << 20), + PartData(number=3, offset=50 << 20, size=60 << 20), + ]) + self.assertEqual(40 << 20, _get_filesystem_size(dev, p2)) + self.assertEqual(60 << 20, _get_filesystem_size(dev, p3)) + + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='200M', ptable='gpt') + config.add_part(size=9 << 20, offset=1 << 20, number=1) + p2 = config.add_part(size='100M', offset=10 << 20, number=2, + fstype='ext4', resize=True) + config.set_preserve() + self.run_bm(config.render()) + with loop_dev(img) as dev: + self.check_data(dev, p2) + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=9 << 20), + PartData(number=2, offset=10 << 20, size=100 << 20), + ]) + self.assertEqual(100 << 20, _get_filesystem_size(dev, p2)) + + def test_mix_of_operations_gpt(self): + # a test that keeps, creates, resizes, and deletes a partition + # 200 MiB disk, using full disk + # init size preserve final size + # p1 - 9 MiB yes 9MiB + # p2 - 90 MiB yes, resize 139MiB + # p3 - 99 MiB no 50MiB + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='200M', ptable='gpt') + config.add_part(size=9 << 20, offset=1 << 20, number=1) + p2 = config.add_part(size='90M', offset=10 << 20, number=2, + fstype='ext4') + p3 = config.add_part(size='99M', offset=100 << 20, number=3, + fstype='ext4') + self.run_bm(config.render()) + with loop_dev(img) as dev: + self.create_data(dev, p2) + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=9 << 20), + PartData(number=2, offset=10 << 20, size=90 << 20), + PartData(number=3, offset=100 << 20, size=99 << 20), + ]) + self.assertEqual(90 << 20, _get_filesystem_size(dev, p2)) + self.assertEqual(99 << 20, _get_filesystem_size(dev, p3)) + + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='200M', ptable='gpt') + config.add_part(size=9 << 20, offset=1 << 20, number=1) + p2 = config.add_part(size='139M', offset=10 << 20, number=2, + fstype='ext4', resize=True) + config.set_preserve() + p3 = config.add_part(size='50M', offset=149 << 20, number=3, + fstype='ext4') + self.run_bm(config.render()) + with loop_dev(img) as dev: + self.check_data(dev, p2) + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=9 << 20), + PartData(number=2, offset=10 << 20, size=139 << 20), + PartData(number=3, offset=149 << 20, size=50 << 20), + ]) + self.assertEqual(139 << 20, _get_filesystem_size(dev, p2)) + self.assertEqual(50 << 20, _get_filesystem_size(dev, p3)) + + def test_mix_of_operations_msdos(self): + # a test that keeps, creates, resizes, and deletes a partition + # including handling of extended/logical + # 200 MiB disk, initially only using front 100MiB + # flag init size preserve final size + # p1 - primary 9MiB yes 9MiB + # p2 - extended 89MiB yes, resize 189MiB + # p3 - logical 37MiB yes, resize 137MiB + # p4 - logical 50MiB no 50MiB + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='200M', ptable='msdos') + p1 = config.add_part(size='9M', offset=1 << 20, number=1, + fstype='ext4') + config.add_part(size='89M', offset=10 << 20, number=2, flag='extended') + p5 = config.add_part(size='36M', offset=11 << 20, number=5, + flag='logical', fstype='ext4') + p6 = config.add_part(size='50M', offset=49 << 20, number=6, + flag='logical', fstype='ext4') + self.run_bm(config.render()) + + with loop_dev(img) as dev: + self.create_data(dev, p1) + self.create_data(dev, p5) + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=9 << 20), + PartData(number=2, offset=10 << 20, size=1 << 10), + PartData(number=5, offset=11 << 20, size=36 << 20), + PartData(number=6, offset=49 << 20, size=50 << 20), + ]) + self.assertEqual(89 << 20, _get_extended_partition_size(dev, 2)) + self.assertEqual(9 << 20, _get_filesystem_size(dev, p1)) + self.assertEqual(36 << 20, _get_filesystem_size(dev, p5)) + self.assertEqual(50 << 20, _get_filesystem_size(dev, p6)) + + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='200M', ptable='msdos') + p1 = config.add_part(size='9M', offset=1 << 20, number=1, + fstype='ext4') + config.add_part(size='189M', offset=10 << 20, number=2, + flag='extended', resize=True) + p5 = config.add_part(size='136M', offset=11 << 20, number=5, + flag='logical', fstype='ext4', resize=True) + config.set_preserve() + p6 = config.add_part(size='50M', offset=149 << 20, number=6, + flag='logical', fstype='ext4') + self.run_bm(config.render()) + + with loop_dev(img) as dev: + self.check_data(dev, p1) + self.check_data(dev, p5) + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=9 << 20), + PartData(number=2, offset=10 << 20, size=1 << 10), + PartData(number=5, offset=11 << 20, size=136 << 20), + PartData(number=6, offset=149 << 20, size=50 << 20), + ]) + self.assertEqual(189 << 20, _get_extended_partition_size(dev, 2)) + self.assertEqual(9 << 20, _get_filesystem_size(dev, p1)) + self.assertEqual(136 << 20, _get_filesystem_size(dev, p5)) + self.assertEqual(50 << 20, _get_filesystem_size(dev, p6)) + + def test_split_and_wiping(self): + # regression test for a bug where a partition wipe would happen before + # a resize was performed, resulting in data loss. + img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='100M', ptable='gpt') + p1 = config.add_part(size=98 << 20, offset=1 << 20, number=1, + fstype='ext4') + self.run_bm(config.render()) + with loop_dev(img) as dev: + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=98 << 20), + ]) + with self.mount(dev, p1) as mnt_point: + # Attempt to create files across the partition with gaps + for i in range(1, 41): + with open(f'{mnt_point}/{str(i)}', 'wb') as fp: + fp.write(bytes([i]) * (2 << 20)) + for i in range(1, 41): + if i % 5 != 0: + os.remove(f'{mnt_point}/{str(i)}') + + config = StorageConfigBuilder(version=2) + config.add_image(path=img, size='100M', ptable='gpt') + p1 = config.add_part(size=49 << 20, offset=1 << 20, number=1, + fstype='ext4', resize=True) + config.set_preserve() + config.add_part(size=49 << 20, offset=50 << 20, number=2, + fstype='ext4') + self.run_bm(config.render()) + with loop_dev(img) as dev: + self.assertEqual( + summarize_partitions(dev), [ + PartData(number=1, offset=1 << 20, size=49 << 20), + PartData(number=2, offset=50 << 20, size=49 << 20), + ]) + with self.mount(dev, p1) as mnt_point: + for i in range(5, 41, 5): + with open(f'{mnt_point}/{i}', 'rb') as fp: + self.assertEqual(bytes([i]) * (2 << 20), fp.read()) + + def test_parttype_dos(self): + # msdos partition table partitions shall retain their type + # create initial situation similar to this + # Device Boot Start End Sectors Size Id Type + # /dev/sda1 * 2048 104447 102400 50M 7 HPFS/NTFS/exFA + # /dev/sda2 104448 208668781 208564334 99.5G 7 HPFS/NTFS/exFA + # /dev/sda3 208670720 209711103 1040384 508M 27 Hidden NTFS Wi + self.img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=self.img, size='200M', ptable='msdos') + config.add_part(size=50 << 20, offset=1 << 20, number=1, + fstype='ntfs', flag='boot', partition_type='0x7') + config.add_part(size=100 << 20, offset=51 << 20, number=2, + fstype='ntfs', partition_type='0x7') + config.add_part(size=48 << 20, offset=151 << 20, number=3, + fstype='ntfs', partition_type='0x27') + self.run_bm(config.render()) + self.assertPartitions( + PartData(number=1, offset=1 << 20, size=50 << 20, + partition_type='7', boot=True), + PartData(number=2, offset=51 << 20, size=100 << 20, + partition_type='7', boot=False), + PartData(number=3, offset=151 << 20, size=48 << 20, + partition_type='27', boot=False)) + + def test_parttype_gpt(self): + # gpt partition table partitions shall retain their type + # create initial situation similar to this + # # Start (sector) End (sector) Size Code Name + # 1 2048 206847 100.0 MiB EF00 EFI system part + # 2 206848 239615 16.0 MiB 0C01 Microsoft reser + # 3 239616 103811181 49.4 GiB 0700 Basic data part + # 4 103813120 104853503 508.0 MiB 2700 + esp = 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B' + msreserved = 'E3C9E316-0B5C-4DB8-817D-F92DF00215AE' + msdata = 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7' + winre = 'DE94BBA4-06D1-4D40-A16A-BFD50179D6AC' + self.img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=self.img, size='100M', ptable='gpt') + config.add_part(number=1, offset=1 << 20, size=9 << 20, + flag='boot', fstype='ntfs') + config.add_part(number=2, offset=10 << 20, size=20 << 20, + partition_type=msreserved) + config.add_part(number=3, offset=30 << 20, size=50 << 20, + partition_type=msdata, fstype='ntfs') + config.add_part(number=4, offset=80 << 20, size=19 << 20, + partition_type=winre, fstype='ntfs') + self.run_bm(config.render()) + self.assertPartitions( + PartData(number=1, offset=1 << 20, size=9 << 20, + partition_type=esp), + PartData(number=2, offset=10 << 20, size=20 << 20, + partition_type=msreserved), + PartData(number=3, offset=30 << 20, size=50 << 20, + partition_type=msdata), + PartData(number=4, offset=80 << 20, size=19 << 20, + partition_type=winre)) + + @parameterized.expand([('msdos',), ('gpt',)]) + def test_disk_label_id_persistent(self, ptable): + # when the disk is preserved, the disk label id shall also be preserved + self.img = self.tmp_path('image.img') + config = StorageConfigBuilder(version=2) + config.add_image(path=self.img, size='20M', ptable=ptable) + config.add_part(number=1, offset=1 << 20, size=18 << 20) + self.run_bm(config.render()) + self.assertPartitions( + PartData(number=1, offset=1 << 20, size=18 << 20)) + with loop_dev(self.img) as dev: + orig_label_id = _get_disk_label_id(dev) + + config.set_preserve() + self.run_bm(config.render()) + self.assertPartitions( + PartData(number=1, offset=1 << 20, size=18 << 20)) + with loop_dev(self.img) as dev: + self.assertEqual(orig_label_id, _get_disk_label_id(dev)) diff -Nru curtin-21.3/tests/unittests/test_apt_source.py curtin-22.1/tests/unittests/test_apt_source.py --- curtin-21.3/tests/unittests/test_apt_source.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/unittests/test_apt_source.py 2022-06-27 00:46:09.000000000 +0000 @@ -572,6 +572,55 @@ 'Acquire::ftp::Proxy "foobar3";\n' 'Acquire::https::Proxy "foobar4";\n')) + def test_preference_to_str(self): + """ test_preference_to_str - Test converting a preference dict to + textual representation. + """ + preference = { + "package": "*", + "pin": "release a=unstable", + "pin-priority": 50, + } + + expected = """\ +Package: * +Pin: release a=unstable +Pin-Priority: 50 +""" + self.assertEqual(expected, apt_config.preference_to_str(preference)) + + @staticmethod + def test_apply_apt_preferences(): + """ test_apply_apt_preferences - Test apt preferences configuration + """ + cfg = { + "preferences": [ + { + "package": "*", + "pin": "release a=unstable", + "pin-priority": 50, + }, { + "package": "dummy-unwanted-package", + "pin": "origin *ubuntu.com*", + "pin-priority": -1, + } + ] + } + + expected_content = """\ +Package: * +Pin: release a=unstable +Pin-Priority: 50 + +Package: dummy-unwanted-package +Pin: origin *ubuntu.com* +Pin-Priority: -1 +""" + with mock.patch.object(util, "write_file") as mockobj: + apt_config.apply_apt_preferences(cfg, "preferencesfn") + + mockobj.assert_called_with("preferencesfn", expected_content) + def test_mirror(self): """test_mirror - Test defining a mirror""" pmir = "http://us.archive.ubuntu.com/ubuntu/" diff -Nru curtin-21.3/tests/unittests/test_block.py curtin-22.1/tests/unittests/test_block.py --- curtin-21.3/tests/unittests/test_block.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/unittests/test_block.py 2022-06-27 00:46:09.000000000 +0000 @@ -927,4 +927,12 @@ self.assertEqual([], self.m_load_json.call_args_list) +class TestResize(CiTestCase): + def test_basic(self): + resizers = 'curtin.commands.block_meta_v2.resizers' + values = {'a': 1, 'b': 2} + with mock.patch.dict(resizers, values, clear=True): + self.assertEqual({'a', 'b'}, block.get_resize_fstypes()) + + # vi: ts=4 expandtab syntax=python diff -Nru curtin-21.3/tests/unittests/test_commands_block_meta.py curtin-22.1/tests/unittests/test_commands_block_meta.py --- curtin-21.3/tests/unittests/test_commands_block_meta.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/unittests/test_commands_block_meta.py 2022-06-27 00:46:09.000000000 +0000 @@ -3,16 +3,25 @@ from argparse import Namespace from collections import OrderedDict import copy -from mock import patch, call +from mock import ( + call, + Mock, + patch, +) import os import random +import uuid from curtin.block import dasd -from curtin.commands import block_meta +from curtin.commands import block_meta, block_meta_v2 from curtin import paths, util from .helpers import CiTestCase +def random_uuid(): + return uuid.uuid4() + + class TestGetPathToStorageVolume(CiTestCase): def setUp(self): @@ -2572,6 +2581,8 @@ base = 'curtin.commands.block_meta.' self.add_patch(base + 'verify_size', 'm_verify_size') self.add_patch(base + 'verify_ptable_flag', 'm_verify_ptable_flag') + self.add_patch(base + 'os.path.realpath', 'm_realpath') + self.m_realpath.side_effect = lambda x: x self.info = { 'id': 'disk-sda-part-2', 'type': 'partition', @@ -2611,6 +2622,257 @@ self.assertEqual([], self.m_verify_ptable_flag.call_args_list) +class TestPartitionVerifySfdiskV2(CiTestCase): + + def setUp(self): + super(TestPartitionVerifySfdiskV2, self).setUp() + base = 'curtin.commands.block_meta_v2.' + self.add_patch(base + 'verify_size', 'm_verify_size') + self.add_patch(base + 'verify_ptable_flag', 'm_verify_ptable_flag') + self.add_patch(base + 'os.path.realpath', 'm_realpath') + self.m_realpath.side_effect = lambda x: x + self.info = { + 'id': 'disk-sda-part-2', + 'type': 'partition', + 'offset': '1GB', + 'device': 'sda', + 'number': 2, + 'size': '5GB', + 'flag': 'boot', + } + self.part_size = int(util.human2bytes(self.info['size'])) + self.devpath = self.random_string() + self.sfdisk_part_info = { + 'node': self.devpath, + 'start': (1 << 30) // 512, + } + self.storage_config = {self.info['id']: self.info} + self.label = self.random_string() + self.table = Mock() + self.table.sectors2bytes = lambda x: x * 512 + + def test_partition_verify_sfdisk(self): + block_meta_v2.partition_verify_sfdisk_v2(self.info, self.label, + self.sfdisk_part_info, + self.storage_config, + self.table) + self.assertEqual( + [call(self.devpath, self.part_size, self.sfdisk_part_info)], + self.m_verify_size.call_args_list) + self.assertEqual( + [call(self.devpath, self.info['flag'], self.label, + self.sfdisk_part_info)], + self.m_verify_ptable_flag.call_args_list) + + def test_partition_verify_no_moves(self): + self.info['preserve'] = True + self.info['resize'] = True + self.info['offset'] = '2GB' + with self.assertRaises(RuntimeError): + block_meta_v2.partition_verify_sfdisk_v2( + self.info, self.label, self.sfdisk_part_info, + self.storage_config, self.table) + + +class TestSfdiskV2(CiTestCase): + def test_gpt_basic(self): + table = block_meta_v2.GPTPartTable(512) + expected = '''\ +label: gpt +''' + self.assertEqual(expected, table.render()) + + def test_gpt_boot(self): + table = block_meta_v2.GPTPartTable(512) + table.add(dict(number=1, offset=1 << 20, size=9 << 20, flag='boot')) + expected = '''\ +label: gpt + +1: start=2048 size=18432 type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B''' + self.assertEqual(expected, table.render()) + + def test_gpt_boot_raw_type(self): + esp = 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B' + table = block_meta_v2.GPTPartTable(512) + table.add(dict(number=1, offset=1 << 20, size=9 << 20, + partition_type=esp)) + expected = '''\ +label: gpt + +1: start=2048 size=18432 type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B''' + self.assertEqual(expected, table.render()) + + def test_gpt_random_uuid(self): + ptype = str(random_uuid()).lower() + table = block_meta_v2.GPTPartTable(512) + table.add(dict(number=1, offset=1 << 20, size=9 << 20, + flag='boot', partition_type=ptype)) + expected = f'''\ +label: gpt + +1: start=2048 size=18432 type={ptype}''' + self.assertEqual(expected, table.render()) + + def test_dos_basic(self): + table = block_meta_v2.DOSPartTable(512) + expected = '''\ +label: dos +''' + self.assertEqual(expected, table.render()) + + def test_dos_boot(self): + table = block_meta_v2.DOSPartTable(512) + table.add(dict(number=1, offset=1 << 20, size=9 << 20, flag='boot')) + expected = '''\ +label: dos + +1: start=2048 size=18432 type=EF bootable''' + self.assertEqual(expected, table.render()) + + def test_dos_random_code(self): + ptype = hex(random.randint(0, 0xff))[2:] + table = block_meta_v2.DOSPartTable(512) + table.add(dict(number=1, offset=1 << 20, size=9 << 20, + flag='boot', partition_type=ptype)) + expected = f'''\ +label: dos + +1: start=2048 size=18432 type={ptype} bootable''' + self.assertEqual(expected, table.render()) + + +class TestPartitionNeedsResize(CiTestCase): + + def setUp(self): + super(TestPartitionNeedsResize, self).setUp() + base = 'curtin.commands.block_meta_v2.' + self.add_patch(base + 'os.path.realpath', 'm_realpath') + self.add_patch(base + '_get_volume_fstype', 'm_get_volume_fstype') + self.m_realpath.side_effect = lambda x: x + self.partition = { + 'id': 'disk-sda-part-2', + 'type': 'partition', + 'offset': '1GB', + 'device': 'sda', + 'number': 2, + 'size': '5GB', + 'flag': 'boot', + } + self.devpath = self.random_string() + self.sfdisk_part_info = { + 'node': self.devpath, + 'start': (1 << 30) // 512, + 'size': (1 << 30) // 512, + } + self.format = { + 'id': 'id-format', + 'type': 'format', + 'fstype': 'ext4', + 'volume': self.partition['id'], + } + self.storage_config = { + self.partition['id']: self.partition, + self.format['id']: self.format, + } + self.table = Mock() + self.table.sectors2bytes = lambda x: x * 512 + + def test_partition_resize_happy_path(self): + self.partition['preserve'] = True + self.partition['resize'] = True + self.format['preserve'] = True + self.format['fstype'] = 'ext4' + self.m_get_volume_fstype.return_value = 'ext4' + expected = { + 'fstype': 'ext4', + 'size': 5 << 30, + 'direction': 'up', + } + actual = block_meta_v2._prepare_resize( + self.storage_config, self.partition, self.table, + self.sfdisk_part_info) + self.assertEqual(expected, actual) + + def test_partition_resize_no_format_action(self): + self.partition['preserve'] = True + self.partition['resize'] = True + self.storage_config = {self.partition['id']: self.partition} + self.m_get_volume_fstype.return_value = 'ext4' + expected = { + 'fstype': 'ext4', + 'size': 5 << 30, + 'direction': 'up', + } + actual = block_meta_v2._prepare_resize( + self.storage_config, self.partition, self.table, + self.sfdisk_part_info) + self.assertEqual(expected, actual) + + def test_partition_resize_change_fs(self): + self.partition['preserve'] = True + self.partition['resize'] = True + self.format['preserve'] = True + self.format['fstype'] = 'ext3' + self.m_get_volume_fstype.return_value = 'ext4' + with self.assertRaises(RuntimeError): + block_meta_v2._prepare_resize(self.storage_config, self.partition, + self.table, self.sfdisk_part_info) + + def test_partition_resize_unsupported_fs(self): + self.partition['preserve'] = True + self.partition['resize'] = True + self.format['preserve'] = True + self.format['fstype'] = 'reiserfs' + self.m_get_volume_fstype.return_value = 'resierfs' + with self.assertRaises(RuntimeError): + block_meta_v2._prepare_resize(self.storage_config, self.partition, + self.table, self.sfdisk_part_info) + + def test_partition_resize_format_preserve_false(self): + # though the filesystem type is not supported for resize, it's ok + # because with format preserve=False, we're recreating anyhow + self.partition['preserve'] = True + self.partition['resize'] = True + self.format['preserve'] = False + self.format['fstype'] = 'reiserfs' + self.m_get_volume_fstype.return_value = 'reiserfs' + self.assertIsNone( + block_meta_v2._prepare_resize(self.storage_config, self.partition, + self.table, self.sfdisk_part_info)) + + def test_partition_resize_partition_preserve_false(self): + # not a resize - partition is recreated + self.partition['preserve'] = False + self.partition['resize'] = True + self.format['preserve'] = False + self.format['fstype'] = 'reiserfs' + self.m_get_volume_fstype.return_value = 'reiserfs' + self.assertIsNone( + block_meta_v2._prepare_resize(self.storage_config, self.partition, + self.table, self.sfdisk_part_info)) + + def test_partition_resize_equal_size(self): + # not a resize - the size is the same so leave it alone + self.partition['preserve'] = True + self.partition['resize'] = True + self.partition['size'] = '1GB' + self.format['preserve'] = True + self.m_get_volume_fstype.return_value = 'ext4' + self.assertIsNone( + block_meta_v2._prepare_resize(self.storage_config, self.partition, + self.table, self.sfdisk_part_info)) + + def test_partition_resize_unformatted(self): + # not a resize - an unformatted partition has nothing to preserve + self.partition['preserve'] = True + self.partition['resize'] = True + self.storage_config = {self.partition['id']: self.partition} + self.m_get_volume_fstype.return_value = '' + self.assertIsNone( + block_meta_v2._prepare_resize(self.storage_config, self.partition, + self.table, self.sfdisk_part_info)) + + class TestPartitionVerifyFdasd(CiTestCase): def setUp(self): diff -Nru curtin-21.3/tests/unittests/test_storage_config.py curtin-22.1/tests/unittests/test_storage_config.py --- curtin-21.3/tests/unittests/test_storage_config.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/unittests/test_storage_config.py 2022-06-27 00:46:09.000000000 +0000 @@ -7,7 +7,7 @@ from curtin.storage_config import (BcacheParser, BlockdevParser, DasdParser, DmcryptParser, FilesystemParser, LvmParser, RaidParser, MountParser, ZfsParser) -from curtin.storage_config import ptable_uuid_to_flag_entry +from curtin.storage_config import ptable_uuid_to_flag_entry, select_configs from curtin import util @@ -285,7 +285,7 @@ """ BlockdevParser skips invalid ID_WWN_* values. """ self.bdevp.blockdev_data['/dev/sda'] = { 'DEVTYPE': 'disk', - 'DEVNAME': 'sda', + 'DEVNAME': '/dev/sda', 'ID_SERIAL': 'Corsair_Force_GS_1785234921906', 'ID_SERIAL_SHORT': '1785234921906', 'ID_WWN': '0x0000000000000000', @@ -300,7 +300,7 @@ """ BlockdevParser skips invalid ID_SERIAL_* values. """ self.bdevp.blockdev_data['/dev/sda'] = { 'DEVTYPE': 'disk', - 'DEVNAME': 'sda', + 'DEVNAME': '/dev/sda', 'ID_SERIAL': ' ', 'ID_SERIAL_SHORT': 'My Serial is My PassPort', } @@ -345,10 +345,12 @@ 'id': 'partition-sda1', 'type': 'partition', 'device': 'disk-sda', + 'path': '/dev/sda1', 'number': 1, 'offset': 1048576, 'size': 499122176, 'flag': 'linux', + 'partition_type': '0fc63daf-8483-4772-8e79-3d69d8477de4', } self.assertDictEqual(expected_dict, self.bdevp.asdict(blockdev)) @@ -375,12 +377,12 @@ """ BlockdevParser ignores partition with zero start value.""" self.bdevp.blockdev_data['/dev/vda'] = { 'DEVTYPE': 'disk', - 'DEVNAME': 'vda', + 'DEVNAME': '/dev/vda', } test_value = { 'DEVTYPE': 'partition', 'MAJOR': "252", - 'DEVNAME': 'vda1', + 'DEVNAME': '/dev/vda1', "DEVPATH": "/devices/pci0000:00/0000:00:04.0/virtio0/block/vda/vda1", "ID_PART_ENTRY_TYPE": "0x0", @@ -390,8 +392,10 @@ 'id': 'partition-vda1', 'type': 'partition', 'device': 'disk-vda', + 'path': '/dev/vda1', 'number': 1, 'size': 784334848, + 'partition_type': '0x0', } self.assertDictEqual(expected_dict, self.bdevp.asdict(test_value)) @@ -403,7 +407,8 @@ # XXX: Parameterize me def test_blockdev_to_id_raises_valueerror_on_empty_devtype(self): - test_value = {'DEVTYPE': '', 'DEVNAME': 'bar', 'DEVPATH': 'foobar'} + test_value = {'DEVTYPE': '', 'DEVNAME': '/dev/bar', + 'DEVPATH': 'foobar'} with self.assertRaises(ValueError): self.bdevp.blockdev_to_id(test_value) @@ -415,7 +420,7 @@ # XXX: Parameterize me def test_blockdev_to_id_raises_valueerror_on_missing_devtype(self): - test_value = {'DEVNAME': 'bar', 'DEVPATH': 'foobar'} + test_value = {'DEVNAME': '/dev/bar', 'DEVPATH': 'foobar'} with self.assertRaises(ValueError): self.bdevp.blockdev_to_id(test_value) @@ -424,9 +429,10 @@ self.probe_data = _get_data('probert_storage_lvm.json') self.bdevp = BlockdevParser(self.probe_data) blockdev = self.bdevp.blockdev_data['/dev/vda2'] - expected_dict = { + base_expected_dict = { 'id': 'partition-vda2', 'type': 'partition', + 'path': '/dev/vda2', 'device': 'disk-vda', 'number': 2, 'offset': 3222274048, @@ -435,6 +441,8 @@ } for ext_part_entry in ['0xf', '0x5', '0x85', '0xc5']: blockdev['ID_PART_ENTRY_TYPE'] = ext_part_entry + expected_dict = base_expected_dict.copy() + expected_dict['partition_type'] = ext_part_entry self.assertDictEqual(expected_dict, self.bdevp.asdict(blockdev)) @@ -446,10 +454,12 @@ 'id': 'partition-vda5', 'type': 'partition', 'device': 'disk-vda', + 'path': '/dev/vda5', 'number': 5, 'offset': 3223322624, 'size': 2147483648, 'flag': 'logical', + 'partition_type': '0x83', } self.assertDictEqual(expected_dict, self.bdevp.asdict(blockdev)) @@ -463,10 +473,12 @@ 'id': 'partition-vdb1', 'type': 'partition', 'device': 'disk-vdb', + 'path': '/dev/vdb1', 'number': 1, 'offset': 1048576, 'size': 536870912, 'flag': 'boot', + 'partition_type': '0xb', } self.assertDictEqual(expected_dict, self.bdevp.asdict(blockdev)) @@ -480,10 +492,12 @@ 'id': 'partition-vda5', 'type': 'partition', 'device': 'disk-vda', + 'path': '/dev/vda5', 'number': 5, 'offset': 3223322624, 'size': 2147483648, 'flag': 'boot', + 'partition_type': '0x83', } self.assertDictEqual(expected_dict, self.bdevp.asdict(blockdev)) @@ -536,6 +550,7 @@ blockdev = self.bdevp.blockdev_data['/dev/dm-2'] expected_dict = { 'device': 'mpath-disk-mpatha', + 'path': '/dev/dm-2', 'flag': 'linux', 'id': 'mpath-partition-mpatha-part2', 'multipath': 'mpatha', @@ -543,6 +558,7 @@ 'offset': 2097152, 'size': 10734272512, 'type': 'partition', + 'partition_type': '0fc63daf-8483-4772-8e79-3d69d8477de4', } self.assertDictEqual(expected_dict, self.bdevp.asdict(blockdev)) @@ -970,7 +986,7 @@ """ verify live-iso extracted storage-config finds target disk. """ extracted = storage_config.extract_storage_config(self.probe_data) self.assertEqual( - {'storage': {'version': 1, + {'storage': {'version': 2, 'config': [{'id': 'disk-sda', 'path': '/dev/sda', 'serial': 'QEMU_HARDDISK_QM00001', 'type': 'disk'}]}}, extracted) @@ -985,13 +1001,13 @@ if missing_key != 'blockdev': self.assertEqual( {'storage': - {'version': 1, + {'version': 2, 'config': [{'id': 'disk-sda', 'path': '/dev/sda', 'serial': 'QEMU_HARDDISK_QM00001', 'type': 'disk'}]}}, extracted) else: # empty config without blockdev data - self.assertEqual({'storage': {'config': [], 'version': 1}}, + self.assertEqual({'storage': {'config': [], 'version': 2}}, extracted) @skipUnlessJsonSchema() @@ -1010,10 +1026,17 @@ 'raidlevel': 'raid1', 'name': 'md1', 'devices': ['partition-vdb1', 'partition-vdc1'], 'spare_devices': []}, raids[0]) - self.assertEqual({'id': 'raid-md1p1', 'type': 'partition', - 'size': 4285530112, 'flag': 'linux', 'number': 1, - 'device': 'raid-md1', 'offset': 1048576}, - raid_partitions[0]) + self.assertEqual({ + 'id': 'raid-md1p1', + 'type': 'partition', + 'path': '/dev/md1p1', + 'size': 4285530112, + 'flag': 'linux', + 'number': 1, + 'partition_type': '0fc63daf-8483-4772-8e79-3d69d8477de4', + 'device': 'raid-md1', + 'offset': 1048576}, + raid_partitions[0]) @skipUnlessJsonSchema() def test_find_extended_partition(self): @@ -1108,4 +1131,26 @@ self.assertEqual(expected_dict, bitlocker[0]) +class TestSelectConfigs(CiTestCase): + def test_basic(self): + id0 = {'a': 1, 'b': 2} + id1 = {'a': 1, 'c': 3} + sc = {'id0': id0, 'id1': id1} + + self.assertEqual([id0, id1], select_configs(sc, a=1)) + + def test_not_found(self): + id0 = {'a': 1, 'b': 2} + id1 = {'a': 1, 'c': 3} + sc = {'id0': id0, 'id1': id1} + + self.assertEqual([], select_configs(sc, a=4)) + + def test_multi_criteria(self): + id0 = {'a': 1, 'b': 2} + id1 = {'a': 1, 'c': 3} + sc = {'id0': id0, 'id1': id1} + + self.assertEqual([id0], select_configs(sc, a=1, b=2)) + # vi: ts=4 expandtab syntax=python diff -Nru curtin-21.3/tests/vmtests/__init__.py curtin-22.1/tests/vmtests/__init__.py --- curtin-21.3/tests/vmtests/__init__.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/__init__.py 2022-06-27 00:46:09.000000000 +0000 @@ -1930,29 +1930,6 @@ self.assertIn(kpackage, self.debian_packages) @skip_if_flag('expected_failure') - def test_clear_holders_ran(self): - """ Test curtin install runs block-meta/clear-holders. """ - if not self.has_storage_config(): - raise SkipTest("This test does not use storage config.") - - install_logfile = 'root/curtin-install.log' - self.output_files_exist([install_logfile]) - install_log = self.load_collect_file(install_logfile) - - # validate block-meta called clear-holders at least once - # We match both 'start' and 'finish' strings, so for each - # call we'll have 2 matches. - clear_holders_re = 'cmd-install/.*cmd-block-meta/clear-holders' - events = re.findall(clear_holders_re, install_log) - print('Matched clear-holder events:\n%s' % events) - self.assertGreaterEqual(len(events), 2) - - # dirty_disks mode runs an early block-meta command which - # also runs clear-holders - if self.dirty_disks is True: - self.assertGreaterEqual(len(events), 4) - - @skip_if_flag('expected_failure') def test_kernel_img_conf(self): """ Test curtin install kernel-img.conf correctly. """ if self.target_distro != 'ubuntu': @@ -2058,17 +2035,6 @@ return swaps - # we don't yet have a skip_by_date on specific releases - if is_devel_release(self.target_release): - name = "test_swaps_used" - bug = "1894910" - fixby = "2020-10-15" - removeby = "2020-11-01" - raise SkipTest( - "skip_by_date({name}) LP: #{bug} " - "fixby={fixby} removeby={removeby}: ".format( - name=name, bug=bug, fixby=fixby, removeby=removeby)) - expected_swaps = find_fstab_swaps() proc_swaps = self.load_collect_file("proc-swaps") for swap in expected_swaps: @@ -2573,7 +2539,6 @@ 'size': '8M', 'flag': 'prep', 'guid': '9e1a2d38-c612-4316-aa26-8b49521e5a8b', - 'offset': '1M', 'wipe': 'zero', 'grub_device': True, 'device': device} diff -Nru curtin-21.3/tests/vmtests/releases.py curtin-22.1/tests/vmtests/releases.py --- curtin-21.3/tests/vmtests/releases.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/releases.py 2022-06-27 00:46:09.000000000 +0000 @@ -7,7 +7,7 @@ repo = "maas-daily" arch = get_platform_arch() target_arch = arch - mem = "1024" + mem = "2048" class _UbuntuBase(_ReleaseBase): @@ -72,22 +72,6 @@ release = "focal" # release for target target_release = "ubuntu-core-20" - mem = "2048" - - -class _Centos66FromXenialBase(_CentosFromUbuntuBase): - release = "xenial" - target_release = "centos66" - - -class _Centos66FromBionicBase(_CentosFromUbuntuBase): - release = "bionic" - target_release = "centos66" - - -class _Centos66FromFocalBase(_CentosFromUbuntuBase): - release = "focal" - target_release = "centos66" class _PreciseBase(_UbuntuBase): @@ -148,7 +132,6 @@ class _BionicBase(_UbuntuBase): release = "bionic" target_release = "bionic" - mem = "2048" if _UbuntuBase.arch == "arm64": subarch = "ga-18.04" @@ -164,7 +147,6 @@ release = "disco" target_release = "disco" # squashfs is over 300MB, need more ram - mem = "2048" if _UbuntuBase.arch == "arm64": subarch = "ga-19.04" @@ -172,7 +154,6 @@ class _EoanBase(_UbuntuBase): release = "eoan" target_release = "eoan" - mem = "2048" if _UbuntuBase.arch == "arm64": subarch = "ga-19.10" @@ -180,7 +161,6 @@ class _FocalBase(_UbuntuBase): release = "focal" target_release = "focal" - mem = "2048" if _UbuntuBase.arch == "arm64": subarch = "ga-20.04" @@ -188,7 +168,6 @@ class _HirsuteBase(_UbuntuBase): release = "hirsute" target_release = "hirsute" - mem = "2048" if _UbuntuBase.arch == "arm64": subarch = "ga-21.04" @@ -196,7 +175,6 @@ class _ImpishBase(_UbuntuBase): release = "impish" target_release = "impish" - mem = "2048" if _UbuntuBase.arch == "arm64": subarch = "ga-21.10" @@ -225,11 +203,8 @@ class _CentosReleases(object): centos70_xenial = _Centos70FromXenialBase - centos66_xenial = _Centos66FromXenialBase centos70_bionic = _Centos70FromBionicBase - centos66_bionic = _Centos66FromBionicBase centos70_focal = _Centos70FromFocalBase - centos66_focal = _Centos66FromFocalBase class _UbuntuCoreReleases(object): diff -Nru curtin-21.3/tests/vmtests/test_basic.py curtin-22.1/tests/vmtests/test_basic.py --- curtin-21.3/tests/vmtests/test_basic.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_basic.py 2022-06-27 00:46:09.000000000 +0000 @@ -41,10 +41,6 @@ f="btrfs_uuid_diskc" if command -v btrfs-debug-tree >/dev/null; then btrfs-debug-tree -r $dev | awk '/^uuid/ {print $2}' | grep "-" - # btrfs-debug-tree fails in centos66, use btrfs-show instead - if [ "$?" != "0" ]; then - btrfs-show $dev | awk '/uuid/ {print $4}' - fi else btrfs inspect-internal dump-super $dev | awk '/^dev_item.fsid/ {print $2}' @@ -61,9 +57,6 @@ """)] def _test_ptable(self, blkid_output, expected): - if self.target_release == "centos66": - raise SkipTest("No PTTYPE blkid output on Centos66") - if not blkid_output: raise RuntimeError('_test_ptable requires blkid output file') @@ -100,8 +93,6 @@ self.assertEqual(kname_uuid, btrfs_uuid) def _test_partition_is_prep(self, info_file): - if self.target_release == "centos66": - raise SkipTest("Cannot detect PReP partitions in Centos66") udev_info = self.load_collect_file(info_file).rstrip() if not udev_info: raise ValueError('Empty udev_info collect file') @@ -132,10 +123,7 @@ def test_partition_numbers(self): # pnum_disk should have partitions 1 2, and 10 - if self.target_release != 'centos66': - disk = self._dname_to_kname('pnum_disk') - else: - disk = self._serial_to_kname('disk-d') + disk = self._dname_to_kname('pnum_disk') expected = [disk + s for s in ["", "1", "2", "10"]] self._test_partition_numbers(disk, expected) @@ -220,19 +208,6 @@ __test__ = True -class Centos66XenialTestBasic(centos_relbase.centos66_xenial, - CentosTestBasicAbs): - __test__ = True - - -class Centos66BionicTestBasic(centos_relbase.centos66_bionic, - CentosTestBasicAbs): - # Centos66 cannot handle ext4 defaults in Bionic (64bit,meta_csum) - # this conf defaults to ext3 - conf_file = "examples/tests/centos6_basic.yaml" - __test__ = True - - class XenialGAi386TestBasic(relbase.xenial_ga, TestBasicAbs): __test__ = True arch_skip = ["arm64", "ppc64el", "s390x"] diff -Nru curtin-21.3/tests/vmtests/test_network_alias.py curtin-22.1/tests/vmtests/test_network_alias.py --- curtin-21.3/tests/vmtests/test_network_alias.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_network_alias.py 2022-06-27 00:46:09.000000000 +0000 @@ -34,11 +34,6 @@ pass -class Centos66TestNetworkAlias(centos_relbase.centos66_xenial, - CentosTestNetworkAliasAbs): - __test__ = True - - class Centos70TestNetworkAlias(centos_relbase.centos70_xenial, CentosTestNetworkAliasAbs): __test__ = True diff -Nru curtin-21.3/tests/vmtests/test_network_bonding.py curtin-22.1/tests/vmtests/test_network_bonding.py --- curtin-21.3/tests/vmtests/test_network_bonding.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_network_bonding.py 2022-06-27 00:46:09.000000000 +0000 @@ -69,11 +69,6 @@ __test__ = True -class Centos66TestNetworkBonding(centos_relbase.centos66_xenial, - CentosTestNetworkBondingAbs): - __test__ = True - - class Centos70TestNetworkBonding(centos_relbase.centos70_xenial, CentosTestNetworkBondingAbs): __test__ = True diff -Nru curtin-21.3/tests/vmtests/test_network_bridging.py curtin-22.1/tests/vmtests/test_network_bridging.py --- curtin-21.3/tests/vmtests/test_network_bridging.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_network_bridging.py 2022-06-27 00:46:09.000000000 +0000 @@ -41,8 +41,6 @@ # attrs we cannot validate release_to_bridge_params_uncheckable = { - 'centos66': ['bridge_fd', 'bridge_hello', 'bridge_hw', 'bridge_maxage', - 'bridge_pathcost', 'bridge_portprio'], 'centos70': ['bridge_fd', 'bridge_hello', 'bridge_hw', 'bridge_maxage', 'bridge_pathcost', 'bridge_portprio'], 'xenial': ['bridge_ageing'], @@ -220,11 +218,6 @@ self.assertTrue('bridge' in status) -class Centos66TestBridgeNetwork(centos_relbase.centos66_xenial, - CentosTestBridgeNetworkAbs): - __test__ = True - - class Centos70TestBridgeNetwork(centos_relbase.centos70_xenial, CentosTestBridgeNetworkAbs): __test__ = True diff -Nru curtin-21.3/tests/vmtests/test_network_ipv6.py curtin-22.1/tests/vmtests/test_network_ipv6.py --- curtin-21.3/tests/vmtests/test_network_ipv6.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_network_ipv6.py 2022-06-27 00:46:09.000000000 +0000 @@ -65,11 +65,6 @@ __test__ = True -class Centos66TestNetworkIPV6(centos_relbase.centos66_xenial, - CentosTestNetworkIPV6Abs): - __test__ = True - - class Centos70TestNetworkIPV6(centos_relbase.centos70_xenial, CentosTestNetworkIPV6Abs): __test__ = True diff -Nru curtin-21.3/tests/vmtests/test_network_ipv6_static.py curtin-22.1/tests/vmtests/test_network_ipv6_static.py --- curtin-21.3/tests/vmtests/test_network_ipv6_static.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_network_ipv6_static.py 2022-06-27 00:46:09.000000000 +0000 @@ -35,11 +35,6 @@ __test__ = True -class Centos66TestNetworkIPV6Static(centos_relbase.centos66_xenial, - CentosTestNetworkIPV6StaticAbs): - __test__ = True - - class Centos70TestNetworkIPV6Static(centos_relbase.centos70_xenial, CentosTestNetworkIPV6StaticAbs): __test__ = True diff -Nru curtin-21.3/tests/vmtests/test_network_ipv6_vlan.py curtin-22.1/tests/vmtests/test_network_ipv6_vlan.py --- curtin-21.3/tests/vmtests/test_network_ipv6_vlan.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_network_ipv6_vlan.py 2022-06-27 00:46:09.000000000 +0000 @@ -34,11 +34,6 @@ __test__ = True -class Centos66TestNetworkIPV6Vlan(centos_relbase.centos66_xenial, - CentosTestNetworkIPV6VlanAbs): - __test__ = True - - class Centos70TestNetworkIPV6Vlan(centos_relbase.centos70_xenial, CentosTestNetworkIPV6VlanAbs): __test__ = True diff -Nru curtin-21.3/tests/vmtests/test_network_mtu.py curtin-22.1/tests/vmtests/test_network_mtu.py --- curtin-21.3/tests/vmtests/test_network_mtu.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_network_mtu.py 2022-06-27 00:46:09.000000000 +0000 @@ -201,11 +201,6 @@ __test__ = True -class Centos66TestNetworkMtu(centos_relbase.centos66_xenial, - CentosTestNetworkMtuAbs): - __test__ = True - - class Centos70TestNetworkMtu(centos_relbase.centos70_xenial, CentosTestNetworkMtuAbs): __test__ = True diff -Nru curtin-21.3/tests/vmtests/test_network.py curtin-22.1/tests/vmtests/test_network.py --- curtin-21.3/tests/vmtests/test_network.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_network.py 2022-06-27 00:46:09.000000000 +0000 @@ -486,11 +486,6 @@ __test__ = True -class Centos66TestNetworkBasic(centos_relbase.centos66_xenial, - CentosTestNetworkBasicAbs): - __test__ = True - - class Centos70TestNetworkBasic(centos_relbase.centos70_xenial, CentosTestNetworkBasicAbs): __test__ = True diff -Nru curtin-21.3/tests/vmtests/test_network_static.py curtin-22.1/tests/vmtests/test_network_static.py --- curtin-21.3/tests/vmtests/test_network_static.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_network_static.py 2022-06-27 00:46:09.000000000 +0000 @@ -40,11 +40,6 @@ __test__ = True -class Centos66TestNetworkStatic(centos_relbase.centos66_xenial, - CentosTestNetworkStaticAbs): - __test__ = True - - class Centos70TestNetworkStatic(centos_relbase.centos70_xenial, CentosTestNetworkStaticAbs): __test__ = True diff -Nru curtin-21.3/tests/vmtests/test_network_static_routes.py curtin-22.1/tests/vmtests/test_network_static_routes.py --- curtin-21.3/tests/vmtests/test_network_static_routes.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_network_static_routes.py 2022-06-27 00:46:09.000000000 +0000 @@ -43,11 +43,6 @@ __test__ = True -class Centos66TestNetworkStaticRoutes(centos_relbase.centos66_xenial, - CentosTestNetworkStaticRoutesAbs): - __test__ = False - - class Centos70TestNetworkStaticRoutes(centos_relbase.centos70_xenial, CentosTestNetworkStaticRoutesAbs): __test__ = False diff -Nru curtin-21.3/tests/vmtests/test_network_vlan.py curtin-22.1/tests/vmtests/test_network_vlan.py --- curtin-21.3/tests/vmtests/test_network_vlan.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_network_vlan.py 2022-06-27 00:46:09.000000000 +0000 @@ -88,11 +88,6 @@ __test__ = True -class Centos66TestNetworkVlan(centos_relbase.centos66_xenial, - CentosTestNetworkVlanAbs): - __test__ = True - - class Centos70TestNetworkVlan(centos_relbase.centos70_xenial, CentosTestNetworkVlanAbs): __test__ = True diff -Nru curtin-21.3/tests/vmtests/test_preserve_raid.py curtin-22.1/tests/vmtests/test_preserve_raid.py --- curtin-21.3/tests/vmtests/test_preserve_raid.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_preserve_raid.py 2022-06-27 00:46:09.000000000 +0000 @@ -56,6 +56,9 @@ relbase.bionic, TestPartitionExistingRAID): __test__ = True + def test_correct_ptype(self): + self.skipTest("lsblk on bionic does not support PTTYPE") + class FocalTestPartitionExistingRAID( relbase.focal, TestPartitionExistingRAID): diff -Nru curtin-21.3/tests/vmtests/test_simple.py curtin-22.1/tests/vmtests/test_simple.py --- curtin-21.3/tests/vmtests/test_simple.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_simple.py 2022-06-27 00:46:09.000000000 +0000 @@ -29,15 +29,6 @@ __test__ = True -class Centos66XenialTestSimple(centos_relbase.centos66_xenial, TestSimple): - __test__ = True - - -class Centos66BionicTestSimple(centos_relbase.centos66_bionic, TestSimple): - __test__ = False - # LP: #1775424 Centos66 fails with Bionic Ephemeral ext4 features - - class XenialTestSimple(relbase.xenial, TestSimple): __test__ = True diff -Nru curtin-21.3/tests/vmtests/test_uefi_basic.py curtin-22.1/tests/vmtests/test_uefi_basic.py --- curtin-21.3/tests/vmtests/test_uefi_basic.py 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tests/vmtests/test_uefi_basic.py 2022-06-27 00:46:09.000000000 +0000 @@ -17,7 +17,7 @@ disk_to_check = [('main_disk', 1), ('main_disk', 2)] extra_collect_scripts = [textwrap.dedent(""" cd OUTPUT_COLLECT_D - ls /sys/firmware/efi/ | cat >ls_sys_firmware_efi + test -d /sys/firmware/efi ; echo $? >is_efi cp /sys/class/block/vda/queue/logical_block_size vda_lbs cp /sys/class/block/vda/queue/physical_block_size vda_pbs blockdev --getsz /dev/vda | cat >vda_blockdev_getsz @@ -28,24 +28,10 @@ exit 0 """)] - def test_sys_firmware_efi(self): - self.output_files_exist(["ls_sys_firmware_efi"]) - sys_efi_possible = [ - 'config_table', - 'efivars', - 'fw_platform_size', - 'fw_vendor', - 'runtime', - 'runtime-map', - 'systab', - 'vars', - ] - efi_lines = self.load_collect_file( - "ls_sys_firmware_efi").strip().split('\n') - - # sys/firmware/efi contents differ based on kernel and configuration - for efi_line in efi_lines: - self.assertIn(efi_line, sys_efi_possible) + def test_is_efi(self): + self.output_files_exist(["is_efi"]) + efi_lines = self.load_collect_file("is_efi").strip().split('\n') + self.assertEqual(['0'], efi_lines) def test_disk_block_sizes(self): """ Test disk logical and physical block size are match diff -Nru curtin-21.3/tools/build-deb curtin-22.1/tools/build-deb --- curtin-21.3/tools/build-deb 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tools/build-deb 2022-06-27 00:46:09.000000000 +0000 @@ -1,7 +1,7 @@ #!/bin/sh # This file is part of curtin. See LICENSE file for copyright and license info. -set -e +set -eu sourcename="curtin" TEMP_D="" @@ -13,7 +13,7 @@ [ -z "$TEMP_D" ] || rm -Rf "$TEMP_D" } -if [ "$1" = "-h" -o "$1" = "--help" ]; then +if [ "${1:-}" = "-h" -o "${1:-}" = "--help" ]; then cat <&2 - dir=$d - else - echo "did not find a directory created by make-tarball. sorry." 1>&2 - exit - fi + echo "did not find a directory created by make-tarball. sorry." 1>&2 + exit fi cd "$dir" || fail "failed cd $dir" -# move files ending in .trunk to name without .trunk -# ie, this copies debian/changelog.trunk to debian/changelog -for f in debian/*.trunk; do - mv "$f" "${f%.trunk}" -done - -# first line of debian/changelog looks like -# curtin () UNRELEASED; urgency=low -# fix the version and UNRELEASED -sed -i -e "1s,([^)]*),(${clogver_new})," \ - -e "1s,UNRELEASED,${RELEASE}," debian/changelog || - fail "failed to write debian/changelog" +dch --create --package curtin --newversion "$clogver_new" \ + --distribution "$RELEASE" "Development release" debuild "$@" || fail "debuild failed" cd "$TEMP_D" diff -Nru curtin-21.3/tox.ini curtin-22.1/tox.ini --- curtin-21.3/tox.ini 2021-12-06 22:08:54.000000000 +0000 +++ curtin-22.1/tox.ini 2022-06-27 00:46:09.000000000 +0000 @@ -3,14 +3,10 @@ skipsdist = True envlist = py3-flake8, - py27, py3, py3-pyflakes, py3-pylint, - py27-pylint, - trusty-py27, - block-schema, - xenial-py3 + block-schema [tox:jenkins] downloadcache = ~/cache/pip @@ -31,23 +27,6 @@ basepython = python3 sitepackages = true -[testenv:py27] -basepython = python2.7 -sitepackages = true -# https://github.com/pypa/setuptools/issues/1963 -deps = {[testenv]deps} - setuptools<45 - -# tox uses '--pre' by default to pip install. We don't want that, and -# 'pip_pre=False' isn't available until tox version 1.9. -install_command = pip install {opts} {packages} - -[testenv:py2-flake8] -basepython = python2 -deps = {[testenv]deps} - flake8 -commands = {envpython} -m flake8 {posargs:curtin} - [testenv:py3-flake8] basepython = python3 deps = {[testenv]deps} @@ -64,19 +43,10 @@ basepython = python3 sitepackages = true deps = {[testenv]deps} - pylint==2.6.0 + pylint==2.12.2 git+https://git.launchpad.net/simplestreams commands = {envpython} -m pylint --errors-only {posargs:curtin tests/vmtests} -[testenv:py27-pylint] -# set basepython because tox 1.6 (trusty) does not support generated environments -basepython = python2.7 -sitepackages = true -deps = {[testenv]deps} - {[testenv:py27]deps} - pylint==1.8.1 -commands = {envpython} -m pylint --errors-only {posargs:curtin} - [testenv:docs] deps = {[testenv]deps} sphinx @@ -107,15 +77,6 @@ commands = {toxinidir}/tools/run-pyflakes3 {posargs} -[testenv:trusty-py27] -deps = {[testenv:trusty]deps} - setuptools<45 - -basepython = python2.7 -sitepackages = true -commands = {envpython} {toxinidir}/tools/noproxy {envpython} -m nose \ - {posargs:tests/unittests} - [testenv:trusty-py3] deps = {[testenv:trusty]deps} basepython = python3 @@ -129,13 +90,6 @@ pyyaml==3.11 oauthlib==1.0.3 -[testenv:xenial-py27] -basepython = python27 -deps = {[testenv:xenial]deps} - {[testenv:py27]deps} -commands = {envpython} {toxinidir}/tools/noproxy {envpython} -m nose \ - {posargs:tests/unittests} - [testenv:xenial-py3] basepython = python3 sitepackages = true