From a59518ef4d33572fc734cec399bac78a2d4debf9 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Aug 08 2022 14:32:58 +0000 Subject: [PATCH 1/7] comps-sync.py: Minor Python lint fixes --- diff --git a/comps-sync.py b/comps-sync.py index aaabd93..dc94db3 100755 --- a/comps-sync.py +++ b/comps-sync.py @@ -1,27 +1,28 @@ #!/usr/bin/python3 # Usage: ./comps-sync.py /path/to/comps-f37.xml.in # -# Can both remove packages from the manifest -# which are not mentioned in comps, and add packages from -# comps. +# Can both remove packages from the manifest which are not mentioned in comps, +# and add packages from comps. -import os, sys, subprocess, argparse, shlex, json, yaml, re +import argparse +import re +import sys +import yaml import libcomps def fatal(msg): - print >>sys.stderr, msg + print(msg, file = sys.stderr) sys.exit(1) def format_pkgtype(n): if n == libcomps.PACKAGE_TYPE_DEFAULT: return 'default' - elif n == libcomps.PACKAGE_TYPE_MANDATORY: + if n == libcomps.PACKAGE_TYPE_MANDATORY: return 'mandatory' - else: - assert False + assert False def write_manifest(fpath, pkgs, include=None): - with open(fpath, 'w') as f: + with open(fpath, 'w', encoding='UTF-8') as f: f.write("# DO NOT EDIT! This content is generated from comps-sync.py\n") if include is not None: f.write("include: {}\n".format(include)) @@ -39,11 +40,11 @@ args = parser.parse_args() print("Syncing packages common to all desktops:") base_pkgs_path = 'fedora-common-ostree-pkgs.yaml' -with open(base_pkgs_path) as f: +with open(base_pkgs_path, encoding='UTF-8') as f: manifest = yaml.safe_load(f) manifest_packages = set(manifest['packages']) -with open('comps-sync-exclude-list.yml') as f: +with open('comps-sync-exclude-list.yml', encoding='UTF-8') as f: doc = yaml.safe_load(f) comps_exclude_list = doc['exclude_list'] comps_include_list = doc['include_list'] @@ -137,11 +138,12 @@ for desktop in [ 'gnome-desktop', 'kde-desktop', 'xfce-desktop', print("Syncing packages for {}:".format(desktop)) manifest_path = '{}-pkgs.yaml'.format(desktop) - with open(manifest_path) as f: + with open(manifest_path, encoding='UTF-8') as f: manifest = yaml.safe_load(f) manifest_packages = set(manifest['packages']) - # Filter packages in the comps desktop group using the exclude_list + # Filter packages in the comps groups associated with a given desktop using + # the per group exclude_list comps_group_pkgs = set() comps_group_pkgs = set() for pkg in comps.groups_match(id=desktop)[0].packages: pkgname = pkg.name From 72a6738957333932363ee00962c57a5f8e4e6d16 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: Aug 08 2022 14:32:59 +0000 Subject: [PATCH 2/7] comps-sync.py: Support multiple comps groups for a desktop Enable fetching packages from multiple comps groups for each desktop. This is currently unused but this removes the constraints around having all base packages for a given desktop in a single comps group. This could also be used to get packages from other groups such as the input-methods group for example. --- diff --git a/comps-sync.py b/comps-sync.py index dc94db3..6bb7514 100755 --- a/comps-sync.py +++ b/comps-sync.py @@ -131,13 +131,23 @@ else: if (n_manifest_new > 0 or n_comps_new > 0) and args.save: write_manifest(base_pkgs_path, manifest_packages) +# List of comps groups used for each desktop +desktops_comps_groups = { + "gnome": ["gnome-desktop"], + "kde": ["kde-desktop"], + "xfce": ["xfce-desktop"], + "lxqt": ["lxqt-desktop"], + "deepin": ["deepin-desktop"], + "pantheon": ["pantheon-desktop"], + "mate": ["mate-desktop"] +} + # Generate treefiles for all desktops -for desktop in [ 'gnome-desktop', 'kde-desktop', 'xfce-desktop', - 'lxqt-desktop', 'deepin-desktop', 'pantheon-desktop', 'mate-desktop']: +for desktop, groups in desktops_comps_groups.items(): print() print("Syncing packages for {}:".format(desktop)) - manifest_path = '{}-pkgs.yaml'.format(desktop) + manifest_path = '{}-desktop-pkgs.yaml'.format(desktop) with open(manifest_path, encoding='UTF-8') as f: manifest = yaml.safe_load(f) manifest_packages = set(manifest['packages']) @@ -145,12 +155,15 @@ for desktop in [ 'gnome-desktop', 'kde-desktop', 'xfce-desktop', # Filter packages in the comps groups associated with a given desktop using # the per group exclude_list comps_group_pkgs = set() comps_group_pkgs = set() - for pkg in comps.groups_match(id=desktop)[0].packages: - pkgname = pkg.name - exclude_list = comps_desktop_exclude_list.get(desktop, set()) - if pkgname in exclude_list or is_exclude_listed(pkgname): - continue - comps_group_pkgs.add(pkg.name) + for group in groups: + for pkg in comps.groups_match(id=group)[0].packages: + pkgname = pkg.name + exclude_list = comps_desktop_exclude_list.get(group, set()) + if exclude_list is None: + exclude_list = set() + if pkgname in exclude_list or is_exclude_listed(pkgname): + continue + comps_group_pkgs.add(pkg.name) # Look for packages in the manifest but not in comps group comps_unknown = set() From d37f9f60ff76bc6c66fb1d5296cf4f5766159995 Mon Sep 17 00:00:00 2001 From: Aleksei Bavshin Date: Aug 08 2022 14:33:00 +0000 Subject: [PATCH 3/7] comps-sync.py: move desktop list to comps-sync-desktop-list.yml Move desktop options - lists of groups and excluded packages - to a new configuration file. The change completes the split of comps-sync logic and configuration and allows having all desktop-specific setup in a single place. Apply group exclude lists to a groups listed in a desktop environment (previously these were applied only to a workstation-product-environment groups). This ensures that we could safely move some groups from fedora-common to individual desktops and continue using common `exclude_list`. --- diff --git a/comps-sync-desktop-list.yml b/comps-sync-desktop-list.yml new file mode 100644 index 0000000..d970d0d --- /dev/null +++ b/comps-sync-desktop-list.yml @@ -0,0 +1,169 @@ +--- +# Desktop environment specific configurations +# +# Keys: +# groups list of groups to be synced +# exclude-packages filter packages collected from the groups before +# writing {name}-desktop-pkgs.yaml + +gnome: + groups: + - gnome-desktop + exclude-packages: + # Non-critical apps -> Flatpak + - baobab + - cheese + - eog + - evince + - evince-djvu + - evince-nautilus + - file-roller + - file-roller-nautilus + - gnome-boxes + - gnome-calculator + - gnome-calendar + - gnome-characters + - gnome-clocks + - gnome-connections + - gnome-contacts + - gnome-documents + - gnome-font-viewer + - gnome-logs + - gnome-maps + - gnome-photos + - gnome-screenshot + - gnome-text-editor + - gnome-weather + - jwhois + - rdist + - sane-backends-drivers-scanners + - symlinks + - simple-scan + - sushi + - tcpdump + - telnet + - totem + - totem-nautilus + - traceroute + +kde: + groups: + - kde-desktop + exclude-packages: + # Incompatible with ostree for various reasons + - abrt-desktop + - cups-pk-helper + - dnfdragora + - plasma-pk-updates + # Non-critical apps -> Flatpak + - akregator + - cagibi + - elisa-player + - gwenview + - kaddressbook + - kamera + - kcalc + - kcharselect + - kcolorchooser + - kdnssd + - kf5-kipi-plugins + - kfind + - kget + - kgpg + - kmail + - kmouth + - knode + - konqueror + - kontact + - korganizer + - kruler + - ksshaskpass + - kwrite + - libreoffice-kde + - okular + # Misc + - adwaita-gtk2-theme + - colord-kde + - keditbookmarks + - kwebkitpart + - plasma-nm-l2tp + - plasma-nm-openswan + - plasma-nm-pptp + +xfce: + groups: + - xfce-desktop + exclude-packages: + # Incompatible with ostree for various reasons + - abrt-desktop + - dnfdragora-updater + # Non-critical apps -> Flatpak + - fros-recordmydesktop + - tumbler + # Misc + - alsa-utils + - firewall-config + - openssh-askpass + - vim-enhanced + # Remove uncommon NetworkManager plugins + - NetworkManager-fortisslvpn-gnome + - NetworkManager-iodine-gnome + - NetworkManager-l2tp-gnome + - NetworkManager-libreswan-gnome + - NetworkManager-sstp-gnome + - NetworkManager-strongswan-gnome + +lxqt: + groups: + - lxqt-desktop + exclude-packages: + # Incompatible with ostree for various reasons + - dnfdragora-updater + +deepin: + groups: + - deepin-desktop + exclude-packages: + # Incompatible with ostree for various reasons + - dnfdragora-updater + +pantheon: + groups: + - pantheon-desktop + +mate: + groups: + - mate-desktop + exclude-packages: + # Incompatible with ostree for various reasons + - abrt-desktop + - abrt-java-connector + - dnfdragora-updater + # Non-critical apps -> Flatpak + - blivet-gui + - filezilla + - gnome-disk-utility + - gnome-logs + - gnote + - gparted + - hexchat + - p7zip + - p7zip-plugins + - simple-scan + - thunderbird + - transmission-gtk + - vim-enhanced + - xfburn + - yelp + # Non critical NetworkManager plugins + - NetworkManager-bluetooth + - NetworkManager-iodine-gnome + - NetworkManager-l2tp-gnome + - NetworkManager-libreswan-gnome + - NetworkManager-ovs + - NetworkManager-sstp-gnome + - NetworkManager-strongswan-gnome + - NetworkManager-team + - NetworkManager-wifi + # Already in the common set + - wireplumber diff --git a/comps-sync-exclude-list.yml b/comps-sync-exclude-list.yml index f2e053b..428b18d 100644 --- a/comps-sync-exclude-list.yml +++ b/comps-sync-exclude-list.yml @@ -128,140 +128,3 @@ exclude_list: - cups-pk-helper # For now... - ghostscript - -# Desktop environment specific exclude lists -desktop_exclude_list: - gnome-desktop: - # Non-critical apps -> Flatpak - - baobab - - cheese - - eog - - evince - - evince-djvu - - evince-nautilus - - file-roller - - file-roller-nautilus - - gnome-boxes - - gnome-calculator - - gnome-calendar - - gnome-characters - - gnome-clocks - - gnome-connections - - gnome-contacts - - gnome-documents - - gnome-font-viewer - - gnome-logs - - gnome-maps - - gnome-photos - - gnome-screenshot - - gnome-text-editor - - gnome-weather - - jwhois - - rdist - - sane-backends-drivers-scanners - - symlinks - - simple-scan - - sushi - - tcpdump - - telnet - - totem - - totem-nautilus - - traceroute - kde-desktop: - # Incompatible with ostree for various reasons - - abrt-desktop - - cups-pk-helper - - dnfdragora - - plasma-pk-updates - # Non-critical apps -> Flatpak - - akregator - - cagibi - - elisa-player - - gwenview - - kaddressbook - - kamera - - kcalc - - kcharselect - - kcolorchooser - - kdnssd - - kf5-kipi-plugins - - kfind - - kget - - kgpg - - kmail - - kmouth - - knode - - konqueror - - kontact - - korganizer - - kruler - - ksshaskpass - - kwrite - - libreoffice-kde - - okular - # Misc - - adwaita-gtk2-theme - - colord-kde - - keditbookmarks - - kwebkitpart - - plasma-nm-l2tp - - plasma-nm-openswan - - plasma-nm-pptp - xfce-desktop: - # Incompatible with ostree for various reasons - - abrt-desktop - - dnfdragora-updater - # Non-critical apps -> Flatpak - - fros-recordmydesktop - - tumbler - # Misc - - alsa-utils - - firewall-config - - openssh-askpass - - vim-enhanced - # Remove uncommon NetworkManager plugins - - NetworkManager-fortisslvpn-gnome - - NetworkManager-iodine-gnome - - NetworkManager-l2tp-gnome - - NetworkManager-libreswan-gnome - - NetworkManager-sstp-gnome - - NetworkManager-strongswan-gnome - lxqt-desktop: - # Incompatible with ostree for various reasons - - dnfdragora-updater - deepin-desktop: - # Incompatible with ostree for various reasons - - dnfdragora-updater - mate-desktop: - # Incompatible with ostree for various reasons - - abrt-desktop - - abrt-java-connector - - dnfdragora-updater - # Non-critical apps -> Flatpak - - blivet-gui - - filezilla - - gnome-disk-utility - - gnome-logs - - gnote - - gparted - - hexchat - - p7zip - - p7zip-plugins - - simple-scan - - thunderbird - - transmission-gtk - - vim-enhanced - - xfburn - - yelp - # Non critical NetworkManager plugins - - NetworkManager-bluetooth - - NetworkManager-iodine-gnome - - NetworkManager-l2tp-gnome - - NetworkManager-libreswan-gnome - - NetworkManager-ovs - - NetworkManager-sstp-gnome - - NetworkManager-strongswan-gnome - - NetworkManager-team - - NetworkManager-wifi - # Already in the common set - - wireplumber diff --git a/comps-sync.py b/comps-sync.py index 6bb7514..8f90881 100755 --- a/comps-sync.py +++ b/comps-sync.py @@ -10,6 +10,27 @@ import sys import yaml import libcomps +from dataclasses import dataclass, field + + +@dataclass +class DesktopEnvironment: + """ + Represents a desktop environment configuration which could include + contents of multiple groups and filter out specified packages. + """ + + groups: set[str] = field(default_factory=set) + exclude_packages: set[str] = field(default_factory=set) + + @classmethod + def from_dict(cls, obj: dict): + """Create DesktopEnvironment object from deserialized yaml dict""" + groups = set(obj.get("groups", [])) + exclude_packages = set(obj.get("exclude-packages", [])) + return cls(groups=groups, exclude_packages=exclude_packages) + + def fatal(msg): print(msg, file = sys.stderr) sys.exit(1) @@ -49,7 +70,6 @@ with open('comps-sync-exclude-list.yml', encoding='UTF-8') as f: comps_exclude_list = doc['exclude_list'] comps_include_list = doc['include_list'] comps_exclude_list_groups = doc['exclude_list_groups'] - comps_desktop_exclude_list = doc['desktop_exclude_list'] comps_exclude_list_all = [re.compile(x) for x in doc['exclude_list_all_regexp']] def is_exclude_listed(pkgname): @@ -132,18 +152,15 @@ if (n_manifest_new > 0 or n_comps_new > 0) and args.save: write_manifest(base_pkgs_path, manifest_packages) # List of comps groups used for each desktop -desktops_comps_groups = { - "gnome": ["gnome-desktop"], - "kde": ["kde-desktop"], - "xfce": ["xfce-desktop"], - "lxqt": ["lxqt-desktop"], - "deepin": ["deepin-desktop"], - "pantheon": ["pantheon-desktop"], - "mate": ["mate-desktop"] -} +with open('comps-sync-desktop-list.yml', encoding='UTF-8') as f: + doc = yaml.safe_load(f) + desktops_comps_groups = { + name: DesktopEnvironment.from_dict(obj) + for name, obj in doc.items() if obj is not None + } # Generate treefiles for all desktops -for desktop, groups in desktops_comps_groups.items(): +for desktop, conf in desktops_comps_groups.items(): print() print("Syncing packages for {}:".format(desktop)) @@ -152,16 +169,19 @@ for desktop, groups in desktops_comps_groups.items(): manifest = yaml.safe_load(f) manifest_packages = set(manifest['packages']) + desktop_exclude_list = conf.exclude_packages + # Filter packages in the comps groups associated with a given desktop using # the per group exclude_list comps_group_pkgs = set() comps_group_pkgs = set() - for group in groups: + for group in conf.groups: + group_exclude_list = comps_exclude_list.get(group, set()) + if group_exclude_list is None: + group_exclude_list = set() for pkg in comps.groups_match(id=group)[0].packages: pkgname = pkg.name - exclude_list = comps_desktop_exclude_list.get(group, set()) - if exclude_list is None: - exclude_list = set() - if pkgname in exclude_list or is_exclude_listed(pkgname): + if (pkgname in desktop_exclude_list or pkgname in group_exclude_list + or is_exclude_listed(pkgname)): continue comps_group_pkgs.add(pkg.name) From 69cdf0db64d38e244ab90e77cf7be60b19a746c8 Mon Sep 17 00:00:00 2001 From: Aleksei Bavshin Date: Aug 08 2022 14:43:43 +0000 Subject: [PATCH 4/7] comps-sync.py: support specifying list of architecture-specific packages This change allows to keep a list of architecture-specific packages and generate `packages-{arch}` sections instead of excluding the arched packages and re-adding these manually in fedora-common-ostree. With the change in place we can correctly handle architecture-specific packages in the desktop environment groups. --- diff --git a/comps-sync.py b/comps-sync.py index 8f90881..8258184 100755 --- a/comps-sync.py +++ b/comps-sync.py @@ -42,14 +42,36 @@ def format_pkgtype(n): return 'mandatory' assert False + +def get_pkg_arches(pkgname): + """lookup package architecture restrictions from arch_specific_list""" + return set( + (arch for arch, pkgs in comps_arch_specific_list.items() if pkgname in pkgs)) + + +def get_manifest_packages(manifest): + manifest_packages = set(manifest['packages']) + for key, pkgs in manifest.items(): + if re.match(r'packages-\w+$', key): + manifest_packages.update(pkgs) + return manifest_packages + + def write_manifest(fpath, pkgs, include=None): with open(fpath, 'w', encoding='UTF-8') as f: f.write("# DO NOT EDIT! This content is generated from comps-sync.py\n") if include is not None: f.write("include: {}\n".format(include)) - f.write("packages:\n") + + doc = {'packages': []} for pkg in sorted(pkgs): - f.write(" - {}\n".format(pkg)) + arches = get_pkg_arches(pkg) + if len(arches) > 0: + for arch in arches: + doc.setdefault(f'packages-{arch}', []).append(pkg) + else: + doc['packages'].append(pkg) + yaml.safe_dump(doc, f) print("Wrote {}".format(fpath)) parser = argparse.ArgumentParser() @@ -63,12 +85,13 @@ print("Syncing packages common to all desktops:") base_pkgs_path = 'fedora-common-ostree-pkgs.yaml' with open(base_pkgs_path, encoding='UTF-8') as f: manifest = yaml.safe_load(f) -manifest_packages = set(manifest['packages']) +manifest_packages = get_manifest_packages(manifest) with open('comps-sync-exclude-list.yml', encoding='UTF-8') as f: doc = yaml.safe_load(f) comps_exclude_list = doc['exclude_list'] comps_include_list = doc['include_list'] + comps_arch_specific_list = doc.get('arch_specific_list', {}) comps_exclude_list_groups = doc['exclude_list_groups'] comps_exclude_list_all = [re.compile(x) for x in doc['exclude_list_all_regexp']] @@ -167,7 +190,7 @@ for desktop, conf in desktops_comps_groups.items(): manifest_path = '{}-desktop-pkgs.yaml'.format(desktop) with open(manifest_path, encoding='UTF-8') as f: manifest = yaml.safe_load(f) - manifest_packages = set(manifest['packages']) + manifest_packages = get_manifest_packages(manifest) desktop_exclude_list = conf.exclude_packages From 882f61ec1d3edf029035c183d644096ca6cfceaa Mon Sep 17 00:00:00 2001 From: Aleksei Bavshin Date: Aug 08 2022 14:52:35 +0000 Subject: [PATCH 5/7] comps-sync.py: make package list diffing architecture-aware Rewrite all package operations using a class Package containing all additional metadata: architectures, groups, dependency type. Make package list comparison aware of that and treat architecture list modifications as a change that requires regenerating the manifest. Also, wrap all the manifest logic into a class Manifest. --- diff --git a/comps-sync.py b/comps-sync.py index 8258184..4465454 100755 --- a/comps-sync.py +++ b/comps-sync.py @@ -31,6 +31,99 @@ class DesktopEnvironment: return cls(groups=groups, exclude_packages=exclude_packages) +@dataclass(order=True) +class Package(): + name: str + type: int = libcomps.PACKAGE_TYPE_DEFAULT + # restricts the package to the listed architectures if specified + arch: set[str] = field(default_factory=set) + # groups that require the package + groups: set[str] = field(default_factory=set) + + def __str__(self): + arches = '' + if len(self.arch) > 0: + arches = ', arch: ' + ', '.join(self.arch) + groups = '' + if len(self.groups) > 0: + groups = ', groups: ' + ', '.join(self.groups) + return f"{self.name} ({format_pkgtype(self.type)}{groups}{arches})" + + +class Manifest(): + + def __init__(self): + self.includes = [] + self.packages = {} + + def add(self, pkgname, type, group=None): + # try to update existing list entry + if pkg := self.packages.get(pkgname, None): + if group is not None: + pkg.groups.add(group) + if (pkg.type == libcomps.PACKAGE_TYPE_DEFAULT + and type == libcomps.PACKAGE_TYPE_MANDATORY): + pkg.type = type + return + + groups = set() + if group is not None: + groups.add(group) + self.packages[pkgname] = Package(pkgname, + type=type, + arch=get_pkg_arches(pkgname), + groups=groups) + + def compare(self, other): + """Compare two manifests and return lists of added/removed entries""" + added = [] + removed = [] + all_pkgnames = set(self.packages.keys()) | set(other.packages.keys()) + for name in all_pkgnames: + pkg1 = self.packages.get(name, None) + pkg2 = other.packages.get(name, None) + if pkg1 is None: + removed.append(pkg2) + elif pkg2 is None: + added.append(pkg1) + elif pkg1.arch != pkg2.arch: + # package architectures has changed, + # treat this as a package list change + added.append(pkg1) + removed.append(pkg2) + return added, removed + + def load(self, stream): + """Load from yaml stream or string""" + temp = yaml.safe_load(stream) or {} + includes = temp.get('include', []) + if isinstance(includes, str): + includes = set([includes]) + + packages = {pkg: set() for pkg in temp.get('packages', [])} + for key, values in temp.items(): + if m := re.match(r'^packages-(\w+)$', key): + for pkg in values: + packages.setdefault(pkg, set()).add(m[1]) + + self.includes = includes + self.packages = { + name: Package(name, arch=arch, groups={'manifest'}) + for name, arch in packages.items() + } + + def write(self, stream): + # assemble temporary dict matching the treefile structure + temp = {'packages': []} + for pkg in sorted(self.packages.values()): + if len(pkg.arch) > 0: + for arch in pkg.arch: + temp.setdefault(f'packages-{arch}', []).append(pkg.name) + else: + temp['packages'].append(pkg.name) + yaml.safe_dump(temp, stream) + + def fatal(msg): print(msg, file = sys.stderr) sys.exit(1) @@ -40,6 +133,10 @@ def format_pkgtype(n): return 'default' if n == libcomps.PACKAGE_TYPE_MANDATORY: return 'mandatory' + if n == libcomps.PACKAGE_TYPE_OPTIONAL: + return 'optional' + if n == libcomps.PACKAGE_TYPE_CONDITIONAL: + return 'conditional' assert False @@ -49,29 +146,12 @@ def get_pkg_arches(pkgname): (arch for arch, pkgs in comps_arch_specific_list.items() if pkgname in pkgs)) -def get_manifest_packages(manifest): - manifest_packages = set(manifest['packages']) - for key, pkgs in manifest.items(): - if re.match(r'packages-\w+$', key): - manifest_packages.update(pkgs) - return manifest_packages - - def write_manifest(fpath, pkgs, include=None): with open(fpath, 'w', encoding='UTF-8') as f: f.write("# DO NOT EDIT! This content is generated from comps-sync.py\n") if include is not None: f.write("include: {}\n".format(include)) - - doc = {'packages': []} - for pkg in sorted(pkgs): - arches = get_pkg_arches(pkg) - if len(arches) > 0: - for arch in arches: - doc.setdefault(f'packages-{arch}', []).append(pkg) - else: - doc['packages'].append(pkg) - yaml.safe_dump(doc, f) + pkgs.write(f) print("Wrote {}".format(fpath)) parser = argparse.ArgumentParser() @@ -83,9 +163,9 @@ args = parser.parse_args() print("Syncing packages common to all desktops:") base_pkgs_path = 'fedora-common-ostree-pkgs.yaml' +manifest = Manifest() with open(base_pkgs_path, encoding='UTF-8') as f: - manifest = yaml.safe_load(f) -manifest_packages = get_manifest_packages(manifest) + manifest.load(f) with open('comps-sync-exclude-list.yml', encoding='UTF-8') as f: doc = yaml.safe_load(f) @@ -112,7 +192,7 @@ comps.fromxml_f(args.src) ws_env_name = 'workstation-product-environment' ws_ostree_name = 'workstation-ostree-support' ws_environ = comps.environments[ws_env_name] -ws_pkgs = {} +new_manifest = Manifest() for gid in ws_environ.group_ids: group = comps.groups_match(id=gid.name)[0] if gid.name in comps_exclude_list_groups: @@ -125,26 +205,20 @@ for gid in ws_environ.group_ids: continue if pkgname in exclude_list or is_exclude_listed(pkgname): continue - pkgdata = ws_pkgs.get(pkgname) - if pkgdata is None: - ws_pkgs[pkgname] = pkgdata = (pkg.type, set([gid.name])) - if (pkgdata[0] == libcomps.PACKAGE_TYPE_DEFAULT and - pkg.type == libcomps.PACKAGE_TYPE_MANDATORY): - ws_pkgs[pkgname] = pkgdata = (pkg.type, pkgdata[1]) - pkgdata[1].add(gid.name) + new_manifest.add(pkgname, type=pkg.type, group=gid.name) ws_ostree_pkgs = set() for pkg in comps.groups_match(id=ws_ostree_name)[0].packages: if not is_exclude_listed(pkg.name): ws_ostree_pkgs.add(pkg.name) -comps_unknown = set() -for pkg in manifest_packages: - if (pkg not in comps_include_list and - pkg not in ws_pkgs and - pkg not in ws_ostree_pkgs): - comps_unknown.add(pkg) - +ws_added, comps_unknown = new_manifest.compare(manifest) +# deal with the packages that should magically migrate from the old manifest +# while having no other reason to exist in the freshly generated one +comps_unknown = [ + x for x in comps_unknown + if x.name not in comps_include_list and x.name not in ws_ostree_pkgs +] # Look for packages in the manifest but not in comps at all n_manifest_new = len(comps_unknown) if n_manifest_new == 0: @@ -153,26 +227,20 @@ else: print(" - {} packages not in {}:".format(n_manifest_new, ws_env_name)) for pkg in sorted(comps_unknown): print(' {}'.format(pkg)) - manifest_packages.remove(pkg) + del manifest.packages[pkg.name] # Look for packages in workstation but not in the manifest -ws_added = {} -for (pkg,data) in ws_pkgs.items(): - if pkg not in manifest_packages: - ws_added[pkg] = data - manifest_packages.add(pkg) - n_comps_new = len(ws_added) if n_comps_new == 0: print(" - All comps packages are already listed in manifest.") else: print(" - {} packages not in manifest:".format(n_comps_new)) for pkg in sorted(ws_added): - (req, groups) = ws_added[pkg] - print(' {} ({}, groups: {})'.format(pkg, format_pkgtype(req), ', '.join(groups))) + print(' {}'.format(pkg)) + manifest.packages[pkg.name] = pkg if (n_manifest_new > 0 or n_comps_new > 0) and args.save: - write_manifest(base_pkgs_path, manifest_packages) + write_manifest(base_pkgs_path, manifest) # List of comps groups used for each desktop with open('comps-sync-desktop-list.yml', encoding='UTF-8') as f: @@ -188,15 +256,15 @@ for desktop, conf in desktops_comps_groups.items(): print("Syncing packages for {}:".format(desktop)) manifest_path = '{}-desktop-pkgs.yaml'.format(desktop) + manifest = Manifest() with open(manifest_path, encoding='UTF-8') as f: - manifest = yaml.safe_load(f) - manifest_packages = get_manifest_packages(manifest) + manifest.load(f) desktop_exclude_list = conf.exclude_packages # Filter packages in the comps groups associated with a given desktop using # the per group exclude_list comps_group_pkgs = set() - comps_group_pkgs = set() + new_manifest = Manifest() for group in conf.groups: group_exclude_list = comps_exclude_list.get(group, set()) if group_exclude_list is None: @@ -206,14 +274,10 @@ for desktop, conf in desktops_comps_groups.items(): if (pkgname in desktop_exclude_list or pkgname in group_exclude_list or is_exclude_listed(pkgname)): continue - comps_group_pkgs.add(pkg.name) + new_manifest.add(pkg.name, type=pkg.type, group=group) # Look for packages in the manifest but not in comps group - comps_unknown = set() - for pkg in manifest_packages: - if pkg not in comps_group_pkgs: - comps_unknown.add(pkg) - + desktop_pkgs_added, comps_unknown = new_manifest.compare(manifest) n_manifest_new = len(comps_unknown) if n_manifest_new == 0: print(" - All manifest packages are already listed in comps.") @@ -221,14 +285,8 @@ for desktop, conf in desktops_comps_groups.items(): print(" - {} packages not in {} comps group:".format(n_manifest_new, desktop)) for pkg in sorted(comps_unknown): print(' {}'.format(pkg)) - manifest_packages.remove(pkg) # Look for packages in comps but not in the manifest - desktop_pkgs_added = set() - for pkg in comps_group_pkgs: - if pkg not in manifest_packages: - desktop_pkgs_added.add(pkg) - n_comps_new = len(desktop_pkgs_added) if n_comps_new == 0: print(" - All comps packages are already listed in manifest.") @@ -236,8 +294,7 @@ for desktop, conf in desktops_comps_groups.items(): print(" - {} packages not in {} manifest:".format(n_comps_new, desktop)) for pkg in sorted(desktop_pkgs_added): print(' {}'.format(pkg)) - manifest_packages.add(pkg) # Update manifest if (n_manifest_new > 0 or n_comps_new > 0) and args.save: - write_manifest(manifest_path, manifest_packages, include="fedora-common-ostree.yaml") + write_manifest(manifest_path, new_manifest, include="fedora-common-ostree.yaml") From d57cb2ae837bd6894842d24555ccb6e53ebe991f Mon Sep 17 00:00:00 2001 From: Aleksei Bavshin Date: Aug 08 2022 14:52:36 +0000 Subject: [PATCH 6/7] comps-sync.py: restore lists indentation style --- diff --git a/comps-sync.py b/comps-sync.py index 4465454..cd4522f 100755 --- a/comps-sync.py +++ b/comps-sync.py @@ -50,6 +50,13 @@ class Package(): return f"{self.name} ({format_pkgtype(self.type)}{groups}{arches})" +class Dumper(yaml.SafeDumper): + """Workaround for yaml/pyyaml#234 - preserve current indentation style""" + + def increase_indent(self, flow=False, *args, **kwargs): + return super().increase_indent(flow=flow, indentless=False) + + class Manifest(): def __init__(self): @@ -121,7 +128,7 @@ class Manifest(): temp.setdefault(f'packages-{arch}', []).append(pkg.name) else: temp['packages'].append(pkg.name) - yaml.safe_dump(temp, stream) + yaml.dump(temp, stream, Dumper=Dumper) def fatal(msg): From 23c6c97400f2d648e12c893e96ae33a26a1385fb Mon Sep 17 00:00:00 2001 From: Aleksei Bavshin Date: Aug 08 2022 14:52:38 +0000 Subject: [PATCH 7/7] Add arch_specific_list and sync comps --- diff --git a/comps-sync-exclude-list.yml b/comps-sync-exclude-list.yml index 428b18d..aebed27 100644 --- a/comps-sync-exclude-list.yml +++ b/comps-sync-exclude-list.yml @@ -21,6 +21,26 @@ exclude_list_all_regexp: - "mozilla-openh264" - "openh264" +# Architecture-specific package list. +# The packages listed here will be added to `packages-{arch}` instead of the +# default `packages` list. +# Architecture information is not exposed in libcomps python bindings and is +# not even always specified. We have to maintain our own list instead. +arch_specific_list: + armhfp: + - xorg-x11-drv-armada + x86_64: + - hyperv-daemons + - microcode_ctl + - mcelog + - open-vm-tools-desktop + - thermald + - virtualbox-guest-additions + - xorg-x11-drv-intel + - xorg-x11-drv-openchrome + - xorg-x11-drv-vesa + - xorg-x11-drv-vmware + # Common exclude list for all ostree desktop versions exclude_list: core: @@ -43,20 +63,9 @@ exclude_list: - lsvpd - s390utils-base base-x: - # x86 specific packages. Added back by fedora-common-ostree.yaml - - xorg-x11-drv-intel - - xorg-x11-drv-openchrome - - xorg-x11-drv-vesa - - xorg-x11-drv-vmware - # ARM specific packages - - xorg-x11-drv-armada + # Unused ARM specific packages - xorg-x11-drv-armsoc - xorg-x11-drv-omap - guest-desktop-agents: - # x86 specific packages. Added back by fedora-common-ostree.yaml - - hyperv-daemons - - open-vm-tools-desktop - - virtualbox-guest-additions workstation-product: # We use rpm-ostree for the host - dnf @@ -116,10 +125,6 @@ exclude_list: - gnome-shell-extension-background-logo - pinentry-gnome3 - qgnomeplatform - # x86 specific packages. Added back by fedora-common-ostree.yaml - - mcelog - - microcode_ctl - - thermald networkmanager-submodules: # Let's use the builtin one by default - dhcp-client diff --git a/fedora-common-ostree-pkgs.yaml b/fedora-common-ostree-pkgs.yaml index c213766..4c84632 100644 --- a/fedora-common-ostree-pkgs.yaml +++ b/fedora-common-ostree-pkgs.yaml @@ -234,3 +234,16 @@ packages: - zd1211-firmware - zip - zram-generator-defaults +packages-armhfp: + - xorg-x11-drv-armada +packages-x86_64: + - hyperv-daemons + - mcelog + - microcode_ctl + - open-vm-tools-desktop + - thermald + - virtualbox-guest-additions + - xorg-x11-drv-intel + - xorg-x11-drv-openchrome + - xorg-x11-drv-vesa + - xorg-x11-drv-vmware diff --git a/fedora-common-ostree.yaml b/fedora-common-ostree.yaml index e174ac6..95e0a27 100644 --- a/fedora-common-ostree.yaml +++ b/fedora-common-ostree.yaml @@ -57,7 +57,6 @@ packages-aarch64: - shim packages-armhfp: - extlinux-bootloader - - xorg-x11-drv-armada packages-ppc64: - grub2 - ostree-grub2 @@ -72,16 +71,6 @@ packages-x86_64: - efibootmgr - shim-ia32 - shim-x64 - - microcode_ctl - - mcelog - - thermald - - hyperv-daemons - - open-vm-tools-desktop - - virtualbox-guest-additions - - xorg-x11-drv-intel - - xorg-x11-drv-openchrome - - xorg-x11-drv-vesa - - xorg-x11-drv-vmware # Make sure the following are not pulled in when Recommended by other packages exclude-packages: