From b6b02411519a8c811995009d417a696b89e0a2f2 Mon Sep 17 00:00:00 2001
From: Stephen Gallagher <sgallagh@redhat.com>
Date: Aug 31 2023 17:27:01 +0000
Subject: prepopulate: Add script to trim packages


This can only remove entries that are no longer in Content Resolver.
Package additions still need to be handled manually.

Signed-off-by: Stephen Gallagher <sgallagh@redhat.com>

---

diff --git a/update-prepopulate-json.py b/update-prepopulate-json.py
new file mode 100755
index 0000000..e6b0aa4
--- /dev/null
+++ b/update-prepopulate-json.py
@@ -0,0 +1,104 @@
+#!/usr/bin/python3
+
+import click
+import json
+import logging
+import requests
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_distro_packages(
+    distro_view=["eln"],
+):
+    """
+    Fetches the list of desired sources from Content Resolver
+    for each of the given 'arches'.
+    """
+
+    view = "eln"
+    arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
+
+    all_packages = dict()
+
+    for rpmtype in ["binaries", "sources"]:
+        all_packages[rpmtype] = dict()
+
+        if rpmtype == "binaries":
+            list_types = ["buildroot", "binary"]
+        else:
+            list_types = ["builroot-source", "source"]
+
+        for arch in arches:
+            merged_packages = dict()
+
+            for this_list in list_types:
+                url = f"https://tiny.distro.builders/view-{this_list}-package-name-list--view-{view}--{arch}.txt"
+
+                logger.debug(f"downloading {url}")
+
+                r = requests.get(url, allow_redirects=True)
+                for line in r.text.splitlines():
+                    merged_packages[line] = {
+                        "view": view,
+                        "content_type": this_list,
+                    }
+
+                # There may be an empty line in the file, ignore it.
+                if "" in merged_packages:
+                    del merged_packages[""]
+                logger.debug(f"Found {len(merged_packages)} {arch} packages")
+
+                all_packages[rpmtype][arch] = merged_packages
+
+    return all_packages
+
+
+@click.command()
+@click.option(
+    "--log-level",
+    type=click.Choice(
+        ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False
+    ),
+    default="WARNING",
+    show_default=True,
+)
+def main(log_level):
+    logging.basicConfig(
+        format="%(asctime)s : %(name)s : %(levelname)s : %(message)s",
+        level=log_level,
+    )
+    logger.debug("Debug logging enabled")
+
+    eln_packages = get_distro_packages()
+
+    with open("eln_packages.json", "w") as f:
+        json.dump(eln_packages, f, sort_keys=True, indent=2)
+
+    with open("prepopulate.json", "r") as f:
+        prepopulation = json.load(f)
+
+    for variant in prepopulation:
+        for arch in prepopulation[variant]:
+            for srpm_name in sorted(prepopulation[variant][arch].keys()):
+                logger.debug(f"Checking SRPM {srpm_name}")
+                if srpm_name not in eln_packages["sources"][arch]:
+                    logger.warning(f"Removing {srpm_name} from {variant}({arch})")
+                    del prepopulation[variant][arch][srpm_name]
+                    continue
+                # Next check the binary RPMs
+                for rpm_name in sorted(prepopulation[variant][arch][srpm_name]):
+                    trunc_name = rpm_name.rsplit(".", maxsplit=1)[0]
+                    if trunc_name not in eln_packages["binaries"][arch]:
+                        logger.warning(
+                            f"Removing {rpm_name} from SRPM {srpm_name} on {variant}({arch})"
+                        )
+                        prepopulation[variant][arch][srpm_name].remove(rpm_name)
+
+    with open("prepopulate.json", "w") as f:
+        json.dump(prepopulation, f, sort_keys=True, indent=2)
+
+
+if __name__ == "__main__":
+    main()