From 6428a58b063e8d64fc02d5fee7b994b0053d30ce Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 20 2020 10:48:14 +0000 Subject: [PATCH 1/4] Drop the bz_product_name override Instead of having a product name and an override via the bz_product_name just use always the product name. This fixes the issue around "Fedora Container Images" which were somehow not cached from bugzilla. This led to the script trying to create them instead of updating them, which of course failed since they did exist in bugzilla. Signed-off-by: Pierre-Yves Chibon --- diff --git a/distgit_bugzilla_sync/default-config-files/configuration.toml b/distgit_bugzilla_sync/default-config-files/configuration.toml index 3cd5dd2..709a443 100644 --- a/distgit_bugzilla_sync/default-config-files/configuration.toml +++ b/distgit_bugzilla_sync/default-config-files/configuration.toml @@ -14,9 +14,9 @@ ignorable_accounts = ["packagerbot", "zuul"] namespace = "rpms" versions = ["rawhide", "31", "30", "29"] [products."Fedora Container"] + [products."Fedora Container Images"] namespace = "container" versions = ["rawhide", "29"] - bz_product_name = "Fedora Container Images" [products."Fedora Modules"] namespace = "modules" versions = [] diff --git a/distgit_bugzilla_sync/script.py b/distgit_bugzilla_sync/script.py index 7678515..db64e92 100644 --- a/distgit_bugzilla_sync/script.py +++ b/distgit_bugzilla_sync/script.py @@ -177,22 +177,21 @@ class BugzillaProxy: # Old API -- in python-bugzilla. But with current server, this # gives ProxyError for collection, product in self.config["products"].items(): - bz_product_name = product.get("bz_product_name", collection) self.product_cache[collection] = self.server.getcomponentsdetails( - bz_product_name + collection ) elif self.config["bugzilla"]["compat_api"] == "component.get": # Way that's undocumented in the partner-bugzilla api but works # currently chunk = self.config["bugzilla"]["req_segment"] for collection, product in self.config["products"].items(): - bz_product_name = product.get("bz_product_name", collection) + # restrict the list of info returned to only the packages of # interest pkglist = [ project["name"] for project in pagure_projects - if bz_product_name in project["products"] + if collection in project["products"] ] product_info_by_pkg = {} estimated_size = math.ceil(len(pkglist) / chunk) @@ -200,13 +199,13 @@ class BugzillaProxy: # Format that bugzilla will understand. Strip None's that # segment() pads out the final data segment() with query = [ - {"product": bz_product_name, "component": p} + {"product": collection, "component": p} for p in pkg_segment if p is not None ] if self.config["verbose"]: print( - f" - Querying product `{bz_product_name}`, " + f" - Querying product `{collection}`, " f"query {cnt} of {estimated_size}" ) raw_data = _query_component(query) @@ -384,9 +383,6 @@ class BugzillaProxy: e.args = ("ProtocolError", e.errcode, e.errmsg) raise - bz_product_name = self.config["products"][collection].get( - "bz_product_name", collection - ) # Set the qacontact_email and name default_qa_contact_email = self.config["default_qa_contact_email"] @@ -431,7 +427,7 @@ class BugzillaProxy: if data: # Changes occurred. Submit a request to change via xmlrpc - data["product"] = bz_product_name + data["product"] = collection data["component"] = package if self.config["verbose"]: @@ -535,21 +531,21 @@ class BugzillaProxy: new_poc=owner_email, prev_poc=product[pkg_key]["initialowner"], name=package, - product=bz_product_name, + product=collection, print_fas_names=print_fas_names, ) else: if self.config.get("print-no-change"): - print(f"[NOCHANGE] {package}/{bz_product_name}") + print(f"[NOCHANGE] {package}/{collection}") else: if retired: if self.config["verbose"]: - print(f"[NOADD] {bz_product_name}/{package} (is retired)") + print(f"[NOADD] {collection}/{package} (is retired)") return # Add component data = { - "product": bz_product_name, + "product": collection, "component": package, "description": description or "NA", "initialowner": owner_email, @@ -579,12 +575,12 @@ class BugzillaProxy: else: value = initial_cc_fasnames print( - f"[ADDCOMP] {bz_product_name}/{package}" + f"[ADDCOMP] {collection}/{package}" f" {key} set to FAS name(s) `{value}`" ) else: print( - f"[ADDCOMP] {bz_product_name}/{package}" + f"[ADDCOMP] {collection}/{package}" f" {key} set to {data.get(key)}" ) From 809b88e328e2151e569581741c7a1a7d44d15870 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 20 2020 10:54:18 +0000 Subject: [PATCH 2/4] Wait for 10 seconds instead of 20 when querying bugzilla failed This should make the script a little faster and 10 seconds should be enough. Signed-off-by: Pierre-Yves Chibon --- diff --git a/distgit_bugzilla_sync/script.py b/distgit_bugzilla_sync/script.py index db64e92..8d545f5 100644 --- a/distgit_bugzilla_sync/script.py +++ b/distgit_bugzilla_sync/script.py @@ -114,6 +114,7 @@ def segment(iterable, chunk, fill=None): class BugzillaProxy: + def __init__(self, bz_server, username, password, config, pre_cache_users=True): self.bz_xmlrpc_server = bz_server self.username = username @@ -168,9 +169,9 @@ class BugzillaProxy: raise if self.config["verbose"]: print(f" ERROR {e}") - print(" - Query failed, going to try again in 20 seconds") - # Wait 20 seconds and try again - time.sleep(20) + print(" - Query failed, going to try again in 10 seconds") + # Wait 10 seconds and try again + time.sleep(10) return raw_data if self.config["bugzilla"]["compat_api"] == "getcomponentsdetails": From 74a550ecd3823cd3a6a7a19d90a9b244e9fc2fcb Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 20 2020 10:54:51 +0000 Subject: [PATCH 3/4] Only report namespace mis-configuration once and not always Basically, instead of reporting that the namespace did not exist for every package/component in that namespace, we now only report it once per namespace (but we still print the full info in the logs if verbose is enabled). Then, instead of always emailing the admins for mis-configurations, we'll only email them if there are other errors then these which will happen all the time due to the flatpaks and tests namespaces which do not exist in PDC and bugzilla. Signed-off-by: Pierre-Yves Chibon --- diff --git a/distgit_bugzilla_sync/script.py b/distgit_bugzilla_sync/script.py index 8d545f5..00d3d26 100644 --- a/distgit_bugzilla_sync/script.py +++ b/distgit_bugzilla_sync/script.py @@ -937,11 +937,16 @@ class DistgitBugzillaSync: if project["namespace"] not in self.env["pdc_types"]: project["branches"] = [] project["products"] = [] - self.errors["configuration"].append( - f'Namespace `{project["namespace"]}` not found in the pdc_type ' - f'configuration key, project {project["namespace"]}/{project["name"]} ' - "ignored" - ) + error = f'Namespace `{project["namespace"]}` not found in the pdc_type ' \ + f'configuration key -- ignoring it' + if not error in self.errors["configuration"]: + self.errors["configuration"].append(error) + if self.env["verbose"]: + print( + f'Namespace `{project["namespace"]}` not found in the pdc_type ' + f'configuration key, project {project["namespace"]}/{project["name"]} ' + "ignored" + ) continue pdc_type = self.env["pdc_types"][project["namespace"]] @@ -1156,7 +1161,10 @@ class DistgitBugzillaSync: if self.env["verbose"] or self.env["dryrun"]: print("*" * 80) print("\n".join(report)) - if not self.env["dryrun"]: + + # Do not send the email in dryrun or when the error only relates to + # configuration (which will always happen for flatpaks and tests) + if not self.env["dryrun"] and list(self.errors.keys()) != ["configuration"]: self.send_email( self.env["email"]["from"], self.env["email"]["notify_admins"], From 0083b8ee664f3a69b371e144c26273b1100a11e2 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 20 2020 12:13:29 +0000 Subject: [PATCH 4/4] Update the versions available on bugzilla for the rpms and the containers Signed-off-by: Pierre-Yves Chibon --- diff --git a/distgit_bugzilla_sync/default-config-files/configuration.toml b/distgit_bugzilla_sync/default-config-files/configuration.toml index 709a443..fcfbd97 100644 --- a/distgit_bugzilla_sync/default-config-files/configuration.toml +++ b/distgit_bugzilla_sync/default-config-files/configuration.toml @@ -12,11 +12,10 @@ ignorable_accounts = ["packagerbot", "zuul"] [products] [products.Fedora] namespace = "rpms" - versions = ["rawhide", "31", "30", "29"] - [products."Fedora Container"] + versions = ["rawhide", "33", "32", "31"] [products."Fedora Container Images"] namespace = "container" - versions = ["rawhide", "29"] + versions = ["rawhide", "33", "32", "31"] [products."Fedora Modules"] namespace = "modules" versions = [] diff --git a/distgit_bugzilla_sync/script.py b/distgit_bugzilla_sync/script.py index 00d3d26..258144a 100644 --- a/distgit_bugzilla_sync/script.py +++ b/distgit_bugzilla_sync/script.py @@ -1164,7 +1164,7 @@ class DistgitBugzillaSync: # Do not send the email in dryrun or when the error only relates to # configuration (which will always happen for flatpaks and tests) - if not self.env["dryrun"] and list(self.errors.keys()) != ["configuration"]: + if not self.env["dryrun"] and tuple(self.errors) != ("configuration",): self.send_email( self.env["email"]["from"], self.env["email"]["notify_admins"],