From 4d37489f6f152ba07099ad7e7ec21eba50994a09 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 04 2020 12:11:00 +0000 Subject: bodhi/backend: Port the koji-sync script from fedmsg to fedora-messaging Fixes https://pagure.io/fedora-infrastructure/issue/8967 Signed-off-by: Pierre-Yves Chibon --- diff --git a/playbooks/groups/bodhi-backend.yml b/playbooks/groups/bodhi-backend.yml index 5d608bf..ce0a9a4 100644 --- a/playbooks/groups/bodhi-backend.yml +++ b/playbooks/groups/bodhi-backend.yml @@ -84,20 +84,25 @@ service: bodhi host: "bodhi.fedoraproject.org" when: env == "production" + - role: keytab/service owner_user: apache owner_group: apache service: bodhi host: "bodhi.stg.fedoraproject.org" when: env == "staging" + - bodhi2/backend + - role: push-container-registry cert_dest_dir: "/etc/docker/certs.d/registry{{ env_suffix }}.fedoraproject.org" cert_src: "{{private}}/files/docker-registry/{{env}}/pki/issued/containerstable.crt" key_src: "{{private}}/files/docker-registry/{{env}}/pki/private/containerstable.key" certs_group: apache + - role: rabbit/user username: "bodhi{{ env_suffix }}" + - role: rabbit/queue username: "bodhi{{ env_suffix }}" queue_name: "{{ bodhi_message_queue_name }}" @@ -106,6 +111,17 @@ warning: 10 critical: 100 + - role: rabbit/user + username: "koji_sync_listener{{ env_suffix }}" + + - role: rabbit/queue + username: "koji_sync_listener{{ env_suffix }}" + queue_name: "koji_sync_listener{{ env_suffix }}" + routing_keys: + - "io.pagure.*.pagure.issue.edit" + thresholds: + warning: 10 + critical: 100 tasks: - name: create secondary volume dir for stg bodhi diff --git a/roles/bodhi2/backend/files/koji-sync-listener.py b/roles/bodhi2/backend/files/koji-sync-listener.py deleted file mode 100644 index 70690f1..0000000 --- a/roles/bodhi2/backend/files/koji-sync-listener.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python -""" This is a glue script to run /usr/local/bin/owner-sync-pagure on a given -package anytime a ticket gets closed at -https://pagure.io/releng/fedora-scm-requests - -Author: Ralph Bean -""" - -import json -import logging.config -import subprocess as sp -import sys - -import fedmsg -import fedmsg.meta - - -def handle(content): - body = content.strip('`').strip() - body = json.loads(body) - package = body['repo'] - print("Operating on {package}".format(package=package)) - sys.stdout.flush() - # XXX If you modify this taglist. Please also modify the other copy in - # bodhi2/backend/tasks/main.yml - taglist = 'f33 f33-container f33-modular f33-flatpak f32 f32-container f32-modular f32-flatpak f31 f31-container f31-flatpak f31-modular epel8 epel8-playground epel8-modular epel7 dist-6E-epel module-package-list modular' - cmd = [ - '/usr/local/bin/owner-sync-pagure', - '--package', package, - '--verbose', - ] + taglist.split() - print("Running %r" % cmd) - proc = sp.Popen(cmd) - status = proc.wait() - if status: - raise RuntimeError("%r gave return code %r" % (cmd, status)) - - -def main(fullname, fields, content): - if fullname != 'releng/fedora-scm-requests': - print("Dropping %r. Not scm request." % fullname) - return False - if 'close_status' not in fields: - print("Dropping %r %r. Not closed." % (fullname, fields)) - return False - - handle(content) - - -if __name__ == '__main__': - config = fedmsg.config.load_config() - logging.config.dictConfig(config['logging']) - fedmsg.meta.make_processors(**config) - topic = 'io.pagure.prod.pagure.issue.edit' - for _, _, topic, msg in fedmsg.tail_messages(topic=topic): - # Extract some useful information for debugging - title, subtitle, link, idx = [None] * 4 - try: - title = fedmsg.meta.msg2title(msg, **config) - subtitle = fedmsg.meta.msg2subtitle(msg, **config) - link = fedmsg.meta.msg2link(msg, **config) - idx = msg.get('msg_id') - except Exception as e: - print("!! Failed to determine title, subtitle, link") - print("Inspecting {title}, {subtitle}, {link}, {idx}".format( - title=title, subtitle=subtitle, link=link, idx=idx)) - - # Extract values we need to actually process the message - fullname = msg['msg']['project']['fullname'] - fields = msg['msg']['fields'] - content = msg['msg']['issue']['content'] - - # Do the work. - main(fullname, fields, content) diff --git a/roles/bodhi2/backend/files/koji-sync-listener.service b/roles/bodhi2/backend/files/koji-sync-listener.service deleted file mode 100644 index 07f5f64..0000000 --- a/roles/bodhi2/backend/files/koji-sync-listener.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Sync the package list to koji tags as new repo tickets close -After=syslog.target network.target - -[Service] -ExecStart=/usr/local/bin/koji-sync-listener.py -User=fedmsg -Group=fedmsg -Restart=on-failure - -# Use a different ccname so we don't share tickets with bodhi. -Environment=KRB5CCNAME=/var/tmp/owner-sync-krbcc - -[Install] -WantedBy=multi-user.target diff --git a/roles/bodhi2/backend/files/koji_sync_listener.py b/roles/bodhi2/backend/files/koji_sync_listener.py new file mode 100644 index 0000000..86765c9 --- /dev/null +++ b/roles/bodhi2/backend/files/koji_sync_listener.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +""" This is a glue script to run /usr/local/bin/owner-sync-pagure on a given +package anytime a ticket gets closed at +https://pagure.io/releng/fedora-scm-requests + +Author: Ralph Bean + Pierre-Yves Chibon +""" + +import logging +import json +import subprocess as sp +import sys + +from fedora_messaging import config + +_log = logging.getLogger("koji_sync_listener") + + +class KojiSyncListener(object): + """ + A fedora-messaging consumer that calls the owner-sync-pagure script + upon notifications coming from pagure. + + A single configuration key is used from fedora-messaging's + "consumer_config" key, "path", which is where the consumer will save + the messages:: + + [consumer_config] + path = "/tmp/fedora-messaging/messages.txt" + """ + def __init__(self): + """Perform some one-time initialization for the consumer.""" + self.taglist = config.conf["consumer_config"]["taglist"] + + def __call__(self, message): + """ + Invoked when a message is received by the consumer. + + Args: + message (fedora_messaging.api.Message): The message from AMQP. + """ + body = message.body + + fullname = body.get("project", {}).get("fullname") + fields = body.get("fields") + content = body.get('issue', {}).get('content') + + if fullname != 'releng/fedora-scm-requests': + _log.info("Dropping %r. Not scm request." % fullname) + return False + if 'close_status' not in fields: + _log.info("Dropping %r %r. Not closed." % (fullname, fields)) + return False + + try: + body = content.strip('`').strip() + body = json.loads(body) + except Exception: + _log.info("Failed to decode JSON in the issue's initial comment") + return False + + package = content['repo'] + _log.info("Operating on {package}".format(package=package)) + sys.stdout.flush() + + cmd = [ + '/usr/local/bin/owner-sync-pagure', + '--package', package, + '--verbose', + ] + self.taglist + + _log.info("Running %r" % cmd) + proc = sp.Popen(cmd) + status = proc.wait() + if status: + raise RuntimeError("%r gave return code %r" % (cmd, status)) + + _log.info("Done.") diff --git a/roles/bodhi2/backend/tasks/main.yml b/roles/bodhi2/backend/tasks/main.yml index 31cf14c..822fc6d 100644 --- a/roles/bodhi2/backend/tasks/main.yml +++ b/roles/bodhi2/backend/tasks/main.yml @@ -18,6 +18,7 @@ - bodhi-composer - python3-pyramid_sawing - sigul + - fedora-messaging # This next one is not strictly needed since bodhi-composer depends on bodhi-server which # depends on python3-celery, but since we're depoying a service file calling celery in this # role I think it makes sense to require it here instead of assuming a dependency chain. @@ -83,20 +84,19 @@ - name: put the koji sync listener script in place copy: src: koji-sync-listener.py - dest: /usr/local/bin/koji-sync-listener.py - mode: 0755 + dest: /usr/lib/python3.8/site-packages/koji_sync_listener.py + mode: 0644 when: env == "production" notify: - - restart koji-sync-listener + - restart fedora-messaging@koji_sync_listener tags: - bodhi - koji-sync -- name: put the koji sync listener systemd file in place. - copy: - src: koji-sync-listener.service - dest: /usr/lib/systemd/system/koji-sync-listener.service - mode: 0755 +- name: put the koji sync listener config file in place + template: + src: koji_sync_listener.toml + dest: /etc/fedora-messaging/koji_sync_listener.toml when: env == "production" notify: - reload systemd @@ -106,7 +106,7 @@ - koji-sync - name: start the koji sync listener - service: name=koji-sync-listener enabled=yes state=started + service: name=fedora-messaging@koji_sync_listener enabled=yes state=started when: env == "production" tags: - bodhi diff --git a/roles/bodhi2/backend/templates/koji-sync-listener.toml b/roles/bodhi2/backend/templates/koji-sync-listener.toml new file mode 100644 index 0000000..139ae21 --- /dev/null +++ b/roles/bodhi2/backend/templates/koji-sync-listener.toml @@ -0,0 +1,101 @@ +amqp_url = "amqps://koji_sync_listener{{ env_suffix }}:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub" + +publish_exchange = "amq.topic" +passive_declares = true + +callback = "koji_sync_listener:KojiSyncListener" + +# Don't use topic_prefix, since outgoing message topics are derived from incoming messages. +# topic_prefix = "" + +[[bindings]] +queue = "koji_sync_listener{{ env_suffix }}" + +exchange = "amq.topic" +routing_keys = [ + "io.pagure.*.pagure.issue.edit", +] + +[tls] +ca_cert = "/etc/pki/fedora-messaging/cacert.pem" +keyfile = "/etc/pki/fedora-messaging/bodhi-key.pem" +certfile = "/etc/pki/fedora-messaging/bodhi-cert.pem" + +[client_properties] +app = "koji_sync_listener" +app_contacts_email = ["pingou@fedoraproject.org"] + +[queues."koji_sync_listener{{ env_suffix }}"] +durable = true +auto_delete = false +exclusive = false +arguments = {} + +[consumer_config] + +# XXX If you modify this taglist. Please also modify the other copy in +# bodhi2/backend/tasks/main.yml +taglist = [ + "f33", + "f33-container", + "f33-modular", + "f33-flatpak", + "f32", + "f32-container", + "f32-modular", + "f32-flatpak", + "f31", + "f31-container", + "f31-flatpak", + "f31-modular", + "epel8", + "epel8-playground", + "epel8-modular", + "epel7", + "dist-6E-epel", + "module-package-list", + "modular", +] + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "[%(levelname)s %(name)s] %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.twisted] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.pika] +level = "WARNING" +propagate = false +handlers = ["console"] + +# If your consumer sets up a logger, you must add a configuration for it +# here in order for the messages to show up. e.g. if it set up a logger +# called 'example_printer', you could do: +[log_config.loggers.koji_sync_listener] +level = "DEBUG" +propagate = false +handlers = ["console"] + +[log_config.root] +level = "ERROR" +handlers = ["console"]