#192 Add distgit_commit_processor plugin (and misc fixes)
Merged a year ago by nphilipp. Opened a year ago by nphilipp.
fedora-infra/ nphilipp/toddlers main--distgit-commit-processor  into  main

file modified
+1 -1
@@ -96,7 +96,7 @@ 

     and this toddler needs to listen to a new topic, you will have to add it

     to the ansible playbook for toddlers, the dynamic loading of the topics that

     toddlers has works for local development but will not work in production.

-    The ansible playbook is at: https://pagure.io/fedora-infra/ansible/blob/master/f/playbooks/openshift-apps/toddlers.yml

+    The ansible playbook is at: https://pagure.io/fedora-infra/ansible/blob/main/f/playbooks/openshift-apps/toddlers.yml

  

  

  How to run the tests?

file modified
+1 -1
@@ -4,8 +4,8 @@ 

  defusedxml

  fasjson-client

  fedora-messaging

+ fedora-messaging-git-hook-messages

  GitPython

- jsonschema<4.0.0 # https://github.com/Yelp/bravado/issues/478 bravado is used by fasjson-client

  koji

  requests

  pagure-messages

@@ -0,0 +1,214 @@ 

+ from contextlib import nullcontext

+ import datetime as dt

+ import random

+ from typing import ContextManager, Optional

+ from unittest import mock

+ 

+ from fedora_messaging.message import Message

+ from fedora_messaging_git_hook_messages import CommitV1

+ import pytest

+ 

+ from toddlers.exceptions import ConfigError

+ from toddlers.plugins import distgit_commit_processor

+ 

+ 

+ class TestDistGitCommitProcessor:

+     """Test the DistGitCommitProcessor toddler plugin."""

+ 

+     toddler_cls = distgit_commit_processor.DistGitCommitProcessor

+ 

+     @pytest.mark.parametrize(

+         "topic, valid",

+         (

+             pytest.param("org.fedoraproject.prod.git.receive", True, id="prod-valid"),

+             pytest.param("org.fedoraproject.stg.git.receive", True, id="stg-valid"),

+             pytest.param(

+                 "org.fedoraproject.prod.pagure.git.receive",

+                 False,

+                 id="pagure-prod-invalid",

+             ),

+             pytest.param(

+                 "org.fedoraproject.stg.pagure.git.receive",

+                 False,

+                 id="pagure-stg-invalid",

+             ),

+             pytest.param(

+                 "pagure.io.prod.pagure.git.receive", False, id="pagure-io-invalid"

+             ),

+         ),

+     )

+     def test_accepts_topic(

+         self,

+         topic: str,

+         valid: bool,

+         toddler: distgit_commit_processor.DistGitCommitProcessor,

+     ) -> None:

+         assert toddler.accepts_topic(topic) is valid

+ 

+     @pytest.mark.parametrize(

+         "namespace, repo, path, result",

+         (

+             ("rpms", "kernel", "/some/root/rpms/kernel.git/", False),

+             ("rpms", "kernel", "/some/root/rpms/kernel/", False),

+             ("frobozzniks", "blomp", "/some/root/rpms/kernel.git/", False),

+             ("rpms", "kernel", "/some/root/fork/foo/rpms/kernel.git/", True),

+             ("rpms", "kernel", "/some/root/fork/foo/rpms/kernel", True),

+             (None, "project", "/some/root/project.git/", False),

+         ),

+     )

+     def test_ignore_commit(

+         self,

+         namespace: Optional[str],

+         repo: str,

+         path: str,

+         result: bool,

+         toddler: distgit_commit_processor.DistGitCommitProcessor,

+         caplog: pytest.LogCaptureFixture,

+     ):

+         if namespace:

+             repo_with_ns = namespace + "/" + repo

+         else:

+             repo_with_ns = repo

+ 

+         commit = {

+             "namespace": namespace,

+             "repo": repo,

+             "path": path,

+         }

+         body = {"id": "BOO", "agent": "m0rk", "commit": commit}

+ 

+         message = CommitV1(body=body)

+ 

+         assert toddler.ignore_commit(message) is result

+ 

+         if repo_with_ns in path:

+             assert f"Message {message.id} mismatch" not in caplog.text

+         else:

+             assert f"Message {message.id} mismatch" in caplog.text

+ 

+     @pytest.mark.parametrize(

+         "testcase",

+         (

+             "success",

+             "success-loglevel-info",

+             "success-default-subject",

+             "success-default-content",

+             "failure-wrong-msg-type",

+             "failure-wrong-msg-type-loglevel-info",

+             "failure-fork-commit",

+             "failure-config-error",

+         ),

+     )

+     def test_process(

+         self,

+         testcase: str,

+         toddler: distgit_commit_processor.DistGitCommitProcessor,

+         caplog: pytest.LogCaptureFixture,

+     ) -> None:

+         success = "success" in testcase

+         wrong_msg_type = "wrong-msg-type" in testcase

+         fork_commit = "fork-commit" in testcase

+         config_error = "config-error" in testcase

+         loglevel_info = "loglevel-info" in testcase

+         default_subject = "default-subject" in testcase

+         default_content = "default-content" in testcase

+ 

+         # Appease mypy

+         exception_ctx: ContextManager

+ 

+         now = dt.datetime.now(tz=dt.timezone.utc).isoformat()

+ 

+         commit = {

+             "namespace": "rpms",

+             "repo": "kernel",

+             "path": "/some/root/rpms/kernel.git/",

+             "branch": "rawhide",

+             "rev": "deadbeef",

+             "name": "Mork",

+             "email": "mork@ork.org",

+             "date": now,

+             "summary": "Did the thing",

+             "message": "Did the thing\n\nAmazing.",

+             "patch": "No, I won’t fake a diff here.",

+         }

+         body = {"agent": "m0rk", "commit": commit}

+ 

+         if fork_commit:

+             commit["path"] = "/some/root/fork/foo/rpms/kernel.git/"

+ 

+         msg = CommitV1(body=body)

+         if wrong_msg_type:

+             msg = Message(body=body)

+ 

+         # Config items which must be set

+         config = {

+             "mail_server": "bastion.fedoraproject.org",

+             "mail_from": "notifications@fedoraproject.org",

+             "mail_to": "scm-commits@lists.fedoraproject.org",

+         }

+         if config_error:

+             # Nuke a random configuration item

+             del config[list(config)[random.randint(0, len(config) - 1)]]

+             exception_ctx = pytest.raises(ConfigError)

+         else:

+             exception_ctx = nullcontext()

+ 

+         if not default_subject:

+             config["mail_subject_tmpl"] = "SUBJECT-IS-SET: {message.summary}"

+ 

+         if not default_content:

+             config["mail_content_tmpl"] = "CONTENT-IS-SET\n{message}"

+ 

+         with caplog.at_level(

+             "INFO" if loglevel_info else "DEBUG"

+         ), exception_ctx, mock.patch.object(

+             distgit_commit_processor, "send_email"

+         ) as send_email:

+             toddler.process(config, msg)

+ 

+         if not loglevel_info:

+             assert "Processing message:" in caplog.text

+ 

+         if success:

+             send_email.assert_called_with(

+                 to_addresses=[config["mail_to"]],

+                 from_address=config["mail_from"],

+                 subject=mock.ANY,

+                 content=mock.ANY,

+                 mail_server=config["mail_server"],

+             )

+ 

+             subject = send_email.call_args.kwargs["subject"]

+             assert (

+                 f"{body['agent']} pushed to {commit['namespace']}/{commit['repo']}"

+                 in subject

+             )

+             assert f"({commit['branch']})" in subject

+             assert f"\"{commit['summary']}\"" in subject

+             if default_subject:

+                 assert "SUBJECT-IS-SET" not in subject

+             else:

+                 assert "SUBJECT-IS-SET" in subject

+ 

+             content = send_email.call_args.kwargs["content"]

+             assert f"From {commit['rev']}" in content

+             assert f"From: {commit['name']} <{commit['email']}>" in content

+             assert f"Date: {now}" in content

+             assert f"Subject: {commit['summary']}" in content

+             assert commit["message"] in content

+             if default_content:

+                 assert "CONTENT-IS-SET" not in content

+             else:

+                 assert "CONTENT-IS-SET" in content

+         else:

+             send_email.assert_not_called()

+ 

+         if wrong_msg_type and not loglevel_info:

+             assert "Skipping message" in caplog.text

+         else:

+             assert "Skipping message" not in caplog.text

+ 

+         if fork_commit:

+             assert "Ignoring message" in caplog.text

+         else:

+             assert "Ignoring message" not in caplog.text

file modified
+12
@@ -342,3 +342,15 @@ 

  notify_emails = [

      "root@localhost.localdomain",

  ]

+ 

+ # Configuration section for distgit_commit_processor

+ [consumer_config.distgit_commit_processor]

+ mail_from = "notifications@fedoraproject.org"

+ mail_to = "scm-commits@lists.fedoraproject.org"

+ mail_subject_tmpl = "{message.summary}"

That's already the default in the code, no?

+ mail_content_tmpl = """Notification time stamped {headers['sent-at']}

+ 

+ {message}

+ 

+     {commit['url']}

+ """

@@ -1,2 +1,3 @@ 

+ from .config_error import ConfigError  # noqa: F401

  from .pagure_error import PagureError  # noqa: F401

  from .validation_error import ValidationError  # noqa: F401

@@ -0,0 +1,7 @@ 

+ """

+ Exception that is raised when configuration is broken.

+ """

+ 

+ 

+ class ConfigError(Exception):

+     pass

@@ -0,0 +1,137 @@ 

+ """

+ This plugin takes as input the Fedora messages published under the topic

+ ``pagure.git.receive`` (for dist-git) and sends out emails to scm-commits-list.

+ """

+ 

+ import json

+ import logging

+ from typing import Any, Optional

+ 

+ from fedora_messaging.api import Message

+ from fedora_messaging_git_hook_messages import CommitV1

+ 

+ from toddlers.base import ToddlerBase

+ from toddlers.exceptions import ConfigError

+ from toddlers.utils.notify import send_email

+ 

+ log = logging.getLogger(__name__)

+ 

+ 

+ class DistGitCommitProcessor(ToddlerBase):

+     """A processor which sends emails about commits to dist-git repositories to

+     scm-commits-list.

+     """

+ 

+     name: str = "distgit_commit_processor"

+ 

+     amqp_topics: list[str] = ["org.fedoraproject.*.git.receive"]

+ 

+     def accepts_topic(self, topic: str) -> bool:

+         """Check whether this toddler is interested in messages of a topic.

+ 

+         :arg topic: Topic to check.

+ 

+         :returns: Whether or not this toddler accepts the topic.

+         """

+         return (

+             topic.startswith("org.fedoraproject.")

+             and topic.endswith(".git.receive")

+             and not topic.endswith(".pagure.git.receive")  # different type of message

+         )

+ 

+     def ignore_commit(self, message: CommitV1) -> bool:

+         """Determine if a message pertains to a forked repo.

+ 

+         :arg message: a CommitV1 message from the bus

+         :returns: True if the message is about a fork, not upstream

+         """

+         commit = message.body["commit"]

+         namespace = commit["namespace"]

+         repo = commit["repo"]

+ 

+         if namespace:

+             repo_with_ns = namespace + "/" + repo

+         else:

+             repo_with_ns = repo

+ 

+         orig_path = commit["path"]

+         path = orig_path.rstrip("/")

+ 

+         if path.endswith(".git"):

+             path = path[:-4]

+ 

+         path = path.rstrip("/")

+ 

+         if path.endswith(f"/{repo_with_ns}"):

+             return "/fork/" in path[: -len(repo_with_ns)]

+ 

+         log.warning(

+             "Message %s mismatch: path=%r namespace=%r repo=%r",

+             message.id,

+             orig_path,

+             namespace,

+             repo,

+         )

+ 

+         # Something is off, rather notify superfluously.

I'm not sure I understand this comment :-)

+         return False

+ 

+     def process(self, config: dict[str, Any], message: Message) -> None:

+         """Process a given message.

+ 

+         :arg config: Toddlers configuration

+         :arg message: Message to process

+         """

+         if log.getEffectiveLevel() <= logging.DEBUG:

+             log.debug("Processing message:\n%s", json.dumps(message.body, indent=2))

+ 

+         if not isinstance(message, CommitV1):

+             log.debug(

+                 "Skipping message %s of incompatible type: %s",

+                 message.id,

+                 type(message),

+             )

+             return

+ 

+         if self.ignore_commit(message):

+             log.debug("Ignoring message %r", message)

+             return

+ 

+         # These must be set

+         mail_from: Optional[str] = config.get("mail_from")

+         mail_to: Optional[str] = config.get("mail_to")

+         mail_server: Optional[str] = config.get("mail_server")

+ 

+         if not mail_from or not mail_to or not mail_server:

+             raise ConfigError(

+                 "Invalid toddler configuration: mail_from, mail_to and mail_server have to be set."

+             )

+ 

+         # These have good default fallbacks

+         mail_subject_tmpl: Optional[str] = config.get("mail_subject_tmpl")

+         mail_content_tmpl: Optional[str] = config.get("mail_content_tmpl")

+ 

+         tmpl_params = {

+             "message": message,

+             "headers": message._headers,

+             "body": message.body,

+             "commit": message.body["commit"],

+         }

+ 

+         if mail_subject_tmpl:

+             subject = mail_subject_tmpl.format(**tmpl_params)

+         else:

+             subject = message.summary

+ 

+         if mail_content_tmpl:

+             content = mail_content_tmpl.format(**tmpl_params)

+         else:

+             content = str(message)

+ 

+         send_email(

+             to_addresses=[mail_to],

+             from_address=mail_from,

+             subject=subject,

+             content=content,

+             mail_server=mail_server,

+         )

This plugin listens for *.git.receive messages on the bus sent by
fedora_messaging_git_hook and sends mails to scm-commit-list.

Fixes: #183

This new plugin needs a configuration update for deployment, see the changes to toddlers.toml.example. Also, it doesn’t send out mails for uploads to the lookaside cache – these messages currently aren’t being sent and we don’t have schemas for them yet as well.

WIP: The plugin doesn’t check if the messages are about a “canonical” repository or a fork.

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci
https://fedora.softwarefactory-project.io/zuul/buildset/16a116b15e7343688f9f76a20454ed89

3 new commits added

  • Add distgit_commit_processor plugin
  • Don’t pin jsonschema version
  • Fix link to Ansible playbook
a year ago

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/dbfc4d5e30c24872adcb70362c07ea3a

3 new commits added

  • Add distgit_commit_processor plugin
  • Don’t pin jsonschema version
  • Fix link to Ansible playbook
a year ago

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/b25d32f774c440788acd6c729fbc8b9d

I'm not sure I understand this comment :-)

That's already the default in the code, no?

I'm not sure I understand this comment :-)

Line 67 would return if everything is as expected, the comment is just "This looks sus, send a mail anyway" 😀.

That's already the default in the code, no?

Yeah, it’s just to document the configuration key.

Pull-Request has been merged by nphilipp

a year ago