From a089716f92cf2b09aff3dba2b6c1e71b890e8a75 Mon Sep 17 00:00:00 2001
From: Lubos Kocman <lkocman@redhat.com>
Date: Aug 01 2016 11:09:50 +0000
Subject: [PATCH 1/6] update pungi-trigger to work with current fm-orchestrator


* listen to build.state_name "ready" rather than commit to repo
* get rid of publish which will be done separately

---

diff --git a/pungi-trigger.py b/pungi-trigger.py
index 0a1f96d..122991f 100755
--- a/pungi-trigger.py
+++ b/pungi-trigger.py
@@ -8,23 +8,18 @@ from __future__ import print_function
 import os
 import sys
 import subprocess as sp
-import tempfile
-import shutil
-
-import yaml
 
 import fedmsg
 
 import fedmsg.config
 import fedmsg.meta
+import pdc_client
 
 config = fedmsg.config.load_config()
 fedmsg.meta.make_processors(**config)
-
-distgit_url = 'http://pkgs.stg.fedoraproject.org'
-target = 'org.fedoraproject.stg.git.receive'
-pubdir = os.environ['PUBDIR']
-puburl = os.environ['PUBURL']
+target_topic = 'org.fedoraproject.dev.rida.module.state.change'
+pdcurl = os.environ['PDCURL']
+repodir = os.environ['REPODIR']
 
 
 def log(*args):
@@ -55,15 +50,6 @@ def run(cmd):
 
     return lines
 
-
-def get_static_manifest_location(lines):
-    telltale = "Wrote rpm static manifest into"
-    for line in lines:
-        if telltale in line:
-            return line.split(telltale)[-1].strip()
-    raise ValueError("Could not find %r in output." % telltale)
-
-
 def get_built_repo_location(lines):
     telltale = "as destination dir"
     for line in lines:
@@ -71,122 +57,45 @@ def get_built_repo_location(lines):
             return line.split(telltale)[0].strip().split()[-1]
     raise ValueError("Could not find %r in output." % telltale)
 
-def publish_directory(path, link=None):
-    log("     Publishing directory:", path)
-    tail = path.split('/')[-1]
-    dest = os.path.join(pubdir, tail)
-
-    log("     Moving content to", dest)
-    if os.path.exists(dest):
-        log("         ...first removing old content at", dest)
-        shutil.rmtree(dest)
-    shutil.move(path, dest)
-
-    if link:
-        link = os.path.join(pubdir, link)
-        log("     Managing link at ", link)
-        try:
-            os.remove(link)
-        except OSError:
-            pass
-        os.symlink(dest, link)
-
-
-def is_modularity_event(msg, config):
-    # XXX - HACK - ideally, we would check a 'namespace' field in the git
-    # message here, but we don't have such a field yet.
-    # See -- https://fedorahosted.org/fedora-infrastructure/ticket/5245
-
-    # Instead, for now, just operate on a hardcoded set.
-    known_modules = set([
-        "testmodule",
-        "core",
-    ])
-
-    modules = fedmsg.meta.msg2packages(msg, **config)
-    return bool(set.intersection(modules, known_modules))
-
+def get_tag_from_pdc(pdc_proxy, name, version, release):
+    tv = pdc_proxy.unreleasedvariants(page_size=-1, variant_id=name, variant_version=version, variant_release=release)
+    if not tv:
+        return None
+    return tv[0]['koji_tag']
 
 def main():
-    log("* Listening for messages on topic %r" % target)
-    for _, _, topic, msg in fedmsg.tail_messages(topic=target, **config):
+    log("* Connecting to PDC %s" % pdcurl)
+    pdc_proxy = pdc_client.PDCClient(pdcurl, insecure=True, develop=True) 
+
+    log("* Listening for messages on topic %r" % target_topic)
+    for _, _, topic, msg in fedmsg.tail_messages(topic=target_topic, **config):
+        print(msg)
         idx = msg['msg_id']
         log("Saw a message", topic, idx)
-
-        if not is_modularity_event(msg, config):
-            # Throw it away
-            log("Dropping", topic, idx)
+        if msg['msg']['state_name'] != "ready":
             continue
 
+        # Run signed repo
         log("Working with", topic, idx)
-
-        # There should only be one package (aka, 'module') here..
-        module = list(fedmsg.meta.msg2packages(msg, **config))[0]
-
-        # Create a temp space to start the work..
-        checkout = tempfile.mktemp()
-        try:
-            # Get the metadata that changed
-            link = '%s/cgit/modules/%s.git' % (distgit_url, module)
-            lines = run(['git', 'clone', link, checkout])
-
-            files = os.listdir(checkout)
-            files = [f for f in files if f.endswith('.yaml')]
-
-            for filename in files:
-                # Run depsolver
-                lines = run([
-                    'pungi-gather',
-                    '--arch', 'x86_64',
-                    '--config', checkout + '/' + filename,
-                    '--target-dir', os.environ['TARGET_DIR'],
-                    '--source-repo-from-path', os.environ['SOURCE_REPO'],
-                ])
-
-                # Run linker/createrepo with static manifest
-                manifest = get_static_manifest_location(lines)
-                log("    Got this manifest for pungi-createrepo", manifest)
-                lines = run([
-                    'pungi-createrepo',
-                    '--target-dir', os.environ['TARGET_DIR'],
-                    '--source-repo-from-path', os.environ['SOURCE_REPO'],
-                    '--static-content-manifest', manifest,
-                    '--arch', 'x86_64',
-                    '--extra-file', 'modulemd.yaml,' + manifest + '/fm-metadata.yaml'
-                ])
-
-                # Lastly, move the built module into a web accessible location.
-                repopath = get_built_repo_location(lines)
-                log("    Got this repopath for web publication", repopath)
-
-                publish_directory(manifest)
-                publish_directory(repopath, link=module)
-
-                # And, final step, update the modules.yaml index with a note.
-                # Read it
-                indexfile = os.path.join(pubdir, 'modules.yaml')
-                log("    Modifying indexfile at", indexfile)
-                try:
-                    with open(indexfile, 'rb') as f:
-                        index = yaml.load(f.read().decode('utf-8'))
-                        index = index or {}  # Protect against 'None'
-                except IOError:
-                    index = {}
-
-                # Modify it
-                suffix = 'rpms/repodata/modulemd.yaml'
-                index[module.decode('utf-8')] = \
-                    os.path.join(puburl, module, suffix).decode('utf-8')
-
-                # And write it back.
-                with open(indexfile, 'wb') as f:
-                    f.write(yaml.dump(index).encode('utf-8'))
-        finally:
-            try:
-                shutil.rmtree(checkout)
-            except:
-                pass
-
+        sigkeys = ['unsigned', ] # XXX: fix this
+        name = msg['msg']['name']
+        version = msg['msg']['version']
+        release = msg['msg']['release']
+        tag = get_tag_from_pdc(pdc_proxy, name, version, release)
+        if not tag:
+            log("ERROR: Could get tag from PDC for module %s-%s-%s. Skipping repo creation." % (name, version, release))
+            continue
+        # Should be exception safe
+        cmd = [
+            'pungi-signed-repo-prototype',
+            '--target-dir', os.environ['REPODIR'],
+            '--koji-profile', os.environ['KOJI_PROFILE'],
+            tag,
+        ]
+        cmd.extend(sigkeys) # multiple
+        print(cmd)
+        log("Attemping to execute %s" % " ".join(cmd))
+        lines = run(cmd)
 
 if __name__ == '__main__':
     sanity_check()

From d1bce7210c7d45f951e92ea50d5832410eb9ab73 Mon Sep 17 00:00:00 2001
From: Lubos Kocman <lkocman@redhat.com>
Date: Aug 01 2016 11:11:42 +0000
Subject: [PATCH 2/6] Update default systemd.config to match our staging area


---

diff --git a/systemd/pungi-modularity.env b/systemd/pungi-modularity.env
index 8adfc09..482b17d 100644
--- a/systemd/pungi-modularity.env
+++ b/systemd/pungi-modularity.env
@@ -1,8 +1,11 @@
 # Just an example env file.  In prod/stg, we'll use some real values here.
 # See https://pagure.io/pungi-modularity/
 
-export MODULARITY_DIR=/home/threebean/devel/pungi-modularity/modularity
-export PYTHONPATH=$MODULARITY_DIR/productmd:$MODULARITY_DIR/pungi:$MODULARITY_DIR/pungi-gather:$MODULARITY_DIR/fm-metadata
-export PATH=$MODULARITY_DIR/pungi-gather/bin:$MODULARITY_DIR/pungi/bin:$PATH
-export TARGET_DIR=~/RHWorkdir
-export SOURCE_REPO=fedora-24-alpha-x86_64
+MODULARITY_DIR="/srv/modularity/pungi-trigger-repos"
+# Libhif is required by pungi-signed-repo-prototype
+export PYTHONPATH="$MODULARITY_DIR/pungi-prototype:$MODULARITY_DIR/libhif/build/python/hawkey:$MODULARITY_DIR/dnf:$MODULARITY_DIR/modulemd"
+export LD_LIBRARY_PATH="$MODULARITY_DIR/libhif/build/libhif"
+export PATH=$MODULARITY_DIR/pungi-prototype/bin:$PATH
+export REPODIR=/srv/modularity/repos
+export PDCURL=http://modularity.fedorainfracloud.org:8080/rest_api/v1/
+export KOJI_PROFILE=koji

From 741073b7a7d66267b31415e2cc776a2865348288 Mon Sep 17 00:00:00 2001
From: Lubos Kocman <lkocman@redhat.com>
Date: Aug 01 2016 11:16:44 +0000
Subject: [PATCH 3/6] Merge with lkocman/code on stage


---

diff --git a/pungi-trigger.py b/pungi-trigger.py
index 0a1f96d..122991f 100755
--- a/pungi-trigger.py
+++ b/pungi-trigger.py
@@ -8,23 +8,18 @@ from __future__ import print_function
 import os
 import sys
 import subprocess as sp
-import tempfile
-import shutil
-
-import yaml
 
 import fedmsg
 
 import fedmsg.config
 import fedmsg.meta
+import pdc_client
 
 config = fedmsg.config.load_config()
 fedmsg.meta.make_processors(**config)
-
-distgit_url = 'http://pkgs.stg.fedoraproject.org'
-target = 'org.fedoraproject.stg.git.receive'
-pubdir = os.environ['PUBDIR']
-puburl = os.environ['PUBURL']
+target_topic = 'org.fedoraproject.dev.rida.module.state.change'
+pdcurl = os.environ['PDCURL']
+repodir = os.environ['REPODIR']
 
 
 def log(*args):
@@ -55,15 +50,6 @@ def run(cmd):
 
     return lines
 
-
-def get_static_manifest_location(lines):
-    telltale = "Wrote rpm static manifest into"
-    for line in lines:
-        if telltale in line:
-            return line.split(telltale)[-1].strip()
-    raise ValueError("Could not find %r in output." % telltale)
-
-
 def get_built_repo_location(lines):
     telltale = "as destination dir"
     for line in lines:
@@ -71,122 +57,45 @@ def get_built_repo_location(lines):
             return line.split(telltale)[0].strip().split()[-1]
     raise ValueError("Could not find %r in output." % telltale)
 
-def publish_directory(path, link=None):
-    log("     Publishing directory:", path)
-    tail = path.split('/')[-1]
-    dest = os.path.join(pubdir, tail)
-
-    log("     Moving content to", dest)
-    if os.path.exists(dest):
-        log("         ...first removing old content at", dest)
-        shutil.rmtree(dest)
-    shutil.move(path, dest)
-
-    if link:
-        link = os.path.join(pubdir, link)
-        log("     Managing link at ", link)
-        try:
-            os.remove(link)
-        except OSError:
-            pass
-        os.symlink(dest, link)
-
-
-def is_modularity_event(msg, config):
-    # XXX - HACK - ideally, we would check a 'namespace' field in the git
-    # message here, but we don't have such a field yet.
-    # See -- https://fedorahosted.org/fedora-infrastructure/ticket/5245
-
-    # Instead, for now, just operate on a hardcoded set.
-    known_modules = set([
-        "testmodule",
-        "core",
-    ])
-
-    modules = fedmsg.meta.msg2packages(msg, **config)
-    return bool(set.intersection(modules, known_modules))
-
+def get_tag_from_pdc(pdc_proxy, name, version, release):
+    tv = pdc_proxy.unreleasedvariants(page_size=-1, variant_id=name, variant_version=version, variant_release=release)
+    if not tv:
+        return None
+    return tv[0]['koji_tag']
 
 def main():
-    log("* Listening for messages on topic %r" % target)
-    for _, _, topic, msg in fedmsg.tail_messages(topic=target, **config):
+    log("* Connecting to PDC %s" % pdcurl)
+    pdc_proxy = pdc_client.PDCClient(pdcurl, insecure=True, develop=True) 
+
+    log("* Listening for messages on topic %r" % target_topic)
+    for _, _, topic, msg in fedmsg.tail_messages(topic=target_topic, **config):
+        print(msg)
         idx = msg['msg_id']
         log("Saw a message", topic, idx)
-
-        if not is_modularity_event(msg, config):
-            # Throw it away
-            log("Dropping", topic, idx)
+        if msg['msg']['state_name'] != "ready":
             continue
 
+        # Run signed repo
         log("Working with", topic, idx)
-
-        # There should only be one package (aka, 'module') here..
-        module = list(fedmsg.meta.msg2packages(msg, **config))[0]
-
-        # Create a temp space to start the work..
-        checkout = tempfile.mktemp()
-        try:
-            # Get the metadata that changed
-            link = '%s/cgit/modules/%s.git' % (distgit_url, module)
-            lines = run(['git', 'clone', link, checkout])
-
-            files = os.listdir(checkout)
-            files = [f for f in files if f.endswith('.yaml')]
-
-            for filename in files:
-                # Run depsolver
-                lines = run([
-                    'pungi-gather',
-                    '--arch', 'x86_64',
-                    '--config', checkout + '/' + filename,
-                    '--target-dir', os.environ['TARGET_DIR'],
-                    '--source-repo-from-path', os.environ['SOURCE_REPO'],
-                ])
-
-                # Run linker/createrepo with static manifest
-                manifest = get_static_manifest_location(lines)
-                log("    Got this manifest for pungi-createrepo", manifest)
-                lines = run([
-                    'pungi-createrepo',
-                    '--target-dir', os.environ['TARGET_DIR'],
-                    '--source-repo-from-path', os.environ['SOURCE_REPO'],
-                    '--static-content-manifest', manifest,
-                    '--arch', 'x86_64',
-                    '--extra-file', 'modulemd.yaml,' + manifest + '/fm-metadata.yaml'
-                ])
-
-                # Lastly, move the built module into a web accessible location.
-                repopath = get_built_repo_location(lines)
-                log("    Got this repopath for web publication", repopath)
-
-                publish_directory(manifest)
-                publish_directory(repopath, link=module)
-
-                # And, final step, update the modules.yaml index with a note.
-                # Read it
-                indexfile = os.path.join(pubdir, 'modules.yaml')
-                log("    Modifying indexfile at", indexfile)
-                try:
-                    with open(indexfile, 'rb') as f:
-                        index = yaml.load(f.read().decode('utf-8'))
-                        index = index or {}  # Protect against 'None'
-                except IOError:
-                    index = {}
-
-                # Modify it
-                suffix = 'rpms/repodata/modulemd.yaml'
-                index[module.decode('utf-8')] = \
-                    os.path.join(puburl, module, suffix).decode('utf-8')
-
-                # And write it back.
-                with open(indexfile, 'wb') as f:
-                    f.write(yaml.dump(index).encode('utf-8'))
-        finally:
-            try:
-                shutil.rmtree(checkout)
-            except:
-                pass
-
+        sigkeys = ['unsigned', ] # XXX: fix this
+        name = msg['msg']['name']
+        version = msg['msg']['version']
+        release = msg['msg']['release']
+        tag = get_tag_from_pdc(pdc_proxy, name, version, release)
+        if not tag:
+            log("ERROR: Could get tag from PDC for module %s-%s-%s. Skipping repo creation." % (name, version, release))
+            continue
+        # Should be exception safe
+        cmd = [
+            'pungi-signed-repo-prototype',
+            '--target-dir', os.environ['REPODIR'],
+            '--koji-profile', os.environ['KOJI_PROFILE'],
+            tag,
+        ]
+        cmd.extend(sigkeys) # multiple
+        print(cmd)
+        log("Attemping to execute %s" % " ".join(cmd))
+        lines = run(cmd)
 
 if __name__ == '__main__':
     sanity_check()
diff --git a/systemd/pungi-modularity.env b/systemd/pungi-modularity.env
index 8adfc09..482b17d 100644
--- a/systemd/pungi-modularity.env
+++ b/systemd/pungi-modularity.env
@@ -1,8 +1,11 @@
 # Just an example env file.  In prod/stg, we'll use some real values here.
 # See https://pagure.io/pungi-modularity/
 
-export MODULARITY_DIR=/home/threebean/devel/pungi-modularity/modularity
-export PYTHONPATH=$MODULARITY_DIR/productmd:$MODULARITY_DIR/pungi:$MODULARITY_DIR/pungi-gather:$MODULARITY_DIR/fm-metadata
-export PATH=$MODULARITY_DIR/pungi-gather/bin:$MODULARITY_DIR/pungi/bin:$PATH
-export TARGET_DIR=~/RHWorkdir
-export SOURCE_REPO=fedora-24-alpha-x86_64
+MODULARITY_DIR="/srv/modularity/pungi-trigger-repos"
+# Libhif is required by pungi-signed-repo-prototype
+export PYTHONPATH="$MODULARITY_DIR/pungi-prototype:$MODULARITY_DIR/libhif/build/python/hawkey:$MODULARITY_DIR/dnf:$MODULARITY_DIR/modulemd"
+export LD_LIBRARY_PATH="$MODULARITY_DIR/libhif/build/libhif"
+export PATH=$MODULARITY_DIR/pungi-prototype/bin:$PATH
+export REPODIR=/srv/modularity/repos
+export PDCURL=http://modularity.fedorainfracloud.org:8080/rest_api/v1/
+export KOJI_PROFILE=koji

From ee753775bb9244534a97d267242a2d861f28650d Mon Sep 17 00:00:00 2001
From: Lubos Kocman <lkocman@redhat.com>
Date: Aug 02 2016 14:25:23 +0000
Subject: [PATCH 4/6] tweak pungi-modularity.env for current staging infra and pungi-signed-repo


---

diff --git a/systemd/pungi-modularity.env b/systemd/pungi-modularity.env
index 482b17d..11a6bdc 100644
--- a/systemd/pungi-modularity.env
+++ b/systemd/pungi-modularity.env
@@ -9,3 +9,5 @@ export PATH=$MODULARITY_DIR/pungi-prototype/bin:$PATH
 export REPODIR=/srv/modularity/repos
 export PDCURL=http://modularity.fedorainfracloud.org:8080/rest_api/v1/
 export KOJI_PROFILE=koji
+export NOTIFICATION_SCRIPT="pungi-signed-repo-fedmsg-notification"
+export LINK_TYPE="download"

From d4867dc9c7a2102daf5e723053d490074928e26c Mon Sep 17 00:00:00 2001
From: Lubos Kocman <lkocman@redhat.com>
Date: Aug 02 2016 14:26:34 +0000
Subject: [PATCH 5/6] pungi-trigger: supply modulemd from scmurl/fedmsg


---

diff --git a/pungi-trigger.py b/pungi-trigger.py
index 122991f..74ffaf1 100755
--- a/pungi-trigger.py
+++ b/pungi-trigger.py
@@ -15,6 +15,11 @@ import fedmsg.config
 import fedmsg.meta
 import pdc_client
 
+import tempfile
+import glob
+import rida.scm # XXX: replace
+import shutil
+
 config = fedmsg.config.load_config()
 fedmsg.meta.make_processors(**config)
 target_topic = 'org.fedoraproject.dev.rida.module.state.change'
@@ -63,39 +68,58 @@ def get_tag_from_pdc(pdc_proxy, name, version, release):
         return None
     return tv[0]['koji_tag']
 
+def checkout_modulemd(scmurl):
+    scm_obj = rida.scm.SCM(url=scmurl)
+
+    src_dir = scm_obj.checkout(tempfile.mkdtemp(prefix="pungi-trigger"))
+    mmds = glob.glob("%s/*.yaml" % src_dir)
+    assert len(mmds) == 1, "Expected exactly 1 modulemd file in git checkout. Got: %s" % mmds
+    return mmds[0]
+
 def main():
     log("* Connecting to PDC %s" % pdcurl)
     pdc_proxy = pdc_client.PDCClient(pdcurl, insecure=True, develop=True) 
 
     log("* Listening for messages on topic %r" % target_topic)
     for _, _, topic, msg in fedmsg.tail_messages(topic=target_topic, **config):
-        print(msg)
-        idx = msg['msg_id']
-        log("Saw a message", topic, idx)
-        if msg['msg']['state_name'] != "ready":
-            continue
-
-        # Run signed repo
-        log("Working with", topic, idx)
-        sigkeys = ['unsigned', ] # XXX: fix this
-        name = msg['msg']['name']
-        version = msg['msg']['version']
-        release = msg['msg']['release']
-        tag = get_tag_from_pdc(pdc_proxy, name, version, release)
-        if not tag:
-            log("ERROR: Could get tag from PDC for module %s-%s-%s. Skipping repo creation." % (name, version, release))
-            continue
-        # Should be exception safe
-        cmd = [
-            'pungi-signed-repo-prototype',
-            '--target-dir', os.environ['REPODIR'],
-            '--koji-profile', os.environ['KOJI_PROFILE'],
-            tag,
-        ]
-        cmd.extend(sigkeys) # multiple
-        print(cmd)
-        log("Attemping to execute %s" % " ".join(cmd))
-        lines = run(cmd)
+        try:
+            print(msg)
+            idx = msg['msg_id']
+            log("Saw a message", topic, idx)
+            if msg['msg']['state_name'] != "ready":
+                continue
+
+            # Run signed repo
+            log("Working with", topic, idx)
+            sigkeys = ['unsigned', ] # XXX: fix this
+            name = msg['msg']['name']
+            version = msg['msg']['version']
+            release = msg['msg']['release']
+            tag = get_tag_from_pdc(pdc_proxy, name, version, release)
+            if not tag:
+                log("ERROR: Could get tag from PDC for module %s-%s-%s. Skipping repo creation." % (name, version, release))
+                continue
+            # Should be exception safe
+            modulemd_path = checkout_modulemd(msg['msg']['scmurl'])
+            cmd = [
+                'pungi-signed-repo-prototype',
+                '--target-dir', os.environ['REPODIR'],
+                '--koji-profile', os.environ['KOJI_PROFILE'],
+                '--notification-script', os.environ['NOTIFICATION_SCRIPT'],
+                '--link-type', os.environ['LINK_TYPE'],
+                '--modulemd', modulemd_path,
+                tag,
+            ]
+            cmd.extend(sigkeys) # multiple
+            print(cmd)
+            log("Attemping to execute %s" % " ".join(cmd))
+            lines = run(cmd)
+
+            # Remove checkout of module definition
+            shutil.rmtree(os.path.dirname(modulemd_path), ignore_errors=True)
+        except Exception, e:
+            log("ERROR: %s" % e)
+            pass
 
 if __name__ == '__main__':
     sanity_check()

From 1399f96834fafe186924ba6c4dccc3ed8e41f406 Mon Sep 17 00:00:00 2001
From: Lubos Kocman <lkocman@redhat.com>
Date: Aug 02 2016 14:33:12 +0000
Subject: [PATCH 6/6] set endpoint to rida on modularity.fedorainfracloud.org


---

diff --git a/fedmsg.d/fed-mod-config.py b/fedmsg.d/fed-mod-config.py
index 9c2f4c0..ac9124b 100644
--- a/fedmsg.d/fed-mod-config.py
+++ b/fedmsg.d/fed-mod-config.py
@@ -3,10 +3,10 @@ config = {
     'validate_signatures': False,
 
     'endpoints': {
-        'modularity-test-server': [
+        'rida.modularity': [
             # This assumes that we fedmsg-relay running with its relay_outbound
             # endpoint set at port 4001 and that that port is opened in os1.
-            'tcp://fed-mod.org:4001',
+            'tcp://modularity.fedorainfracloud.org:4001',
         ],
     },
 }

