From f61fbeb6aae325c6a8495b808801d9ca65e09665 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: May 02 2022 09:18:20 +0000
Subject: PR#3217: Adding Driver Update Disk building support


Merges #3217
https://pagure.io/koji/pull-request/3217

Fixes #2998
https://pagure.io/koji/issue/2998

---

diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index 3bcc798..925ad4d 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -281,3 +281,51 @@ Selecting other than default kiwi profile can be done by ``--kiwi-profile``
 option. Similarly to other image tasks, alternative architecture failures can be
 ignored for successful build by ``--can-fail`` option. ``--arch`` can be used to
 limit build tag architectures.
+
+
+Driver Update Disks building
+===========================
+
+**This is just a tech-preview. API/usage can drastically change in upcoming
+releases**
+
+Plugin for creating Driver Update Disks with ``xorrisofs``.
+
+All three parts (cli/hub/builder) needs to be installed. There is currently no
+configuration except allowing the plugins (name is 'dud' for all components).
+
+Builders don't need to have any specific library installed (xorrisofs
+invocation/usage is only in buildroots not on builder itself).
+
+Buildtag needs to be configured by adding special group ``dud-build`` which should contain
+the following packages:
+
+.. code-block:: shell
+
+
+   $ koji add-group dud-build-tag dud-build
+   $ koji add-group-pkg dud-build-tag dud-build xorriso
+   $ koji add-group-pkg dud-build-tag dud-build createrepo_c
+   $ koji add-group-pkg dud-build-tag dud-build dnf
+   $ koji add-group-pkg dud-build-tag dud-build dnf-plugins-core
+
+Another thing we need to ensure is that we're building in chroot and not in
+container.
+
+.. code-block:: shell
+
+   $ koji edit-tag dud-build-tag -x mock.new_chroot=False
+
+Calling the build itself is a matter of simple CLI call:
+
+.. code-block: shell
+
+   $ koji dud-build dud-target --scmurl=git+https://my.git/image-descriptions#master myamazingdud 1 package1 package2
+
+The command options allows to bring all the package dependencies into the DUD
+ISO with ``--alldeps``. ``--scmurl`` allows to include non-RPM related content
+inside the produced ISO.
+
+Similarly to other image tasks, alternative architecture failures can be
+ignored for successful build by ``--can-fail`` option. ``--arch`` can be used
+to limit build tag architectures.
diff --git a/plugins/builder/dud.py b/plugins/builder/dud.py
new file mode 100644
index 0000000..6841430
--- /dev/null
+++ b/plugins/builder/dud.py
@@ -0,0 +1,322 @@
+import os
+import koji
+from fnmatch import fnmatch
+from koji.util import to_list
+from koji.tasks import ServerExit
+from __main__ import BaseBuildTask, BuildImageTask, BuildRoot, SCM
+
+# /usr/lib/koji-builder-plugins/
+
+
+class DudBuildTask(BuildImageTask):
+    Methods = ['dudBuild']
+    _taskWeight = 1.0
+
+    def handler(self, dud_name, dud_version, arches, target, pkg_list, opts=None):
+        target_info = self.session.getBuildTarget(target, strict=True)
+        build_tag = target_info['build_tag']
+        repo_info = self.getRepo(build_tag)
+        # check requested arches against build tag
+        buildconfig = self.session.getBuildConfig(build_tag)
+        if not buildconfig['arches']:
+            raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
+        tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
+        if arches:
+            for arch in arches:
+                if koji.canonArch(arch) not in tag_archlist:
+                    raise koji.BuildError("Invalid arch for build tag: %s" % arch)
+        else:
+            arches = tag_archlist
+
+        if not opts:
+            opts = {}
+        if not opts.get('scratch'):
+            opts['scratch'] = False
+        if not opts.get('alldeps'):
+            opts['alldeps'] = False
+        if not opts.get('scmurl'):
+            opts['scmurl'] = None
+        if not opts.get('optional_arches'):
+            opts['optional_arches'] = []
+        self.opts = opts
+
+        name, version, release = dud_name, dud_version, None
+
+        bld_info = None
+        if opts.get('release'):
+            release = opts['release']
+        else:
+            release = self.session.getNextRelease({'name': name, 'version': version})
+        if '-' in version:
+            raise koji.ApplianceError('The Version may not have a hyphen')
+        if not opts['scratch']:
+            bld_info = self.initImageBuild(name, version, release, target_info, opts)
+            release = bld_info['release']
+
+        try:
+            subtasks = {}
+            canfail = []
+            self.logger.info("Spawning jobs for image arches: %r" % (arches))
+            for arch in arches:
+                subtasks[arch] = self.session.host.subtask(
+                    method='createDudIso',
+                    arglist=[name, version, release, arch,
+                             target_info, build_tag, repo_info,
+                             pkg_list, opts], label=arch, parent=self.id, arch=arch)
+                if arch in self.opts['optional_arches']:
+                    canfail.append(subtasks[arch])
+            self.logger.info("Got image subtasks: %r" % (subtasks))
+            self.logger.info("Waiting on image subtasks (%s can fail)..." % canfail)
+            results = self.wait(to_list(subtasks.values()), all=True, failany=True,
+                                canfail=canfail)
+
+            # if everything failed, fail even if all subtasks are in canfail
+            self.logger.info('subtask results: %r', results)
+            all_failed = True
+            for result in results.values():
+                if not isinstance(result, dict) or 'faultCode' not in result:
+                    all_failed = False
+                    break
+            if all_failed:
+                raise koji.GenericError("all subtasks failed")
+
+            # determine ignored arch failures
+            ignored_arches = set()
+            for arch in arches:
+                if arch in self.opts['optional_arches']:
+                    task_id = subtasks[arch]
+                    result = results[task_id]
+                    if isinstance(result, dict) and 'faultCode' in result:
+                        ignored_arches.add(arch)
+
+            self.logger.info('Image Results for hub: %s' % results)
+            results = {str(k): v for k, v in results.items()}
+            if opts['scratch']:
+                self.session.host.moveImageBuildToScratch(self.id, results)
+            else:
+                self.session.host.completeImageBuild(self.id, bld_info['id'], results)
+        except (SystemExit, ServerExit, KeyboardInterrupt):
+            # we do not trap these
+            raise
+        except Exception:
+            if not opts['scratch']:
+                if bld_info:
+                    self.session.host.failBuild(self.id, bld_info['id'])
+            raise
+
+        # tag it
+        if not opts['scratch'] and not opts.get('skip_tag'):
+            tag_task_id = self.session.host.subtask(method='tagBuild',
+                                                    arglist=[target_info['dest_tag'],
+                                                             bld_info['id'], False, None, True],
+                                                    label='tag', parent=self.id, arch='noarch')
+            self.wait(tag_task_id)
+
+        # report results
+        report = ''
+
+        if opts['scratch']:
+            respath = ', '.join(
+                [os.path.join(koji.pathinfo.work(),
+                              koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()])
+            report += 'Scratch '
+
+        else:
+            respath = koji.pathinfo.imagebuild(bld_info)
+        report += 'image build results in: %s' % respath
+        return report
+
+
+class DudCreateImageTask(BaseBuildTask):
+    Methods = ['createDudIso']
+    _taskWeight = 1.0
+
+    def getImagePackagesFromPath(self, path):
+        """
+        Read RPM header information from the yum cache available in the
+        given path. Returns a list of dictionaries for each RPM included.
+        """
+        found = False
+        hdrlist = {}
+        # For non scratch builds this is a must or it will not work
+        fields = ['name', 'version', 'release', 'epoch', 'arch',
+                  'buildtime', 'sigmd5']
+        for root, dirs, files in os.walk(path):
+            for f in files:
+                if fnmatch(f, '*.rpm'):
+                    pkgfile = os.path.join(root, f)
+                    hdr = koji.get_header_fields(pkgfile, fields)
+                    hdr['size'] = os.path.getsize(pkgfile)
+                    hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
+                    del hdr['sigmd5']
+                    hdrlist[os.path.basename(pkgfile)] = hdr
+                    found = True
+        if not found:
+            raise koji.LiveCDError('No rpms found in root dir!')
+        return list(hdrlist.values())
+
+    def handler(self, dud_name, dud_version, dud_release, arch,
+                target_info, build_tag, repo_info,
+                pkg_list, opts=None):
+        self.opts = opts
+        self.logger.info("Running my dud task...")
+        build_tag = target_info['build_tag']
+        broot = BuildRoot(self.session, self.options,
+                          tag=build_tag,
+                          arch=arch,
+                          task_id=self.id,
+                          repo_id=repo_info['id'],
+                          # Replace with a group that includes createrepo and xorrisofs
+                          install_group='dud-build',
+                          setup_dns=True,
+                          bind_opts={'dirs': {'/dev': '/dev', }})
+        broot.workdir = self.workdir
+
+        # create the mock chroot
+        self.logger.info("Initializing dud buildroot")
+        broot.init()
+        self.logger.info("dud buildroot ready: " + broot.rootdir())
+
+        # user repos
+        repos = self.opts.get('repos', [])
+        # buildroot repo
+        path_info = koji.PathInfo(topdir=self.options.topurl)
+        repopath = path_info.repo(repo_info['id'], target_info['build_tag_name'])
+        baseurl = '%s/%s' % (repopath, arch)
+        self.logger.info('BASEURL: %s' % baseurl)
+        repos.append(baseurl)
+
+        imgdata = {
+            'arch': arch,
+            'task_id': self.id,
+            'name': dud_name,
+            'version': dud_version,
+            'release': dud_release,
+            'logs': [],
+            'rpmlist': [],
+            'files': [],
+        }
+
+        # Donwload each and every one of the packages on the list. We allow more than one
+        # rpms per DUD ISO. Do them one by one to report which one may fail
+        for rpm in pkg_list:
+            cmd = ['/usr/bin/dnf']
+            if self.opts.get('alldeps'):
+                cmd.extend([
+                    'download', '--resolve', '--alldeps', rpm,
+                ])
+            else:
+                cmd.extend([
+                    'download', rpm,
+                ])
+
+            rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
+            if rv:
+                raise koji.GenericError("DUD build failed while getting the involved rpm '{}': {}"
+                      .format(rpm, str(rv)))
+
+        # Create the dd directory structure.
+        cmd = ['/usr/bin/mkdir']
+        cmd.extend([
+            '-p', './dd/rpms/{arch}/repodata/'.format(arch=arch),
+            '-p', './dd/src/',
+        ])
+        rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
+        if rv:
+            raise koji.GenericError("DUD build failed while preparing the dir struct for "
+                                    "the ISO: " + str(rv))
+
+        # Inspiration from https://pagure.io/koji/blob/master/f/plugins/builder/runroot.py#_201
+        # for this dirty hack
+        cmd = ['/usr/bin/sh', '-c']
+        cmd.extend([
+            '/usr/bin/echo -e "Driver Update Disk version 3\c" > ./dd/rhdd3',
+        ])
+        rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
+        if rv:
+            raise koji.GenericError("DUD build failed while writing the rhdd3 file in "
+                                    "the ISO: " + str(rv))
+
+        # Get the SCM content into the ISO root
+        # Retrieve SCM content if it exists
+        if self.opts.get('scmurl'):
+            # get configuration
+            scm = SCM(self.opts.get('scmurl'))
+            scm.assert_allowed(allowed=self.options.allowed_scms,
+                               session=self.session,
+                               by_config=self.options.allowed_scms_use_config,
+                               by_policy=self.options.allowed_scms_use_policy,
+                               policy_data={
+                                   'user_id': self.taskinfo['owner'],
+                                   'channel': self.session.getChannel(self.taskinfo['channel_id'],
+                                                                      strict=True)['name'],
+                                   'scratch': self.opts.get('scratch')
+                               })
+            logfile = os.path.join(self.workdir, 'checkout-%s.log' % arch)
+            self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
+                               build_tag=build_tag, scratch=self.opts.get('scratch'))
+            scmdir = broot.tmpdir()
+            koji.ensuredir(scmdir)
+            scmsrcdir = scm.checkout(scmdir, self.session,
+                                     self.getUploadDir(), logfile)
+            self.run_callbacks("postSCMCheckout",
+                               scminfo=scm.get_info(),
+                               build_tag=build_tag,
+                               scratch=self.opts.get('scratch'),
+                               srcdir=scmsrcdir)
+            cmd = ['/usr/bin/cp']
+            cmd.extend([
+                '-aR', os.path.basename(scmsrcdir), './dd/',
+            ])
+            rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
+            if rv:
+                raise koji.GenericError("DUD build failed while copying SCM repo content into dir"
+                                        "struct: " + str(rv))
+
+        # Get the RPMs inside the corresponding dir struct for the ISO
+        cmd = ['/usr/bin/sh', '-c']
+        # Could not get it to work with a more elegant syntax, as it would not find the *.rpm
+        # files otherwise
+        cmd.extend([
+            '/usr/bin/cp *.rpm ./dd/rpms/{arch}/'.format(arch=arch),
+        ])
+        rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
+        if rv:
+            raise koji.GenericError("DUD build failed while copying RPMs into dir struct: " +
+                                    str(rv))
+
+        cmd = ['/usr/bin/createrepo']
+        cmd.extend([
+            '-q', '--workers=1', './dd/rpms/{arch}/'.format(arch=arch),
+        ])
+        rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
+        if rv:
+            raise koji.GenericError("DUD build failed while creating ISO repodata: " + str(rv))
+
+        # xorrisofs -quiet -lR -V OEMDRV -input-charset utf8 -o $PACKNAME ./dd
+        cmd = ['/usr/bin/sh', '-c']
+        iso_name = 'dd-{name}-{version}-{release}.{arch}.iso'.format(name=dud_name,
+                                                                     version=dud_version,
+                                                                     release=dud_release,
+                                                                     arch=arch)
+        cmd.extend([
+            "/usr/bin/xorrisofs -quiet -lR -V OEMDRV -input-charset utf8 -o {} ".format(iso_name) +
+            "./dd -v"
+        ])
+        rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
+        if rv:
+            raise koji.GenericError("DUD build failed while xorrisofs: " + str(rv))
+
+        fpath = os.path.join(broot.tmpdir(), iso_name)
+        img_file = os.path.basename(fpath)
+        self.uploadFile(fpath, remoteName=os.path.basename(img_file))
+        imgdata['files'].append(img_file)
+
+        if not self.opts.get('scratch'):
+            hdrlist = self.getImagePackagesFromPath(broot.tmpdir())
+            broot.markExternalRPMs(hdrlist)
+            imgdata['rpmlist'] = hdrlist
+
+        broot.expire()
+        self.logger.error("Uploading image data: %s", imgdata)
+        return imgdata
diff --git a/plugins/cli/dud.py b/plugins/cli/dud.py
new file mode 100644
index 0000000..e09ebce
--- /dev/null
+++ b/plugins/cli/dud.py
@@ -0,0 +1,78 @@
+from optparse import OptionParser
+
+from koji import canonArch
+
+from koji.plugin import export_cli
+from koji_cli.lib import (
+    _running_in_bg,
+    activate_session,
+    watch_tasks,
+)
+
+# All client related stuff, to be located in ~/.koji/plugins/dud.py
+
+
+@export_cli
+def handle_dud_build(goptions, session, args):
+    "[build] Run a command in a buildroot"
+    usage = ("usage: %prog dud-build [options] <koji_target> --scmurl=<git_repo> <dud_name>"
+             "<version> <pkg(nvr optional)> [<pkg> ...]")
+    usage += "\n(Specify the --help global option for a list of other help options)"
+    parser = OptionParser(usage=usage)
+    parser.add_option("--scratch", action="store_true", default=False,
+                      help="Perform a scratch build")
+    parser.add_option("--scmurl", metavar="SCMURL", default=None,
+                      help="SCM repository URL for non-rpm related content to be included "
+                      "in the ISO")
+    parser.add_option("--alldeps", action="store_true", default=False,
+                      help="Download all involved rpm dependencies and put them inside "
+                      "the DUD ISO as well")
+    parser.add_option("--arch", action="append", dest="arches", default=[],
+                      help="Limit arches to this subset")
+    parser.add_option("--can-fail", action="store", dest="optional_arches",
+                      metavar="ARCH1,ARCH2,...", default="",
+                      help="List of archs which are not blocking for build "
+                           "(separated by commas.")
+    parser.add_option("--nowait", action="store_false", dest="wait", default=True)
+    parser.add_option("--wait", action="store_true",
+                      help="Wait on the image creation, even if running in the background")
+    (options, args) = parser.parse_args(args)
+
+    if len(args) < 4:
+        parser.error("Incorrect number of arguments")
+        assert False  # pragma: no cover
+    pkg_list = []
+
+    # Can't use * operator with unpacking in Python 2.7, but this works for both Python 2 and 3
+    target, dud_name, dud_version, pkg_list = args[0], args[1], args[2], args[3:]
+
+    activate_session(session, goptions)
+
+    kwargs = {
+        'scratch': options.scratch,
+        'alldeps': options.alldeps,
+        'scmurl': options.scmurl,
+        'optional_arches': [canonArch(arch)
+                            for arch in options.optional_arches.split(',')
+                            if arch],
+    }
+
+    arches = []
+    if options.arches:
+        arches = [canonArch(arch) for arch in options.arches]
+
+    task_id = session.dudBuild(
+        target=target,
+        arches=arches,
+        dud_version=dud_version,
+        dud_name=dud_name,
+        pkg_list=pkg_list,
+        **kwargs)
+
+    if not goptions.quiet:
+        print("Created task: %d" % task_id)
+        print("Task info: %s/taskinfo?taskID=%s" % (goptions.weburl, task_id))
+    if options.wait or (options.wait is None and not _running_in_bg()):
+        session.logout()
+        return watch_tasks(session, [task_id], quiet=goptions.quiet,
+                           poll_interval=goptions.poll_interval, topurl=goptions.topurl)
diff --git a/plugins/hub/dud.py b/plugins/hub/dud.py
new file mode 100644
index 0000000..e31f241
--- /dev/null
+++ b/plugins/hub/dud.py
@@ -0,0 +1,42 @@
+import koji
+import koji.tasks
+import kojihub
+
+from koji.context import context
+from koji.plugin import export
+
+koji.tasks.LEGACY_SIGNATURES['dudBuild'] = [
+    [['dud_name', 'dud_version', 'arches', 'target', 'pkg_list', 'opts'],
+     None, None, (None,)]]
+koji.tasks.LEGACY_SIGNATURES['createDudIso'] = [
+    [['dud_name', 'dud_version', 'dud_release', 'arch',
+      'target_info', 'build_tag', 'repo_info', 'pkg_list', 'opts'],
+     None, None, (None,)]]
+
+# /usr/lib/koji-hub-plugins/
+
+
+@export
+def dudBuild(dud_name, dud_version, arches, target, pkg_list, optional_arches=None, scratch=False,
+             alldeps=False, scmurl=None, priority=None):
+    context.session.assertPerm('image')
+    taskOpts = {
+        'channel': 'image',
+    }
+    if priority:
+        if priority < 0:
+            if not context.session.hasPerm('admin'):
+                raise koji.ActionNotAllowed(
+                    'only admins may create high-priority tasks')
+            taskOpts['priority'] = koji.PRIO_DEFAULT + priority
+
+    opts = {
+        'scratch': scratch,
+        'alldeps': alldeps,
+        'scmurl': scmurl,
+        'optional_arches': optional_arches,
+    }
+
+    return kojihub.make_task('dudBuild',
+                             [dud_name, dud_version, arches, target, pkg_list, opts],
+                             **taskOpts)