From d31306e2f673a89a7791cf209720f52318502816 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 1/15] basic kiwi support


---

diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py
new file mode 100644
index 0000000..42f55fd
--- /dev/null
+++ b/plugins/builder/kiwi.py
@@ -0,0 +1,387 @@
+import glob
+import pickle
+import os
+import xml.dom.minidom
+from fnmatch import fnmatch
+
+import koji
+from koji.util import joinpath, to_list
+from koji.tasks import ServerExit
+from __main__ import BaseBuildTask, BuildImageTask, BuildRoot, SCM
+
+
+class KiwiBuildTask(BuildImageTask):
+    Methods = ['kiwiBuild']
+    _taskWeight = 4.0
+
+    def get_nvr(self, desc_path):
+        # TODO: update release in desc
+        kiwi_files = glob.glob('%s/*.kiwi' % desc_path)
+        if len(kiwi_files) != 1:
+            raise koji.GenericError("Repo must contain only one .kiwi file.")
+
+        cfg = kiwi_files[0]
+
+        newxml = xml.dom.minidom.parse(cfg)
+        image = newxml.getElementsByTagName('image')[0]
+
+        name = image.getAttribute('name')
+        version = None
+        release = None
+        for preferences in image.getElementsByTagName('preferences'):
+            try:
+                version = preferences.getElementsByTagName('version')[0].childNodes[0].data
+            except Exception:
+                pass
+            try:
+                release = preferences.getElementsByTagName('release')[0].childNodes[0].data
+            except Exception:
+                release = None
+        if not version:
+            raise koji.BuildError("Description file doesn't contain preferences/version")
+        return name, version, release
+
+    def handler(self, target, arches, desc_url, desc_path, opts=None):
+        target_info = self.session.getBuildTarget(target, strict=True)
+        build_tag = target_info['build_tag']
+        repo_info = self.getRepo(build_tag)
+        # check requested arches against build tag
+        buildconfig = self.session.getBuildConfig(build_tag)
+        if not buildconfig['arches']:
+            raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
+        tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
+        if arches:
+            for arch in arches:
+                if koji.canonArch(arch) not in tag_archlist:
+                    raise koji.BuildError("Invalid arch for build tag: %s" % arch)
+        else:
+            arches = tag_archlist
+
+        if not opts:
+            opts = {}
+        if not opts.get('scratch'):
+            opts['scratch'] = False
+        if not opts.get('optional_arches'):
+            opts['optional_arches'] = []
+        self.opts = opts
+
+        # get configuration
+        scm = SCM(desc_url)
+        scm.assert_allowed(allowed=self.options.allowed_scms,
+                           session=self.session,
+                           by_config=self.options.allowed_scms_use_config,
+                           by_policy=self.options.allowed_scms_use_policy,
+                           policy_data={
+                               'user_id': self.taskinfo['owner'],
+                               'channel': self.session.getChannel(self.taskinfo['channel_id'],
+                                                                  strict=True)['name'],
+                               'scratch': opts['scratch'],
+                           })
+        logfile = os.path.join(self.workdir, 'checkout.log')
+        self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
+                           build_tag=build_tag, scratch=opts['scratch'])
+        scmdir = self.workdir
+        koji.ensuredir(scmdir)
+        scmsrcdir = scm.checkout(scmdir, self.session,
+                                 self.getUploadDir(), logfile)
+        self.run_callbacks("postSCMCheckout",
+                           scminfo=scm.get_info(),
+                           build_tag=build_tag,
+                           scratch=opts['scratch'],
+                           srcdir=scmsrcdir)
+
+        path = os.path.join(scmsrcdir, desc_path)
+        name, version, release = self.get_nvr(path)
+
+        bld_info = {}
+        if not opts['scratch']:
+            bld_info = self.initImageBuild(name, version, release, target_info, opts)
+            release = bld_info['release']
+        elif not release:
+            release = self.session.getNextRelease({'name': name, 'version': version})
+
+        try:
+            subtasks = {}
+            canfail = []
+            self.logger.debug("Spawning jobs for image arches: %r" % (arches))
+            for arch in arches:
+                subtasks[arch] = self.session.host.subtask(
+                    method='createKiwiImage',
+                    arglist=[name, version, release, arch,
+                             target_info, build_tag, repo_info,
+                             desc_url, desc_path, opts],
+                    label=arch, parent=self.id, arch=arch)
+                if arch in self.opts['optional_arches']:
+                    canfail.append(subtasks[arch])
+            self.logger.debug("Got image subtasks: %r" % (subtasks))
+            self.logger.debug("Waiting on image subtasks (%s can fail)..." % canfail)
+            results = self.wait(to_list(subtasks.values()), all=True,
+                                failany=True, canfail=canfail)
+
+            # if everything failed, fail even if all subtasks are in canfail
+            self.logger.debug('subtask results: %r', results)
+            all_failed = True
+            for result in results.values():
+                if not isinstance(result, dict) or 'faultCode' not in result:
+                    all_failed = False
+                    break
+            if all_failed:
+                raise koji.GenericError("all subtasks failed")
+
+            # determine ignored arch failures
+            ignored_arches = set()
+            for arch in arches:
+                if arch in self.opts['optional_arches']:
+                    task_id = subtasks[arch]
+                    result = results[task_id]
+                    if isinstance(result, dict) and 'faultCode' in result:
+                        ignored_arches.add(arch)
+
+            self.logger.debug('Image Results for hub: %s' % results)
+            results = dict([(str(k), v) for k, v in results.items()])
+            if opts['scratch']:
+                self.session.host.moveImageBuildToScratch(self.id, results)
+            else:
+                self.session.host.completeImageBuild(self.id, bld_info['id'], results)
+        except (SystemExit, ServerExit, KeyboardInterrupt):
+            # we do not trap these
+            raise
+        except Exception:
+            if not opts['scratch']:
+                if bld_info:
+                    self.session.host.failBuild(self.id, bld_info['id'])
+            raise
+
+        # tag it
+        if not opts['scratch'] and not opts.get('skip_tag'):
+            tag_task_id = self.session.host.subtask(method='tagBuild',
+                                                    arglist=[target_info['dest_tag'],
+                                                             bld_info['id'], False, None, True],
+                                                    label='tag', parent=self.id, arch='noarch')
+            self.wait(tag_task_id)
+
+        # report results
+        report = ''
+        if opts['scratch']:
+            respath = ', '.join(
+                [os.path.join(koji.pathinfo.work(),
+                              koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()])
+            report += 'Scratch '
+        else:
+            respath = koji.pathinfo.imagebuild(bld_info)
+        report += 'image build results in: %s' % respath
+        return report
+
+
+class KiwiCreateImageTask(BaseBuildTask):
+    Methods = ['createKiwiImage']
+    _taskWeight = 2.0
+
+    def prepareDescription(self, desc_path, release, repos):
+        # TODO: update release in desc
+        kiwi_files = glob.glob('%s/*.kiwi' % desc_path)
+        if len(kiwi_files) != 1:
+            raise koji.GenericError("Repo must contain only one .kiwi file.")
+
+        cfg = kiwi_files[0]
+
+        newxml = xml.dom.minidom.parse(cfg)
+        image = newxml.getElementsByTagName('image')[0]
+
+        # remove old repos
+        for old_repo in image.getElementsByTagName('repository'):
+            image.removeChild(old_repo)
+
+        # add new ones
+        for repo in sorted(set(repos)):
+            repo_node = newxml.createElement('repository')
+            repo_node.setAttribute('type', 'rpm-md')
+            source = newxml.createElement('source')
+            source.setAttribute('path', repo)
+            repo_node.appendChild(source)
+            image.appendChild(repo_node)
+
+        # TODO: release is part of version (major.minor.release)
+        # preferences = image.getElementsByTagName('preferences')[0]
+        # try:
+        #    preferences.getElementsByTagName('release')[0].childNodes[0].data = release
+        # except Exception:
+        #    rel_node = newxml.createElement('release')
+        #    rel_node.data = release
+        #    preferences.appendChild(rel_node)
+
+        types = []
+        for pref in image.getElementsByTagName('preferences'):
+            for type in pref.getElementsByTagName('type'):
+                # TODO: if type.getAttribute('primary') == 'true':
+                types.append(type.getAttribute('image'))
+
+        # write file back
+        with open(cfg, 'wt') as f:
+            f.write(newxml.toprettyxml())
+ 
+        return cfg, types
+
+    def getImagePackagesFromCache(self, cachepath):
+        """
+        Read RPM header information from the yum cache available in the
+        given path. Returns a list of dictionaries for each RPM included.
+        """
+        found = False
+        hdrlist = {}
+        fields = ['name', 'version', 'release', 'epoch', 'arch',
+                  'buildtime', 'sigmd5']
+        for root, dirs, files in os.walk(cachepath):
+            for f in files:
+                if fnmatch(f, '*.rpm'):
+                    pkgfile = os.path.join(root, f)
+                    hdr = koji.get_header_fields(pkgfile, fields)
+                    hdr['size'] = os.path.getsize(pkgfile)
+                    hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
+                    del hdr['sigmd5']
+                    hdrlist[os.path.basename(pkgfile)] = hdr
+                    found = True
+        if not found:
+            raise koji.LiveCDError('No repos found in yum cache!')
+        return list(hdrlist.values())
+
+    def getImagePackages(self, result):
+        """Proper handler for getting rpminfo from result list,
+        it need result list to contain payloadhash, etc. to work correctly"""
+        hdrlist = []
+        for line in open(result, 'rt'):
+            line = line.strip()
+            name, epoch, version, release, arch, disturl, license = line.split('|')
+            try:
+                # "(none)" for None epochs
+                epoch = int(epoch)
+            except ValueError:
+                epoch = None
+            hdrlist.append({
+                'name': name,
+                'epoch': epoch,
+                'version': version,
+                'release': release,
+                'arch': arch,
+                'payloadhash': '',
+                'size': 0,
+                'buildtime': 0,
+            })
+
+        return hdrlist
+
+    def handler(self, name, version, release, arch,
+                target_info, build_tag, repo_info,
+                desc_url, desc_path, opts=None):
+        self.opts = opts
+        build_tag = target_info['build_tag']
+        broot = BuildRoot(self.session, self.options,
+                          tag=build_tag,
+                          arch=arch,
+                          task_id=self.id,
+                          repo_id=repo_info['id'],
+                          install_group='kiwi',
+                          setup_dns=True,
+                          bind_opts={'dirs': {'/dev': '/dev', }})
+        broot.workdir = self.workdir
+
+        # create the mock chroot
+        self.logger.debug("Initializing kiwi buildroot")
+        broot.init()
+        self.logger.debug("Kiwi buildroot ready: " + broot.rootdir())
+
+        # get configuration
+        scm = SCM(desc_url)
+        scm.assert_allowed(allowed=self.options.allowed_scms,
+                           session=self.session,
+                           by_config=self.options.allowed_scms_use_config,
+                           by_policy=self.options.allowed_scms_use_policy,
+                           policy_data={
+                               'user_id': self.taskinfo['owner'],
+                               'channel': self.session.getChannel(self.taskinfo['channel_id'],
+                                                                  strict=True)['name'],
+                               'scratch': self.opts.get('scratch')
+                           })
+        logfile = os.path.join(self.workdir, 'checkout-%s.log' % arch)
+        self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
+                           build_tag=build_tag, scratch=self.opts.get('scratch'))
+        scmdir = broot.tmpdir()
+        koji.ensuredir(scmdir)
+        scmsrcdir = scm.checkout(scmdir, self.session,
+                                 self.getUploadDir(), logfile)
+        self.run_callbacks("postSCMCheckout",
+                           scminfo=scm.get_info(),
+                           build_tag=build_tag,
+                           scratch=self.opts.get('scratch'),
+                           srcdir=scmsrcdir)
+
+        # user repos
+        repos = self.opts.get('repos', [])
+        # buildroot repo
+        path_info = koji.PathInfo(topdir=self.options.topurl)
+        repopath = path_info.repo(repo_info['id'], target_info['build_tag_name'])
+        baseurl = '%s/%s' % (repopath, arch)
+        self.logger.debug('BASEURL: %s' % baseurl)
+        repos.append(baseurl)
+
+        path = os.path.join(scmsrcdir, desc_path)
+        desc, types = self.prepareDescription(path, release, repos)
+        self.uploadFile(desc)
+
+        cmd = ['kiwi-ng']
+        if self.opts.get('profile'):
+            cmd.extend(['--profile', self.opts['profile']])
+        target_dir = '/builddir/result/image'
+        cmd.extend([
+            'system', 'build',
+            '--description', os.path.join(os.path.basename(scmsrcdir), desc_path),
+            '--target-dir', target_dir,
+        ])
+        rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
+        if rv:
+            raise koji.GenericError("Kiwi failed")
+
+        result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
+
+        imgdata = {
+            'arch': arch,
+            'task_id': self.id,
+            'logs': [
+                os.path.basename(desc)
+            ],
+            'name': name,
+            'version': version,
+            'release': release,
+            'rpmlist': [],
+            'files': [],
+        }
+        # TODO: upload detailed log?
+        # build/image-root.log
+        # os.path.join(broot.tmpdir(), target_dir[1:], "build/image-root.log")
+
+        # for type in types:
+        #     img_file = '%s.%s-%s.%s' % (name, version, arch, type)
+        #     self.uploadFile(os.path.join(broot.rootdir()), remoteName=img_file)
+        #     imgdata['files'].append(img_file)
+        fpath = os.path.join(broot.rootdir(),
+                             result.result_files['disk_format_image'].filename[1:])
+        img_file = os.path.basename(fpath)
+        self.uploadFile(fpath, remoteName=os.path.basename(img_file))
+        imgdata['files'].append(img_file)
+
+        if not self.opts.get('scratch'):
+            if False:
+                # should be used after kiwi update
+                fpath = os.path.join(broot.rootdir(),
+                                     result.result_files['image_packages'].filename[1:])
+                hdrlist = self.getImagePackages(fpath)
+            else:
+                cachepath = os.path.join(broot.rootdir(), 'var/cache/kiwi/dnf')
+                hdrlist = self.getImagePackagesFromCache(cachepath)
+            broot.markExternalRPMs(hdrlist)
+            imgdata['rpmlist'] = hdrlist
+
+        broot.expire()
+
+        self.logger.error("Uploading image data: %s", imgdata)
+        return imgdata
diff --git a/plugins/cli/kiwi.py b/plugins/cli/kiwi.py
new file mode 100644
index 0000000..8076124
--- /dev/null
+++ b/plugins/cli/kiwi.py
@@ -0,0 +1,75 @@
+import os
+from optparse import OptionParser
+
+from koji import canonArch
+
+from koji.plugin import export_cli
+from koji_cli.lib import (
+    _,
+    _running_in_bg,
+    activate_session,
+    watch_tasks,
+)
+
+
+@export_cli
+def handle_kiwi_build(goptions, session, args):
+    "[build] Run a command in a buildroot"
+    usage = _("usage: %prog kiwi-build [options] "
+              "<target> <description_scm> <description_path>")
+    usage += _("\n(Specify the --help global option for a list of other help options)")
+    parser = OptionParser(usage=usage)
+    parser.add_option("--scratch", action="store_true", default=False,
+                      help=_("Perform a scratch build"))
+    parser.add_option("--repo", action="append",
+                      help=_("Specify a repo that will override the repo used to install "
+                             "RPMs in the image. May be used multiple times. The "
+                             "build tag repo associated with the target is the default."))
+    parser.add_option("--noprogress", action="store_true",
+                      help=_("Do not display progress of the upload"))
+    parser.add_option("--kiwi-profile", action="store", default=None,
+                      help=_("Select profile from description file"))
+    parser.add_option("--can-fail", action="store", dest="optional_arches",
+                      metavar="ARCH1,ARCH2,...", default="",
+                      help=_("List of archs which are not blocking for build "
+                             "(separated by commas."))
+    parser.add_option("--arch", action="append", dest="arches", default=[],
+                      help=_("Limit arches to this subset"))
+    parser.add_option("--nowait", action="store_false", dest="wait", default=True)
+    parser.add_option("--wait", action="store_true",
+                      help=_("Wait on the image creation, even if running in the background"))
+    (options, args) = parser.parse_args(args)
+
+    if len(args) != 3:
+        parser.error(_("Incorrect number of arguments"))
+        assert False  # pragma: no cover
+    target, scm, path = args
+
+    activate_session(session, goptions)
+
+    kwargs = {
+        'scratch': options.scratch,
+        'optional_arches': [canonArch(arch)
+                            for arch in options.optional_arches.split(',')
+                            if arch],
+        'profile': options.kiwi_profile,
+    }
+
+    arches = []
+    if options.arches:
+        arches = [canonArch(arch) for arch in options.arches]
+
+    task_id = session.kiwiBuild(
+        target=target,
+        arches=arches,
+        desc_url=scm,
+        desc_path=path,
+        **kwargs)
+
+    if not goptions.quiet:
+        print("Created task: %d" % task_id)
+        print("Task info: %s/taskinfo?taskID=%s" % (goptions.weburl, task_id))
+    if options.wait or (options.wait is None and not _running_in_bg()):
+        session.logout()
+        return watch_tasks(session, [task_id], quiet=goptions.quiet,
+                           poll_interval=goptions.poll_interval, topurl=goptions.topurl)
diff --git a/plugins/hub/kiwi.py b/plugins/hub/kiwi.py
new file mode 100644
index 0000000..911612f
--- /dev/null
+++ b/plugins/hub/kiwi.py
@@ -0,0 +1,38 @@
+import koji
+import koji.tasks
+import kojihub
+
+from koji.context import context
+from koji.plugin import export
+
+koji.tasks.LEGACY_SIGNATURES['kiwiBuild'] = [
+    [['target', 'arches', 'desc_url', 'desc_path', 'opts'],
+     None, None, (None,)]]
+koji.tasks.LEGACY_SIGNATURES['createKiwiImage'] = [
+    [['name', 'version', 'release', 'arch',
+      'target_info', 'build_tag', 'repo_info', 'desc_url', 'opts'],
+     None, None, (None,)]]
+
+
+@export
+def kiwiBuild(target, arches, desc_url, desc_path, optional_arches=None, profile=None,
+              scratch=False, priority=None):
+    context.session.assertPerm('image')
+    taskOpts = {
+        'channel': 'image',
+    }
+    if priority:
+        if priority < 0:
+            if not context.session.hasPerm('admin'):
+                raise koji.ActionNotAllowed(
+                    'only admins may create high-priority tasks')
+        taskOpts['priority'] = koji.PRIO_DEFAULT + priority
+
+    opts = {
+        'optional_arches': optional_arches,
+        'profile': profile,
+        'scratch': scratch,
+    }
+    return kojihub.make_task('kiwiBuild',
+                             [target, arches, desc_url, desc_path, opts],
+                             **taskOpts)

From 86ee135dd182d9354a91ebcf0abb635d8688cac5 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 2/15] json results


---

diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py
index 42f55fd..3cfe632 100644
--- a/plugins/builder/kiwi.py
+++ b/plugins/builder/kiwi.py
@@ -1,4 +1,5 @@
 import glob
+#import json
 import pickle
 import os
 import xml.dom.minidom
@@ -342,6 +343,7 @@ class KiwiCreateImageTask(BaseBuildTask):
             raise koji.GenericError("Kiwi failed")
 
         result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
+        #result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
 
         imgdata = {
             'arch': arch,

From d07c0d400eef60aa8b8c28ede7bf9d34fb092879 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 3/15] basic kiwi docs


---

diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index 1ec332f..4e4d999 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -226,3 +226,41 @@ sent on the bus (e.g. if the amqps server is offline).
 Admins should consider the balance between the ``batch_size`` and
 ``extra_limit`` options, as both can affect the total amount of data that the
 plugin could attempt to send during a single call.
+
+
+Kiwi images
+===========
+
+**This is just a tech-preview. API/usage can drastically change in upcoming
+releases**
+
+Plugin for creating images via `kiwi <http://osinside.github.io/kiwi/>`_
+project.
+
+All three parts (cli/hub/builder) needs to be installed. There is currently no
+configuration except allowing the plugins.
+
+Builders have to be part of ``image`` channel and don't need to have any
+specific library installed (kiwi invocation/usage is only in buildroots not on
+builder itself).
+
+Buildtag needs to be configured by adding special group ``kiwi`` which should
+contain at least ``kiwi-cli``, potentially ``jing`` for better description files
+validation and any ``kiwi-systemdeps-*`` packages for requested image types. So,
+most simple configuration will look like:
+
+.. code-block:: shell
+
+   $ koji add-group kiwi-build-tag kiwi
+   $ koji add-group-pkg kiwi-build-tag kiwi-cli
+
+Calling the build itself is a matter of simple CLI call:
+
+.. code-block: shell
+
+   $ koji kiwi-build kiwi-target git+https://my.git/image-descriptions#master my_image_path
+
+Selecting other than default kiwi profile can be done by ``--kiwi-profile``
+option. Similarly to other image tasks, alternative architecture failures can be
+ignored for successful build by ``--can-fail`` option. ``--arch`` can be used to
+limit build tag architectures.

From 8c1a7337abc8476af40ccc412db908a1941cdc71 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 4/15] kiwi: check include directive


---

diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py
index 3cfe632..3403423 100644
--- a/plugins/builder/kiwi.py
+++ b/plugins/builder/kiwi.py
@@ -189,11 +189,22 @@ class KiwiCreateImageTask(BaseBuildTask):
         newxml = xml.dom.minidom.parse(cfg)
         image = newxml.getElementsByTagName('image')[0]
 
-        # remove old repos
+        # apply includes - kiwi can include only top-level nodes, so we can simply
+        # go through "include" elements and replace them with referred content (without
+        # doing it recursively)
+        for inc_node in image.getElementsByTagName('include'):
+            path = inc_node.getAttribute('from')
+            inc = xml.dom.minidom.parse(path)
+            # every included xml has image root element again
+            for node in inc.getElementsByTagName('image').childNodes:
+                if node.nodeName != 'repository':
+                    image.appendChild(node)
+
+        # remove remaining old repos
         for old_repo in image.getElementsByTagName('repository'):
             image.removeChild(old_repo)
 
-        # add new ones
+        # add koji ones
         for repo in sorted(set(repos)):
             repo_node = newxml.createElement('repository')
             repo_node.setAttribute('type', 'rpm-md')
@@ -220,7 +231,7 @@ class KiwiCreateImageTask(BaseBuildTask):
         # write file back
         with open(cfg, 'wt') as f:
             f.write(newxml.toprettyxml())
- 
+
         return cfg, types
 
     def getImagePackagesFromCache(self, cachepath):

From 3c15c7a34f7db08b2a4e5c445e2bb39a10f7b558 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 5/15] update docs


---

diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index 4e4d999..39de555 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -228,17 +228,17 @@ Admins should consider the balance between the ``batch_size`` and
 plugin could attempt to send during a single call.
 
 
-Kiwi images
-===========
+Image builds using Kiwi
+=======================
 
 **This is just a tech-preview. API/usage can drastically change in upcoming
 releases**
 
 Plugin for creating images via `kiwi <http://osinside.github.io/kiwi/>`_
-project.
+project. Minimal supported version of kiwi is ``kiwi-9.24.2``.
 
 All three parts (cli/hub/builder) needs to be installed. There is currently no
-configuration except allowing the plugins.
+configuration except allowing the plugins (name is 'kiwi' for all components).
 
 Builders have to be part of ``image`` channel and don't need to have any
 specific library installed (kiwi invocation/usage is only in buildroots not on

From 946a24cca8dd5619ac948a61bc2b060dec37c8fc Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 6/15] remove translation


---

diff --git a/plugins/cli/kiwi.py b/plugins/cli/kiwi.py
index 8076124..0de6559 100644
--- a/plugins/cli/kiwi.py
+++ b/plugins/cli/kiwi.py
@@ -5,7 +5,6 @@ from koji import canonArch
 
 from koji.plugin import export_cli
 from koji_cli.lib import (
-    _,
     _running_in_bg,
     activate_session,
     watch_tasks,
@@ -15,33 +14,32 @@ from koji_cli.lib import (
 @export_cli
 def handle_kiwi_build(goptions, session, args):
     "[build] Run a command in a buildroot"
-    usage = _("usage: %prog kiwi-build [options] "
-              "<target> <description_scm> <description_path>")
-    usage += _("\n(Specify the --help global option for a list of other help options)")
+    usage = "usage: %prog kiwi-build [options] <target> <description_scm> <description_path>"
+    usage += "\n(Specify the --help global option for a list of other help options)"
     parser = OptionParser(usage=usage)
     parser.add_option("--scratch", action="store_true", default=False,
-                      help=_("Perform a scratch build"))
+                      help="Perform a scratch build")
     parser.add_option("--repo", action="append",
-                      help=_("Specify a repo that will override the repo used to install "
-                             "RPMs in the image. May be used multiple times. The "
-                             "build tag repo associated with the target is the default."))
+                      help="Specify a repo that will override the repo used to install "
+                           "RPMs in the image. May be used multiple times. The "
+                           "build tag repo associated with the target is the default.")
     parser.add_option("--noprogress", action="store_true",
-                      help=_("Do not display progress of the upload"))
+                      help="Do not display progress of the upload")
     parser.add_option("--kiwi-profile", action="store", default=None,
-                      help=_("Select profile from description file"))
+                      help="Select profile from description file")
     parser.add_option("--can-fail", action="store", dest="optional_arches",
                       metavar="ARCH1,ARCH2,...", default="",
-                      help=_("List of archs which are not blocking for build "
-                             "(separated by commas."))
+                      help="List of archs which are not blocking for build "
+                           "(separated by commas.")
     parser.add_option("--arch", action="append", dest="arches", default=[],
-                      help=_("Limit arches to this subset"))
+                      help="Limit arches to this subset")
     parser.add_option("--nowait", action="store_false", dest="wait", default=True)
     parser.add_option("--wait", action="store_true",
-                      help=_("Wait on the image creation, even if running in the background"))
+                      help="Wait on the image creation, even if running in the background")
     (options, args) = parser.parse_args(args)
 
     if len(args) != 3:
-        parser.error(_("Incorrect number of arguments"))
+        parser.error("Incorrect number of arguments")
         assert False  # pragma: no cover
     target, scm, path = args
 

From 86c8fdc17720eccf915c4031cc349bcbfa0e0d04 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 7/15] use json results


---

diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py
index 3403423..58f25d1 100644
--- a/plugins/builder/kiwi.py
+++ b/plugins/builder/kiwi.py
@@ -1,6 +1,5 @@
 import glob
-#import json
-import pickle
+import json
 import os
 import xml.dom.minidom
 from fnmatch import fnmatch
@@ -353,8 +352,7 @@ class KiwiCreateImageTask(BaseBuildTask):
         if rv:
             raise koji.GenericError("Kiwi failed")
 
-        result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
-        #result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
+        result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
 
         imgdata = {
             'arch': arch,
@@ -377,7 +375,7 @@ class KiwiCreateImageTask(BaseBuildTask):
         #     self.uploadFile(os.path.join(broot.rootdir()), remoteName=img_file)
         #     imgdata['files'].append(img_file)
         fpath = os.path.join(broot.rootdir(),
-                             result.result_files['disk_format_image'].filename[1:])
+                             result['result_files']['disk_format_image'].filename[1:])
         img_file = os.path.basename(fpath)
         self.uploadFile(fpath, remoteName=os.path.basename(img_file))
         imgdata['files'].append(img_file)
@@ -386,7 +384,7 @@ class KiwiCreateImageTask(BaseBuildTask):
             if False:
                 # should be used after kiwi update
                 fpath = os.path.join(broot.rootdir(),
-                                     result.result_files['image_packages'].filename[1:])
+                                     result['result_files']['image_packages'].filename[1:])
                 hdrlist = self.getImagePackages(fpath)
             else:
                 cachepath = os.path.join(broot.rootdir(), 'var/cache/kiwi/dnf')
diff --git a/plugins/cli/kiwi.py b/plugins/cli/kiwi.py
index 0de6559..c2fb6af 100644
--- a/plugins/cli/kiwi.py
+++ b/plugins/cli/kiwi.py
@@ -1,4 +1,3 @@
-import os
 from optparse import OptionParser
 
 from koji import canonArch

From 336573c0fd430d37b87d99a7bc92dc02973e6a9c Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 8/15] code simplification


---

diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py
index 58f25d1..7310a28 100644
--- a/plugins/builder/kiwi.py
+++ b/plugins/builder/kiwi.py
@@ -138,7 +138,7 @@ class KiwiBuildTask(BuildImageTask):
                         ignored_arches.add(arch)
 
             self.logger.debug('Image Results for hub: %s' % results)
-            results = dict([(str(k), v) for k, v in results.items()])
+            results = {str(k): v for k, v in results.items()}
             if opts['scratch']:
                 self.session.host.moveImageBuildToScratch(self.id, results)
             else:
@@ -263,11 +263,10 @@ class KiwiCreateImageTask(BaseBuildTask):
         for line in open(result, 'rt'):
             line = line.strip()
             name, epoch, version, release, arch, disturl, license = line.split('|')
-            try:
-                # "(none)" for None epochs
-                epoch = int(epoch)
-            except ValueError:
+            if epoch == '(none)':
                 epoch = None
+            else:
+                epoch = int(epoch)
             hdrlist.append({
                 'name': name,
                 'epoch': epoch,

From 2d6b03eebaa1a9aeb7a0150f3a85e40a2fca380b Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 9/15] fix signature


---

diff --git a/plugins/hub/kiwi.py b/plugins/hub/kiwi.py
index 911612f..be51bb0 100644
--- a/plugins/hub/kiwi.py
+++ b/plugins/hub/kiwi.py
@@ -10,7 +10,7 @@ koji.tasks.LEGACY_SIGNATURES['kiwiBuild'] = [
      None, None, (None,)]]
 koji.tasks.LEGACY_SIGNATURES['createKiwiImage'] = [
     [['name', 'version', 'release', 'arch',
-      'target_info', 'build_tag', 'repo_info', 'desc_url', 'opts'],
+      'target_info', 'build_tag', 'repo_info', 'desc_url', 'desc_path', 'opts'],
      None, None, (None,)]]
 
 

From 2d6a14845f8ac8207697f3969aeb90e0508ddc93 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 10/15] alter name by selected profile


---

diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py
index 7310a28..3f06163 100644
--- a/plugins/builder/kiwi.py
+++ b/plugins/builder/kiwi.py
@@ -14,7 +14,7 @@ class KiwiBuildTask(BuildImageTask):
     Methods = ['kiwiBuild']
     _taskWeight = 4.0
 
-    def get_nvr(self, desc_path):
+    def get_nvrp(self, desc_path):
         # TODO: update release in desc
         kiwi_files = glob.glob('%s/*.kiwi' % desc_path)
         if len(kiwi_files) != 1:
@@ -37,9 +37,17 @@ class KiwiBuildTask(BuildImageTask):
                 release = preferences.getElementsByTagName('release')[0].childNodes[0].data
             except Exception:
                 release = None
+        profile = None
+        try:
+            for p in image.getElementsByTagName('profiles')[0].getElementsByTagName('profile'):
+                if p.getAttribute('image') == 'true':
+                    profile = p.getAttribute('name')
+        except IndexError:
+            # missing profiles section
+            pass
         if not version:
             raise koji.BuildError("Description file doesn't contain preferences/version")
-        return name, version, release
+        return name, version, release, profile
 
     def handler(self, target, arches, desc_url, desc_path, opts=None):
         target_info = self.session.getBuildTarget(target, strict=True)
@@ -91,7 +99,12 @@ class KiwiBuildTask(BuildImageTask):
                            srcdir=scmsrcdir)
 
         path = os.path.join(scmsrcdir, desc_path)
-        name, version, release = self.get_nvr(path)
+
+        name, version, release, default_profile = self.get_nvrp(path)
+        if opts.get('profile') or default_profile:
+            # package name is a combination of name + profile
+            # in case profiles are not used, let's use the standalone name
+            name = "%s-%s" % (name, opts.get('profile', default_profile))
 
         bld_info = {}
         if not opts['scratch']:
@@ -177,7 +190,7 @@ class KiwiCreateImageTask(BaseBuildTask):
     Methods = ['createKiwiImage']
     _taskWeight = 2.0
 
-    def prepareDescription(self, desc_path, release, repos):
+    def prepareDescription(self, desc_path, name, release, repos):
         # TODO: update release in desc
         kiwi_files = glob.glob('%s/*.kiwi' % desc_path)
         if len(kiwi_files) != 1:
@@ -212,6 +225,7 @@ class KiwiCreateImageTask(BaseBuildTask):
             repo_node.appendChild(source)
             image.appendChild(repo_node)
 
+        image.setAttribute('name', name)
         # TODO: release is part of version (major.minor.release)
         # preferences = image.getElementsByTagName('preferences')[0]
         # try:
@@ -229,7 +243,10 @@ class KiwiCreateImageTask(BaseBuildTask):
 
         # write file back
         with open(cfg, 'wt') as f:
-            f.write(newxml.toprettyxml())
+            s = newxml.toprettyxml()
+            # toprettyxml adds too many whitespaces/newlines
+            s = '\n'.join([x for x in s.splitlines() if x.strip()])
+            f.write(s)
 
         return cfg, types
 
@@ -335,7 +352,7 @@ class KiwiCreateImageTask(BaseBuildTask):
         repos.append(baseurl)
 
         path = os.path.join(scmsrcdir, desc_path)
-        desc, types = self.prepareDescription(path, release, repos)
+        desc, types = self.prepareDescription(path, name, release, repos)
         self.uploadFile(desc)
 
         cmd = ['kiwi-ng']
@@ -351,13 +368,15 @@ class KiwiCreateImageTask(BaseBuildTask):
         if rv:
             raise koji.GenericError("Kiwi failed")
 
-        result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
+        #result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
+        import pickle
+        result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
 
         imgdata = {
             'arch': arch,
             'task_id': self.id,
             'logs': [
-                os.path.basename(desc)
+                os.path.basename(desc),
             ],
             'name': name,
             'version': version,
@@ -365,19 +384,25 @@ class KiwiCreateImageTask(BaseBuildTask):
             'rpmlist': [],
             'files': [],
         }
+
         # TODO: upload detailed log?
         # build/image-root.log
-        # os.path.join(broot.tmpdir(), target_dir[1:], "build/image-root.log")
+        root_log_path = os.path.join(broot.tmpdir(), target_dir[1:], "build/image-root.log")
+        if os.path.exists(root_log_path):
+            self.uploadFile(root_log_path, remoteName="image-root.log")
 
         # for type in types:
         #     img_file = '%s.%s-%s.%s' % (name, version, arch, type)
         #     self.uploadFile(os.path.join(broot.rootdir()), remoteName=img_file)
         #     imgdata['files'].append(img_file)
-        fpath = os.path.join(broot.rootdir(),
-                             result['result_files']['disk_format_image'].filename[1:])
-        img_file = os.path.basename(fpath)
-        self.uploadFile(fpath, remoteName=os.path.basename(img_file))
-        imgdata['files'].append(img_file)
+        for ftype in ('disk_format_image', 'installation_image'):
+            fdata = result.result_files.get(ftype)
+            if not fdata:
+                continue
+            fpath = os.path.join(broot.rootdir(), fdata.filename[1:])
+            img_file = os.path.basename(fpath)
+            self.uploadFile(fpath, remoteName=os.path.basename(img_file))
+            imgdata['files'].append(img_file)
 
         if not self.opts.get('scratch'):
             if False:

From 1e971affef300e41c2d3ff3db535ef1efbba865e Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 11/15] fix doc typo


---

diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index 39de555..c2fbb87 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -252,7 +252,7 @@ most simple configuration will look like:
 .. code-block:: shell
 
    $ koji add-group kiwi-build-tag kiwi
-   $ koji add-group-pkg kiwi-build-tag kiwi-cli
+   $ koji add-group-pkg kiwi-build-tag kiwi kiwi-cli
 
 Calling the build itself is a matter of simple CLI call:
 

From 79e1841c96a1aa2a3de31f8de08b9b400960d0ce Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 12/15] fix bandit


---

diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py
index 3f06163..64d6bcf 100644
--- a/plugins/builder/kiwi.py
+++ b/plugins/builder/kiwi.py
@@ -22,7 +22,7 @@ class KiwiBuildTask(BuildImageTask):
 
         cfg = kiwi_files[0]
 
-        newxml = xml.dom.minidom.parse(cfg)
+        newxml = xml.dom.minidom.parse(cfg)  # nosec
         image = newxml.getElementsByTagName('image')[0]
 
         name = image.getAttribute('name')
@@ -198,7 +198,7 @@ class KiwiCreateImageTask(BaseBuildTask):
 
         cfg = kiwi_files[0]
 
-        newxml = xml.dom.minidom.parse(cfg)
+        newxml = xml.dom.minidom.parse(cfg)  # nosec
         image = newxml.getElementsByTagName('image')[0]
 
         # apply includes - kiwi can include only top-level nodes, so we can simply
@@ -206,7 +206,7 @@ class KiwiCreateImageTask(BaseBuildTask):
         # doing it recursively)
         for inc_node in image.getElementsByTagName('include'):
             path = inc_node.getAttribute('from')
-            inc = xml.dom.minidom.parse(path)
+            inc = xml.dom.minidom.parse(path)  # nosec
             # every included xml has image root element again
             for node in inc.getElementsByTagName('image').childNodes:
                 if node.nodeName != 'repository':
@@ -369,8 +369,10 @@ class KiwiCreateImageTask(BaseBuildTask):
             raise koji.GenericError("Kiwi failed")
 
         #result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
+        # nosec comment - we will replace it with json ASAP
         import pickle
-        result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
+        result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:],  # nosec
+                                           'kiwi.result'), 'rb'))
 
         imgdata = {
             'arch': arch,

From eb56f2bec437f4f7a102ff4153be5e69ac4c8c69 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 13/15] expand docs


---

diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index c2fbb87..c6c8f43 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -254,6 +254,13 @@ most simple configuration will look like:
    $ koji add-group kiwi-build-tag kiwi
    $ koji add-group-pkg kiwi-build-tag kiwi kiwi-cli
 
+Another thing we need to ensure is that we're building in chroot and not in
+container.
+
+.. code-block:: shell
+
+   $ koji edit-tag kiwi-build-tag -x mock.new_chroot=False
+
 Calling the build itself is a matter of simple CLI call:
 
 .. code-block: shell

From f0f227f0ab7fbf9643f0533c585e9f585575ee9e Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 14/15] fix flake8


---

diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py
index 64d6bcf..83cffc1 100644
--- a/plugins/builder/kiwi.py
+++ b/plugins/builder/kiwi.py
@@ -1,5 +1,5 @@
 import glob
-import json
+# import json
 import os
 import xml.dom.minidom
 from fnmatch import fnmatch
@@ -368,7 +368,7 @@ class KiwiCreateImageTask(BaseBuildTask):
         if rv:
             raise koji.GenericError("Kiwi failed")
 
-        #result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
+        # result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
         # nosec comment - we will replace it with json ASAP
         import pickle
         result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:],  # nosec

From 468656d8a384fcf08e1a83beb9ed3864ee5a7b8b Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Nov 18 2021 10:12:48 +0000
Subject: [PATCH 15/15] fix docs


---

diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index c6c8f43..8e5aa5d 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -242,7 +242,9 @@ configuration except allowing the plugins (name is 'kiwi' for all components).
 
 Builders have to be part of ``image`` channel and don't need to have any
 specific library installed (kiwi invocation/usage is only in buildroots not on
-builder itself).
+builder itself). (Temporarily ``python3-kiwi`` needs to be installed on builder
+for kojid to be able to parse kiwi output. It will be changed to json in next
+version and this requirement will be dropped.)
 
 Buildtag needs to be configured by adding special group ``kiwi`` which should
 contain at least ``kiwi-cli``, potentially ``jing`` for better description files

