From 3e08789edc2de5ae3c4d1ebbfbc68df1daab528f Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Aug 14 2019 10:53:56 +0000 Subject: use only createrepo_c, drop createrepo createrepo_c/mergerepo_c already supports everything we needs in koji. There is no reason to support internal implementation of mergerepo and all distributions in sight already have createrepo_c. Commit is dropping all createrepo/mergerepo uses and replace them with their _c variants. Config option use_creterepo_c is deprecated and prints warning to kojid.log. It should be removed completely in koji 1.21. Fixes: https://pagure.io/koji/issue/1336 --- diff --git a/builder/Makefile b/builder/Makefile index 1de0a9c..8ac6a4a 100644 --- a/builder/Makefile +++ b/builder/Makefile @@ -1,5 +1,4 @@ BINFILES = kojid -LIBEXECFILES = mergerepos SYSTEMDSYSTEMUNITDIR = $(shell pkg-config systemd --variable=systemdsystemunitdir) TYPE = systemd @@ -20,9 +19,6 @@ _install: mkdir -p $(DESTDIR)/usr/sbin install -p -m 755 $(BINFILES) $(DESTDIR)/usr/sbin - mkdir -p $(DESTDIR)/usr/libexec/kojid - install -p -m 755 $(LIBEXECFILES) $(DESTDIR)/usr/libexec/kojid - mkdir -p $(DESTDIR)/etc/mock/koji mkdir -p $(DESTDIR)/etc/kojid diff --git a/builder/kojid b/builder/kojid index 142a440..2048691 100755 --- a/builder/kojid +++ b/builder/kojid @@ -801,7 +801,7 @@ class BuildRoot(object): nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % koji.parse_NVRA(parts[0]) origin_idx[nvra] = parts[1] fo2.close() - # mergerepo starts from a local repo in the task workdir, so internal + # mergerepo_c starts from a local repo in the task workdir, so internal # rpms have an odd-looking origin that we need to look for localtail = '/repo_%s_premerge/' % self.repo_info['id'] for rpm_info in rpmlist: @@ -5240,11 +5240,7 @@ class CreaterepoTask(BaseTaskHandler): def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo): koji.ensuredir(self.outdir) - if self.options.use_createrepo_c: - cmd = ['/usr/bin/createrepo_c'] - else: - cmd = ['/usr/bin/createrepo'] - cmd.extend(['-vd', '-o', self.outdir]) + cmd = ['/usr/bin/createrepo_c', '-vd', '-o', self.outdir] if pkglist is not None: cmd.extend(['-i', pkglist]) if os.path.isfile(groupdata): @@ -5310,21 +5306,14 @@ class CreaterepoTask(BaseTaskHandler): repos.append(ext_url) # construct command + cmd = ['/usr/bin/mergerepo_c'] if merge_mode == 'simple': - # currently only supported by our own mergerepos script - # (we need it to write pkgorigins) - cmd = ['/usr/libexec/kojid/mergerepos', - '--mode', 'simple', - '--tempdir', self.workdir] + cmd += ['--mode', 'simple'] elif merge_mode == 'bare': # "bare" merge mode for repos with modular metadata - # forces use of mergerepo_c - cmd = ['/usr/bin/mergerepo_c', '--pkgorigins', '--all'] - elif self.options.use_createrepo_c: - cmd = ['/usr/bin/mergerepo_c', '--koji'] + cmd += ['--pkgorigins', '--all'] else: - cmd = ['/usr/libexec/kojid/mergerepos'] - cmd.extend(['--tempdir', self.workdir]) + cmd += ['--koji'] if merge_mode != 'bare': blocklist = self.repodir + '/blocklist' cmd.extend(['-b', blocklist]) @@ -5530,11 +5519,7 @@ class createDistRepoTask(BaseTaskHandler): our requirements here """ koji.ensuredir(repodir) - if self.options.use_createrepo_c: - cmd = ['/usr/bin/createrepo_c'] - else: - cmd = ['/usr/bin/createrepo'] - cmd.extend(['-vd', '-i', pkglist]) + cmd = ['/usr/bin/createrepo_c', '-vd', '-i', pkglist] if groupdata and os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) # TODO: can we recycle data (with --update) as in create_local_repo? @@ -6149,7 +6134,11 @@ def get_options(): 'serverca': None} if config.has_section('kojid'): for name, value in config.items('kojid'): - if name in ['sleeptime', 'maxjobs', 'minspace', 'retry_interval', + if name == 'use_createrepo_c': + logger.warning("use_createrepo_c is deprecated (createrepo_c" + " is always used) and option will be removed" + " in koji 1.21)") + elif name in ['sleeptime', 'maxjobs', 'minspace', 'retry_interval', 'max_retries', 'offline_retry_interval', 'failed_buildroot_lifetime', 'timeout', 'rpmbuild_timeout', 'oz_install_timeout', 'task_avail_delay']: @@ -6157,9 +6146,9 @@ def get_options(): defaults[name] = int(value) except ValueError: quit("value for %s option must be a valid integer" % name) - elif name in ['offline_retry', 'use_createrepo_c', 'createrepo_skip_stat', - 'createrepo_update', 'use_fast_upload', 'support_rpm_source_layout', - 'krb_rdns', 'krb_canon_host', 'build_arch_can_fail', 'no_ssl_verify', + elif name in ['offline_retry', 'createrepo_skip_stat', 'createrepo_update', + 'use_fast_upload', 'support_rpm_source_layout', 'krb_rdns', + 'krb_canon_host', 'build_arch_can_fail', 'no_ssl_verify', 'log_timestamps']: defaults[name] = config.getboolean('kojid', name) elif name in ['plugin', 'plugins']: diff --git a/builder/kojid.conf b/builder/kojid.conf index b9f0851..316e6df 100644 --- a/builder/kojid.conf +++ b/builder/kojid.conf @@ -49,9 +49,6 @@ server=http://hub.example.com/kojihub ; The URL for the file access topurl=http://hub.example.com/kojifiles -; use createrepo_c rather than createrepo -; use_createrepo_c=True - ; A space-separated list of tuples from which kojid is allowed to checkout. ; The format of those tuples is: ; diff --git a/builder/mergerepos b/builder/mergerepos deleted file mode 100755 index dcd7647..0000000 --- a/builder/mergerepos +++ /dev/null @@ -1,360 +0,0 @@ -#!/usr/bin/python2 - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -# Copyright 2009-2014 Red Hat, Inc. -# Written by Mike Bonnet - -# Merge repos using rules specific to Koji -# Largely borrowed from the mergerepo script included in createrepo and -# written by Seth Vidal - -from __future__ import absolute_import -import createrepo -import os.path -import rpmUtils.miscutils -import shutil -import sys -import tempfile -import yum -import yum.misc -from optparse import OptionParser - -# Expand a canonical arch to the full list of -# arches that should be included in the repo. -# Basically the inverse of koji.canonArch(). -# Lists taken from rpmUtils.arch. -EXPAND_ARCHES = { - 'i386': ['i486', 'i586', 'geode', 'i686', 'athlon'], - 'x86_64': ['ia32e', 'amd64'], - 'ppc64': ['ppc64pseries', 'ppc64iseries'], - 'sparc64': ['sparc64v', 'sparc64v2'], - 'sparc': ['sparcv8', 'sparcv9', 'sparcv9v', 'sparcv9v2'], - 'alpha': ['alphaev4', 'alphaev45', 'alphaev5', 'alphaev56', - 'alphapca56', 'alphaev6', 'alphaev67', 'alphaev68', 'alphaev7'], - 'armhfp': ['armv7hl', 'armv7hnl', 'armv6hl', 'armv6hnl'], - 'arm': ['armv5tel', 'armv5tejl', 'armv6l','armv7l'], - 'sh4': ['sh4a'] - } - -MULTILIB_ARCHES = { - 'x86_64': 'i386', - 'ppc64': 'ppc', - 's390x': 's390' - } - -def parse_args(args): - """Parse our opts/args""" - usage = """ - mergerepos: take 2 or more repositories and merge their metadata into a new repo using Koji semantics - - mergerepos --repo=url --repo=url --outputdir=/some/path""" - - parser = OptionParser(version = "mergerepos 0.1", usage=usage) - # query options - parser.add_option("-r", "--repo", dest="repos", default=[], action="append", - help="repo url") - parser.add_option("-g", "--groupfile", default=None, - help="path to groupfile to include in metadata") - parser.add_option("-a", "--arch", dest="arches", default=[], action="append", - help="List of arches to include in the repo") - parser.add_option("-b", "--blocked", default=None, - help="A file containing a list of srpm names to exclude from the merged repo") - parser.add_option("--mode", default='koji', help="Select the merge mode") - parser.add_option("-o", "--outputdir", default=None, - help="Location to create the repository") - parser.add_option("--tempdir", default=None, - help="Location for temporary files") - (opts, argsleft) = parser.parse_args(args) - - if len(opts.repos) < 1: - parser.print_usage() - sys.exit(1) - - # expand arches - for arch in opts.arches[:]: - if arch in EXPAND_ARCHES: - opts.arches.extend(EXPAND_ARCHES[arch]) - - # support multilib repos - for arch in opts.arches[:]: - multilib_arch = MULTILIB_ARCHES.get(arch) - if multilib_arch: - opts.arches.append(multilib_arch) - if multilib_arch in EXPAND_ARCHES: - opts.arches.extend(EXPAND_ARCHES[multilib_arch]) - - # always include noarch - if not 'noarch' in opts.arches: - opts.arches.append('noarch') - - if not opts.outputdir: - parser.error('You must specify an outputdir with -o') - sys.exit(1) - - return opts - - -def make_const_func(value): - """Return a function that returns the given value""" - return lambda *a: value - - -class RepoMerge(object): - def __init__(self, repolist, arches, groupfile, blocked, outputdir, - tempdir=None, mode='koji'): - self.repolist = repolist - self.outputdir = outputdir - self.tempdir = tempdir - self.mdconf = createrepo.MetaDataConfig() - # explicitly request sha1 for backward compatibility with older yum - self.mdconf.sumtype = 'sha1' - self.mdconf.database = True - self.mdconf.verbose = True - self.mdconf.changelog_limit = 3 - self.yumbase = yum.YumBase() - if hasattr(self.yumbase, 'preconf'): - self.yumbase.preconf.fn = '/dev/null' - self.yumbase.preconf.init_plugins = False - self.yumbase.preconf.debuglevel = 2 - else: - self.yumbase._getConfig('/dev/null', init_plugins=False, debuglevel=2) - self.yumbase.conf.cachedir = tempfile.mkdtemp(dir=self.tempdir) - self.yumbase.conf.cache = 0 - self.archlist = arches - self.mdconf.groupfile = groupfile - self.blocked = blocked - self.mode = mode - - def close(self): - if self.yumbase is not None: - cachedir = self.yumbase.conf.cachedir - self.yumbase.close() - self.yumbase = None - self.mdconf = None - if os.path.isdir(cachedir): - shutil.rmtree(cachedir) - - def __del__(self): - self.close() - - def merge_repos(self): - self.yumbase.repos.disableRepo('*') - # add our repos and give them a merge rank in the order they appear in - # in the repolist - count = 0 - for r in self.repolist: - count +=1 - rid = 'repo%s' % count - sys.stderr.write('Adding repo: %s\n' % r) - n = self.yumbase.add_enable_repo(rid, baseurls=[r]) - n._merge_rank = count - - #setup our sacks - self.yumbase._getSacks(archlist=self.archlist) - - self.sort_and_filter() - - def sort_and_filter(self): - """ - For each package object, check if the srpm name has ever been seen before. - If is has not, keep the package. If it has, check if the srpm name was first seen - in the same repo as the current package. If so, keep the package from the srpm with the - highest NVR. If not, keep the packages from the first srpm we found, and delete packages from - all other srpms. - - Packages with matching NVRs in multiple repos will be taken from the first repo. - - If the srpm name appears in the blocked package list, any packages generated from the srpm - will be deleted from the package sack as well. - - This method will also generate a file called "pkgorigins" and add it to the repo metadata. This - is a tab-separated map of package E:N-V-R.A to repo URL (as specified on the command-line). This - allows a package to be tracked back to its origin, even if the location field in the repodata does - not match the original repo location. - """ - - if self.mode == 'simple': - return self.do_simple_sort() - - # sort the repos by _merge_rank - # lowest number is the highest rank (1st place, 2nd place, etc.) - repos = self.yumbase.repos.listEnabled() - repos.sort(key=lambda o: o._merge_rank) - - include_srpms = {} - - # calculating what "builds" (srpms) we're allowing into the repo - seen_srpms = {} - for reponum, repo in enumerate(repos): - for pkg in repo.sack: - if reponum == 0 and not pkg.basepath: - # this is the first repo (i.e. the koji repo) and appears - # to be using relative urls - #XXX - kind of a hack, but yum leaves us little choice - #force the pkg object to report a relative location - loc = """\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True) - pkg._return_remote_location = make_const_func(loc) - if pkg.sourcerpm in seen_srpms: - # we're just looking at sourcerpms this pass and we've - # already seen this one - continue - seen_srpms[pkg.sourcerpm] = 1 - srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm) - if srpm_name in include_srpms: - other_srpm, other_repoid = include_srpms[srpm_name] - if pkg.repoid != other_repoid: - # We found a rpm built from an srpm with the same name in a previous repo. - # The previous repo takes precedence, so ignore the srpm found here. - sys.stderr.write('Package %s already provided by repo %s' - ' (at %s in repo %s)\n' - % (srpm_name, other_repoid, str(pkg), pkg.repoid)) - continue - else: - # We're in the same repo, so compare srpm NVRs - other_srpm_name, other_ver, other_rel, other_epoch, other_arch = \ - rpmUtils.miscutils.splitFilename(other_srpm) - cmp = rpmUtils.miscutils.compareEVR((epoch, ver, rel), - (other_epoch, other_ver, other_rel)) - if cmp > 0: - # The current package we're processing is from a newer srpm than the - # existing srpm in the dict, so update the dict - include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid) - sys.stderr.write('Replacing older source nvr: ' - '%s > %s\n' % (pkg.sourcerpm, other_srpm)) - elif cmp < 0: - sys.stderr.write('Ignoring older source nvr: ' - '%s < %s\n' % (pkg.sourcerpm, other_srpm)) - # otherwise same, so we already have it - elif srpm_name in self.blocked: - sys.stderr.write('Ignoring blocked package: %s\n\n' % \ - pkg.sourcerpm) - continue - else: - include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid) - - pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins') - origins = open(pkgorigins, 'w') - - seen_rpms = {} - for repo in repos: - for pkg in repo.sack: - srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm) - incl_srpm, incl_repoid = include_srpms.get(srpm_name, (None, None)) - pkg_nvra = str(pkg) - if pkg_nvra in seen_rpms: - sys.stderr.write('Duplicate rpm: %s\n' % pkg_nvra) - elif incl_srpm is None: - sys.stderr.write('Excluding %s (%s is blocked)\n' - % (pkg_nvra, srpm_name)) - repo.sack.delPackage(pkg) - elif incl_srpm == pkg.sourcerpm: - origins.write('%s\t%s\n' % (pkg_nvra, repo.urls[0])) - seen_rpms[pkg_nvra] = 1 - else: - sys.stderr.write('Excluding %s (wrong srpm version ' - '%s != %s)\n' % (pkg_nvra, pkg.sourcerpm, incl_srpm)) - repo.sack.delPackage(pkg) - - origins.close() - self.mdconf.additional_metadata['origin'] = pkgorigins - - def do_simple_sort(self): - """ - Handle the 'sort_and_filter' case when mode=simple - - As the name implies, this is a much simpler approach. Mainly, we need - to generate the pkgorigins file. - """ - - # sort the repos by _merge_rank - # lowest number is the highest rank (1st place, 2nd place, etc.) - repos = self.yumbase.repos.listEnabled() - repos.sort(key=lambda o: o._merge_rank) - - # TODO: reduce duplication between this function and sort_and_filter() - - # We lack the complex filtration of mode=koji, but we still need to: - # - fix urls for primary repo - # - enforce blocked list - for reponum, repo in enumerate(repos): - for pkg in repo.sack: - if reponum == 0 and not pkg.basepath: - # this is the first repo (i.e. the koji repo) and appears - # to be using relative urls - #XXX - kind of a hack, but yum leaves us little choice - #force the pkg object to report a relative location - loc = """\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True) - pkg._return_remote_location = make_const_func(loc) - - pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins') - origins = open(pkgorigins, 'w') - - seen_rpms = {} - for repo in repos: - for pkg in repo.sack: - srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm) - pkg_nvra = str(pkg) - if pkg_nvra in seen_rpms: - sys.stderr.write('Duplicate rpm: %s\n' % pkg_nvra) - # note: we warn, but do not omit it - if srpm_name in self.blocked: - sys.stderr.write('Ignoring blocked package: %s\n\n' % - pkg.sourcerpm) - repo.sack.delPackage(pkg) - if pkg_nvra not in seen_rpms: - origins.write('%s\t%s\n' % (pkg_nvra, repo.urls[0])) - seen_rpms[pkg_nvra] = 1 - origins.close() - self.mdconf.additional_metadata['origin'] = pkgorigins - - def write_metadata(self): - self.mdconf.pkglist = self.yumbase.pkgSack - self.mdconf.directory = self.outputdir - # clean out what was there - if os.path.exists(self.mdconf.directory + '/repodata'): - shutil.rmtree(self.mdconf.directory + '/repodata') - - if not os.path.exists(self.mdconf.directory): - os.makedirs(self.mdconf.directory) - - mdgen = createrepo.MetaDataGenerator(config_obj=self.mdconf) - mdgen.doPkgMetadata() - mdgen.doRepoMetadata() - mdgen.doFinalMove() - -def main(args): - """main""" - opts = parse_args(args) - - if opts.blocked: - blocked_fo = open(opts.blocked) - blocked_list = blocked_fo.readlines() - blocked_fo.close() - blocked = dict([(b.strip(), 1) for b in blocked_list]) - else: - blocked = {} - - merge = RepoMerge(opts.repos, opts.arches, opts.groupfile, blocked, - opts.outputdir, opts.tempdir, opts.mode) - - try: - merge.merge_repos() - merge.write_metadata() - finally: - merge.close() - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/docs/source/kojid_conf.rst b/docs/source/kojid_conf.rst index fd5aa6f..297f63d 100644 --- a/docs/source/kojid_conf.rst +++ b/docs/source/kojid_conf.rst @@ -147,12 +147,6 @@ Building Install timeout in seconds for image build. Default value is 0, which means using the number in ``/etc/oz/oz.cfg``. Supported since oz-0.16.0. - use_createrepo_c=False - Use ``createrepo_c`` rather than ``createrepo`` command. There is - generally no reason to not use createrepo_c in modern depolyments. It - is disabled by default only to be backward-compatible. This default - would change in future. - task_avail_delay=300 [Added in 1.17.0] diff --git a/docs/source/server_howto.rst b/docs/source/server_howto.rst index b070b94..ae36e81 100644 --- a/docs/source/server_howto.rst +++ b/docs/source/server_howto.rst @@ -41,7 +41,7 @@ On the builder (koji-builder) * mock * setarch (for some archs you'll require a patched version) * rpm-build -* createrepo +* createrepo_c A note on filesystem space ========================== @@ -1272,7 +1272,7 @@ operating a koji server. .. _dnf: https://fedoraproject.org/wiki/Dnf .. _yum: https://fedoraproject.org/wiki/Yum -.. _createrepo: http://createrepo.baseurl.org/ +.. _createrepo_c: https://rpm-software-management.github.io/createrepo_c/ .. _mock: https://fedoraproject.org/wiki/Mock .. _Apache mod_ssl documentation: https://httpd.apache.org/docs/trunk/mod/mod_ssl.html#ssloptions diff --git a/koji.spec b/koji.spec index 05eefe3..6eb4dcf 100644 --- a/koji.spec +++ b/koji.spec @@ -282,8 +282,7 @@ Plugins for the koji build daemon %package builder Summary: Koji RPM builder daemon Group: Applications/System -License: LGPLv2 and GPLv2+ -#mergerepos (from createrepo) is GPLv2+ +License: LGPLv2 Requires: mock >= 0.9.14 Requires(pre): /usr/sbin/useradd Requires: squashfs-tools @@ -300,7 +299,7 @@ Requires(preun): /sbin/service Requires: /usr/bin/cvs Requires: /usr/bin/svn Requires: /usr/bin/git -Requires: createrepo >= 0.9.2 +Requires: createrepo_c >= 0.14 %if 0%{py3_support} > 1 Requires: python%{python3_pkgversion}-%{name} = %{version}-%{release} Requires: python%{python3_pkgversion}-librepo @@ -615,8 +614,6 @@ rm -rf $RPM_BUILD_ROOT %files builder %defattr(-,root,root) %{_sbindir}/kojid -%dir %{_libexecdir}/kojid -%{_libexecdir}/kojid/mergerepos %if %{use_systemd} %{_unitdir}/kojid.service %else