- LIBEXECFILES = mergerepos

- 	mkdir -p $(DESTDIR)/usr/libexec/kojid

- 	install -p -m 755 $(LIBEXECFILES) $(DESTDIR)/usr/libexec/kojid

- 

-         # mergerepo starts from a local repo in the task workdir, so internal

-         if self.options.use_createrepo_c:

-             cmd = ['/usr/bin/createrepo_c']

-         else:

-             cmd = ['/usr/bin/createrepo']

-         cmd.extend(['-vd', '-o', self.outdir])

-             # currently only supported by our own mergerepos script

-             # (we need it to write pkgorigins)

-             cmd = ['/usr/libexec/kojid/mergerepos',

-                    '--mode', 'simple',

-                    '--tempdir', self.workdir]

-             # forces use of mergerepo_c

-             cmd = ['/usr/bin/mergerepo_c', '--pkgorigins', '--all']

-         elif self.options.use_createrepo_c:

-             cmd = ['/usr/bin/mergerepo_c', '--koji']

-             cmd = ['/usr/libexec/kojid/mergerepos']

-             cmd.extend(['--tempdir', self.workdir])

-         if self.options.use_createrepo_c:

-             cmd = ['/usr/bin/createrepo_c']

-         else:

-             cmd = ['/usr/bin/createrepo']

-         cmd.extend(['-vd', '-i', pkglist])

-             if name in ['sleeptime', 'maxjobs', 'minspace', 'retry_interval',

-             elif name in ['offline_retry', 'use_createrepo_c',  'createrepo_skip_stat',

-                           'createrepo_update', 'use_fast_upload', 'support_rpm_source_layout',

-                           'krb_rdns', 'krb_canon_host', 'build_arch_can_fail', 'no_ssl_verify',

- ; use createrepo_c rather than createrepo

- ; use_createrepo_c=True

- 

- #!/usr/bin/python2

- 

- # This program is free software; you can redistribute it and/or modify

- # it under the terms of the GNU General Public License as published by

- # the Free Software Foundation; either version 2 of the License, or

- # (at your option) any later version.

- #

- # This program is distributed in the hope that it will be useful,

- # but WITHOUT ANY WARRANTY; without even the implied warranty of

- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

- # GNU Library General Public License for more details.

- #

- # You should have received a copy of the GNU General Public License

- # along with this program; if not, write to the Free Software

- # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

- 

- # Copyright 2009-2014 Red Hat, Inc.

- # Written by Mike Bonnet <mikeb@redhat.com>

- 

- # Merge repos using rules specific to Koji

- # Largely borrowed from the mergerepo script included in createrepo and

- # written by Seth Vidal

- 

- from __future__ import absolute_import

- import createrepo

- import os.path

- import rpmUtils.miscutils

- import shutil

- import sys

- import tempfile

- import yum

- import yum.misc

- from optparse import OptionParser

- 

- # Expand a canonical arch to the full list of

- # arches that should be included in the repo.

- # Basically the inverse of koji.canonArch().

- # Lists taken from rpmUtils.arch.

- EXPAND_ARCHES = {

-     'i386': ['i486', 'i586', 'geode', 'i686', 'athlon'],

-     'x86_64': ['ia32e', 'amd64'],

-     'ppc64': ['ppc64pseries', 'ppc64iseries'],

-     'sparc64': ['sparc64v', 'sparc64v2'],

-     'sparc': ['sparcv8', 'sparcv9', 'sparcv9v', 'sparcv9v2'],

-     'alpha': ['alphaev4', 'alphaev45', 'alphaev5', 'alphaev56',

-               'alphapca56', 'alphaev6', 'alphaev67', 'alphaev68', 'alphaev7'],

-     'armhfp': ['armv7hl', 'armv7hnl', 'armv6hl', 'armv6hnl'],

-     'arm': ['armv5tel', 'armv5tejl', 'armv6l','armv7l'],

-     'sh4': ['sh4a']

-     }

- 

- MULTILIB_ARCHES = {

-     'x86_64': 'i386',

-     'ppc64': 'ppc',

-     's390x': 's390'

-     }

- 

- def parse_args(args):

-     """Parse our opts/args"""

-     usage = """

-     mergerepos: take 2 or more repositories and merge their metadata into a new repo using Koji semantics

- 

-     mergerepos --repo=url --repo=url --outputdir=/some/path"""

- 

-     parser = OptionParser(version = "mergerepos 0.1", usage=usage)

-     # query options

-     parser.add_option("-r", "--repo", dest="repos", default=[], action="append",

-                       help="repo url")

-     parser.add_option("-g", "--groupfile", default=None,

-                       help="path to groupfile to include in metadata")

-     parser.add_option("-a", "--arch", dest="arches", default=[], action="append",

-                       help="List of arches to include in the repo")

-     parser.add_option("-b", "--blocked", default=None,

-                       help="A file containing a list of srpm names to exclude from the merged repo")

-     parser.add_option("--mode", default='koji', help="Select the merge mode")

-     parser.add_option("-o", "--outputdir", default=None,

-                       help="Location to create the repository")

-     parser.add_option("--tempdir", default=None,

-                       help="Location for temporary files")

-     (opts, argsleft) = parser.parse_args(args)

- 

-     if len(opts.repos) < 1:

-         parser.print_usage()

-         sys.exit(1)

- 

-     # expand arches

-     for arch in opts.arches[:]:

-         if arch in EXPAND_ARCHES:

-             opts.arches.extend(EXPAND_ARCHES[arch])

- 

-     # support multilib repos

-     for arch in opts.arches[:]:

-         multilib_arch = MULTILIB_ARCHES.get(arch)

-         if multilib_arch:

-             opts.arches.append(multilib_arch)

-             if multilib_arch in EXPAND_ARCHES:

-                 opts.arches.extend(EXPAND_ARCHES[multilib_arch])

- 

-     # always include noarch

-     if not 'noarch' in opts.arches:

-         opts.arches.append('noarch')

- 

-     if not opts.outputdir:

-         parser.error('You must specify an outputdir with -o')

-         sys.exit(1)

- 

-     return opts

- 

- 

- def make_const_func(value):

-     """Return a function that returns the given value"""

-     return lambda *a: value

- 

- 

- class RepoMerge(object):

-     def __init__(self, repolist, arches, groupfile, blocked, outputdir,

-                  tempdir=None, mode='koji'):

-         self.repolist = repolist

-         self.outputdir = outputdir

-         self.tempdir = tempdir

-         self.mdconf = createrepo.MetaDataConfig()

-         # explicitly request sha1 for backward compatibility with older yum

-         self.mdconf.sumtype = 'sha1'

-         self.mdconf.database = True

-         self.mdconf.verbose = True

-         self.mdconf.changelog_limit = 3

-         self.yumbase = yum.YumBase()

-         if hasattr(self.yumbase, 'preconf'):

-             self.yumbase.preconf.fn = '/dev/null'

-             self.yumbase.preconf.init_plugins = False

-             self.yumbase.preconf.debuglevel = 2

-         else:

-             self.yumbase._getConfig('/dev/null', init_plugins=False, debuglevel=2)

-         self.yumbase.conf.cachedir = tempfile.mkdtemp(dir=self.tempdir)

-         self.yumbase.conf.cache = 0

-         self.archlist = arches

-         self.mdconf.groupfile = groupfile

-         self.blocked = blocked

-         self.mode = mode

- 

-     def close(self):

-         if self.yumbase is not None:

-             cachedir = self.yumbase.conf.cachedir

-             self.yumbase.close()

-             self.yumbase = None

-             self.mdconf = None

-             if os.path.isdir(cachedir):

-                 shutil.rmtree(cachedir)

- 

-     def __del__(self):

-         self.close()

- 

-     def merge_repos(self):

-         self.yumbase.repos.disableRepo('*')

-         # add our repos and give them a merge rank in the order they appear in

-         # in the repolist

-         count = 0

-         for r in self.repolist:

-             count +=1

-             rid = 'repo%s' % count

-             sys.stderr.write('Adding repo: %s\n' % r)

-             n = self.yumbase.add_enable_repo(rid, baseurls=[r])

-             n._merge_rank = count

- 

-         #setup our sacks

-         self.yumbase._getSacks(archlist=self.archlist)

- 

-         self.sort_and_filter()

- 

-     def sort_and_filter(self):

-         """

-         For each package object, check if the srpm name has ever been seen before.

-         If is has not, keep the package.  If it has, check if the srpm name was first seen

-         in the same repo as the current package.  If so, keep the package from the srpm with the

-         highest NVR.  If not, keep the packages from the first srpm we found, and delete packages from

-         all other srpms.

- 

-         Packages with matching NVRs in multiple repos will be taken from the first repo.

- 

-         If the srpm name appears in the blocked package list, any packages generated from the srpm

-         will be deleted from the package sack as well.

- 

-         This method will also generate a file called "pkgorigins" and add it to the repo metadata.  This

-         is a tab-separated map of package E:N-V-R.A to repo URL (as specified on the command-line).  This

-         allows a package to be tracked back to its origin, even if the location field in the repodata does

-         not match the original repo location.

-         """

- 

-         if self.mode == 'simple':

-             return self.do_simple_sort()

- 

-         # sort the repos by _merge_rank

-         # lowest number is the highest rank (1st place, 2nd place, etc.)

-         repos = self.yumbase.repos.listEnabled()

-         repos.sort(key=lambda o: o._merge_rank)

- 

-         include_srpms = {}

- 

-         # calculating what "builds" (srpms) we're allowing into the repo

-         seen_srpms = {}

-         for reponum, repo in enumerate(repos):

-             for pkg in repo.sack:

-                 if reponum == 0 and not pkg.basepath:

-                     # this is the first repo (i.e. the koji repo) and appears

-                     # to be using relative urls

-                     #XXX - kind of a hack, but yum leaves us little choice

-                     #force the pkg object to report a relative location

-                     loc = """<location href="%s"/>\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True)

-                     pkg._return_remote_location = make_const_func(loc)

-                 if pkg.sourcerpm in seen_srpms:

-                     # we're just looking at sourcerpms this pass and we've

-                     # already seen this one

-                     continue

-                 seen_srpms[pkg.sourcerpm] = 1

-                 srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm)

-                 if srpm_name in include_srpms:

-                     other_srpm, other_repoid = include_srpms[srpm_name]

-                     if pkg.repoid != other_repoid:

-                         # We found a rpm built from an srpm with the same name in a previous repo.

-                         # The previous repo takes precedence, so ignore the srpm found here.

-                         sys.stderr.write('Package %s already provided by repo %s'

-                                 ' (at %s in repo %s)\n'

-                                 % (srpm_name, other_repoid, str(pkg), pkg.repoid))

-                         continue

-                     else:

-                         # We're in the same repo, so compare srpm NVRs

-                         other_srpm_name, other_ver, other_rel, other_epoch, other_arch = \

-                                          rpmUtils.miscutils.splitFilename(other_srpm)

-                         cmp = rpmUtils.miscutils.compareEVR((epoch, ver, rel),

-                                                             (other_epoch, other_ver, other_rel))

-                         if cmp > 0:

-                             # The current package we're processing is from a newer srpm than the

-                             # existing srpm in the dict, so update the dict

-                             include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid)

-                             sys.stderr.write('Replacing older source nvr: '

-                                     '%s > %s\n' % (pkg.sourcerpm, other_srpm))

-                         elif cmp < 0:

-                             sys.stderr.write('Ignoring older source nvr: '

-                                     '%s < %s\n' % (pkg.sourcerpm, other_srpm))

-                         # otherwise same, so we already have it

-                 elif srpm_name in self.blocked:

-                     sys.stderr.write('Ignoring blocked package: %s\n\n' % \

-                                      pkg.sourcerpm)

-                     continue

-                 else:

-                     include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid)

- 

-         pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins')

-         origins = open(pkgorigins, 'w')

- 

-         seen_rpms = {}

-         for repo in repos:

-             for pkg in repo.sack:

-                 srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm)

-                 incl_srpm, incl_repoid = include_srpms.get(srpm_name, (None, None))

-                 pkg_nvra = str(pkg)

-                 if pkg_nvra in seen_rpms:

-                     sys.stderr.write('Duplicate rpm: %s\n' % pkg_nvra)

-                 elif incl_srpm is None:

-                     sys.stderr.write('Excluding %s (%s is blocked)\n'

-                             % (pkg_nvra, srpm_name))

-                     repo.sack.delPackage(pkg)

-                 elif incl_srpm == pkg.sourcerpm:

-                     origins.write('%s\t%s\n' % (pkg_nvra, repo.urls[0]))

-                     seen_rpms[pkg_nvra] = 1

-                 else:

-                     sys.stderr.write('Excluding %s (wrong srpm version '

-                             '%s != %s)\n' % (pkg_nvra, pkg.sourcerpm, incl_srpm))

-                     repo.sack.delPackage(pkg)

- 

-         origins.close()

-         self.mdconf.additional_metadata['origin'] = pkgorigins

- 

-     def do_simple_sort(self):

-         """

-         Handle the 'sort_and_filter' case when mode=simple

- 

-         As the name implies, this is a much simpler approach. Mainly, we need

-         to generate the pkgorigins file.

-         """

- 

-         # sort the repos by _merge_rank

-         # lowest number is the highest rank (1st place, 2nd place, etc.)

-         repos = self.yumbase.repos.listEnabled()

-         repos.sort(key=lambda o: o._merge_rank)

- 

-         # TODO: reduce duplication between this function and sort_and_filter()

- 

-         # We lack the complex filtration of mode=koji, but we still need to:

-         # - fix urls for primary repo

-         # - enforce blocked list

-         for reponum, repo in enumerate(repos):

-             for pkg in repo.sack:

-                 if reponum == 0 and not pkg.basepath:

-                     # this is the first repo (i.e. the koji repo) and appears

-                     # to be using relative urls

-                     #XXX - kind of a hack, but yum leaves us little choice

-                     #force the pkg object to report a relative location

-                     loc = """<location href="%s"/>\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True)

-                     pkg._return_remote_location = make_const_func(loc)

- 

-         pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins')

-         origins = open(pkgorigins, 'w')

- 

-         seen_rpms = {}

-         for repo in repos:

-             for pkg in repo.sack:

-                 srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm)

-                 pkg_nvra = str(pkg)

-                 if pkg_nvra in seen_rpms:

-                     sys.stderr.write('Duplicate rpm: %s\n' % pkg_nvra)

-                     # note: we warn, but do not omit it

-                 if srpm_name in self.blocked:

-                     sys.stderr.write('Ignoring blocked package: %s\n\n' %

-                                      pkg.sourcerpm)

-                     repo.sack.delPackage(pkg)

-                 if pkg_nvra not in seen_rpms:

-                     origins.write('%s\t%s\n' % (pkg_nvra, repo.urls[0]))

-                 seen_rpms[pkg_nvra] = 1

-         origins.close()

-         self.mdconf.additional_metadata['origin'] = pkgorigins

- 

-     def write_metadata(self):

-         self.mdconf.pkglist = self.yumbase.pkgSack

-         self.mdconf.directory = self.outputdir

-         # clean out what was there

-         if os.path.exists(self.mdconf.directory + '/repodata'):

-             shutil.rmtree(self.mdconf.directory + '/repodata')

- 

-         if not os.path.exists(self.mdconf.directory):

-             os.makedirs(self.mdconf.directory)

- 

-         mdgen = createrepo.MetaDataGenerator(config_obj=self.mdconf)

-         mdgen.doPkgMetadata()

-         mdgen.doRepoMetadata()

-         mdgen.doFinalMove()

- 

- def main(args):

-     """main"""

-     opts = parse_args(args)

- 

-     if opts.blocked:

-         blocked_fo = open(opts.blocked)

-         blocked_list = blocked_fo.readlines()

-         blocked_fo.close()

-         blocked = dict([(b.strip(), 1) for b in blocked_list])

-     else:

-         blocked = {}

- 

-     merge = RepoMerge(opts.repos, opts.arches, opts.groupfile, blocked,

-                       opts.outputdir, opts.tempdir, opts.mode)

- 

-     try:

-         merge.merge_repos()

-         merge.write_metadata()

-     finally:

-         merge.close()

- 

- if __name__ == "__main__":

-     main(sys.argv[1:])

-    use_createrepo_c=False

-       Use ``createrepo_c`` rather than ``createrepo`` command. There is

-       generally no reason to not use createrepo_c in modern depolyments. It

-       is disabled by default only to be backward-compatible. This default

-       would change in future.

- 

- * createrepo

- .. _createrepo: http://createrepo.baseurl.org/

- License: LGPLv2 and GPLv2+

- #mergerepos (from createrepo) is GPLv2+

- Requires: createrepo >= 0.9.2

- %dir %{_libexecdir}/kojid

- %{_libexecdir}/kojid/mergerepos