From 0f56798fa95594514b37ef29a8bdcf7c08c33a44 Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Jul 20 2016 19:58:39 +0000 Subject: [PATCH 1/70] deprecate spin-livecd with spin-livemedia Signed-off-by: Dennis Gilmore --- diff --git a/cli/koji b/cli/koji index 0abe244..23ac97e 100755 --- a/cli/koji +++ b/cli/koji @@ -5267,6 +5267,7 @@ def handle_spin_livemedia(options, session, args): (task_options, args) = parser.parse_args(args) # Make sure the target and kickstart is specified. + print 'spin-livecd is deprecated and will be replaced with spin-livemedia' if len(args) != 5: parser.error(_("Five arguments are required: a name, a version, an" + " architecture, a build target, and a relative path to" + From 42398209bbea44568f8c8ca4c1cf6f35973a9444 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Jul 20 2016 20:03:53 +0000 Subject: [PATCH 2/70] Allow hiding a user from the frontpage task list. This adds new query arguments to the taskList hub xmlrpc endpoint, and then makes use of those arguments in koji-web. A new optional configuration value is added for koji-web: `HiddenUser`, which can be used to specify which user account should be hidden. This could be useful for deployments that have a continuous-integration account, the spam from which makes the frontpage difficult to read. Unit test cases are also added for some functions of the hub taskList endpoint. Signed-off-by: Ralph Bean --- diff --git a/Makefile b/Makefile index 9a0bd34..709400d 100644 --- a/Makefile +++ b/Makefile @@ -65,10 +65,7 @@ git-clean: @git clean -d -q -x test: - coverage erase - PYTHONPATH=hub/.:plugins/hub/. nosetests --with-coverage --cover-package . - coverage html - @echo Coverage report in htmlcov/index.html + PYTHONPATH=hub/. nosetests --with-coverage --cover-package . subdirs: for d in $(SUBDIRS); do make -C $$d; [ $$? = 0 ] || exit 1; done diff --git a/hub/kojihub.py b/hub/kojihub.py index 64ef1fd..0740db4 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9722,14 +9722,14 @@ class RootExports(object): not_arch[list]: limit to tasks without the given arches state[list]: limit to tasks of given state not_state[list]: limit to tasks not of the given state - owner[int|list]: limit to tasks owned by the user with the given ID - not_owner[int|list]: limit to tasks not owned by the user with the given ID - host_id[int|list]: limit to tasks running on the host with the given ID - not_host_id[int|list]: limit to tasks running on the hosts with IDs other than the given ID - channel_id[int|list]: limit to tasks in the specified channel - not_channel_id[int|list]: limit to tasks not in the specified channel - parent[int|list]: limit to tasks with the given parent - not_parent[int|list]: limit to tasks without the given parent + owner[int]: limit to tasks owned by the user with the given ID + not_owner[int]: limit to tasks not owned by the user with the given ID + host_id[int]: limit to tasks running on the host with the given ID + not_host_id[int]: limit to tasks running on the hosts with IDs other than the given ID + channel_id[int]: limit to tasks in the specified channel + not_channel_id[int]: limit to tasks not in the specified channel + parent[int]: limit to tasks with the given parent + not_parent[int]: limit to tasks without the given parent decode[bool]: whether or not xmlrpc data in the 'request' and 'result' fields should be decoded; defaults to False method[str]: limit to tasks of the given method @@ -9791,8 +9791,6 @@ class RootExports(object): if opts.has_key('not_' + f): if opts['not_' + f] is None: conditions.append('%s IS NOT NULL' % f) - elif isinstance(opts['not_' + f], types.ListType): - conditions.append('%s NOT IN %%(not_%s)s' % (f, f)) else: conditions.append('%s != %%(not_%s)i' % (f, f)) diff --git a/tests/test_hub/test_listing.py b/tests/test_hub/test_listing.py index a39f2b0..0f7bac4 100644 --- a/tests/test_hub/test_listing.py +++ b/tests/test_hub/test_listing.py @@ -20,11 +20,11 @@ class TestListing(unittest.TestCase): @mock.patch('kojihub.QueryProcessor') def test_list_tasks_basic_invocation(self, processor): generator = self.hub.listTasks() - list(generator) # Exhaust the generator + results = list(generator) # Exhaust the generator processor.assert_called_once_with(**self.standard_processor_kwargs) @mock.patch('kojihub.QueryProcessor') - def test_list_tasks_by_owner_as_int(self, processor): + def test_list_tasks_by_owner(self, processor): generator = self.hub.listTasks(opts={'owner': 1}) results = list(generator) # Exhaust the generator arguments = self.standard_processor_kwargs.copy() @@ -33,7 +33,7 @@ class TestListing(unittest.TestCase): self.assertEqual(results, []) @mock.patch('kojihub.QueryProcessor') - def test_list_tasks_by_not_owner_as_int(self, processor): + def test_list_tasks_by_not_owner(self, processor): generator = self.hub.listTasks(opts={'not_owner': 1}) results = list(generator) # Exhaust the generator arguments = self.standard_processor_kwargs.copy() @@ -59,20 +59,3 @@ class TestListing(unittest.TestCase): processor.assert_called_once_with(**arguments) self.assertEqual(results, []) - @mock.patch('kojihub.QueryProcessor') - def test_list_tasks_by_owner_as_list(self, processor): - generator = self.hub.listTasks(opts={'owner': [1, 2]}) - results = list(generator) # Exhaust the generator - arguments = self.standard_processor_kwargs.copy() - arguments['clauses'] = ['owner IN %(owner)s'] - processor.assert_called_once_with(**arguments) - self.assertEqual(results, []) - - @mock.patch('kojihub.QueryProcessor') - def test_list_tasks_by_not_owner_as_list(self, processor): - generator = self.hub.listTasks(opts={'not_owner': [1, 2]}) - results = list(generator) # Exhaust the generator - arguments = self.standard_processor_kwargs.copy() - arguments['clauses'] = ['owner NOT IN %(not_owner)s'] - processor.assert_called_once_with(**arguments) - self.assertEqual(results, []) diff --git a/www/conf/web.conf b/www/conf/web.conf index 4069258..1b167be 100644 --- a/www/conf/web.conf +++ b/www/conf/web.conf @@ -29,7 +29,7 @@ LibPath = /usr/share/koji-web/lib # Defaults to True LiteralFooter = True -# This can be a space-delimited list of the numeric IDs of users that you want -# to hide from tasks listed on the front page. You might want to, for instance, -# hide the activity of an account used for continuous integration. -#HiddenUsers = 5372 1234 +# This can be the numeric ID of a user that you want to hide from tasks listed +# on the front page. You might want to, for instance, hide the activity of an +# account used for continuous integration. +#HiddenUser = 5372 diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 420c606..87b9288 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -283,10 +283,8 @@ def index(environ, packageOrder='package_name', packageStart=None): taskOpts = {'parent': None, 'decode': True} if user: taskOpts['owner'] = user['id'] - if opts.get('HiddenUsers'): - taskOpts['not_owner'] = [ - int(userid) for userid in opts['HiddenUsers'].split() - ] + if opts.get('HiddenUser'): + taskOpts['not_owner'] = opts['HiddenUser'] values['tasks'] = server.listTasks( opts=taskOpts, queryOpts={'order': '-id', 'limit': 10} From cda958731e2f36fe98f24946ea25a1584515ef48 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Jul 20 2016 20:03:53 +0000 Subject: [PATCH 3/70] Remove unused var. --- diff --git a/tests/test_hub/test_listing.py b/tests/test_hub/test_listing.py index 0f7bac4..eee76d6 100644 --- a/tests/test_hub/test_listing.py +++ b/tests/test_hub/test_listing.py @@ -20,7 +20,7 @@ class TestListing(unittest.TestCase): @mock.patch('kojihub.QueryProcessor') def test_list_tasks_basic_invocation(self, processor): generator = self.hub.listTasks() - results = list(generator) # Exhaust the generator + list(generator) # Exhaust the generator processor.assert_called_once_with(**self.standard_processor_kwargs) @mock.patch('kojihub.QueryProcessor') From a78ce924eb3a9ad21bbc1ac7e284b8855746ce53 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Jul 20 2016 20:05:25 +0000 Subject: [PATCH 4/70] Make HiddenUser into HiddenUsers. At @mikeb's suggestion in the code review, this makes HiddenUsers plural. This makes the whole changeset a little more invasive than it was before, so please review carefully. --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 0740db4..64ef1fd 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9722,14 +9722,14 @@ class RootExports(object): not_arch[list]: limit to tasks without the given arches state[list]: limit to tasks of given state not_state[list]: limit to tasks not of the given state - owner[int]: limit to tasks owned by the user with the given ID - not_owner[int]: limit to tasks not owned by the user with the given ID - host_id[int]: limit to tasks running on the host with the given ID - not_host_id[int]: limit to tasks running on the hosts with IDs other than the given ID - channel_id[int]: limit to tasks in the specified channel - not_channel_id[int]: limit to tasks not in the specified channel - parent[int]: limit to tasks with the given parent - not_parent[int]: limit to tasks without the given parent + owner[int|list]: limit to tasks owned by the user with the given ID + not_owner[int|list]: limit to tasks not owned by the user with the given ID + host_id[int|list]: limit to tasks running on the host with the given ID + not_host_id[int|list]: limit to tasks running on the hosts with IDs other than the given ID + channel_id[int|list]: limit to tasks in the specified channel + not_channel_id[int|list]: limit to tasks not in the specified channel + parent[int|list]: limit to tasks with the given parent + not_parent[int|list]: limit to tasks without the given parent decode[bool]: whether or not xmlrpc data in the 'request' and 'result' fields should be decoded; defaults to False method[str]: limit to tasks of the given method @@ -9791,6 +9791,8 @@ class RootExports(object): if opts.has_key('not_' + f): if opts['not_' + f] is None: conditions.append('%s IS NOT NULL' % f) + elif isinstance(opts['not_' + f], types.ListType): + conditions.append('%s NOT IN %%(not_%s)s' % (f, f)) else: conditions.append('%s != %%(not_%s)i' % (f, f)) diff --git a/tests/test_hub/test_listing.py b/tests/test_hub/test_listing.py index eee76d6..a39f2b0 100644 --- a/tests/test_hub/test_listing.py +++ b/tests/test_hub/test_listing.py @@ -24,7 +24,7 @@ class TestListing(unittest.TestCase): processor.assert_called_once_with(**self.standard_processor_kwargs) @mock.patch('kojihub.QueryProcessor') - def test_list_tasks_by_owner(self, processor): + def test_list_tasks_by_owner_as_int(self, processor): generator = self.hub.listTasks(opts={'owner': 1}) results = list(generator) # Exhaust the generator arguments = self.standard_processor_kwargs.copy() @@ -33,7 +33,7 @@ class TestListing(unittest.TestCase): self.assertEqual(results, []) @mock.patch('kojihub.QueryProcessor') - def test_list_tasks_by_not_owner(self, processor): + def test_list_tasks_by_not_owner_as_int(self, processor): generator = self.hub.listTasks(opts={'not_owner': 1}) results = list(generator) # Exhaust the generator arguments = self.standard_processor_kwargs.copy() @@ -59,3 +59,20 @@ class TestListing(unittest.TestCase): processor.assert_called_once_with(**arguments) self.assertEqual(results, []) + @mock.patch('kojihub.QueryProcessor') + def test_list_tasks_by_owner_as_list(self, processor): + generator = self.hub.listTasks(opts={'owner': [1, 2]}) + results = list(generator) # Exhaust the generator + arguments = self.standard_processor_kwargs.copy() + arguments['clauses'] = ['owner IN %(owner)s'] + processor.assert_called_once_with(**arguments) + self.assertEqual(results, []) + + @mock.patch('kojihub.QueryProcessor') + def test_list_tasks_by_not_owner_as_list(self, processor): + generator = self.hub.listTasks(opts={'not_owner': [1, 2]}) + results = list(generator) # Exhaust the generator + arguments = self.standard_processor_kwargs.copy() + arguments['clauses'] = ['owner NOT IN %(not_owner)s'] + processor.assert_called_once_with(**arguments) + self.assertEqual(results, []) diff --git a/www/conf/web.conf b/www/conf/web.conf index 1b167be..f7dd34d 100644 --- a/www/conf/web.conf +++ b/www/conf/web.conf @@ -29,7 +29,7 @@ LibPath = /usr/share/koji-web/lib # Defaults to True LiteralFooter = True -# This can be the numeric ID of a user that you want to hide from tasks listed -# on the front page. You might want to, for instance, hide the activity of an -# account used for continuous integration. -#HiddenUser = 5372 +# This can be a comma-delimited list of the numeric IDs of users that you want +# to hide from tasks listed on the front page. You might want to, for instance, +# hide the activity of an account used for continuous integration. +#HiddenUsers = 5372,1234 diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 87b9288..0387eba 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -283,8 +283,8 @@ def index(environ, packageOrder='package_name', packageStart=None): taskOpts = {'parent': None, 'decode': True} if user: taskOpts['owner'] = user['id'] - if opts.get('HiddenUser'): - taskOpts['not_owner'] = opts['HiddenUser'] + if opts.get('HiddenUsers'): + taskOpts['not_owner'] = opts['HiddenUsers'] values['tasks'] = server.listTasks( opts=taskOpts, queryOpts={'order': '-id', 'limit': 10} From 2bdcd9793e7987518e33ae8bbd2dc4e3270cfc25 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Jul 20 2016 20:05:25 +0000 Subject: [PATCH 5/70] Split the HiddenUsers string pulled from our config. --- diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 0387eba..d85ac89 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -284,7 +284,9 @@ def index(environ, packageOrder='package_name', packageStart=None): if user: taskOpts['owner'] = user['id'] if opts.get('HiddenUsers'): - taskOpts['not_owner'] = opts['HiddenUsers'] + taskOpts['not_owner'] = [ + int(userid.strip()) for userid in opts['HiddenUsers'].split() + ] values['tasks'] = server.listTasks( opts=taskOpts, queryOpts={'order': '-id', 'limit': 10} From e4d85688ae81a128c559ad4bb375631b9ac575d2 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Jul 20 2016 20:05:25 +0000 Subject: [PATCH 6/70] Space-delimited. --- diff --git a/www/conf/web.conf b/www/conf/web.conf index f7dd34d..4069258 100644 --- a/www/conf/web.conf +++ b/www/conf/web.conf @@ -29,7 +29,7 @@ LibPath = /usr/share/koji-web/lib # Defaults to True LiteralFooter = True -# This can be a comma-delimited list of the numeric IDs of users that you want +# This can be a space-delimited list of the numeric IDs of users that you want # to hide from tasks listed on the front page. You might want to, for instance, # hide the activity of an account used for continuous integration. -#HiddenUsers = 5372,1234 +#HiddenUsers = 5372 1234 diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index d85ac89..420c606 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -285,7 +285,7 @@ def index(environ, packageOrder='package_name', packageStart=None): taskOpts['owner'] = user['id'] if opts.get('HiddenUsers'): taskOpts['not_owner'] = [ - int(userid.strip()) for userid in opts['HiddenUsers'].split() + int(userid) for userid in opts['HiddenUsers'].split() ] values['tasks'] = server.listTasks( opts=taskOpts, From e88bac85173b9fc06ad633880b78f237542a6a51 Mon Sep 17 00:00:00 2001 From: Jon Disnard Date: Jul 20 2016 20:06:38 +0000 Subject: [PATCH 7/70] Optparsing for LMC Initial LMC optparsing post for koji cli, and probably needs improvement as backside arguments are fleshed out. Code taken iand modified slightly from handle_spin_appliance() by Jay Greguski. Signed-off-by: Jon Disnard --- diff --git a/cli/koji b/cli/koji index 23ac97e..444d9e8 100755 --- a/cli/koji +++ b/cli/koji @@ -5248,26 +5248,32 @@ def handle_spin_livemedia(options, session, args): help=_("Run the livemedia creation task at a lower priority")) parser.add_option("--ksurl", metavar="SCMURL", help=_("The URL to the SCM containing the kickstart file")) - parser.add_option("--install-tree-url", metavar="URL", - help=_("Provide the URL for the install tree")) parser.add_option("--ksversion", metavar="VERSION", help=_("The syntax version used in the kickstart file")) parser.add_option("--scratch", action="store_true", - help=_("Create a scratch LiveMedia image")) + help=_("Create a scratch LiveMEDIA image")) parser.add_option("--repo", action="append", help=_("Specify a repo that will override the repo used to install " + - "RPMs in the LiveMedia. May be used multiple times. The " + + "RPMs in the LiveMEDIA. May be used multiple times. The " + "build tag repo associated with the target is the default.")) parser.add_option("--release", help=_("Forcibly set the release field")) - parser.add_option("--title", help=_("Set the image title (defaults to )")) parser.add_option("--specfile", metavar="URL", help=_("SCM URL to spec file fragment to use to generate wrapper RPMs")) parser.add_option("--skip-tag", action="store_true", help=_("Do not attempt to tag package")) (task_options, args) = parser.parse_args(args) + # LMC operations + parser.add_option("--disk-img", action="store_true", + help=_("Spin bootable disk media")) + + parser.add_option("--live-ostree-pxe", action="store_true", + help=_("Spin live ostree PXE media")) + + parser.add_option("--live-iso", action="store_true", + help=_("Spin live ISO media")) + # Make sure the target and kickstart is specified. - print 'spin-livecd is deprecated and will be replaced with spin-livemedia' if len(args) != 5: parser.error(_("Five arguments are required: a name, a version, an" + " architecture, a build target, and a relative path to" + From bdb016b3cb50a5414fa49cee046b7a564fc19ed8 Mon Sep 17 00:00:00 2001 From: Jon Disnard Date: Jul 20 2016 20:07:59 +0000 Subject: [PATCH 8/70] builder: Add LMC task handler A copy of Jay Greguski's livecd-creator task. It replaces livecd-creator for livemedia-creator. For now it does live-iso, ostree-live, and disk-img. This initial commit needs more work to be functional. Signed-off-by: Jon Disnard --- diff --git a/builder/kojid b/builder/kojid index 240b5ef..d5e55ac 100755 --- a/builder/kojid +++ b/builder/kojid @@ -60,7 +60,7 @@ from optparse import OptionParser, SUPPRESS_HELP from StringIO import StringIO from yum import repoMDObject -#imports for LiveCD, LiveMedia, and Appliance handler +#imports for LiveCD, LiveMEDIA, and Appliance handler image_enabled = False try: import pykickstart.parser as ksparser @@ -3011,73 +3011,51 @@ class LiveMediaTask(ImageTask): kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts) cachedir = '/tmp/koji-livemedia' # arbitrary paths in chroot - livemedia_log = '/tmp/lmc-logs/livemedia-out.log' - resultdir = '/tmp/lmc' + livemedia_log = '/tmp/livemedia.log' - - # Common LMC command setup, needs extending + # Common LMC command setup, needs extending cmd = ['/sbin/livemedia-creator', - '--ks', kskoji, + '-ks', kskoji, '--logfile', livemedia_log, '--no-virt', - '--resultdir', resultdir, - '--project', name, - #'--tmp', '/tmp' + '--tmp', '/tmp' ] - # note: at the moment, we are only generating live isos. We may add support - # for other types in the future - - cmd.extend(['--make-iso', - '--volid', self._shortenVolID(name, version, release), - '--iso-only', - ]) - - isoname='%s-%s-%s-%s.iso' % (name, arch, version, release) - title = self.opts.get('title', name) - cmd.extend(['--iso-name', isoname, - '--releasever', version, - '--title', title, - ]) - - - if arch == 'x86_64': - cmd.append('--macboot') - + # Determine what LMC opperation we are doing: live-iso, live-pxe, disk-img. + if live-iso: + # we set the fs label to the same as the isoname if it exists, + # taking at most 32 characters + isoname = '%s-%s-%s' % (name, version, release) + cmd.extend(['--make-iso', '--volid', isoname[:32]]) + + elif live-ostree-pxe: + cmd.extend(['--make-ostree-live']) + + elif disk-img: + img_name='disk.img' + prj_name='Fedora' + cmd.extend(['--make-disk', + '--image-name', img_name, + '--project', prj_name, + '--releasever', release + ]) + + else + # bail + raise koji.LiveMediaError, \ + 'This task needs an option: --live-iso or --live-ostree-pxe or --disk-img' # Run livemedia-creator rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd) - - # upload logs - logdirs = [ - os.path.join(broot.rootdir(), 'tmp/lmc-logs'), - os.path.join(broot.rootdir(), 'tmp/lmc-logs/anaconda'), - ] - for logdir in logdirs: - if not os.path.isdir(logdir): - continue - for filename in os.listdir(logdir): - if not filename.endswith('.log'): - continue - filepath = os.path.join(logdir, filename) - if os.stat(filepath).st_size == 0: - continue - # avoid file duplication between directories by prefixing anaconda logs - if logdir.endswith('anaconda'): - self.uploadFile(os.path.join(filepath), remoteName='anaconda-%s' % filename) - continue - - self.uploadFile(os.path.join(filepath)) - + self.uploadFile(os.path.join(broot.rootdir(), livemedia_log[1:])) if rv: raise koji.LiveMediaError, \ - 'Could not create LiveMedia: %s' % parseStatus(rv, 'livemedia-creator') + '; see root.log or livemedia-out.log for more information' + 'Could not create LiveMedia: %s' % parseStatus(rv, 'livemedia-creator') + '; see root.log or livemedia.log for more information' # Find the resultant iso # The cwd of the livemedia-creator process is /tmp in the chroot, so # that is where it writes the .iso - rootresultsdir = os.path.join(broot.rootdir(), resultdir.lstrip('/')) - files = os.listdir(rootresultsdir) + files = os.listdir(os.path.join(broot.rootdir(), 'tmp')) isofile = None for afile in files: if afile.endswith('.iso'): @@ -3087,14 +3065,17 @@ class LiveMediaTask(ImageTask): raise koji.LiveMediaError, 'multiple .iso files found: %s and %s' % (isofile, afile) if not isofile: raise koji.LiveMediaError, 'could not find iso file in chroot' - isosrc = os.path.join(rootresultsdir, isofile) + isosrc = os.path.join(broot.rootdir(), 'tmp', isofile) + # copy the iso out of the chroot. If we were given an isoname, + # this is where the renaming happens. + self.logger.debug('uploading image: %s' % isosrc) + isoname += '.iso' # Generate the file manifest of the image, upload the results manifest = os.path.join(broot.resultdir(), 'manifest.log') self.genISOManifest(isosrc, manifest) self.uploadFile(manifest) - self.logger.debug('uploading image: %s' % isosrc) self.uploadFile(isosrc, remoteName=isoname) imgdata = {'arch': arch, @@ -3102,19 +3083,17 @@ class LiveMediaTask(ImageTask): 'rootdev': None, 'task_id': self.id, 'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log', - 'livemedia-out.log', os.path.basename(ksfile), + 'livemedia.log', os.path.basename(ksfile), os.path.basename(kskoji)], 'name': name, 'version': version, 'release': release } if not opts.get('scratch'): - # TODO - generate list of rpms in image - # (getImagePackages doesn't work here) - #hdrlist = self.getImagePackages(os.path.join(broot.rootdir(), - # cachedir[1:])) - imgdata ['rpmlist'] = [] - #broot.markExternalRPMs(hdrlist) + hdrlist = self.getImagePackages(os.path.join(broot.rootdir(), + cachedir[1:])) + imgdata ['rpmlist'] = hdrlist + broot.markExternalRPMs(hdrlist) broot.expire() return imgdata From 2f094e94ee2aa72226879ca5b6560cad2498fbc9 Mon Sep 17 00:00:00 2001 From: Jon Disnard Date: Jul 20 2016 20:08:36 +0000 Subject: [PATCH 9/70] koji: Add LMC error handler, and misc glue. Signed-off-by: Jon Disnard --- diff --git a/koji/__init__.py b/koji/__init__.py index b453017..5d9697f 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -338,7 +338,7 @@ class ConfigurationError(GenericError): faultCode = 1021 class LiveMediaError(GenericError): - """Raised when LiveMedia Image creation fails""" + """Raised when LiveMEDIA Image creation fails""" faultCode = 1022 class MultiCallInProgress(object): @@ -2555,7 +2555,7 @@ def _taskLabel(taskInfo): else: kickstart = os.path.basename(stuff[4]) extra = '%s, %s-%s, %s' % (stuff[3], stuff[0], stuff[1], kickstart) - elif method in ('createLiveCD', 'createAppliance', 'createImage', 'createLiveMedia'): + elif method in ('createLiveCD', 'createAppliance', 'createImage', 'createLiveMEDIA'): if taskInfo.has_key('request'): stuff = taskInfo['request'] if method == 'createImage': From 595b725615c104366f6f4e309d6cf293cdaf531f Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Jul 20 2016 20:08:52 +0000 Subject: [PATCH 10/70] - fix whitespace issues with tab characters vs spaces - fix invalid variable names in livemediacreator handler - add parser for ostree pxe2live to cli --- diff --git a/builder/kojid b/builder/kojid index d5e55ac..7b30f58 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3022,16 +3022,16 @@ class LiveMediaTask(ImageTask): ] # Determine what LMC opperation we are doing: live-iso, live-pxe, disk-img. - if live-iso: + if live_iso: # we set the fs label to the same as the isoname if it exists, # taking at most 32 characters isoname = '%s-%s-%s' % (name, version, release) cmd.extend(['--make-iso', '--volid', isoname[:32]]) - elif live-ostree-pxe: + elif live_ostree_pxe: cmd.extend(['--make-ostree-live']) - elif disk-img: + elif disk_img: img_name='disk.img' prj_name='Fedora' cmd.extend(['--make-disk', @@ -3040,10 +3040,10 @@ class LiveMediaTask(ImageTask): '--releasever', release ]) - else - # bail - raise koji.LiveMediaError, \ - 'This task needs an option: --live-iso or --live-ostree-pxe or --disk-img' + else: + # bail + raise koji.LiveMediaError, \ + 'This task needs an option: --live-iso or --live-ostree-pxe or --disk-img' # Run livemedia-creator rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd) diff --git a/cli/koji b/cli/koji index 444d9e8..f798c16 100755 --- a/cli/koji +++ b/cli/koji @@ -5261,6 +5261,8 @@ def handle_spin_livemedia(options, session, args): help=_("SCM URL to spec file fragment to use to generate wrapper RPMs")) parser.add_option("--skip-tag", action="store_true", help=_("Do not attempt to tag package")) + parser.add_option("--live-ostree-pxe", action="store_true", + help=_("Build a live pxe boot squashfs image of Atomic Host")) (task_options, args) = parser.parse_args(args) # LMC operations @@ -5653,7 +5655,9 @@ def _build_image(options, task_opts, session, args, img_type): 'title', 'install_tree_url', ] hub_opts = {} - for opt in passthru_opts: + for opt in ('isoname', 'ksurl', 'ksversion', 'scratch', 'repo', + 'release', 'skip_tag', 'vmem', 'vcpu', 'format', 'specfile', + 'live_ostree_pxe'): val = getattr(task_opts, opt, None) if val is not None: hub_opts[opt] = val From 70c97e61dff87059896cec94e88159444d0cb768 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:09:12 +0000 Subject: [PATCH 11/70] fix whitespace --- diff --git a/builder/kojid b/builder/kojid index 7b30f58..8b4f6c5 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3013,7 +3013,7 @@ class LiveMediaTask(ImageTask): cachedir = '/tmp/koji-livemedia' # arbitrary paths in chroot livemedia_log = '/tmp/livemedia.log' - # Common LMC command setup, needs extending + # Common LMC command setup, needs extending cmd = ['/sbin/livemedia-creator', '-ks', kskoji, '--logfile', livemedia_log, From 354bee384920390b3f133ed1d4bb8e396f56b9e9 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:09:58 +0000 Subject: [PATCH 12/70] first stab at livemedia control task, based on livecd --- diff --git a/builder/kojid b/builder/kojid index 8b4f6c5..d3940da 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2367,7 +2367,7 @@ class BuildLiveCDTask(BuildImageTask): class BuildLiveMediaTask(BuildImageTask): Methods = ['livemedia'] - def handler(self, name, version, arches, target, ksfile, opts=None): + def handler(self, name, version, arch, target, ksfile, opts=None): """Governing task for building live media""" target_info = self.session.getBuildTarget(target, strict=True) @@ -2378,12 +2378,8 @@ class BuildLiveMediaTask(BuildImageTask): if not buildconfig['arches']: raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()] - - # check arches and remove duplicates - arches = set(arches) - for arch in arches: - if koji.canonArch(arch) not in tag_archlist: - raise koji.BuildError, "Invalid arch for build tag: %s" % arch + if koji.canonArch(arch) not in tag_archlist: + raise koji.BuildError, "Invalid arch for build tag: %s" % arch if not opts: opts = {} @@ -2394,52 +2390,31 @@ class BuildLiveMediaTask(BuildImageTask): raise koji.PreBuildError, 'Live Media functions not available' # build the image - bld_info = None try: release = opts.get('release') if not release: release = self.getRelease(name, version) + bld_info = None if not opts.get('scratch'): bld_info = self.initImageBuild(name, version, release, target_info, opts) - subtasks = {} - for arch in arches: - subtasks[arch] = self.subtask('createLiveMedia', - [name, version, release, arch, target_info, build_tag, - repo_info, ksfile, opts], - label='livemedia %s' % arch, arch=arch) - - - self.logger.debug("Got image subtasks: %r", subtasks) - self.logger.debug("Waiting on livemedia subtasks...") - results = self.wait(subtasks.values(), all=True, failany=True) - - self.logger.debug('subtask results: %r', results) + create_task_id = self.subtask('createLiveMedia', + [name, version, release, arch, target_info, build_tag, + repo_info, ksfile, opts], + label='livemedia', arch=arch) + results = self.wait(create_task_id) + self.logger.info('image build task (%s) completed' % create_task_id) + self.logger.info('results: %s' % results) - # wrap each image an RPM if needed + # wrap in an RPM if needed spec_url = opts.get('specfile') + rpm_results = None if spec_url: - wrapper_tasks = {} - for arch in arches: - subtask_id = subtasks[arch] - result = results[subtask_id] - tinfo = self.session.getTaskInfo(subtask_id) - arglist = [spec_url, target_info, bld_info, tinfo, - {'repo_id': repo_info['id']}] - wrapper_tasks[arch] = self.subtask('wrapperRPM', arglist, - label='wrapper %s' % arch, arch='noarch') - - results2 = self.wait(wrapper_tasks.values(), all=True, failany=True) - self.logger.debug('wrapper results: %r', results2) - - # add wrapper rpm results into main results - for arch in arches: - result = results[subtasks[arch]] - result2 = results2[wrapper_tasks[arch]] - result['rpmresults'] = result2 - - # re-key results for xmlrpc friendliness - results = dict([(str(k), results[k]) for k in results]) + results[create_task_id]['rpmresults'] = self.buildWrapperRPM( + spec_url, create_task_id, + target_info, bld_info, repo_info['id']) + results[str(create_task_id)] = results[create_task_id] + del results[create_task_id] # import it (and move) if not opts.get('scratch'): @@ -2467,8 +2442,8 @@ class BuildLiveMediaTask(BuildImageTask): # report the results if opts.get('scratch'): - respath = ', '.join( - [os.path.join(koji.pathinfo.work(), koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()]) + respath = os.path.join(koji.pathinfo.work(), + koji.pathinfo.taskrelpath(create_task_id)) report = 'Scratch ' else: respath = koji.pathinfo.imagebuild(bld_info) From a1ecb3ed8579f9eebec3eebfb5721ec3fd73fc15 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:09:59 +0000 Subject: [PATCH 13/70] misc fixes --- diff --git a/cli/koji b/cli/koji index f798c16..06d7151 100755 --- a/cli/koji +++ b/cli/koji @@ -5263,17 +5263,12 @@ def handle_spin_livemedia(options, session, args): help=_("Do not attempt to tag package")) parser.add_option("--live-ostree-pxe", action="store_true", help=_("Build a live pxe boot squashfs image of Atomic Host")) - (task_options, args) = parser.parse_args(args) - # LMC operations parser.add_option("--disk-img", action="store_true", help=_("Spin bootable disk media")) - - parser.add_option("--live-ostree-pxe", action="store_true", - help=_("Spin live ostree PXE media")) - parser.add_option("--live-iso", action="store_true", help=_("Spin live ISO media")) + (task_options, args) = parser.parse_args(args) # Make sure the target and kickstart is specified. if len(args) != 5: From 002d27e23ea7c73445c1a9ba50a71cb88509b28d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:27 +0000 Subject: [PATCH 14/70] make sure livemedia options are passed through to task --- diff --git a/builder/kojid b/builder/kojid index d3940da..3c0707b 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2997,16 +2997,16 @@ class LiveMediaTask(ImageTask): ] # Determine what LMC opperation we are doing: live-iso, live-pxe, disk-img. - if live_iso: + if opts.get('live_iso'): # we set the fs label to the same as the isoname if it exists, # taking at most 32 characters isoname = '%s-%s-%s' % (name, version, release) cmd.extend(['--make-iso', '--volid', isoname[:32]]) - elif live_ostree_pxe: + elif opts.get('live_ostree_pxe'): cmd.extend(['--make-ostree-live']) - elif disk_img: + elif opts.get('disk_img'): img_name='disk.img' prj_name='Fedora' cmd.extend(['--make-disk', @@ -3017,8 +3017,8 @@ class LiveMediaTask(ImageTask): else: # bail - raise koji.LiveMediaError, \ - 'This task needs an option: --live-iso or --live-ostree-pxe or --disk-img' + raise koji.LiveMediaError('Please specify one of the following ' + 'options: live_iso, live_ostree_pxe, or disk_img') # Run livemedia-creator rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd) diff --git a/cli/koji b/cli/koji index 06d7151..7986d81 100755 --- a/cli/koji +++ b/cli/koji @@ -5270,6 +5270,11 @@ def handle_spin_livemedia(options, session, args): help=_("Spin live ISO media")) (task_options, args) = parser.parse_args(args) + if (not task_options.live_ostree_pxe and not task_options.disk_img + and not task_options.live_iso): + parser.error(_("Please specify one of the following options: " + "--live-ostree-pxe, --disk-img, or --live-iso")) + # Make sure the target and kickstart is specified. if len(args) != 5: parser.error(_("Five arguments are required: a name, a version, an" + @@ -5647,12 +5652,10 @@ def _build_image(options, task_opts, session, args, img_type): passthru_opts = [ 'isoname', 'ksurl', 'ksversion', 'scratch', 'repo', 'release', 'skip_tag', 'vmem', 'vcpu', 'format', 'specfile', - 'title', 'install_tree_url', + 'live_ostree_pxe', 'disk_img', 'live_ostree_pxe', 'live_iso', ] hub_opts = {} - for opt in ('isoname', 'ksurl', 'ksversion', 'scratch', 'repo', - 'release', 'skip_tag', 'vmem', 'vcpu', 'format', 'specfile', - 'live_ostree_pxe'): + for opt in passthru_opts: val = getattr(task_opts, opt, None) if val is not None: hub_opts[opt] = val From 85cd9930d7d596ce2aecd16c1ad21bf490f76638 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:27 +0000 Subject: [PATCH 15/70] fix livemedia-creator kickstart opt --- diff --git a/builder/kojid b/builder/kojid index 3c0707b..8b8d014 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2990,7 +2990,7 @@ class LiveMediaTask(ImageTask): # Common LMC command setup, needs extending cmd = ['/sbin/livemedia-creator', - '-ks', kskoji, + '--ks', kskoji, '--logfile', livemedia_log, '--no-virt', '--tmp', '/tmp' From 2e18818d907d1c077d88e294afc4cee5fa11292a Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:27 +0000 Subject: [PATCH 16/70] refactor lmc args. only making live isos for now --- diff --git a/builder/kojid b/builder/kojid index 8b8d014..b141965 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2986,39 +2986,40 @@ class LiveMediaTask(ImageTask): kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts) cachedir = '/tmp/koji-livemedia' # arbitrary paths in chroot - livemedia_log = '/tmp/livemedia.log' + livemedia_log = '/tmp/lmc-logs/livemedia-out.log' + resultdir = '/tmp/lmc' # Common LMC command setup, needs extending cmd = ['/sbin/livemedia-creator', '--ks', kskoji, '--logfile', livemedia_log, '--no-virt', - '--tmp', '/tmp' + '--resultdir', resultdir, + #'--tmp', '/tmp' ] - # Determine what LMC opperation we are doing: live-iso, live-pxe, disk-img. - if opts.get('live_iso'): - # we set the fs label to the same as the isoname if it exists, - # taking at most 32 characters - isoname = '%s-%s-%s' % (name, version, release) - cmd.extend(['--make-iso', '--volid', isoname[:32]]) - - elif opts.get('live_ostree_pxe'): - cmd.extend(['--make-ostree-live']) - - elif opts.get('disk_img'): - img_name='disk.img' - prj_name='Fedora' - cmd.extend(['--make-disk', - '--image-name', img_name, - '--project', prj_name, - '--releasever', release - ]) + # note: at the moment, we are only generating live isos. We may add support + # for other types in the future + + # we set the fs label to the same as the isoname if it exists, + # taking at most 32 characters + isoname = '%s-%s-%s' % (name, version, release) + cmd.extend(['--make-iso', + '--volid', isoname[:32], + '--iso-only', + ]) + + img_name='%s-%s-%s-%s.iso' % (name, arch, version, release) + title = self.opts.get('title', name) + cmd.extend(['--image-name', img_name, + '--releasever', release, + '--title', title, + ]) + + + if arch == 'x86_64': + cmd.append('--macboot') - else: - # bail - raise koji.LiveMediaError('Please specify one of the following ' - 'options: live_iso, live_ostree_pxe, or disk_img') # Run livemedia-creator rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd) diff --git a/cli/koji b/cli/koji index 7986d81..5479df4 100755 --- a/cli/koji +++ b/cli/koji @@ -5257,24 +5257,13 @@ def handle_spin_livemedia(options, session, args): "RPMs in the LiveMEDIA. May be used multiple times. The " + "build tag repo associated with the target is the default.")) parser.add_option("--release", help=_("Forcibly set the release field")) + parser.add_option("--title", help=_("Set the image title (defaults to )")) parser.add_option("--specfile", metavar="URL", help=_("SCM URL to spec file fragment to use to generate wrapper RPMs")) parser.add_option("--skip-tag", action="store_true", help=_("Do not attempt to tag package")) - parser.add_option("--live-ostree-pxe", action="store_true", - help=_("Build a live pxe boot squashfs image of Atomic Host")) - # LMC operations - parser.add_option("--disk-img", action="store_true", - help=_("Spin bootable disk media")) - parser.add_option("--live-iso", action="store_true", - help=_("Spin live ISO media")) (task_options, args) = parser.parse_args(args) - if (not task_options.live_ostree_pxe and not task_options.disk_img - and not task_options.live_iso): - parser.error(_("Please specify one of the following options: " - "--live-ostree-pxe, --disk-img, or --live-iso")) - # Make sure the target and kickstart is specified. if len(args) != 5: parser.error(_("Five arguments are required: a name, a version, an" + @@ -5652,7 +5641,7 @@ def _build_image(options, task_opts, session, args, img_type): passthru_opts = [ 'isoname', 'ksurl', 'ksversion', 'scratch', 'repo', 'release', 'skip_tag', 'vmem', 'vcpu', 'format', 'specfile', - 'live_ostree_pxe', 'disk_img', 'live_ostree_pxe', 'live_iso', + 'title', ] hub_opts = {} for opt in passthru_opts: From ab22870d12078180f3a4575f86c6b1f5202b4163 Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Jul 20 2016 20:10:54 +0000 Subject: [PATCH 17/70] shorten the livecd iso labels to match fedora's naming policy --- diff --git a/builder/kojid b/builder/kojid index b141965..f5f9e58 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2825,6 +2825,35 @@ class LiveCDTask(ImageTask): return manifest + def _shortenVolID(self, name, version, release): + # Based on code from pungi + substitutions = { + 'MATE_Compiz': 'MATE', + 'Security': 'Sec', + 'Electronic_Lab': 'Elec', + 'Robotics': 'Robo', + 'Scientific_KDE': 'SciK', + 'Design_suite': 'Dsgn', + 'Games': 'Game', + 'Jam_KDE': 'Jam', + 'Workstation': 'WS', + 'Server': 'S', + 'Cloud': 'C', + 'Alpha': 'A', + 'Beta': 'B', + 'TC': 'T', + } + + for k, v in substitutions.iteritems(): + if k in name: + name = name.replace(k, v) + if k in version: + version = version.replace(k, v) + if k in release: + release = release.replace(k, v) + + volid = "%s-%s-%s" % (name, version, release) + return volid[:32] def handler(self, name, version, release, arch, target_info, build_tag, repo_info, ksfile, opts=None): From 19a49b21586b785b00594a3417f0dd27aa470cd7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:54 +0000 Subject: [PATCH 18/70] use _shortenVolID for livemedia handler too --- diff --git a/builder/kojid b/builder/kojid index f5f9e58..5eab40c 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2825,35 +2825,6 @@ class LiveCDTask(ImageTask): return manifest - def _shortenVolID(self, name, version, release): - # Based on code from pungi - substitutions = { - 'MATE_Compiz': 'MATE', - 'Security': 'Sec', - 'Electronic_Lab': 'Elec', - 'Robotics': 'Robo', - 'Scientific_KDE': 'SciK', - 'Design_suite': 'Dsgn', - 'Games': 'Game', - 'Jam_KDE': 'Jam', - 'Workstation': 'WS', - 'Server': 'S', - 'Cloud': 'C', - 'Alpha': 'A', - 'Beta': 'B', - 'TC': 'T', - } - - for k, v in substitutions.iteritems(): - if k in name: - name = name.replace(k, v) - if k in version: - version = version.replace(k, v) - if k in release: - release = release.replace(k, v) - - volid = "%s-%s-%s" % (name, version, release) - return volid[:32] def handler(self, name, version, release, arch, target_info, build_tag, repo_info, ksfile, opts=None): @@ -3032,9 +3003,8 @@ class LiveMediaTask(ImageTask): # we set the fs label to the same as the isoname if it exists, # taking at most 32 characters - isoname = '%s-%s-%s' % (name, version, release) cmd.extend(['--make-iso', - '--volid', isoname[:32], + '--volid', self._shortenVolID(name, version, release), '--iso-only', ]) From 92d42b8c4cef5fc872009a60433b73cc87bcb111 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:55 +0000 Subject: [PATCH 19/70] fix isoname --- diff --git a/builder/kojid b/builder/kojid index 5eab40c..1c6b9b9 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3001,8 +3001,6 @@ class LiveMediaTask(ImageTask): # note: at the moment, we are only generating live isos. We may add support # for other types in the future - # we set the fs label to the same as the isoname if it exists, - # taking at most 32 characters cmd.extend(['--make-iso', '--volid', self._shortenVolID(name, version, release), '--iso-only', @@ -3045,7 +3043,7 @@ class LiveMediaTask(ImageTask): # copy the iso out of the chroot. If we were given an isoname, # this is where the renaming happens. self.logger.debug('uploading image: %s' % isosrc) - isoname += '.iso' + isoname = '%s-%s-%s.iso' % (name, version, release) # Generate the file manifest of the image, upload the results manifest = os.path.join(broot.resultdir(), 'manifest.log') From bbee2b96dbdcb50597c9065bd5abd174d7fcab0c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:55 +0000 Subject: [PATCH 20/70] capture all livemedia logs --- diff --git a/builder/kojid b/builder/kojid index 1c6b9b9..1407647 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3020,7 +3020,17 @@ class LiveMediaTask(ImageTask): # Run livemedia-creator rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd) - self.uploadFile(os.path.join(broot.rootdir(), livemedia_log[1:])) + + # upload logs + logdir = os.path.join(broot.rootdir(), 'tmp/lmc-logs') + for filename in os.listdir(logdir): + if not filename.endswith('.log'): + continue + filepath = os.path.join(logdir, filename) + if os.stat(filepath).st_size == 0: + continue + self.uploadFile(os.path.join(filepath)) + if rv: raise koji.LiveMediaError, \ 'Could not create LiveMedia: %s' % parseStatus(rv, 'livemedia-creator') + '; see root.log or livemedia.log for more information' From b5a4e5fe6091f617fa71c16059f3ce7581fc5f12 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:55 +0000 Subject: [PATCH 21/70] grab livemedia anaconda logs too --- diff --git a/builder/kojid b/builder/kojid index 1407647..c560354 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3022,14 +3022,20 @@ class LiveMediaTask(ImageTask): rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + cmd) # upload logs - logdir = os.path.join(broot.rootdir(), 'tmp/lmc-logs') - for filename in os.listdir(logdir): - if not filename.endswith('.log'): + logdirs = [ + os.path.join(broot.rootdir(), 'tmp/lmc-logs'), + os.path.join(broot.rootdir(), 'tmp/lmc-logs/anaconda'), + ] + for logdir in logdirs: + if not os.path.isdir(logdir): continue - filepath = os.path.join(logdir, filename) - if os.stat(filepath).st_size == 0: - continue - self.uploadFile(os.path.join(filepath)) + for filename in os.listdir(logdir): + if not filename.endswith('.log'): + continue + filepath = os.path.join(logdir, filename) + if os.stat(filepath).st_size == 0: + continue + self.uploadFile(os.path.join(filepath)) if rv: raise koji.LiveMediaError, \ From e53480d6670b89bd2eca37488f1849b6fc00c60e Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:55 +0000 Subject: [PATCH 22/70] use --iso-name livemedia opt --- diff --git a/builder/kojid b/builder/kojid index c560354..e4b5249 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3008,7 +3008,7 @@ class LiveMediaTask(ImageTask): img_name='%s-%s-%s-%s.iso' % (name, arch, version, release) title = self.opts.get('title', name) - cmd.extend(['--image-name', img_name, + cmd.extend(['--iso-name', img_name, '--releasever', release, '--title', title, ]) From 306e072ee3c46d7bd31aef5251926f90a4b7b020 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:55 +0000 Subject: [PATCH 23/70] look in right place for livemedia isos --- diff --git a/builder/kojid b/builder/kojid index e4b5249..6861be8 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3044,7 +3044,7 @@ class LiveMediaTask(ImageTask): # Find the resultant iso # The cwd of the livemedia-creator process is /tmp in the chroot, so # that is where it writes the .iso - files = os.listdir(os.path.join(broot.rootdir(), 'tmp')) + files = os.listdir(os.path.join(broot.rootdir(), resultdir.lstrip('/'))) isofile = None for afile in files: if afile.endswith('.iso'): From e526bf34978e764eea832fb8b638ac0b32af3052 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:55 +0000 Subject: [PATCH 24/70] use correct value for lmc "releasever" arg --- diff --git a/builder/kojid b/builder/kojid index 6861be8..026b6c6 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3009,7 +3009,7 @@ class LiveMediaTask(ImageTask): img_name='%s-%s-%s-%s.iso' % (name, arch, version, release) title = self.opts.get('title', name) cmd.extend(['--iso-name', img_name, - '--releasever', release, + '--releasever', version, '--title', title, ]) From cb4ed8d3bf5674e68d02e1f558806d7bab0a785b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:55 +0000 Subject: [PATCH 25/70] use correct path to result iso --- diff --git a/builder/kojid b/builder/kojid index 026b6c6..6677d7c 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3044,7 +3044,8 @@ class LiveMediaTask(ImageTask): # Find the resultant iso # The cwd of the livemedia-creator process is /tmp in the chroot, so # that is where it writes the .iso - files = os.listdir(os.path.join(broot.rootdir(), resultdir.lstrip('/'))) + rootresultsdir = os.path.join(broot.rootdir(), resultdir.lstrip('/')) + files = os.listdir(rootresultsdir) isofile = None for afile in files: if afile.endswith('.iso'): @@ -3054,7 +3055,7 @@ class LiveMediaTask(ImageTask): raise koji.LiveMediaError, 'multiple .iso files found: %s and %s' % (isofile, afile) if not isofile: raise koji.LiveMediaError, 'could not find iso file in chroot' - isosrc = os.path.join(broot.rootdir(), 'tmp', isofile) + isosrc = os.path.join(rootresultsdir, isofile) # copy the iso out of the chroot. If we were given an isoname, # this is where the renaming happens. From 7151afe636d8adf2b1b0575ff052f7dbafc6af8b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:10:55 +0000 Subject: [PATCH 26/70] partial work on multiarch livemedia builds --- diff --git a/builder/kojid b/builder/kojid index 6677d7c..721001b 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2367,7 +2367,7 @@ class BuildLiveCDTask(BuildImageTask): class BuildLiveMediaTask(BuildImageTask): Methods = ['livemedia'] - def handler(self, name, version, arch, target, ksfile, opts=None): + def handler(self, name, version, arches, target, ksfile, opts=None): """Governing task for building live media""" target_info = self.session.getBuildTarget(target, strict=True) @@ -2378,8 +2378,10 @@ class BuildLiveMediaTask(BuildImageTask): if not buildconfig['arches']: raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()] - if koji.canonArch(arch) not in tag_archlist: - raise koji.BuildError, "Invalid arch for build tag: %s" % arch + arches = set(arches) + for arch in arches: + if koji.canonArch(arch) not in tag_archlist: + raise koji.BuildError, "Invalid arch for build tag: %s" % arch if not opts: opts = {} @@ -2398,23 +2400,47 @@ class BuildLiveMediaTask(BuildImageTask): if not opts.get('scratch'): bld_info = self.initImageBuild(name, version, release, target_info, opts) - create_task_id = self.subtask('createLiveMedia', - [name, version, release, arch, target_info, build_tag, - repo_info, ksfile, opts], - label='livemedia', arch=arch) - results = self.wait(create_task_id) - self.logger.info('image build task (%s) completed' % create_task_id) - self.logger.info('results: %s' % results) + subtasks = {} + for arch in arches: + subtasks[arch] = self.subtask('createLiveMedia', + [name, version, release, arch, target_info, build_tag, + repo_info, ksfile, opts], + label=str(arch), arch=arch) - # wrap in an RPM if needed + + self.logger.debug("Got image subtasks: %r", subtasks) + self.logger.debug("Waiting on livemedia subtasks...") + results = self.wait(subtasks.values(), all=True, failany=True) + + self.logger.debug('subtask results: %r', results) + + # wrap each image an RPM if needed spec_url = opts.get('specfile') - rpm_results = None if spec_url: + wrapper_tasks = {} + for arch in arches: + subtask_id = subtasks[arch] + result = results[subtask_id] + tinfo = self.session.getTaskInfo(subtask_id) + arglist = [spec_url, target_info, bld_info, tinfo, + {'repo_id': repo_info['id']}] + wrapper_tasks[arch] = self.subtask('wrapperRPM', arglist, + label='wrapper %s' % arch, arch='noarch') + + results2 = self.wait(wrapper_tasks.values(), all=True, failany=True) + self.logger.debug('wrapper results: %r', results2) + + # add wrapper rpm results into main results + for arch in arches: + result = results[subtasks[arch]] + result2 = results2[wrapper_tasks[arch]] + result['rpmresults'] = results[create_task_id]['rpmresults'] = self.buildWrapperRPM( spec_url, create_task_id, target_info, bld_info, repo_info['id']) - results[str(create_task_id)] = results[create_task_id] - del results[create_task_id] + + # re-key results for xmlrpc friendliness + results = dict([(str(k), results[k]) for k in results]) # import it (and move) if not opts.get('scratch'): From daadd14d7b1de31511522ad202023842e4fee94a Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:11:13 +0000 Subject: [PATCH 27/70] allow multiple arches on spin-livemedia command line --- diff --git a/cli/koji b/cli/koji index 5479df4..2f25c80 100755 --- a/cli/koji +++ b/cli/koji @@ -5624,7 +5624,7 @@ def _build_image(options, task_opts, session, args, img_type): # Set the architecture if img_type == 'livemedia': # livemedia accepts multiple arches - arch = [koji.canonArch(a) for a in args[3].split(",")] + arch = [koji.canonArch(a) for a in ','.split(args[3])] else: arch = koji.canonArch(args[3]) From e363dd27815ee8ac265e96078f1246212ba1a4ff Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:11:13 +0000 Subject: [PATCH 28/70] more livemedia multiarch --- diff --git a/builder/kojid b/builder/kojid index 721001b..6ea234a 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2378,6 +2378,8 @@ class BuildLiveMediaTask(BuildImageTask): if not buildconfig['arches']: raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()] + + # check arches and remove duplicates arches = set(arches) for arch in arches: if koji.canonArch(arch) not in tag_archlist: @@ -2405,7 +2407,7 @@ class BuildLiveMediaTask(BuildImageTask): subtasks[arch] = self.subtask('createLiveMedia', [name, version, release, arch, target_info, build_tag, repo_info, ksfile, opts], - label=str(arch), arch=arch) + label='livemedia %s' % arch, arch=arch) self.logger.debug("Got image subtasks: %r", subtasks) @@ -2434,10 +2436,7 @@ class BuildLiveMediaTask(BuildImageTask): for arch in arches: result = results[subtasks[arch]] result2 = results2[wrapper_tasks[arch]] - result['rpmresults'] = - results[create_task_id]['rpmresults'] = self.buildWrapperRPM( - spec_url, create_task_id, - target_info, bld_info, repo_info['id']) + result['rpmresults'] = result2 # re-key results for xmlrpc friendliness results = dict([(str(k), results[k]) for k in results]) From b39b624ff6559f170cf5c977a8e0e5e19a28399e Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:11:13 +0000 Subject: [PATCH 29/70] fix arch splitting --- diff --git a/cli/koji b/cli/koji index 2f25c80..5479df4 100755 --- a/cli/koji +++ b/cli/koji @@ -5624,7 +5624,7 @@ def _build_image(options, task_opts, session, args, img_type): # Set the architecture if img_type == 'livemedia': # livemedia accepts multiple arches - arch = [koji.canonArch(a) for a in ','.split(args[3])] + arch = [koji.canonArch(a) for a in args[3].split(",")] else: arch = koji.canonArch(args[3]) From 4ea31a27e0c5b3196d0844800814a4d2b27d2ee1 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:11:13 +0000 Subject: [PATCH 30/70] fix livemedia report text --- diff --git a/builder/kojid b/builder/kojid index 6ea234a..f0cb8fb 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2467,8 +2467,8 @@ class BuildLiveMediaTask(BuildImageTask): # report the results if opts.get('scratch'): - respath = os.path.join(koji.pathinfo.work(), - koji.pathinfo.taskrelpath(create_task_id)) + respath = ', '.join( + [os.path.join(koji.pathinfo.work(), koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()]) report = 'Scratch ' else: respath = koji.pathinfo.imagebuild(bld_info) From 355faca4205f8025c13ab2fceab425ee8ede7183 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:11:13 +0000 Subject: [PATCH 31/70] correct livemedia log name --- diff --git a/builder/kojid b/builder/kojid index f0cb8fb..fc403c3 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3064,7 +3064,7 @@ class LiveMediaTask(ImageTask): if rv: raise koji.LiveMediaError, \ - 'Could not create LiveMedia: %s' % parseStatus(rv, 'livemedia-creator') + '; see root.log or livemedia.log for more information' + 'Could not create LiveMedia: %s' % parseStatus(rv, 'livemedia-creator') + '; see root.log or livemedia-out.log for more information' # Find the resultant iso # The cwd of the livemedia-creator process is /tmp in the chroot, so @@ -3098,7 +3098,7 @@ class LiveMediaTask(ImageTask): 'rootdev': None, 'task_id': self.id, 'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log', - 'livemedia.log', os.path.basename(ksfile), + 'livemedia-out.log', os.path.basename(ksfile), os.path.basename(kskoji)], 'name': name, 'version': version, From 40a16033696c3d763291ff9fa9ab2158109fdbcf Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:11:14 +0000 Subject: [PATCH 32/70] fixing up cli task display --- diff --git a/koji/__init__.py b/koji/__init__.py index 5d9697f..8a7e467 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -2555,7 +2555,7 @@ def _taskLabel(taskInfo): else: kickstart = os.path.basename(stuff[4]) extra = '%s, %s-%s, %s' % (stuff[3], stuff[0], stuff[1], kickstart) - elif method in ('createLiveCD', 'createAppliance', 'createImage', 'createLiveMEDIA'): + elif method in ('createLiveCD', 'createAppliance', 'createImage', 'createLiveMedia'): if taskInfo.has_key('request'): stuff = taskInfo['request'] if method == 'createImage': From 2fdb707a945e724205907e31c0b2b7987f9ddaaf Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:11:32 +0000 Subject: [PATCH 33/70] parse task params for createLiveMedia --- diff --git a/cli/koji b/cli/koji index 5479df4..6b23f0a 100755 --- a/cli/koji +++ b/cli/koji @@ -4329,7 +4329,7 @@ def _do_parseTaskParams(session, method, task_id): if len(params) > 2: _handleOpts(lines, params[2]) elif method in ('createLiveCD', 'createAppliance', 'createLiveMedia'): - argnames = ['Name', 'Version', 'Release', 'Arch', 'Target Info', 'Build Tag', 'Repo', 'Kickstart File'] + argnames = ['Name', 'Version', 'Release', 'Arch', 'Target Info', 'Build Tag', 'Repo', 'Kickstart File:'] for n, v in zip(argnames, params): lines.append("%s: %s" % (n, v)) if len(params) > 8: From 756c74e0567eb3e46e4889eea115bd24a733ba47 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:11:32 +0000 Subject: [PATCH 34/70] use correct livemedia isoname throughout --- diff --git a/builder/kojid b/builder/kojid index fc403c3..160e46c 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3031,9 +3031,9 @@ class LiveMediaTask(ImageTask): '--iso-only', ]) - img_name='%s-%s-%s-%s.iso' % (name, arch, version, release) + isoname='%s-%s-%s-%s.iso' % (name, arch, version, release) title = self.opts.get('title', name) - cmd.extend(['--iso-name', img_name, + cmd.extend(['--iso-name', isoname, '--releasever', version, '--title', title, ]) @@ -3082,15 +3082,12 @@ class LiveMediaTask(ImageTask): raise koji.LiveMediaError, 'could not find iso file in chroot' isosrc = os.path.join(rootresultsdir, isofile) - # copy the iso out of the chroot. If we were given an isoname, - # this is where the renaming happens. - self.logger.debug('uploading image: %s' % isosrc) - isoname = '%s-%s-%s.iso' % (name, version, release) # Generate the file manifest of the image, upload the results manifest = os.path.join(broot.resultdir(), 'manifest.log') self.genISOManifest(isosrc, manifest) self.uploadFile(manifest) + self.logger.debug('uploading image: %s' % isosrc) self.uploadFile(isosrc, remoteName=isoname) imgdata = {'arch': arch, From d4d92ce760f9c2adaff972948b28e93008b60e56 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:12:10 +0000 Subject: [PATCH 35/70] inject install tree url for livemedia spins --- diff --git a/builder/kojid b/builder/kojid index 160e46c..753b270 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2613,7 +2613,7 @@ class ImageTask(BaseTaskHandler): self.ks.handler.repo.repoList.append(repo_class(baseurl=baseurl, name='koji-%s-%i' % (target_info['build_tag_name'], repo_info['id']))) #inject url if provided if opts.get('install_tree_url'): - self.ks.handler.url(url=opts['install_tree_url']) + self.ks.handler.url.url = opts['install_tree_url'] # Write out the new ks file. Note that things may not be in the same # order and comments in the original ks file may be lost. diff --git a/cli/koji b/cli/koji index 6b23f0a..c9b5c74 100755 --- a/cli/koji +++ b/cli/koji @@ -5248,6 +5248,8 @@ def handle_spin_livemedia(options, session, args): help=_("Run the livemedia creation task at a lower priority")) parser.add_option("--ksurl", metavar="SCMURL", help=_("The URL to the SCM containing the kickstart file")) + parser.add_option("--install-tree-url", metavar="URL", + help=_("Provide the URL for the install tree")) parser.add_option("--ksversion", metavar="VERSION", help=_("The syntax version used in the kickstart file")) parser.add_option("--scratch", action="store_true", @@ -5641,7 +5643,7 @@ def _build_image(options, task_opts, session, args, img_type): passthru_opts = [ 'isoname', 'ksurl', 'ksversion', 'scratch', 'repo', 'release', 'skip_tag', 'vmem', 'vcpu', 'format', 'specfile', - 'title', + 'title', 'install_tree_url', ] hub_opts = {} for opt in passthru_opts: From 30c8a9a1dd74d14d01af298195e1445d2ea86875 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:12:10 +0000 Subject: [PATCH 36/70] no rpm list for livemedia images until we can get the data from lmc --- diff --git a/builder/kojid b/builder/kojid index 753b270..199150f 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3102,10 +3102,12 @@ class LiveMediaTask(ImageTask): 'release': release } if not opts.get('scratch'): - hdrlist = self.getImagePackages(os.path.join(broot.rootdir(), - cachedir[1:])) - imgdata ['rpmlist'] = hdrlist - broot.markExternalRPMs(hdrlist) + # TODO - generate list of rpms in image + # (getImagePackages doesn't work here) + #hdrlist = self.getImagePackages(os.path.join(broot.rootdir(), + # cachedir[1:])) + imgdata ['rpmlist'] = [] + #broot.markExternalRPMs(hdrlist) broot.expire() return imgdata From 88166fa73ea9cf282baedc84da934c733fb0bfdd Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:12:10 +0000 Subject: [PATCH 37/70] apply fix for setting kickstart url to livemedia as well see: 86ca706 and e7abc97 --- diff --git a/builder/kojid b/builder/kojid index 199150f..8cb7ae3 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2613,7 +2613,7 @@ class ImageTask(BaseTaskHandler): self.ks.handler.repo.repoList.append(repo_class(baseurl=baseurl, name='koji-%s-%i' % (target_info['build_tag_name'], repo_info['id']))) #inject url if provided if opts.get('install_tree_url'): - self.ks.handler.url.url = opts['install_tree_url'] + ks.handler.url(url=opts['install_tree_url']) # Write out the new ks file. Note that things may not be in the same # order and comments in the original ks file may be lost. From 4e40ebe919d79f674daa1ba052eaee14dcd308ca Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:12:11 +0000 Subject: [PATCH 38/70] remove stray colon in option display --- diff --git a/cli/koji b/cli/koji index c9b5c74..3e0853a 100755 --- a/cli/koji +++ b/cli/koji @@ -4329,7 +4329,7 @@ def _do_parseTaskParams(session, method, task_id): if len(params) > 2: _handleOpts(lines, params[2]) elif method in ('createLiveCD', 'createAppliance', 'createLiveMedia'): - argnames = ['Name', 'Version', 'Release', 'Arch', 'Target Info', 'Build Tag', 'Repo', 'Kickstart File:'] + argnames = ['Name', 'Version', 'Release', 'Arch', 'Target Info', 'Build Tag', 'Repo', 'Kickstart File'] for n, v in zip(argnames, params): lines.append("%s: %s" % (n, v)) if len(params) > 8: From 76347684a24987c76808a35f508c6177fcc0a7f7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:12:11 +0000 Subject: [PATCH 39/70] define bld_info earlier in livemedia/livecd/appliance handlers --- diff --git a/builder/kojid b/builder/kojid index 8cb7ae3..9e58da3 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2394,11 +2394,11 @@ class BuildLiveMediaTask(BuildImageTask): raise koji.PreBuildError, 'Live Media functions not available' # build the image + bld_info = None try: release = opts.get('release') if not release: release = self.getRelease(name, version) - bld_info = None if not opts.get('scratch'): bld_info = self.initImageBuild(name, version, release, target_info, opts) From 521e950b2dc7831e1a4b3a32638b69590a501736 Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Jul 20 2016 20:12:11 +0000 Subject: [PATCH 40/70] add missing self. in ks handling Signed-off-by: Dennis Gilmore --- diff --git a/builder/kojid b/builder/kojid index 9e58da3..e36b0a4 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2613,7 +2613,7 @@ class ImageTask(BaseTaskHandler): self.ks.handler.repo.repoList.append(repo_class(baseurl=baseurl, name='koji-%s-%i' % (target_info['build_tag_name'], repo_info['id']))) #inject url if provided if opts.get('install_tree_url'): - ks.handler.url(url=opts['install_tree_url']) + self.ks.handler.url(url=opts['install_tree_url']) # Write out the new ks file. Note that things may not be in the same # order and comments in the original ks file may be lost. From 714b89dd6481e0a245fcf24a29f5ee27ff3515d8 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 20 2016 20:12:11 +0000 Subject: [PATCH 41/70] s/LiveMEDIA/LiveMedia --- diff --git a/builder/kojid b/builder/kojid index e36b0a4..8cb9bb4 100755 --- a/builder/kojid +++ b/builder/kojid @@ -60,7 +60,7 @@ from optparse import OptionParser, SUPPRESS_HELP from StringIO import StringIO from yum import repoMDObject -#imports for LiveCD, LiveMEDIA, and Appliance handler +#imports for LiveCD, LiveMedia, and Appliance handler image_enabled = False try: import pykickstart.parser as ksparser diff --git a/cli/koji b/cli/koji index 3e0853a..0abe244 100755 --- a/cli/koji +++ b/cli/koji @@ -5253,10 +5253,10 @@ def handle_spin_livemedia(options, session, args): parser.add_option("--ksversion", metavar="VERSION", help=_("The syntax version used in the kickstart file")) parser.add_option("--scratch", action="store_true", - help=_("Create a scratch LiveMEDIA image")) + help=_("Create a scratch LiveMedia image")) parser.add_option("--repo", action="append", help=_("Specify a repo that will override the repo used to install " + - "RPMs in the LiveMEDIA. May be used multiple times. The " + + "RPMs in the LiveMedia. May be used multiple times. The " + "build tag repo associated with the target is the default.")) parser.add_option("--release", help=_("Forcibly set the release field")) parser.add_option("--title", help=_("Set the image title (defaults to )")) diff --git a/koji/__init__.py b/koji/__init__.py index 8a7e467..b453017 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -338,7 +338,7 @@ class ConfigurationError(GenericError): faultCode = 1021 class LiveMediaError(GenericError): - """Raised when LiveMEDIA Image creation fails""" + """Raised when LiveMedia Image creation fails""" faultCode = 1022 class MultiCallInProgress(object): From ff9b13c3e981490ce243095144dc0a41d58ca9b1 Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Jul 20 2016 20:12:11 +0000 Subject: [PATCH 42/70] lmc: add --project to livemedia-creator add --project argument to livemedia-creator take name and replace - with a space. Signed-off-by: Dennis Gilmore --- diff --git a/builder/kojid b/builder/kojid index 8cb9bb4..b1c3d10 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3019,7 +3019,8 @@ class LiveMediaTask(ImageTask): '--ks', kskoji, '--logfile', livemedia_log, '--no-virt', - '--resultdir', resultdir, + '--resultdir', resultdir, + '--project', '"%s"' % name.replace("-", " "), #'--tmp', '/tmp' ] From ad0dfb9b440c5616da1e876d6645bdbc2bfec1b6 Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Jul 20 2016 20:12:12 +0000 Subject: [PATCH 43/70] --project does not like options with spaces, just go with name Signed-off-by: Dennis Gilmore --- diff --git a/builder/kojid b/builder/kojid index b1c3d10..41969c3 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3020,7 +3020,7 @@ class LiveMediaTask(ImageTask): '--logfile', livemedia_log, '--no-virt', '--resultdir', resultdir, - '--project', '"%s"' % name.replace("-", " "), + '--project', name, #'--tmp', '/tmp' ] From 29ef16bdc9d8fc68e27d6188ba416753ceff68bc Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Jul 20 2016 20:12:12 +0000 Subject: [PATCH 44/70] LMC: change the name of the andconda log files anaconda and lmc both have a program.log so we are losing one. This patch adds a anaconda- prefix to the logfile for files in the anaconda directory so that we get all files and know which files are from lmc and which are from anaconda. Signed-off-by: Dennis Gilmore --- diff --git a/builder/kojid b/builder/kojid index 41969c3..ad0b198 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3061,6 +3061,11 @@ class LiveMediaTask(ImageTask): filepath = os.path.join(logdir, filename) if os.stat(filepath).st_size == 0: continue + # avoid file duplication between directories by prefixing anaconda logs + if logdir.endswith('anaconda'): + self.uploadFile(os.path.join(filepath), remoteName='anaconda-%s' % filename) + continue + self.uploadFile(os.path.join(filepath)) if rv: From 5c190dafa7714c77e65ae7cd3b49768c33f17598 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:12 +0000 Subject: [PATCH 45/70] implement CLI for signed-repos --- diff --git a/cli/koji b/cli/koji index 0abe244..30f319c 100755 --- a/cli/koji +++ b/cli/koji @@ -6777,6 +6777,45 @@ def handle_regen_repo(options, session, args): session.logout() return watch_tasks(session, [task_id], quiet=options.quiet) +def handle_signed_repo(options, session, args): + """create a yum repo of GPG signed RPMs""" + usage = _("usage: %prog signed-repo [options] tag keyID [keyID...]") + usage += _("\n(Specify the --help option for a list of other options)") + parser = OptionParser(usage=usage) + parser.add_option("--arch", action='append', default=[], + help=_("Indicate an architecture to consider. The default is all architectures associated with the given tag. This option may be specified multiple times.")) + parser.add_option('--multilib', action='store_true', default=False, + help=_('Include multilib packages in the repository')) + parser.add_option("--noinherit", action='store_true', default=False, + help=_('Do not consider tag inheritance')) + parser.add_option("--nowait", action='store_true', default=False, + help=_('Do not wait for the task to complete')) + task_opts, args = parser.parse_args(args) + if len(args) < 2: + parser.error(_('You must provide a tag and 1 or more GPG key IDs')) + activate_session(session) + tag = args[0] + keys = args[1:] + taginfo = session.getTag(tag) + if not taginfo: + parser.error(_('unknown tag %s' % tag)) + if len(task_opts.arch) == 0: + task_opts.arch = taginfo['arches'] + if task_opts.arch == None: + parser.error(_('No arches given and no arches associated with tag')) + else: + for a in task_opts.arch: + if a not in taginfo['arches']: + print _('Warning: %s is not in the list of tag arches' % a) + task_id = session.signedRepo(tag, keys, **task_opts) + print "Creating signed repo for tag " + tag + if _running_in_bg() or task_opts.nowait: + return + else: + session.logout() + return watch_tasks(session, [task_id], quiet=options.quiet) + + def anon_handle_search(options, session, args): "[search] Search the system" usage = _("usage: %prog search [options] search_type pattern") From 6ed109f2b8bed6a18dbb7ef2aaab8389ecc267ff Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:12 +0000 Subject: [PATCH 46/70] initial hub implementation for signed-repos --- diff --git a/cli/koji b/cli/koji index 30f319c..835fdf5 100755 --- a/cli/koji +++ b/cli/koji @@ -6788,6 +6788,10 @@ def handle_signed_repo(options, session, args): help=_('Include multilib packages in the repository')) parser.add_option("--noinherit", action='store_true', default=False, help=_('Do not consider tag inheritance')) + # TODO: accept comps + # TODO: accept events + # TODO: sources or no? + # TODO: latest? parser.add_option("--nowait", action='store_true', default=False, help=_('Do not wait for the task to complete')) task_opts, args = parser.parse_args(args) @@ -6805,9 +6809,14 @@ def handle_signed_repo(options, session, args): parser.error(_('No arches given and no arches associated with tag')) else: for a in task_opts.arch: - if a not in taginfo['arches']: + if not taginfo['arches'] or a not in taginfo['arches']: print _('Warning: %s is not in the list of tag arches' % a) - task_id = session.signedRepo(tag, keys, **task_opts) + opts = { + 'arch': task_opts.arch, + 'multilib': task_opts.multilib, + 'inherit': not task_opts.noinherit + } + task_id = session.signedRepo(tag, keys, **opts) print "Creating signed repo for tag " + tag if _running_in_bg() or task_opts.nowait: return diff --git a/hub/kojihub.py b/hub/kojihub.py index 64ef1fd..c2ac76e 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2334,6 +2334,79 @@ def _write_maven_repo_metadata(destdir, artifacts): mdfile.close() _generate_maven_metadata(destdir) +def signed_repo_init(tag, keys, task_opts): + """Create a new repo entry in the INIT state, return full repo data""" + logger = logging.getLogger("koji.hub.signed_repo_init") + state = koji.REPO_INIT + tinfo = get_tag(tag, strict=True) + koji.plugin.run_callbacks('preRepoInit', tag=tinfo, keys=keys, repo_id=None) + tag_id = tinfo['id'] + repo_arches = task_opts['arch'] + arches = set([]) + for arch in repo_arches: + arches.add(koji.canonArch(arch)) + repo_id = _singleValue("SELECT nextval('repo_id_seq')") + event_id = _singleValue("SELECT get_event()") + insert = InsertProcessor('repo') + insert.set(id=repo_id, create_event=event_id, tag_id=tag_id, state=state) + insert.execute() + # Need to pass event_id because even though this is a single transaction, + # it is possible to see the results of other committed transactions + rpms, builds = readTaggedRPMS(tag_id, event=event_id, + inherit=task_opts['inherit'], rpmsigs=True) + repodir = koji.pathinfo.signedrepo(tag, str(repo_id)) + os.makedirs(repodir) # should not already exist + + #get build dirs + relpathinfo = koji.PathInfo(topdir='toplink') + builddirs = {} + for build in builds: + relpath = relpathinfo.build(build) + builddirs[build['id']] = relpath.lstrip('/') + #generate pkglist files + pkglist = {} + for repoarch in arches: + archdir = os.path.join(repodir, repoarch) + koji.ensuredir(archdir) + # Make a symlink to our topdir + top_relpath = koji.util.relpath(koji.pathinfo.topdir, archdir) + top_link = os.path.join(archdir, 'toplink') + os.symlink(top_relpath, top_link) + pkglist[repoarch] = file(os.path.join(archdir, 'pkglist'), 'w') + #NOTE - rpms is now an iterator + preferred = {} + for rpminfo in rpms: + if rpminfo['sigkey'] == '': + # skip, this is the unsigned rpminfo + continue + if rpminfo['sigkey'] not in keys: + # skip, not a key we are looking for + continue + arch = koji.canonArch(rpminfo['arch']) + if arch not in arches and arch != 'noarch': + # not an architecture we care about + continue + idx = keys.index(rpminfo['sigkey']) + if preferred.has_key(rpminfo['id']): + if keys.index(preferred[rpminfo['id']]['sigkey']) <= idx: + # key for this is not as preferable as what we have seen before + continue + preferred[rpminfo['id']] = rpminfo + for rpminfo in preferred.values(): + relpath = "%s/%s\n" % (builddirs[rpminfo['build_id']], + relpathinfo.signed(rpminfo, rpminfo['sigkey'])) + if rpminfo['arch'] == 'noarch': + for repoarch in arches: + pkglist[repoarch].write(relpath) + else: + pkglist[rpminfo['arch']].write(relpath) + for repoarch in arches: + pkglist[repoarch].close() + koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=event_id, + repo_id=repo_id) + return [repo_id, event_id] + + def repo_set_state(repo_id, state, check=True): """Set repo state""" if check: @@ -9618,6 +9691,12 @@ class RootExports(object): repoInfo = staticmethod(repo_info) getActiveRepos = staticmethod(get_active_repos) + def signedRepo(self, tag, keys, **task_opts): + """Create a signed-repo task. returns task id""" + context.session.assertPerm('signed-repo') + repo_id = signed_repo_init(tag, keys, task_opts) + return make_task('signedRepo', repo_id, priority=15) + def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" if context.session.hasPerm('regen-repo'): diff --git a/koji/__init__.py b/koji/__init__.py index b453017..b90862c 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1694,6 +1694,10 @@ class PathInfo(object): """Return the directory where a repo belongs""" return self.topdir + ("/repos/%(tag_str)s/%(repo_id)s" % locals()) + def signedrepo(self, repo_id, tag): + """Return the directory with a signed repo lives""" + return os.path.join(self.topdir, 'repos', 'signed', tag, repo_id) + def repocache(self,tag_str): """Return the directory where a repo belongs""" return self.topdir + ("/repos/%(tag_str)s/cache" % locals()) From 68f1bf9498bbe6a165552ca7a361405e15dd5106 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:12 +0000 Subject: [PATCH 47/70] initial builder implementation for signed-repos --- diff --git a/builder/kojid b/builder/kojid index ad0b198..b7583cb 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4701,14 +4701,17 @@ class CreaterepoTask(BaseTaskHandler): Methods = ['createrepo'] _taskWeight = 1.5 - def handler(self, repo_id, arch, oldrepo): + def getRepoPath(self, repo_id, tag): + return self.pathinfo.repo(repo_id, tag) + + def handler(self, repo_id, arch, oldrepo, do_external): #arch is the arch of the repo, not the task rinfo = self.session.repoInfo(repo_id, strict=True) if rinfo['state'] != koji.REPO_INIT: raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo self.repo_id = rinfo['id'] self.pathinfo = koji.PathInfo(self.options.topdir) - toprepodir = self.pathinfo.repo(repo_id, rinfo['tag_name']) + toprepodir = self.getRepoPath(repo_id, rinfo['tag_name']) self.repodir = '%s/%s' % (toprepodir, arch) if not os.path.isdir(self.repodir): raise koji.GenericError, "Repo directory missing: %s" % self.repodir @@ -4722,7 +4725,7 @@ class CreaterepoTask(BaseTaskHandler): self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo) external_repos = self.session.getExternalRepoList(rinfo['tag_id'], event=rinfo['create_event']) - if external_repos: + if external_repos and do_external: self.merge_repos(external_repos, arch, groupdata) elif pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') @@ -4808,6 +4811,45 @@ class CreaterepoTask(BaseTaskHandler): raise koji.GenericError, 'failed to merge repos: %s' \ % parseStatus(status, ' '.join(cmd)) + +class NewSignedRepoTask(BaseTaskHandler): + Methods = ['signedRepo'] + _taskWeight = 0.1 + + def handler(self, repo_id, tag): + # TODO: remember to use an event here + tinfo = self.session.getTag(tag, strict=True) + kwargs = {} + path = koji.pathinfo.signedrepo(repo_id, tinfo['name']) + if not os.path.isdir(path): + raise koji.GenericError, "Repo directory missing: %s" % path + arches = [] + for fn in os.listdir(path): + if os.path.isfile("%s/%s/pkglist" % (path, fn)): + arches.append(fn) + subtasks = {} + for arch in arches: + arglist = [repo_id, arch, None, False] # no old repo or external + subtasks[arch] = self.session.host.subtask( + method='createsignedrepo', arglist=arglist, label=arch, + parent=self.id, arch='noarch') + # wait for subtasks to finish + results = self.wait(subtasks.values(), all=True, failany=True) + data = {} + for (arch, task_id) in subtasks.iteritems(): + data[arch] = results[task_id] + self.logger.debug("DEBUG: %r : %r " % (arch, data[arch],)) + self.session.host.repoDone(repo_id, data, expire=True, signed=True) + return repo_id + + +class createSignedRepoTask(CreaterepoTask): + Methods = ['createsignedrepo'] + _taskWeight = 1.5 + + def getRepoPath(self, repo_id, tag): + return self.pathinfo.signedrepo(repo_id, tag) + class WaitrepoTask(BaseTaskHandler): Methods = ['waitrepo'] diff --git a/hub/kojihub.py b/hub/kojihub.py index c2ac76e..2c0d827 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2354,14 +2354,14 @@ def signed_repo_init(tag, keys, task_opts): # it is possible to see the results of other committed transactions rpms, builds = readTaggedRPMS(tag_id, event=event_id, inherit=task_opts['inherit'], rpmsigs=True) - repodir = koji.pathinfo.signedrepo(tag, str(repo_id)) + repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) os.makedirs(repodir) # should not already exist #get build dirs - relpathinfo = koji.PathInfo(topdir='toplink') + pathinfo = koji.PathInfo() builddirs = {} for build in builds: - relpath = relpathinfo.build(build) + relpath = pathinfo.build(build) builddirs[build['id']] = relpath.lstrip('/') #generate pkglist files pkglist = {} @@ -2393,18 +2393,26 @@ def signed_repo_init(tag, keys, task_opts): continue preferred[rpminfo['id']] = rpminfo for rpminfo in preferred.values(): - relpath = "%s/%s\n" % (builddirs[rpminfo['build_id']], - relpathinfo.signed(rpminfo, rpminfo['sigkey'])) - if rpminfo['arch'] == 'noarch': + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + pathinfo.signed(rpminfo, rpminfo['sigkey'])) + repopath = '/' + pkgpath + repopath = repopath.replace(koji.pathinfo.topdir, 'toplink') + '\n' + arch = koji.canonArch(rpminfo['arch']) + if arch == 'noarch': for repoarch in arches: - pkglist[repoarch].write(relpath) + pkglist[repoarch].write(repopath) + archdir = os.path.join(repodir, repoarch) + os.link(pkgpath, + os.path.join(archdir, os.path.basename(pkgpath))) else: - pkglist[rpminfo['arch']].write(relpath) + pkglist[arch].write(repopath) + dest = os.path.join(repodir, arch, os.path.basename(pkgpath)) + os.link(pkgpath, dest) for repoarch in arches: pkglist[repoarch].close() koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=event_id, repo_id=repo_id) - return [repo_id, event_id] + return repo_id, event_id def repo_set_state(repo_id, state, check=True): @@ -9694,8 +9702,8 @@ class RootExports(object): def signedRepo(self, tag, keys, **task_opts): """Create a signed-repo task. returns task id""" context.session.assertPerm('signed-repo') - repo_id = signed_repo_init(tag, keys, task_opts) - return make_task('signedRepo', repo_id, priority=15) + repo_id, event_id = signed_repo_init(tag, keys, task_opts) + return make_task('signedRepo', [repo_id, tag], priority=15) def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" @@ -11775,7 +11783,7 @@ class HostExports(object): else: os.link(filepath, dst) - def repoDone(self, repo_id, data, expire=False): + def repoDone(self, repo_id, data, expire=False, signed=False): """Move repo data into place, mark as ready, and expire earlier repos repo_id: the id of the repo @@ -11790,7 +11798,10 @@ class HostExports(object): koji.plugin.run_callbacks('preRepoDone', repo=rinfo, data=data, expire=expire) if rinfo['state'] != koji.REPO_INIT: raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo - repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) + if signed: + repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) + else: + repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) workdir = koji.pathinfo.work() for arch, (uploadpath, files) in data.iteritems(): archdir = "%s/%s" % (repodir, arch) diff --git a/koji/__init__.py b/koji/__init__.py index b90862c..b705461 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1696,7 +1696,7 @@ class PathInfo(object): def signedrepo(self, repo_id, tag): """Return the directory with a signed repo lives""" - return os.path.join(self.topdir, 'repos', 'signed', tag, repo_id) + return os.path.join(self.topdir, 'repos', 'signed', tag, str(repo_id)) def repocache(self,tag_str): """Return the directory where a repo belongs""" From 06e1e98b508b785837ab92a686c72bb0fea5b99a Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:12 +0000 Subject: [PATCH 48/70] signed-repo kojiweb tweaks --- diff --git a/builder/kojid b/builder/kojid index b7583cb..145ae0c 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4816,7 +4816,7 @@ class NewSignedRepoTask(BaseTaskHandler): Methods = ['signedRepo'] _taskWeight = 0.1 - def handler(self, repo_id, tag): + def handler(self, tag, repo_id): # TODO: remember to use an event here tinfo = self.session.getTag(tag, strict=True) kwargs = {} diff --git a/hub/kojihub.py b/hub/kojihub.py index 2c0d827..03d8ec9 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9703,7 +9703,7 @@ class RootExports(object): """Create a signed-repo task. returns task id""" context.session.assertPerm('signed-repo') repo_id, event_id = signed_repo_init(tag, keys, task_opts) - return make_task('signedRepo', [repo_id, tag], priority=15) + return make_task('signedRepo', [tag, repo_id], priority=15) def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" diff --git a/koji/__init__.py b/koji/__init__.py index b705461..73faefb 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -2523,7 +2523,7 @@ def _taskLabel(taskInfo): if taskInfo.has_key('request'): build = taskInfo['request'][1] extra = buildLabel(build) - elif method == 'newRepo': + elif method in ('newRepo', 'signedRepo'): if taskInfo.has_key('request'): extra = str(taskInfo['request'][0]) elif method in ('tagBuild', 'tagNotification'): @@ -2534,7 +2534,7 @@ def _taskLabel(taskInfo): if taskInfo.has_key('request'): tagInfo = taskInfo['request'][0] extra = tagInfo['name'] - elif method == 'createrepo': + elif method in ('createrepo', 'createsignedrepo'): if taskInfo.has_key('request'): arch = taskInfo['request'][1] extra = arch diff --git a/www/conf/web.conf b/www/conf/web.conf index 4069258..8f1dbeb 100644 --- a/www/conf/web.conf +++ b/www/conf/web.conf @@ -33,3 +33,8 @@ LiteralFooter = True # to hide from tasks listed on the front page. You might want to, for instance, # hide the activity of an account used for continuous integration. #HiddenUsers = 5372 1234 + +# Uncommenting this will show python tracebacks in the webUI, but they are the +# same as what you will see in apache's error_log. +# Not for production use +#PythonDebug = True diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 420c606..1ab7441 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -436,6 +436,8 @@ _TASKS = ['build', 'tagBuild', 'newRepo', 'createrepo', + 'signedRepo', + 'createsignedrepo', 'buildNotification', 'tagNotification', 'dependantTask', @@ -449,9 +451,9 @@ _TASKS = ['build', 'livemedia', 'createLiveMedia'] # Tasks that can exist without a parent -_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'maven', 'chainmaven', 'wrapperRPM', 'winbuild', 'newRepo', 'tagBuild', 'tagNotification', 'waitrepo', 'livecd', 'appliance', 'image', 'livemedia'] +_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'maven', 'chainmaven', 'wrapperRPM', 'winbuild', 'newRepo', 'signedRepo', 'tagBuild', 'tagNotification', 'waitrepo', 'livecd', 'appliance', 'image', 'livemedia'] # Tasks that can have children -_PARENT_TASKS = ['build', 'chainbuild', 'maven', 'chainmaven', 'winbuild', 'newRepo', 'wrapperRPM', 'livecd', 'appliance', 'image', 'livemedia'] +_PARENT_TASKS = ['build', 'chainbuild', 'maven', 'chainmaven', 'winbuild', 'newRepo', 'signedRepo', 'wrapperRPM', 'livecd', 'appliance', 'image', 'livemedia'] def tasks(environ, owner=None, state='active', view='tree', method='all', hostID=None, channelID=None, start=None, order='-id'): values = _initValues(environ, 'Tasks', 'tasks') @@ -628,7 +630,7 @@ def taskinfo(environ, taskID): build = server.getBuild(params[1]) values['destTag'] = destTag values['build'] = build - elif task['method'] == 'newRepo': + elif task['method'] in ('newRepo', 'signedRepo'): tag = server.getTag(params[0]) values['tag'] = tag elif task['method'] == 'tagNotification': diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 522b9f7..ca7224c 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -218,23 +218,27 @@ $value #if $len($params) > 2 $printOpts($params[2]) #end if - #elif $task.method == 'newRepo' + #elif $task.method in ('newRepo', 'signedRepo') Tag: $tag.name
- #if $len($params) > 1 - $printOpts($params[1]) + #if $task.method == 'signedRepo' + Repo ID: $params[1]
+ #elif $len($params) > 1 + $printOpts($params[1]) #end if #elif $task.method == 'prepRepo' Tag: $params[0].name - #elif $task.method == 'createrepo' + #elif $task.method in ('createrepo', 'createsignedrepo') Repo ID: $params[0]
Arch: $params[1]
- #set $oldrepo = $params[2] - #if $oldrepo - Old Repo ID: $oldrepo.id
- Old Repo Creation: $koji.formatTimeLong($oldrepo.creation_time)
+ #if $len($params) > 2 + #set $oldrepo = $params[2] + #if $oldrepo + Old Repo ID: $oldrepo.id
+ Old Repo Creation: $koji.formatTimeLong($oldrepo.creation_time)
+ #end if #end if - #if $len($params) > 3 - External Repos: $printValue(None, [ext['external_repo_name'] for ext in $params[3]])
+ #if $len($params) > 3 and $params[3] + External Repos: $printValue(None, [ext['external_repo_name'] for ext in $params[3]])
#end if #elif $task.method == 'dependantTask' Dependant Tasks:
From 62ae30a82f15772486fe6db4ddf460c236b42081 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:12 +0000 Subject: [PATCH 49/70] implement --allow-unsigned and --skip-unsigned --- diff --git a/builder/kojid b/builder/kojid index 145ae0c..db268a1 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4816,10 +4816,9 @@ class NewSignedRepoTask(BaseTaskHandler): Methods = ['signedRepo'] _taskWeight = 0.1 - def handler(self, tag, repo_id): + def handler(self, tag, repo_id, task_opts): # TODO: remember to use an event here tinfo = self.session.getTag(tag, strict=True) - kwargs = {} path = koji.pathinfo.signedrepo(repo_id, tinfo['name']) if not os.path.isdir(path): raise koji.GenericError, "Repo directory missing: %s" % path diff --git a/cli/koji b/cli/koji index 835fdf5..2634d3e 100755 --- a/cli/koji +++ b/cli/koji @@ -6782,8 +6782,12 @@ def handle_signed_repo(options, session, args): usage = _("usage: %prog signed-repo [options] tag keyID [keyID...]") usage += _("\n(Specify the --help option for a list of other options)") parser = OptionParser(usage=usage) + parser.add_option('--allow-unsigned', action='store_true', default=False, + help=_('Use unsigned RPMs if none are available with the right key')) parser.add_option("--arch", action='append', default=[], - help=_("Indicate an architecture to consider. The default is all architectures associated with the given tag. This option may be specified multiple times.")) + help=_("Indicate an architecture to consider. The default is all " + + "architectures associated with the given tag. This option may " + + "be specified multiple times.")) parser.add_option('--multilib', action='store_true', default=False, help=_('Include multilib packages in the repository')) parser.add_option("--noinherit", action='store_true', default=False, @@ -6792,11 +6796,16 @@ def handle_signed_repo(options, session, args): # TODO: accept events # TODO: sources or no? # TODO: latest? + # TODO: delta-rpms ugh parser.add_option("--nowait", action='store_true', default=False, help=_('Do not wait for the task to complete')) + parser.add_option('--skip-unsigned', action='store_true', default=False, + help=_('Skip RPMs not signed with the desired key(s)')) task_opts, args = parser.parse_args(args) if len(args) < 2: parser.error(_('You must provide a tag and 1 or more GPG key IDs')) + if task_opts.allow_unsigned and task_opts.skip_unsigned: + parser.error(_('allow_signed and skip_unsigned are mutually exclusive')) activate_session(session) tag = args[0] keys = args[1:] @@ -6811,10 +6820,17 @@ def handle_signed_repo(options, session, args): for a in task_opts.arch: if not taginfo['arches'] or a not in taginfo['arches']: print _('Warning: %s is not in the list of tag arches' % a) + try: + task_opts.arch.remove('noarch') # handled specifically + task_opts.arch.remove('src') # ditto + except ValueError: + pass opts = { 'arch': task_opts.arch, 'multilib': task_opts.multilib, - 'inherit': not task_opts.noinherit + 'inherit': not task_opts.noinherit, + 'skip': task_opts.skip_unsigned, + 'unsigned': task_opts.allow_unsigned } task_id = session.signedRepo(tag, keys, **opts) print "Creating signed repo for tag " + tag diff --git a/hub/kojihub.py b/hub/kojihub.py index 03d8ec9..1fa0e8e 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2352,8 +2352,15 @@ def signed_repo_init(tag, keys, task_opts): insert.execute() # Need to pass event_id because even though this is a single transaction, # it is possible to see the results of other committed transactions - rpms, builds = readTaggedRPMS(tag_id, event=event_id, + rpm_iter, builds = readTaggedRPMS(tag_id, event=event_id, inherit=task_opts['inherit'], rpmsigs=True) + rpms = list(rpm_iter) + for rpm_copy in list(rpms): + arch = koji.canonArch(rpm_copy['arch']) + if arch not in arches: + # not an architecture we care about + rpms.remove(rpm_copy) + need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) os.makedirs(repodir) # should not already exist @@ -2373,28 +2380,32 @@ def signed_repo_init(tag, keys, task_opts): top_link = os.path.join(archdir, 'toplink') os.symlink(top_relpath, top_link) pkglist[repoarch] = file(os.path.join(archdir, 'pkglist'), 'w') - #NOTE - rpms is now an iterator preferred = {} + if task_opts['unsigned']: + keys.append('') # make unsigned rpms the least preferred for rpminfo in rpms: - if rpminfo['sigkey'] == '': + if rpminfo['sigkey'] == '' and not task_opts['unsigned']: # skip, this is the unsigned rpminfo continue if rpminfo['sigkey'] not in keys: # skip, not a key we are looking for continue - arch = koji.canonArch(rpminfo['arch']) - if arch not in arches and arch != 'noarch': - # not an architecture we care about - continue idx = keys.index(rpminfo['sigkey']) if preferred.has_key(rpminfo['id']): if keys.index(preferred[rpminfo['id']]['sigkey']) <= idx: # key for this is not as preferable as what we have seen before continue preferred[rpminfo['id']] = rpminfo + seen = set() for rpminfo in preferred.values(): - pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], - pathinfo.signed(rpminfo, rpminfo['sigkey'])) + if rpminfo['sigkey'] == '': + # we're taking an unsigned rpm (--allow-unsigned) + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + pathinfo.rpm(rpminfo)) + else: + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + pathinfo.signed(rpminfo, rpminfo['sigkey'])) + seen.add(os.path.basename(pkgpath)) repopath = '/' + pkgpath repopath = repopath.replace(koji.pathinfo.topdir, 'toplink') + '\n' arch = koji.canonArch(rpminfo['arch']) @@ -2410,6 +2421,12 @@ def signed_repo_init(tag, keys, task_opts): os.link(pkgpath, dest) for repoarch in arches: pkglist[repoarch].close() + if not task_opts['skip']: + missing = list(need - seen) + if len(missing) != 0: + missing.sort() + raise koji.GenericError('Unsigned packages found: ' + + '\n'.join(missing)) koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=event_id, repo_id=repo_id) return repo_id, event_id @@ -9703,7 +9720,7 @@ class RootExports(object): """Create a signed-repo task. returns task id""" context.session.assertPerm('signed-repo') repo_id, event_id = signed_repo_init(tag, keys, task_opts) - return make_task('signedRepo', [tag, repo_id], priority=15) + return make_task('signedRepo', [tag, repo_id, task_opts], priority=15) def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index ca7224c..3effea4 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -222,6 +222,7 @@ $value Tag: $tag.name
#if $task.method == 'signedRepo' Repo ID: $params[1]
+ $printOpts($params[2]) #elif $len($params) > 1 $printOpts($params[1]) #end if From f5f3e72dede4794c33a66bd7e65c1cc5ad84ae53 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:12 +0000 Subject: [PATCH 50/70] implement --event --- diff --git a/builder/kojid b/builder/kojid index db268a1..9a252bf 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4817,8 +4817,7 @@ class NewSignedRepoTask(BaseTaskHandler): _taskWeight = 0.1 def handler(self, tag, repo_id, task_opts): - # TODO: remember to use an event here - tinfo = self.session.getTag(tag, strict=True) + tinfo = self.session.getTag(tag, strict=True, event=task_opts['event']) path = koji.pathinfo.signedrepo(repo_id, tinfo['name']) if not os.path.isdir(path): raise koji.GenericError, "Repo directory missing: %s" % path @@ -4839,7 +4838,7 @@ class NewSignedRepoTask(BaseTaskHandler): data[arch] = results[task_id] self.logger.debug("DEBUG: %r : %r " % (arch, data[arch],)) self.session.host.repoDone(repo_id, data, expire=True, signed=True) - return repo_id + return repo_id, task_opts['event'] class createSignedRepoTask(CreaterepoTask): diff --git a/cli/koji b/cli/koji index 2634d3e..408861f 100755 --- a/cli/koji +++ b/cli/koji @@ -6788,13 +6788,13 @@ def handle_signed_repo(options, session, args): help=_("Indicate an architecture to consider. The default is all " + "architectures associated with the given tag. This option may " + "be specified multiple times.")) + parser.add_option('--event', type='int', + help=_('create a signed repository based on a Brew event')) parser.add_option('--multilib', action='store_true', default=False, help=_('Include multilib packages in the repository')) parser.add_option("--noinherit", action='store_true', default=False, help=_('Do not consider tag inheritance')) # TODO: accept comps - # TODO: accept events - # TODO: sources or no? # TODO: latest? # TODO: delta-rpms ugh parser.add_option("--nowait", action='store_true', default=False, @@ -6827,6 +6827,7 @@ def handle_signed_repo(options, session, args): pass opts = { 'arch': task_opts.arch, + 'event': task_opts.event, 'multilib': task_opts.multilib, 'inherit': not task_opts.noinherit, 'skip': task_opts.skip_unsigned, diff --git a/hub/kojihub.py b/hub/kojihub.py index 1fa0e8e..2fdf464 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2346,13 +2346,15 @@ def signed_repo_init(tag, keys, task_opts): for arch in repo_arches: arches.add(koji.canonArch(arch)) repo_id = _singleValue("SELECT nextval('repo_id_seq')") - event_id = _singleValue("SELECT get_event()") + if not task_opts['event']: + task_opts['event'] = _singleValue("SELECT get_event()") insert = InsertProcessor('repo') - insert.set(id=repo_id, create_event=event_id, tag_id=tag_id, state=state) + insert.set(id=repo_id, create_event=task_opts['event'], tag_id=tag_id, + state=state) insert.execute() # Need to pass event_id because even though this is a single transaction, # it is possible to see the results of other committed transactions - rpm_iter, builds = readTaggedRPMS(tag_id, event=event_id, + rpm_iter, builds = readTaggedRPMS(tag_id, event=task_opts['event'], inherit=task_opts['inherit'], rpmsigs=True) rpms = list(rpm_iter) for rpm_copy in list(rpms): @@ -2427,9 +2429,9 @@ def signed_repo_init(tag, keys, task_opts): missing.sort() raise koji.GenericError('Unsigned packages found: ' + '\n'.join(missing)) - koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=event_id, - repo_id=repo_id) - return repo_id, event_id + koji.plugin.run_callbacks('postRepoInit', tag=tinfo, + event=task_opts['event'], repo_id=repo_id) + return repo_id, task_opts['event'] def repo_set_state(repo_id, state, check=True): From 8e28929eddfaad2cbe1a2f95e4089f79c219b135 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:13 +0000 Subject: [PATCH 51/70] kojira policy for signed repos --- diff --git a/util/kojira b/util/kojira index c50228a..e07f0b8 100755 --- a/util/kojira +++ b/util/kojira @@ -333,40 +333,51 @@ class RepoManager(object): finally: session.logout() - def pruneLocalRepos(self): + def pruneLocalRepos(self, topdir, timername): """Scan filesystem for repos and remove any deleted ones Also, warn about any oddities""" if self.delete_pids: #skip return - self.logger.debug("Scanning filesystem for repos") - topdir = "%s/repos" % pathinfo.topdir + self.logger.debug("Scanning %s for repos" % topdir) + self.logger.debug('max age allowed: %s seconds (from %s)' % + (getattr(self.options, timername), timername)) for tag in os.listdir(topdir): tagdir = "%s/%s" % (topdir, tag) if not os.path.isdir(tagdir): + self.logger.debug("%s is not a directory, skipping" % tagdir) continue for repo_id in os.listdir(tagdir): try: repo_id = int(repo_id) except ValueError: + self.logger.debug("%s not an int, skipping" % tagdir) + # This condition is how signed repos are not removed by + # the first call to this method. Although, if someone has + # tags that are just integers, that could be a problem. continue repodir = "%s/%s" % (tagdir, repo_id) if not os.path.isdir(repodir): + self.logger.debug("%s not a directory, skipping" % repodir) continue if self.repos.has_key(repo_id): #we're already managing it, no need to deal with it here + self.logger.debug("seen %s already, skipping" % repodir) continue try: dir_ts = os.stat(repodir).st_mtime except OSError: #just in case something deletes the repo out from under us + self.logger.debug("%s deleted already?!" % repodir) continue rinfo = self.session.repoInfo(repo_id) if rinfo is None: if not self.options.ignore_stray_repos: age = time.time() - dir_ts - if age > self.options.deleted_repo_lifetime: + self.logger.debug("did not expect %s; age: %s" % + (repodir, age)) + if age > getattr(self.options, timername): self.logger.info("Removing unexpected directory (no such repo): %s" % repodir) self.rmtree(repodir) continue @@ -375,11 +386,11 @@ class RepoManager(object): continue if rinfo['state'] in (koji.REPO_DELETED, koji.REPO_PROBLEM): age = time.time() - max(rinfo['create_ts'], dir_ts) - if age > self.options.deleted_repo_lifetime: + self.logger.debug("potential removal candidate: %s; age: %s" % (repodir, age)) + if age > getattr(self.options, timername): #XXX should really be called expired_repo_lifetime logger.info("Removing stray repo (state=%s): %s" % (koji.REPO_STATES[rinfo['state']], repodir)) self.rmtree(repodir) - pass def tagUseStats(self, tag_id): stats = self.tag_use_stats.get(tag_id) @@ -632,7 +643,9 @@ def main(options, session): repomgr.updateRepos() repomgr.checkQueue() repomgr.printState() - repomgr.pruneLocalRepos() + repodir = "%s/repos" % pathinfo.topdir + repomgr.pruneLocalRepos(repodir, 'deleted_repo_lifetime') + repomgr.pruneLocalRepos(repodir + '/signed', 'signed_repo_lifetime') if not curr_chk_thread.isAlive(): logger.error("Currency checker thread died. Restarting it.") curr_chk_thread = start_currency_checker(session, repomgr) @@ -726,6 +739,7 @@ def get_options(): 'delete_batch_size' : 3, 'deleted_repo_lifetime': 7*24*3600, #XXX should really be called expired_repo_lifetime + 'signed_repo_lifetime': 7*24*3600, 'sleeptime' : 15, 'cert': '/etc/kojira/client.crt', 'ca': '', # FIXME: unused, remove in next major release @@ -734,7 +748,8 @@ def get_options(): if config.has_section(section): int_opts = ('deleted_repo_lifetime', 'max_repo_tasks', 'repo_tasks_limit', 'retry_interval', 'max_retries', 'offline_retry_interval', - 'max_delete_processes', 'max_repo_tasks_maven', 'delete_batch_size', ) + 'max_delete_processes', 'max_repo_tasks_maven', + 'delete_batch_size', 'signed_repo_lifetime') str_opts = ('topdir', 'server', 'user', 'password', 'logfile', 'principal', 'keytab', 'krbservice', 'cert', 'ca', 'serverca', 'debuginfo_tags', 'source_tags') # FIXME: remove ca here bool_opts = ('with_src','verbose','debug','ignore_stray_repos', 'offline_retry', 'krb_rdns') diff --git a/util/kojira.conf b/util/kojira.conf index def5370..1d361b3 100644 --- a/util/kojira.conf +++ b/util/kojira.conf @@ -39,3 +39,12 @@ with_src=no ;certificate of the CA that issued the HTTP server certificate ;serverca = /etc/kojira/serverca.crt + +;how soon (in seconds) to clean up expired repositories. 1 week default +;deleted_repo_lifetime = 604800 + +;how soon (in seconds) to clean up signed repositories. 1 week default here too +;signed_repo_lifetime = 604800 + +;turn on debugging statements in the log +;debug = false From a9867c570e79024cb34a7add7b64219be6c98473 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:13 +0000 Subject: [PATCH 52/70] fix createrepo task breakage in webui --- diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 3effea4..b38597e 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -228,19 +228,21 @@ $value #end if #elif $task.method == 'prepRepo' Tag: $params[0].name - #elif $task.method in ('createrepo', 'createsignedrepo') + #elif $task.method == 'createrepo' Repo ID: $params[0]
Arch: $params[1]
- #if $len($params) > 2 - #set $oldrepo = $params[2] - #if $oldrepo - Old Repo ID: $oldrepo.id
- Old Repo Creation: $koji.formatTimeLong($oldrepo.creation_time)
- #end if + #set $oldrepo = $params[2] + #if $oldrepo + Old Repo ID: $oldrepo.id
+ Old Repo Creation: $koji.formatTimeLong($oldrepo.creation_time)
#end if - #if $len($params) > 3 and $params[3] + #if $len($params) > 4 and $params[4] External Repos: $printValue(None, [ext['external_repo_name'] for ext in $params[3]])
#end if + #elif $task.method == 'createsignedrepo' + Repo ID: $params[0]
+ Arch: $params[1]
+ Options: $printMap($params[3], '    ') #elif $task.method == 'dependantTask' Dependant Tasks:
#for $dep in $deps From 95493a4682d84927187854f96475a40f03846e7a Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:13 +0000 Subject: [PATCH 53/70] implement --delta-rpms --- diff --git a/builder/kojid b/builder/kojid index 9a252bf..24f6a77 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4672,8 +4672,9 @@ class NewRepoTask(BaseTaskHandler): else: oldrepo = self.session.getRepo(tinfo['id'], state=koji.REPO_READY) subtasks = {} + opts = {'do_external': True, 'deltas': False} for arch in arches: - arglist = [repo_id, arch, oldrepo] + arglist = [repo_id, arch, oldrepo, opts] subtasks[arch] = self.session.host.subtask(method='createrepo', arglist=arglist, label=arch, @@ -4704,7 +4705,7 @@ class CreaterepoTask(BaseTaskHandler): def getRepoPath(self, repo_id, tag): return self.pathinfo.repo(repo_id, tag) - def handler(self, repo_id, arch, oldrepo, do_external): + def handler(self, repo_id, arch, oldrepo, opts): #arch is the arch of the repo, not the task rinfo = self.session.repoInfo(repo_id, strict=True) if rinfo['state'] != koji.REPO_INIT: @@ -4722,11 +4723,13 @@ class CreaterepoTask(BaseTaskHandler): pkglist = os.path.join(self.repodir, 'pkglist') if os.path.getsize(pkglist) == 0: pkglist = None - self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo) - - external_repos = self.session.getExternalRepoList(rinfo['tag_id'], event=rinfo['create_event']) - if external_repos and do_external: - self.merge_repos(external_repos, arch, groupdata) + self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo, + opts['deltas']) + if opts['do_external']: + external_repos = self.session.getExternalRepoList( + rinfo['tag_id'], event=rinfo['create_event']) + if external_repos: + self.merge_repos(external_repos, arch, groupdata) elif pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") @@ -4737,10 +4740,14 @@ class CreaterepoTask(BaseTaskHandler): for f in os.listdir(self.datadir): files.append(f) self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) - + if opts['deltas']: + ddir = os.path.join(self.outdir, 'drpms') + for f in os.listdir(ddir): + files.append(f) + self.session.uploadWrapper('%s/%s' % (ddir, f), uploadpath, f) return [uploadpath, files] - def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo): + def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, drpms): koji.ensuredir(self.outdir) if self.options.use_createrepo_c: cmd = ['/usr/bin/createrepo_c'] @@ -4752,7 +4759,9 @@ class CreaterepoTask(BaseTaskHandler): if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) #attempt to recycle repodata from last repo - if pkglist and oldrepo and self.options.createrepo_update: + if pkglist and oldrepo and self.options.createrepo_update and not drpms: + # signed repos overload the use of "oldrepo", so the conditional + # explicitly make sure this does not get executed with that on oldpath = self.pathinfo.repo(oldrepo['id'], rinfo['tag_name']) olddatadir = '%s/%s/repodata' % (oldpath, arch) if not os.path.isdir(olddatadir): @@ -4767,6 +4776,11 @@ class CreaterepoTask(BaseTaskHandler): cmd.append('--update') if self.options.createrepo_skip_stat: cmd.append('--skip-stat') + if drpms: + # generate delta-rpms + cmd.append('--deltas') + for repo in oldrepo: + cmd.extend(['--oldpackagedirs', repo]) # note: we can't easily use a cachedir because we do not have write # permission. The good news is that with --update we won't need to # be scanning many rpms. @@ -4826,8 +4840,15 @@ class NewSignedRepoTask(BaseTaskHandler): if os.path.isfile("%s/%s/pkglist" % (path, fn)): arches.append(fn) subtasks = {} + if task_opts['delta']: + make_drpms = True + oldrepo = task_opts['delta'] + else: + make_drpms = False + oldrepo = None for arch in arches: - arglist = [repo_id, arch, None, False] # no old repo or external + opts = {'do_external': False, 'deltas': make_drpms} + arglist = [repo_id, arch, oldrepo, opts] subtasks[arch] = self.session.host.subtask( method='createsignedrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') diff --git a/cli/koji b/cli/koji index 408861f..d843268 100755 --- a/cli/koji +++ b/cli/koji @@ -6788,6 +6788,9 @@ def handle_signed_repo(options, session, args): help=_("Indicate an architecture to consider. The default is all " + "architectures associated with the given tag. This option may " + "be specified multiple times.")) + parser.add_option('--delta-rpms', metavar='PATH',default=[], + action='append', + help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times.')) parser.add_option('--event', type='int', help=_('create a signed repository based on a Brew event')) parser.add_option('--multilib', action='store_true', default=False, @@ -6796,7 +6799,6 @@ def handle_signed_repo(options, session, args): help=_('Do not consider tag inheritance')) # TODO: accept comps # TODO: latest? - # TODO: delta-rpms ugh parser.add_option("--nowait", action='store_true', default=False, help=_('Do not wait for the task to complete')) parser.add_option('--skip-unsigned', action='store_true', default=False, @@ -6828,6 +6830,7 @@ def handle_signed_repo(options, session, args): opts = { 'arch': task_opts.arch, 'event': task_opts.event, + 'delta': task_opts.delta_rpms, 'multilib': task_opts.multilib, 'inherit': not task_opts.noinherit, 'skip': task_opts.skip_unsigned, diff --git a/hub/kojihub.py b/hub/kojihub.py index 2fdf464..48d2753 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -11830,7 +11830,11 @@ class HostExports(object): koji.ensuredir(datadir) for fn in files: src = "%s/%s/%s" % (workdir,uploadpath, fn) - dst = "%s/%s" % (datadir, fn) + if fn.endswith('.drpm'): + koji.ensuredir(os.path.join(archdir, 'drpms')) + dst = "%s/drpms/%s" % (archdir, fn) + else: + dst = "%s/%s" % (datadir, fn) if not os.path.exists(src): raise koji.GenericError, "uploaded file missing: %s" % src os.link(src, dst) From 18f90b75d31351222a0a757ecdc903c554ed8e51 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:13 +0000 Subject: [PATCH 54/70] implement comps --- diff --git a/cli/koji b/cli/koji index d843268..e139fa3 100755 --- a/cli/koji +++ b/cli/koji @@ -6788,6 +6788,7 @@ def handle_signed_repo(options, session, args): help=_("Indicate an architecture to consider. The default is all " + "architectures associated with the given tag. This option may " + "be specified multiple times.")) + parser.add_option('--comps', help='Include a comps file in the repodata') parser.add_option('--delta-rpms', metavar='PATH',default=[], action='append', help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times.')) @@ -6797,7 +6798,6 @@ def handle_signed_repo(options, session, args): help=_('Include multilib packages in the repository')) parser.add_option("--noinherit", action='store_true', default=False, help=_('Do not consider tag inheritance')) - # TODO: accept comps # TODO: latest? parser.add_option("--nowait", action='store_true', default=False, help=_('Do not wait for the task to complete')) @@ -6809,6 +6809,13 @@ def handle_signed_repo(options, session, args): if task_opts.allow_unsigned and task_opts.skip_unsigned: parser.error(_('allow_signed and skip_unsigned are mutually exclusive')) activate_session(session) + if task_opts.comps: + if not os.path.exists(task_opts.comps): + parser.error(_('could not find %s' % task_opts.comps)) + compsdir = _unique_path('cli-signed') + session.uploadWrapper(task_opts.comps, compsdir, + callback=_progress_callback) + task_opts.comps = os.path.join(compsdir, os.path.basename(task_opts.comps)) tag = args[0] keys = args[1:] taginfo = session.getTag(tag) @@ -6829,6 +6836,7 @@ def handle_signed_repo(options, session, args): pass opts = { 'arch': task_opts.arch, + 'comps': task_opts.comps, 'event': task_opts.event, 'delta': task_opts.delta_rpms, 'multilib': task_opts.multilib, diff --git a/hub/kojihub.py b/hub/kojihub.py index 48d2753..3f41409 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2429,6 +2429,13 @@ def signed_repo_init(tag, keys, task_opts): missing.sort() raise koji.GenericError('Unsigned packages found: ' + '\n'.join(missing)) + + # handle comps + if task_opts['comps']: + groupsdir = os.path.join(repodir, 'groups') + koji.ensuredir(groupsdir) + shutil.copyfile(os.path.join(koji.pathinfo.work(), task_opts['comps']), + groupsdir + '/comps.xml') koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=task_opts['event'], repo_id=repo_id) return repo_id, task_opts['event'] From 1a217fe3c2368d8a5824c272d4ab740e2ab689f0 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:13 +0000 Subject: [PATCH 55/70] move package addition logic to builder from hub --- diff --git a/builder/kojid b/builder/kojid index 24f6a77..fcfd1d8 100755 --- a/builder/kojid +++ b/builder/kojid @@ -35,6 +35,8 @@ import logging.handlers from koji.daemon import incremental_upload, log_output, TaskManager, SCM from koji.tasks import ServerExit, ServerRestart, BaseTaskHandler, MultiPlatformTask from koji.util import parseStatus, isSuccess, dslice, dslice_ex +import multilib +import multilib.fakepo import os import pwd import grp @@ -59,6 +61,7 @@ from gzip import GzipFile from optparse import OptionParser, SUPPRESS_HELP from StringIO import StringIO from yum import repoMDObject +import yum.packages #imports for LiveCD, LiveMedia, and Appliance handler image_enabled = False @@ -4672,9 +4675,8 @@ class NewRepoTask(BaseTaskHandler): else: oldrepo = self.session.getRepo(tinfo['id'], state=koji.REPO_READY) subtasks = {} - opts = {'do_external': True, 'deltas': False} for arch in arches: - arglist = [repo_id, arch, oldrepo, opts] + arglist = [repo_id, arch, oldrepo] subtasks[arch] = self.session.host.subtask(method='createrepo', arglist=arglist, label=arch, @@ -4702,17 +4704,14 @@ class CreaterepoTask(BaseTaskHandler): Methods = ['createrepo'] _taskWeight = 1.5 - def getRepoPath(self, repo_id, tag): - return self.pathinfo.repo(repo_id, tag) - - def handler(self, repo_id, arch, oldrepo, opts): + def handler(self, repo_id, arch, oldrepo): #arch is the arch of the repo, not the task rinfo = self.session.repoInfo(repo_id, strict=True) if rinfo['state'] != koji.REPO_INIT: raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo self.repo_id = rinfo['id'] self.pathinfo = koji.PathInfo(self.options.topdir) - toprepodir = self.getRepoPath(repo_id, rinfo['tag_name']) + toprepodir = self.pathinfo.repo(repo_id, rinfo['tag_name']) self.repodir = '%s/%s' % (toprepodir, arch) if not os.path.isdir(self.repodir): raise koji.GenericError, "Repo directory missing: %s" % self.repodir @@ -4723,13 +4722,11 @@ class CreaterepoTask(BaseTaskHandler): pkglist = os.path.join(self.repodir, 'pkglist') if os.path.getsize(pkglist) == 0: pkglist = None - self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo, - opts['deltas']) - if opts['do_external']: - external_repos = self.session.getExternalRepoList( - rinfo['tag_id'], event=rinfo['create_event']) - if external_repos: - self.merge_repos(external_repos, arch, groupdata) + self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo) + external_repos = self.session.getExternalRepoList( + rinfo['tag_id'], event=rinfo['create_event']) + if external_repos: + self.merge_repos(external_repos, arch, groupdata) elif pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") @@ -4740,14 +4737,9 @@ class CreaterepoTask(BaseTaskHandler): for f in os.listdir(self.datadir): files.append(f) self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) - if opts['deltas']: - ddir = os.path.join(self.outdir, 'drpms') - for f in os.listdir(ddir): - files.append(f) - self.session.uploadWrapper('%s/%s' % (ddir, f), uploadpath, f) return [uploadpath, files] - def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, drpms): + def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, baseurl=None, drpms=False): koji.ensuredir(self.outdir) if self.options.use_createrepo_c: cmd = ['/usr/bin/createrepo_c'] @@ -4758,6 +4750,8 @@ class CreaterepoTask(BaseTaskHandler): cmd.extend(['-i', pkglist]) if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) + if baseurl: + cmd.extend(['-u', baseurl]) #attempt to recycle repodata from last repo if pkglist and oldrepo and self.options.createrepo_update and not drpms: # signed repos overload the use of "oldrepo", so the conditional @@ -4830,44 +4824,161 @@ class NewSignedRepoTask(BaseTaskHandler): Methods = ['signedRepo'] _taskWeight = 0.1 - def handler(self, tag, repo_id, task_opts): + def handler(self, tag, repo_id, keys, task_opts): tinfo = self.session.getTag(tag, strict=True, event=task_opts['event']) path = koji.pathinfo.signedrepo(repo_id, tinfo['name']) - if not os.path.isdir(path): - raise koji.GenericError, "Repo directory missing: %s" % path - arches = [] - for fn in os.listdir(path): - if os.path.isfile("%s/%s/pkglist" % (path, fn)): - arches.append(fn) + if len(task_opts['arch']) == 0: + task_opts['arch'] = tinfo['arches'].split() + if len(task_opts['arch']) == 0: + raise koji.GenericError('No arches specified nor for the tag!') subtasks = {} - if task_opts['delta']: - make_drpms = True - oldrepo = task_opts['delta'] - else: - make_drpms = False - oldrepo = None - for arch in arches: - opts = {'do_external': False, 'deltas': make_drpms} - arglist = [repo_id, arch, oldrepo, opts] + for arch in task_opts['arch']: + # call canonArch? + arglist = [tag, repo_id, arch, keys, task_opts] # no mergerepo subtasks[arch] = self.session.host.subtask( method='createsignedrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') # wait for subtasks to finish + self.logger.warn("5: %s" % subtasks.values()) results = self.wait(subtasks.values(), all=True, failany=True) + self.logger.warn("6") data = {} for (arch, task_id) in subtasks.iteritems(): data[arch] = results[task_id] self.logger.debug("DEBUG: %r : %r " % (arch, data[arch],)) self.session.host.repoDone(repo_id, data, expire=True, signed=True) - return repo_id, task_opts['event'] + return 'Signed repository #%s successfully generated' % repo_id class createSignedRepoTask(CreaterepoTask): Methods = ['createsignedrepo'] _taskWeight = 1.5 + archmap = {'s390x': 's390', 'ppc64': 'ppc', 'x86_64': 'i686'} + + def handler(self, tag, repo_id, arch, keys, opts): + #arch is the arch of the repo, not the task + rinfo = self.session.repoInfo(repo_id, strict=True) + if rinfo['state'] != koji.REPO_INIT: + raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo + self.repo_id = rinfo['id'] + self.pathinfo = koji.PathInfo(self.options.topdir) + groupdata = os.path.join( + self.pathinfo.signedrepo(repo_id, rinfo['tag_name']), + 'groups', 'comps.xml') + self.repodir = self.options.topdir # workaround for create_local_repo + #set up our output dir + self.outdir = '%s/repo' % self.workdir + self.datadir = '%s/repodata' % self.outdir + if len(opts['delta']) > 0: + for path in opts['delta']: + if not os.path.exists(path): + raise koji.GenericError( + 'drpm path %s does not exist!' % path) + pkglist = self.make_pkglist(tag, arch, keys, opts) + uploadpath = self.getUploadDir() + self.session.uploadWrapper(pkglist, uploadpath, + os.path.basename(pkglist)) + if os.path.getsize(pkglist) == 0: + pkglist = None + if len(opts['delta']) > 0: + do_drpms = True + else: + do_drpms = False + self.create_local_repo(rinfo, arch, pkglist, groupdata, opts['delta'], + drpms=do_drpms, baseurl='toplink') + if pkglist is None: + fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') + fo.write("This repo is empty because its tag has no content for this arch\n") + fo.close() + files = ['pkglist'] + for f in os.listdir(self.datadir): + files.append(f) + self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) + if opts['delta']: + ddir = os.path.join(self.outdir, 'drpms') + for f in os.listdir(ddir): + files.append(f) + self.session.uploadWrapper('%s/%s' % (ddir, f), uploadpath, f) + return [uploadpath, files] + + def get_po(self, rpmpath): + """create a fake yum-like package object given an rpminfo dictionary""" + po = yum.packages.YumLocalPackage(filename=rpmpath) + return multilib.fakepo.FakePackageObject(po=po) + + def make_pkglist(self, tag_id, arch, keys, opts): + + def write_pkg(pkgpath): + self.logger.info('incoming: %s' % pkgpath) + self.logger.info('topdir: %s' % self.options.topdir) + newpath = pkgpath.replace(self.options.topdir, '') + '\n' + self.logger.info('outgoing: %s' % newpath) + pkglist.write(newpath) + + # Need to pass event_id because even though this is a single trans, + # it is possible to see the results of other committed transactions + rpm_iter, builds = self.session.listTaggedRPMS(tag_id, + event=opts['event'], arch=arch, + inherit=opts['inherit'], rpmsigs=True) + rpms = list(rpm_iter) + if opts['multilib']: + mlm = multilib.MultilibDevelMethod(opts['multilib']) + else: + # this method always returns False, no multilib packages added + mlm = multilib.NoMultilibMethod(opts['multilib']) + need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) + #get build dirs + builddirs = {} + for build in builds: + builddirs[build['id']] = self.pathinfo.build(build) + #generate pkglist files + archdir = os.path.join(self.outdir, arch) + koji.ensuredir(archdir) + pkgfile = os.path.join(archdir, 'pkglist') + pkglist = file(pkgfile, 'w') + preferred = {} + if opts['unsigned']: + keys.append('') # make unsigned rpms the least preferred + for rpminfo in rpms: + if rpminfo['sigkey'] == '' and not opts['unsigned']: + # skip, this is the unsigned rpminfo + continue + if rpminfo['sigkey'] not in keys: + # skip, not a key we are looking for + continue + idx = keys.index(rpminfo['sigkey']) + if preferred.has_key(rpminfo['id']): + if keys.index(preferred[rpminfo['id']]['sigkey']) <= idx: + # key for this is not as preferable as what has been seen + continue + preferred[rpminfo['id']] = rpminfo + seen = set() + for rpminfo in preferred.values(): + if rpminfo['sigkey'] == '': + # we're taking an unsigned rpm (--allow-unsigned) + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + self.pathinfo.rpm(rpminfo)) + else: + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + self.pathinfo.signed(rpminfo, rpminfo['sigkey'])) + seen.add(os.path.basename(pkgpath)) + po = self.get_po(pkgpath) + mlppath = None # multilib package path + if mlm.select(po): + # we need a multilib package to be included + # we assume the same signature level is available + write_pkg(pkgpath.replace(arch, archmap[arch])) + write_pkg(pkgpath) + pkglist.close() + if not opts['skip']: + missing = list(need - seen) + if len(missing) != 0: + missing.sort() + raise koji.GenericError('Unsigned packages found: ' + + '\n'.join(missing)) + # TODO: needs to not be in /var/tmp... + return pkgfile - def getRepoPath(self, repo_id, tag): - return self.pathinfo.signedrepo(repo_id, tag) class WaitrepoTask(BaseTaskHandler): diff --git a/cli/koji b/cli/koji index e139fa3..83a11fa 100755 --- a/cli/koji +++ b/cli/koji @@ -6791,14 +6791,13 @@ def handle_signed_repo(options, session, args): parser.add_option('--comps', help='Include a comps file in the repodata') parser.add_option('--delta-rpms', metavar='PATH',default=[], action='append', - help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times.')) + help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times. These have to be reachable by the builder too, so the path needs to reach shared storage.')) parser.add_option('--event', type='int', help=_('create a signed repository based on a Brew event')) - parser.add_option('--multilib', action='store_true', default=False, - help=_('Include multilib packages in the repository')) + parser.add_option('--multilib', action='store_true', default=None, + help=_('Include multilib packages in the repository using a config')) parser.add_option("--noinherit", action='store_true', default=False, help=_('Do not consider tag inheritance')) - # TODO: latest? parser.add_option("--nowait", action='store_true', default=False, help=_('Do not wait for the task to complete')) parser.add_option('--skip-unsigned', action='store_true', default=False, @@ -6809,13 +6808,27 @@ def handle_signed_repo(options, session, args): if task_opts.allow_unsigned and task_opts.skip_unsigned: parser.error(_('allow_signed and skip_unsigned are mutually exclusive')) activate_session(session) + stuffdir = _unique_path('cli-signed') if task_opts.comps: if not os.path.exists(task_opts.comps): parser.error(_('could not find %s' % task_opts.comps)) - compsdir = _unique_path('cli-signed') - session.uploadWrapper(task_opts.comps, compsdir, + session.uploadWrapper(task_opts.comps, stuffdir, callback=_progress_callback) - task_opts.comps = os.path.join(compsdir, os.path.basename(task_opts.comps)) + task_opts.comps = os.path.join(stuffdir, + os.path.basename(task_opts.comps)) + if len(task_opts.delta_rpms) > 0: + for path in task_opts.delta_rpms: + if not os.path.exists(path): + print _("Warning: %s is not reachable locally. If this" % path) + print _(" host does not have access to Koji's shared storage") + print _(" this can be ignored.") + if task_opts.multilib: + if not os.path.exists(task_opts.multilib): + parser.error(_('could not find %s' % task_opts.multilib)) + session.uploadWrapper(task_opts.multilib, stuffdir, + callback=_progress_callback) + task_opts.comps = os.path.join(stuffdir, + os.path.basename(task_opts.multilib)) tag = args[0] keys = args[1:] taginfo = session.getTag(tag) diff --git a/hub/kojihub.py b/hub/kojihub.py index 3f41409..8942105 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2341,101 +2341,31 @@ def signed_repo_init(tag, keys, task_opts): tinfo = get_tag(tag, strict=True) koji.plugin.run_callbacks('preRepoInit', tag=tinfo, keys=keys, repo_id=None) tag_id = tinfo['id'] + repo_id = _singleValue("SELECT nextval('repo_id_seq')") repo_arches = task_opts['arch'] arches = set([]) for arch in repo_arches: arches.add(koji.canonArch(arch)) - repo_id = _singleValue("SELECT nextval('repo_id_seq')") if not task_opts['event']: task_opts['event'] = _singleValue("SELECT get_event()") insert = InsertProcessor('repo') insert.set(id=repo_id, create_event=task_opts['event'], tag_id=tag_id, state=state) insert.execute() - # Need to pass event_id because even though this is a single transaction, - # it is possible to see the results of other committed transactions - rpm_iter, builds = readTaggedRPMS(tag_id, event=task_opts['event'], - inherit=task_opts['inherit'], rpmsigs=True) - rpms = list(rpm_iter) - for rpm_copy in list(rpms): - arch = koji.canonArch(rpm_copy['arch']) - if arch not in arches: - # not an architecture we care about - rpms.remove(rpm_copy) - need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) - os.makedirs(repodir) # should not already exist - - #get build dirs - pathinfo = koji.PathInfo() - builddirs = {} - for build in builds: - relpath = pathinfo.build(build) - builddirs[build['id']] = relpath.lstrip('/') - #generate pkglist files - pkglist = {} - for repoarch in arches: - archdir = os.path.join(repodir, repoarch) - koji.ensuredir(archdir) + for arch in arches: + koji.ensuredir(os.path.join(repodir, arch)) # Make a symlink to our topdir + archdir = os.path.join(repodir, arch) top_relpath = koji.util.relpath(koji.pathinfo.topdir, archdir) top_link = os.path.join(archdir, 'toplink') os.symlink(top_relpath, top_link) - pkglist[repoarch] = file(os.path.join(archdir, 'pkglist'), 'w') - preferred = {} - if task_opts['unsigned']: - keys.append('') # make unsigned rpms the least preferred - for rpminfo in rpms: - if rpminfo['sigkey'] == '' and not task_opts['unsigned']: - # skip, this is the unsigned rpminfo - continue - if rpminfo['sigkey'] not in keys: - # skip, not a key we are looking for - continue - idx = keys.index(rpminfo['sigkey']) - if preferred.has_key(rpminfo['id']): - if keys.index(preferred[rpminfo['id']]['sigkey']) <= idx: - # key for this is not as preferable as what we have seen before - continue - preferred[rpminfo['id']] = rpminfo - seen = set() - for rpminfo in preferred.values(): - if rpminfo['sigkey'] == '': - # we're taking an unsigned rpm (--allow-unsigned) - pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], - pathinfo.rpm(rpminfo)) - else: - pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], - pathinfo.signed(rpminfo, rpminfo['sigkey'])) - seen.add(os.path.basename(pkgpath)) - repopath = '/' + pkgpath - repopath = repopath.replace(koji.pathinfo.topdir, 'toplink') + '\n' - arch = koji.canonArch(rpminfo['arch']) - if arch == 'noarch': - for repoarch in arches: - pkglist[repoarch].write(repopath) - archdir = os.path.join(repodir, repoarch) - os.link(pkgpath, - os.path.join(archdir, os.path.basename(pkgpath))) - else: - pkglist[arch].write(repopath) - dest = os.path.join(repodir, arch, os.path.basename(pkgpath)) - os.link(pkgpath, dest) - for repoarch in arches: - pkglist[repoarch].close() - if not task_opts['skip']: - missing = list(need - seen) - if len(missing) != 0: - missing.sort() - raise koji.GenericError('Unsigned packages found: ' + - '\n'.join(missing)) - # handle comps if task_opts['comps']: groupsdir = os.path.join(repodir, 'groups') koji.ensuredir(groupsdir) - shutil.copyfile(os.path.join(koji.pathinfo.work(), task_opts['comps']), - groupsdir + '/comps.xml') + shutil.copyfile(os.path.join(koji.pathinfo.work(), + task_opts['comps']), groupsdir + '/comps.xml') koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=task_opts['event'], repo_id=repo_id) return repo_id, task_opts['event'] @@ -9729,7 +9659,8 @@ class RootExports(object): """Create a signed-repo task. returns task id""" context.session.assertPerm('signed-repo') repo_id, event_id = signed_repo_init(tag, keys, task_opts) - return make_task('signedRepo', [tag, repo_id, task_opts], priority=15) + task_opts['event'] = event_id + return make_task('signedRepo', [tag, repo_id, keys, task_opts], priority=15) def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" @@ -11815,6 +11746,7 @@ class HostExports(object): repo_id: the id of the repo data: a dictionary of the form { arch: (uploadpath, files), ...} expire(optional): if set to true, mark the repo expired immediately* + signed(optional): if true, hardlink signed rpms in the final directory * This is used when a repo from an older event is generated """ @@ -11840,11 +11772,20 @@ class HostExports(object): if fn.endswith('.drpm'): koji.ensuredir(os.path.join(archdir, 'drpms')) dst = "%s/drpms/%s" % (archdir, fn) + elif fn.endswith('pkglist'): + dst = '%s/%s' % (archdir, fn) else: dst = "%s/%s" % (datadir, fn) if not os.path.exists(src): raise koji.GenericError, "uploaded file missing: %s" % src os.link(src, dst) + if fn.endswith('pkglist') and signed: + # hardlink the found rpms into the final repodir + with open(src) as pkgfile: + for pkg in pkgfile: + pkg = pkg.strip() + rpm = os.path.basename(pkg) + os.link(koji.pathinfo.topdir + pkg, os.path.join(archdir, rpm)) os.unlink(src) if expire: repo_expire(repo_id) diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 1ab7441..3d20626 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -630,7 +630,7 @@ def taskinfo(environ, taskID): build = server.getBuild(params[1]) values['destTag'] = destTag values['build'] = build - elif task['method'] in ('newRepo', 'signedRepo'): + elif task['method'] in ('newRepo', 'signedRepo', 'createsignedrepo'): tag = server.getTag(params[0]) values['tag'] = tag elif task['method'] == 'tagNotification': diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index b38597e..d668f53 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -218,14 +218,11 @@ $value #if $len($params) > 2 $printOpts($params[2]) #end if - #elif $task.method in ('newRepo', 'signedRepo') + #elif $task.method == 'signedRepo' Tag: $tag.name
- #if $task.method == 'signedRepo' - Repo ID: $params[1]
- $printOpts($params[2]) - #elif $len($params) > 1 - $printOpts($params[1]) - #end if + Repo ID: $params[1]
+ Keys: $printValue(0, $params[2])
+ $printOpts($params[3]) #elif $task.method == 'prepRepo' Tag: $params[0].name #elif $task.method == 'createrepo' @@ -240,9 +237,11 @@ $value External Repos: $printValue(None, [ext['external_repo_name'] for ext in $params[3]])
#end if #elif $task.method == 'createsignedrepo' - Repo ID: $params[0]
- Arch: $params[1]
- Options: $printMap($params[3], '    ') + Tag: $tag.name
+ Repo ID: $params[1]
+ Arch: $printValue(0, $params[2])
+ Keys: $printValue(0, $params[3])
+ Options: $printMap($params[4], '    ') #elif $task.method == 'dependantTask' Dependant Tasks:
#for $dep in $deps From 3bdc1a5c22b32bd42b4bf1dd0106d2f769b8d3b6 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:13 +0000 Subject: [PATCH 56/70] implement multilib --- diff --git a/builder/kojid b/builder/kojid index fcfd1d8..1991cb0 100755 --- a/builder/kojid +++ b/builder/kojid @@ -36,13 +36,13 @@ from koji.daemon import incremental_upload, log_output, TaskManager, SCM from koji.tasks import ServerExit, ServerRestart, BaseTaskHandler, MultiPlatformTask from koji.util import parseStatus, isSuccess, dslice, dslice_ex import multilib -import multilib.fakepo import os import pwd import grp import random import re import rpm +import rpmUtils.arch import shutil import signal import smtplib @@ -62,6 +62,7 @@ from optparse import OptionParser, SUPPRESS_HELP from StringIO import StringIO from yum import repoMDObject import yum.packages +import yum.Errors #imports for LiveCD, LiveMedia, and Appliance handler image_enabled = False @@ -4832,20 +4833,41 @@ class NewSignedRepoTask(BaseTaskHandler): if len(task_opts['arch']) == 0: raise koji.GenericError('No arches specified nor for the tag!') subtasks = {} + arch32s = set() for arch in task_opts['arch']: - # call canonArch? - arglist = [tag, repo_id, arch, keys, task_opts] # no mergerepo + if not rpmUtils.arch.isMultiLibArch(arch): + arch32s.add(arch) + for arch in arch32s: + # we do 32-bit multilib arches first so the 64-bit ones can + # get a task ID and wait for them to complete + arglist = [tag, repo_id, arch, keys, task_opts] subtasks[arch] = self.session.host.subtask( method='createsignedrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') - # wait for subtasks to finish - self.logger.warn("5: %s" % subtasks.values()) - results = self.wait(subtasks.values(), all=True, failany=True) - self.logger.warn("6") + if len(subtasks) > 0 and task_opts['multilib']: + results = self.wait(subtasks.values(), all=True, failany=True) + for arch in arch32s: + # move the 32-bit task output to the final resting place + # so the 64-bit arches can use it + upload, files = results[subtasks[arch]] + self.session.host.signedRepoMove(repo_id, upload, files, arch) + for arch in task_opts['arch']: + # do the other arches + if arch not in arch32s: + arglist = [tag, repo_id, arch, keys, task_opts] + subtasks[arch] = self.session.host.subtask( + method='createsignedrepo', arglist=arglist, label=arch, + parent=self.id, arch='noarch') + # wait for 64-bit subtasks to finish data = {} + results = self.wait(subtasks.values(), all=True, failany=True) for (arch, task_id) in subtasks.iteritems(): data[arch] = results[task_id] - self.logger.debug("DEBUG: %r : %r " % (arch, data[arch],)) + self.logger.debug("DEBUG: %r : %r " % (arch, data[arch])) + if arch not in arch32s: + # we moved the 32-bit results before, do the 64-bit + upload, files = results[subtasks[arch]] + self.session.host.signedRepoMove(repo_id, upload, files, arch) self.session.host.repoDone(repo_id, data, expire=True, signed=True) return 'Signed repository #%s successfully generated' % repo_id @@ -4853,17 +4875,36 @@ class NewSignedRepoTask(BaseTaskHandler): class createSignedRepoTask(CreaterepoTask): Methods = ['createsignedrepo'] _taskWeight = 1.5 + archmap = {'s390x': 's390', 'ppc64': 'ppc', 'x86_64': 'i686'} + compat = {"i386": ("athlon", "i686", "i586", "i486", "i386", "noarch"), + "x86_64": ("amd64", "ia32e", "x86_64", "noarch"), + "ia64": ("ia64", "noarch"), + "ppc": ("ppc", "noarch"), + "ppc64": ("ppc64p7", "ppc64pseries", "ppc64iseries", "ppc64", "noarch"), + "ppc64le": ("ppc64le", "noarch"), + "s390": ("s390", "noarch"), + "s390x": ("s390x", "noarch"), + "sparc": ("sparcv9v", "sparcv9", "sparcv8", "sparc", "noarch"), + "sparc64": ("sparc64v", "sparc64", "noarch"), + "alpha": ("alphaev6", "alphaev56", "alphaev5", "alpha", "noarch"), + "arm": ("arm", "armv4l", "armv4tl", "armv5tel", "armv5tejl", "armv6l", "armv7l", "noarch"), + "armhfp": ("armv7hl", "armv7hnl", "noarch"), + "aarch64": ("aarch64", "noarch"), + } + + biarch = {"ppc": "ppc64", "x86_64": "i386", "sparc": + "sparc64", "s390x": "s390", "ppc64": "ppc"} def handler(self, tag, repo_id, arch, keys, opts): #arch is the arch of the repo, not the task - rinfo = self.session.repoInfo(repo_id, strict=True) - if rinfo['state'] != koji.REPO_INIT: - raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo - self.repo_id = rinfo['id'] + self.rinfo = self.session.repoInfo(repo_id, strict=True) + if self.rinfo['state'] != koji.REPO_INIT: + raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo + self.repo_id = self.rinfo['id'] self.pathinfo = koji.PathInfo(self.options.topdir) groupdata = os.path.join( - self.pathinfo.signedrepo(repo_id, rinfo['tag_name']), + self.pathinfo.signedrepo(repo_id, self.rinfo['tag_name']), 'groups', 'comps.xml') self.repodir = self.options.topdir # workaround for create_local_repo #set up our output dir @@ -4874,65 +4915,173 @@ class createSignedRepoTask(CreaterepoTask): if not os.path.exists(path): raise koji.GenericError( 'drpm path %s does not exist!' % path) - pkglist = self.make_pkglist(tag, arch, keys, opts) - uploadpath = self.getUploadDir() - self.session.uploadWrapper(pkglist, uploadpath, - os.path.basename(pkglist)) - if os.path.getsize(pkglist) == 0: - pkglist = None + self.uploadpath = self.getUploadDir() + self.pkglist = self.make_pkglist(tag, arch, keys, opts) + if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch): + self.do_multilib(arch, self.archmap[arch], opts['multilib']) + self.logger.debug('package list is %s' % self.pkglist) + self.session.uploadWrapper(self.pkglist, self.uploadpath, + os.path.basename(self.pkglist)) + if os.path.getsize(self.pkglist) == 0: + self.pkglist = None if len(opts['delta']) > 0: do_drpms = True else: do_drpms = False - self.create_local_repo(rinfo, arch, pkglist, groupdata, opts['delta'], - drpms=do_drpms, baseurl='toplink') - if pkglist is None: + self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, + opts['delta'], drpms=do_drpms, baseurl='toplink') + if self.pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") fo.close() files = ['pkglist'] for f in os.listdir(self.datadir): files.append(f) - self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) + self.session.uploadWrapper('%s/%s' % (self.datadir, f), + self.uploadpath, f) if opts['delta']: ddir = os.path.join(self.outdir, 'drpms') for f in os.listdir(ddir): files.append(f) - self.session.uploadWrapper('%s/%s' % (ddir, f), uploadpath, f) - return [uploadpath, files] - - def get_po(self, rpmpath): - """create a fake yum-like package object given an rpminfo dictionary""" - po = yum.packages.YumLocalPackage(filename=rpmpath) - return multilib.fakepo.FakePackageObject(po=po) + self.session.uploadWrapper('%s/%s' % (ddir, f), + self.uploadpath, f) + return [self.uploadpath, files] + + def do_multilib(self, arch, ml_arch, conf): + self.repo_id = self.rinfo['id'] + pathinfo = koji.PathInfo(self.options.topdir) + repodir = pathinfo.signedrepo(self.rinfo['id'], self.rinfo['tag_name']) + archdir = os.path.join(repodir, arch) + mldir = os.path.join(repodir, koji.canonArch(ml_arch)) + ml_true = set() + ml_conf = os.path.join(self.pathinfo.work(), conf) + + # step 1: figure out which packages are multlib (should already exist) + mlm = multilib.DevelMultilibMethod(ml_conf) + fs_missing = set() + with open(self.pkglist) as pkglist: + for pkg in pkglist: + pkg = pkg.strip() + rpmpath = self.options.topdir + pkg + try: + po = yum.packages.YumLocalPackage(filename=rpmpath) + except yum.Errors.MiscError: + self.logger.error('%s is not on the filesystem' % rpmpath) + fs_missing.add(rpmpath) + continue + if mlm.select(po) and self.archmap.has_key(arch): + # we need a multilib package to be included + # we assume the same signature level is available + pl_path = pkg.replace(arch, self.archmap[arch]) + real_path = rpmpath.replace(arch, self.archmap[arch]) + ml_true.add(pl_path) + if not os.path.exists(real_path): + self.logger.error('%s (multilib) is not on the filesystem' % ml_path) + fs_missing.add(real_path) + + # step 2: set up architectures for yum configuration + self.logger.info("Resolving multilib for %s using method devel" % arch) + yumbase = yum.YumBase() + yumbase.verbose_logger.setLevel(logging.ERROR) + yumdir = os.path.join(self.workdir, 'yum') + # TODO: unwind this arch mess + archlist = (arch, 'noarch') + transaction_arch = arch + archlist = archlist + self.compat[self.biarch[arch]] + best_compat = self.compat[self.biarch[arch]][0] + if rpmUtils.arch.archDifference(best_compat, arch) > 0: + transaction_arch = best_compat + if hasattr(rpmUtils.arch, 'ArchStorage'): + yumbase.preconf.arch = transaction_arch + else: + rpmUtils.arch.canonArch = transaction_arch + + yconfig = """ +[main] +debuglevel=2 +pkgpolicy=newest +exactarch=1 +gpgcheck=0 +reposdir=/dev/null +cachedir=/yumcache +installroot=%s +logfile=/yum.log + +[koji-%s] +name=koji multilib task +baseurl=file://%s +enabled=1 + +""" % (yumdir, self.id, mldir) + os.makedirs(os.path.join(yumdir, "yumcache")) + os.makedirs(os.path.join(yumdir, 'var/lib/rpm')) + + # step 3: proceed with yum config and set up + yconfig_path = os.path.join(yumdir, 'yum.conf-koji-%s' % arch) + f = open(yconfig_path, 'w') + f.write(yconfig) + f.close() + self.session.uploadWrapper(yconfig_path, self.uploadpath, + os.path.basename(yconfig_path)) + yumbase.doConfigSetup(fn=yconfig_path) + yumbase.conf.cache = 0 + yumbase.doRepoSetup() + yumbase.doTsSetup() + yumbase.doRpmDBSetup() + # we trust Koji's files, so skip verifying sigs and digests + yumbase.ts.pushVSFlags( + (rpm._RPMVSF_NOSIGNATURES | rpm._RPMVSF_NODIGESTS)) + yumbase.doSackSetup(archlist=archlist, thisrepo='koji-%s' % arch) + yumbase.doSackFilelistPopulate() + for pkg in ml_true: + # TODO: store packages by first letter + # ppath = os.path.join(pkgdir, pkg.name[0].lower(), pname) + real_path = self.options.topdir + pkg + po = yum.packages.YumLocalPackage(filename=real_path) + yumbase.tsInfo.addInstall(po) + + # step 4: execute yum transaction to get dependencies + self.logger.info("Resolving depenencies for arch %s" % arch) + rc, errors = yumbase.resolveDeps() + ml_needed = set() + for f in yumbase.tsInfo.getMembers(): + dep_path = os.path.join(mldir, os.path.basename(f.po.localPkg())) + rel_path = dep_path.replace(self.options.topdir, '') + ml_needed.add(rel_path) + self.logger.debug("added %s" % rel_path) + if not os.path.exists(dep_path): + self.logger.error('%s (multilib dep) not on filesystem' % dep_path) + fs_missing.add(dep_path) + self.logger.info('yum return code: %s' % rc) + if not rc: + self.logger.error('yum depsolve was unsuccessful') + raise koji.GenericError(errors) + if len(fs_missing) > 0: + raise koji.GenericError('multilib packages missing:\n' + + '\n'.join(fs_missing)) + + # step 5: add dependencies to our package list + pkgwriter = open(self.pkglist, 'a') + for ml_pkg in ml_needed: + pkgwriter.write(ml_pkg + '\n') def make_pkglist(self, tag_id, arch, keys, opts): - def write_pkg(pkgpath): - self.logger.info('incoming: %s' % pkgpath) - self.logger.info('topdir: %s' % self.options.topdir) - newpath = pkgpath.replace(self.options.topdir, '') + '\n' - self.logger.info('outgoing: %s' % newpath) - pkglist.write(newpath) - # Need to pass event_id because even though this is a single trans, # it is possible to see the results of other committed transactions - rpm_iter, builds = self.session.listTaggedRPMS(tag_id, - event=opts['event'], arch=arch, - inherit=opts['inherit'], rpmsigs=True) - rpms = list(rpm_iter) - if opts['multilib']: - mlm = multilib.MultilibDevelMethod(opts['multilib']) - else: - # this method always returns False, no multilib packages added - mlm = multilib.NoMultilibMethod(opts['multilib']) - need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) - #get build dirs + rpms = [] builddirs = {} - for build in builds: - builddirs[build['id']] = self.pathinfo.build(build) + for a in (arch, 'noarch'): + rpm_iter, builds = self.session.listTaggedRPMS(tag_id, + event=opts['event'], arch=a, + inherit=opts['inherit'], rpmsigs=True) + for build in builds: + builddirs[build['id']] = self.pathinfo.build(build) + rpms += list(rpm_iter) + #get build dirs + need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) #generate pkglist files - archdir = os.path.join(self.outdir, arch) + archdir = os.path.join(self.outdir, koji.canonArch(arch)) koji.ensuredir(archdir) pkgfile = os.path.join(archdir, 'pkglist') pkglist = file(pkgfile, 'w') @@ -4953,6 +5102,7 @@ class createSignedRepoTask(CreaterepoTask): continue preferred[rpminfo['id']] = rpminfo seen = set() + fs_missing = set() for rpminfo in preferred.values(): if rpminfo['sigkey'] == '': # we're taking an unsigned rpm (--allow-unsigned) @@ -4962,21 +5112,19 @@ class createSignedRepoTask(CreaterepoTask): pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], self.pathinfo.signed(rpminfo, rpminfo['sigkey'])) seen.add(os.path.basename(pkgpath)) - po = self.get_po(pkgpath) - mlppath = None # multilib package path - if mlm.select(po): - # we need a multilib package to be included - # we assume the same signature level is available - write_pkg(pkgpath.replace(arch, archmap[arch])) - write_pkg(pkgpath) + pkglist.write(pkgpath.replace(self.options.topdir, '') + '\n') + if not os.path.exists(pkgpath): + fs_missing.add(pkgpath) pkglist.close() + if len(fs_missing) > 0: + raise koji.GenericError('Packages missing from the filesystem:\n' + + '\n'.join(fs_missing)) if not opts['skip']: missing = list(need - seen) if len(missing) != 0: missing.sort() raise koji.GenericError('Unsigned packages found: ' + '\n'.join(missing)) - # TODO: needs to not be in /var/tmp... return pkgfile diff --git a/cli/koji b/cli/koji index 83a11fa..a9c4ec3 100755 --- a/cli/koji +++ b/cli/koji @@ -6794,7 +6794,7 @@ def handle_signed_repo(options, session, args): help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times. These have to be reachable by the builder too, so the path needs to reach shared storage.')) parser.add_option('--event', type='int', help=_('create a signed repository based on a Brew event')) - parser.add_option('--multilib', action='store_true', default=None, + parser.add_option('--multilib', default=None, help=_('Include multilib packages in the repository using a config')) parser.add_option("--noinherit", action='store_true', default=False, help=_('Do not consider tag inheritance')) @@ -6814,6 +6814,7 @@ def handle_signed_repo(options, session, args): parser.error(_('could not find %s' % task_opts.comps)) session.uploadWrapper(task_opts.comps, stuffdir, callback=_progress_callback) + print task_opts.comps = os.path.join(stuffdir, os.path.basename(task_opts.comps)) if len(task_opts.delta_rpms) > 0: @@ -6822,13 +6823,6 @@ def handle_signed_repo(options, session, args): print _("Warning: %s is not reachable locally. If this" % path) print _(" host does not have access to Koji's shared storage") print _(" this can be ignored.") - if task_opts.multilib: - if not os.path.exists(task_opts.multilib): - parser.error(_('could not find %s' % task_opts.multilib)) - session.uploadWrapper(task_opts.multilib, stuffdir, - callback=_progress_callback) - task_opts.comps = os.path.join(stuffdir, - os.path.basename(task_opts.multilib)) tag = args[0] keys = args[1:] taginfo = session.getTag(tag) @@ -6842,6 +6836,20 @@ def handle_signed_repo(options, session, args): for a in task_opts.arch: if not taginfo['arches'] or a not in taginfo['arches']: print _('Warning: %s is not in the list of tag arches' % a) + if task_opts.multilib: + if not os.path.exists(task_opts.multilib): + parser.error(_('could not find %s' % task_opts.multilib)) + if 'x86_64' in task_opts.arch and not 'i686' in task_opts.arch: + parser.error(_('The multilib arch (i686) must be included')) + if 's390x' in task_opts.arch and not 's390' in task_opts.arch: + parser.error(_('The multilib arch (s390) must be included')) + if 'ppc64' in task_opts.arch and not 'ppc' in task_opts.arch: + parser.error(_('The multilib arch (ppc) must be included')) + session.uploadWrapper(task_opts.multilib, stuffdir, + callback=_progress_callback) + task_opts.multilib = os.path.join(stuffdir, + os.path.basename(task_opts.multilib)) + print try: task_opts.arch.remove('noarch') # handled specifically task_opts.arch.remove('src') # ditto diff --git a/hub/kojihub.py b/hub/kojihub.py index 8942105..b719990 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -11756,37 +11756,25 @@ class HostExports(object): koji.plugin.run_callbacks('preRepoDone', repo=rinfo, data=data, expire=expire) if rinfo['state'] != koji.REPO_INIT: raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo - if signed: - repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) - else: - repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) + repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) workdir = koji.pathinfo.work() - for arch, (uploadpath, files) in data.iteritems(): - archdir = "%s/%s" % (repodir, arch) - if not os.path.isdir(archdir): - raise koji.GenericError, "Repo arch directory missing: %s" % archdir - datadir = "%s/repodata" % archdir - koji.ensuredir(datadir) - for fn in files: - src = "%s/%s/%s" % (workdir,uploadpath, fn) - if fn.endswith('.drpm'): - koji.ensuredir(os.path.join(archdir, 'drpms')) - dst = "%s/drpms/%s" % (archdir, fn) - elif fn.endswith('pkglist'): - dst = '%s/%s' % (archdir, fn) - else: - dst = "%s/%s" % (datadir, fn) - if not os.path.exists(src): - raise koji.GenericError, "uploaded file missing: %s" % src - os.link(src, dst) - if fn.endswith('pkglist') and signed: - # hardlink the found rpms into the final repodir - with open(src) as pkgfile: - for pkg in pkgfile: - pkg = pkg.strip() - rpm = os.path.basename(pkg) - os.link(koji.pathinfo.topdir + pkg, os.path.join(archdir, rpm)) - os.unlink(src) + if not signed: + for arch, (uploadpath, files) in data.iteritems(): + archdir = "%s/%s" % (repodir, koji.canonArch(arch)) + if not os.path.isdir(archdir): + raise koji.GenericError, "Repo arch directory missing: %s" % archdir + datadir = "%s/repodata" % archdir + koji.ensuredir(datadir) + for fn in files: + src = "%s/%s/%s" % (workdir,uploadpath, fn) + if fn.endswith('pkglist'): + dst = '%s/%s' % (archdir, fn) + else: + dst = "%s/%s" % (datadir, fn) + if not os.path.exists(src): + raise koji.GenericError, "uploaded file missing: %s" % src + os.link(src, dst) + os.unlink(src) if expire: repo_expire(repo_id) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) @@ -11806,6 +11794,37 @@ class HostExports(object): log_error("Unable to create latest link for repo: %s" % repodir) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) + def signedRepoMove(self, repo_id, uploadpath, files, arch): + """very similar to repoDone, except only the uploads are completed""" + workdir = koji.pathinfo.work() + rinfo = repo_info(repo_id, strict=True) + repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) + archdir = "%s/%s" % (repodir, koji.canonArch(arch)) + if not os.path.isdir(archdir): + raise koji.GenericError, "Repo arch directory missing: %s" % archdir + datadir = "%s/repodata" % archdir + koji.ensuredir(datadir) + for fn in files: + src = "%s/%s/%s" % (workdir, uploadpath, fn) + if fn.endswith('.drpm'): + koji.ensuredir(os.path.join(archdir, 'drpms')) + dst = "%s/drpms/%s" % (archdir, fn) + elif fn.endswith('pkglist'): + dst = '%s/%s' % (archdir, fn) + else: + dst = "%s/%s" % (datadir, fn) + if not os.path.exists(src): + raise koji.GenericError, "uploaded file missing: %s" % src + os.link(src, dst) + if fn.endswith('pkglist'): + # hardlink the found rpms into the final repodir + with open(src) as pkgfile: + for pkg in pkgfile: + pkg = pkg.strip() + rpm = os.path.basename(pkg) + os.link(koji.pathinfo.topdir + pkg, os.path.join(archdir, rpm)) + os.unlink(src) + def isEnabled(self): host = Host() host.verify() diff --git a/koji/__init__.py b/koji/__init__.py index 73faefb..ee1e734 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -2534,10 +2534,15 @@ def _taskLabel(taskInfo): if taskInfo.has_key('request'): tagInfo = taskInfo['request'][0] extra = tagInfo['name'] - elif method in ('createrepo', 'createsignedrepo'): + elif method in ('createrepo'): if taskInfo.has_key('request'): arch = taskInfo['request'][1] extra = arch + elif method in ('createsignedrepo'): + if taskInfo.has_key('request'): + repo_id = taskInfo['request'][1] + arch = taskInfo['request'][2] + extra = '%s, %s' % (repo_id, arch) elif method == 'dependantTask': if taskInfo.has_key('request'): extra = ', '.join([subtask[0] for subtask in taskInfo['request'][1]]) From dbfa586e2a57f4babf0cb62003e12a83d2b0825b Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:13 +0000 Subject: [PATCH 57/70] encapsulate repodata references --- diff --git a/builder/kojid b/builder/kojid index 1991cb0..5735daf 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4740,7 +4740,7 @@ class CreaterepoTask(BaseTaskHandler): self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) return [uploadpath, files] - def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, baseurl=None, drpms=False): + def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, drpms=False): koji.ensuredir(self.outdir) if self.options.use_createrepo_c: cmd = ['/usr/bin/createrepo_c'] @@ -4751,8 +4751,6 @@ class CreaterepoTask(BaseTaskHandler): cmd.extend(['-i', pkglist]) if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) - if baseurl: - cmd.extend(['-u', baseurl]) #attempt to recycle repodata from last repo if pkglist and oldrepo and self.options.createrepo_update and not drpms: # signed repos overload the use of "oldrepo", so the conditional @@ -4849,8 +4847,9 @@ class NewSignedRepoTask(BaseTaskHandler): for arch in arch32s: # move the 32-bit task output to the final resting place # so the 64-bit arches can use it - upload, files = results[subtasks[arch]] - self.session.host.signedRepoMove(repo_id, upload, files, arch) + upload, files, keypaths = results[subtasks[arch]] + self.session.host.signedRepoMove( + repo_id, upload, files, arch, keypaths) for arch in task_opts['arch']: # do the other arches if arch not in arch32s: @@ -4866,8 +4865,9 @@ class NewSignedRepoTask(BaseTaskHandler): self.logger.debug("DEBUG: %r : %r " % (arch, data[arch])) if arch not in arch32s: # we moved the 32-bit results before, do the 64-bit - upload, files = results[subtasks[arch]] - self.session.host.signedRepoMove(repo_id, upload, files, arch) + upload, files, keypaths = results[subtasks[arch]] + self.session.host.signedRepoMove( + repo_id, upload, files, arch, keypaths) self.session.host.repoDone(repo_id, data, expire=True, signed=True) return 'Signed repository #%s successfully generated' % repo_id @@ -4906,10 +4906,12 @@ class createSignedRepoTask(CreaterepoTask): groupdata = os.path.join( self.pathinfo.signedrepo(repo_id, self.rinfo['tag_name']), 'groups', 'comps.xml') - self.repodir = self.options.topdir # workaround for create_local_repo #set up our output dir - self.outdir = '%s/repo' % self.workdir - self.datadir = '%s/repodata' % self.outdir + self.repodir = '%s/repo' % self.workdir + koji.ensuredir(self.repodir) + self.outdir = self.repodir # workaround create_local_repo use + self.datadir = '%s/repodata' % self.repodir + self.keypaths = {} if len(opts['delta']) > 0: for path in opts['delta']: if not os.path.exists(path): @@ -4929,7 +4931,7 @@ class createSignedRepoTask(CreaterepoTask): else: do_drpms = False self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, - opts['delta'], drpms=do_drpms, baseurl='toplink') + opts['delta'], drpms=do_drpms) if self.pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") @@ -4940,20 +4942,19 @@ class createSignedRepoTask(CreaterepoTask): self.session.uploadWrapper('%s/%s' % (self.datadir, f), self.uploadpath, f) if opts['delta']: - ddir = os.path.join(self.outdir, 'drpms') + ddir = os.path.join(self.repodir, 'drpms') for f in os.listdir(ddir): files.append(f) self.session.uploadWrapper('%s/%s' % (ddir, f), self.uploadpath, f) - return [self.uploadpath, files] + return [self.uploadpath, files, self.keypaths] def do_multilib(self, arch, ml_arch, conf): self.repo_id = self.rinfo['id'] pathinfo = koji.PathInfo(self.options.topdir) repodir = pathinfo.signedrepo(self.rinfo['id'], self.rinfo['tag_name']) - archdir = os.path.join(repodir, arch) mldir = os.path.join(repodir, koji.canonArch(ml_arch)) - ml_true = set() + ml_true = set() # multilib packages we need to include before depsolve ml_conf = os.path.join(self.pathinfo.work(), conf) # step 1: figure out which packages are multlib (should already exist) @@ -4961,22 +4962,17 @@ class createSignedRepoTask(CreaterepoTask): fs_missing = set() with open(self.pkglist) as pkglist: for pkg in pkglist: - pkg = pkg.strip() - rpmpath = self.options.topdir + pkg - try: - po = yum.packages.YumLocalPackage(filename=rpmpath) - except yum.Errors.MiscError: - self.logger.error('%s is not on the filesystem' % rpmpath) - fs_missing.add(rpmpath) - continue + ppath = os.path.join(self.repodir, pkg.strip()) + po = yum.packages.YumLocalPackage(filename=ppath) if mlm.select(po) and self.archmap.has_key(arch): # we need a multilib package to be included # we assume the same signature level is available - pl_path = pkg.replace(arch, self.archmap[arch]) - real_path = rpmpath.replace(arch, self.archmap[arch]) - ml_true.add(pl_path) + pl_path = pkg.replace(arch, self.archmap[arch]).strip() + # assume this exists in the task results for the ml arch + real_path = os.path.join(mldir, pl_path) + ml_true.add(real_path) if not os.path.exists(real_path): - self.logger.error('%s (multilib) is not on the filesystem' % ml_path) + self.logger.error('%s (multilib) is not on the filesystem' % real_path) fs_missing.add(real_path) # step 2: set up architectures for yum configuration @@ -5036,8 +5032,7 @@ enabled=1 for pkg in ml_true: # TODO: store packages by first letter # ppath = os.path.join(pkgdir, pkg.name[0].lower(), pname) - real_path = self.options.topdir + pkg - po = yum.packages.YumLocalPackage(filename=real_path) + po = yum.packages.YumLocalPackage(filename=pkg) yumbase.tsInfo.addInstall(po) # step 4: execute yum transaction to get dependencies @@ -5046,9 +5041,8 @@ enabled=1 ml_needed = set() for f in yumbase.tsInfo.getMembers(): dep_path = os.path.join(mldir, os.path.basename(f.po.localPkg())) - rel_path = dep_path.replace(self.options.topdir, '') - ml_needed.add(rel_path) - self.logger.debug("added %s" % rel_path) + ml_needed.add(dep_path) + self.logger.debug("added %s" % dep_path) if not os.path.exists(dep_path): self.logger.error('%s (multilib dep) not on filesystem' % dep_path) fs_missing.add(dep_path) @@ -5063,7 +5057,11 @@ enabled=1 # step 5: add dependencies to our package list pkgwriter = open(self.pkglist, 'a') for ml_pkg in ml_needed: - pkgwriter.write(ml_pkg + '\n') + bnp = os.path.basename(ml_pkg) + pkgwriter.write(bnp + '\n') + os.symlink(ml_pkg, os.path.join(self.repodir, bnp)) + self.keypaths[bnp] = ml_pkg + def make_pkglist(self, tag_id, arch, keys, opts): @@ -5081,9 +5079,7 @@ enabled=1 #get build dirs need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) #generate pkglist files - archdir = os.path.join(self.outdir, koji.canonArch(arch)) - koji.ensuredir(archdir) - pkgfile = os.path.join(archdir, 'pkglist') + pkgfile = os.path.join(self.repodir, 'pkglist') pkglist = file(pkgfile, 'w') preferred = {} if opts['unsigned']: @@ -5112,9 +5108,13 @@ enabled=1 pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], self.pathinfo.signed(rpminfo, rpminfo['sigkey'])) seen.add(os.path.basename(pkgpath)) - pkglist.write(pkgpath.replace(self.options.topdir, '') + '\n') if not os.path.exists(pkgpath): fs_missing.add(pkgpath) + else: + bnp = os.path.basename(pkgpath) + pkglist.write(bnp + '\n') + self.keypaths[bnp] = pkgpath + os.symlink(pkgpath, os.path.join(self.repodir, bnp)) pkglist.close() if len(fs_missing) > 0: raise koji.GenericError('Packages missing from the filesystem:\n' + diff --git a/hub/kojihub.py b/hub/kojihub.py index b719990..6732605 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2355,11 +2355,6 @@ def signed_repo_init(tag, keys, task_opts): repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) for arch in arches: koji.ensuredir(os.path.join(repodir, arch)) - # Make a symlink to our topdir - archdir = os.path.join(repodir, arch) - top_relpath = koji.util.relpath(koji.pathinfo.topdir, archdir) - top_link = os.path.join(archdir, 'toplink') - os.symlink(top_relpath, top_link) # handle comps if task_opts['comps']: groupsdir = os.path.join(repodir, 'groups') @@ -11794,8 +11789,10 @@ class HostExports(object): log_error("Unable to create latest link for repo: %s" % repodir) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) - def signedRepoMove(self, repo_id, uploadpath, files, arch): - """very similar to repoDone, except only the uploads are completed""" + def signedRepoMove(self, repo_id, uploadpath, files, arch, fullpaths): + """ + Very similar to repoDone, except only the uploads are completed. + fullpaths is a dict like so: rpm file name -> sig""" workdir = koji.pathinfo.work() rinfo = repo_info(repo_id, strict=True) repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) @@ -11821,8 +11818,8 @@ class HostExports(object): with open(src) as pkgfile: for pkg in pkgfile: pkg = pkg.strip() - rpm = os.path.basename(pkg) - os.link(koji.pathinfo.topdir + pkg, os.path.join(archdir, rpm)) + rpmpath = fullpaths[pkg] + os.link(rpmpath, os.path.join(archdir, os.path.basename(rpmpath))) os.unlink(src) def isEnabled(self): From a85e7c7bfbd3a2df8f8c9ab74f16d4fdf433c958 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:13 +0000 Subject: [PATCH 58/70] lay out rpms by first character --- diff --git a/builder/kojid b/builder/kojid index 5735daf..205dc0a 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5040,7 +5040,8 @@ enabled=1 rc, errors = yumbase.resolveDeps() ml_needed = set() for f in yumbase.tsInfo.getMembers(): - dep_path = os.path.join(mldir, os.path.basename(f.po.localPkg())) + bnp = os.path.basename(f.po.localPkg()) + dep_path = os.path.join(mldir, bnp[0], bnp) ml_needed.add(dep_path) self.logger.debug("added %s" % dep_path) if not os.path.exists(dep_path): @@ -5058,8 +5059,9 @@ enabled=1 pkgwriter = open(self.pkglist, 'a') for ml_pkg in ml_needed: bnp = os.path.basename(ml_pkg) - pkgwriter.write(bnp + '\n') - os.symlink(ml_pkg, os.path.join(self.repodir, bnp)) + pkgwriter.write(bnp[0] + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnp[0])) + os.symlink(ml_pkg, os.path.join(self.repodir, bnp[0], bnp)) self.keypaths[bnp] = ml_pkg @@ -5112,9 +5114,10 @@ enabled=1 fs_missing.add(pkgpath) else: bnp = os.path.basename(pkgpath) - pkglist.write(bnp + '\n') + pkglist.write(bnp[0] + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnp[0])) self.keypaths[bnp] = pkgpath - os.symlink(pkgpath, os.path.join(self.repodir, bnp)) + os.symlink(pkgpath, os.path.join(self.repodir, bnp[0], bnp)) pkglist.close() if len(fs_missing) > 0: raise koji.GenericError('Packages missing from the filesystem:\n' + diff --git a/hub/kojihub.py b/hub/kojihub.py index 6732605..950df2d 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -11817,9 +11817,11 @@ class HostExports(object): # hardlink the found rpms into the final repodir with open(src) as pkgfile: for pkg in pkgfile: - pkg = pkg.strip() + pkg = os.path.basename(pkg.strip()) rpmpath = fullpaths[pkg] - os.link(rpmpath, os.path.join(archdir, os.path.basename(rpmpath))) + bnp = os.path.basename(rpmpath) + koji.ensuredir(os.path.join(archdir, bnp[0])) + os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) os.unlink(src) def isEnabled(self): From 6a0ac3be2c6ffe41d59dbeb43f2031a0c602dd0d Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:13 +0000 Subject: [PATCH 59/70] add signed flag to repo table --- diff --git a/builder/kojid b/builder/kojid index 205dc0a..a698428 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4868,7 +4868,7 @@ class NewSignedRepoTask(BaseTaskHandler): upload, files, keypaths = results[subtasks[arch]] self.session.host.signedRepoMove( repo_id, upload, files, arch, keypaths) - self.session.host.repoDone(repo_id, data, expire=True, signed=True) + self.session.host.repoDone(repo_id, data, expire=False, signed=True) return 'Signed repository #%s successfully generated' % repo_id diff --git a/docs/schema.sql b/docs/schema.sql index 523afd7..5ed1d67 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -51,6 +51,7 @@ CREATE TABLE permissions ( INSERT INTO permissions (name) VALUES ('admin'); INSERT INTO permissions (name) VALUES ('build'); INSERT INTO permissions (name) VALUES ('repo'); +INSERT INTO permissions (name) VALUES ('image'); INSERT INTO permissions (name) VALUES ('livecd'); INSERT INTO permissions (name) VALUES ('maven-import'); INSERT INTO permissions (name) VALUES ('win-import'); @@ -388,7 +389,8 @@ CREATE TABLE repo ( id SERIAL NOT NULL PRIMARY KEY, create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), tag_id INTEGER NOT NULL REFERENCES tag(id), - state INTEGER + state INTEGER, + signed BOOLEAN DEFAULT 'false' ) WITHOUT OIDS; -- external yum repos diff --git a/hub/kojihub.py b/hub/kojihub.py index 950df2d..fec162a 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2350,7 +2350,7 @@ def signed_repo_init(tag, keys, task_opts): task_opts['event'] = _singleValue("SELECT get_event()") insert = InsertProcessor('repo') insert.set(id=repo_id, create_event=task_opts['event'], tag_id=tag_id, - state=state) + state=state, signed=True) insert.execute() repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) for arch in arches: @@ -2387,6 +2387,7 @@ def repo_info(repo_id, strict=False): ('EXTRACT(EPOCH FROM events.time)','create_ts'), ('repo.tag_id', 'tag_id'), ('tag.name', 'tag_name'), + ('repo.signed', 'signed'), ) q = """SELECT %s FROM repo JOIN tag ON tag_id=tag.id @@ -9623,16 +9624,20 @@ class RootExports(object): taginfo['extra'][key] = ancestor['extra'][key] return taginfo - def getRepo(self,tag,state=None,event=None): - if isinstance(tag,int): + def getRepo(self, tag, state=None, event=None, signed=False): + if isinstance(tag, int): id = tag else: - id = get_tag_id(tag,strict=True) + id = get_tag_id(tag, strict=True) - fields = ['repo.id', 'repo.state', 'repo.create_event', 'events.time', 'EXTRACT(EPOCH FROM events.time)'] - aliases = ['id', 'state', 'create_event', 'creation_time', 'create_ts'] + fields = ['repo.id', 'repo.state', 'repo.create_event', 'events.time', 'EXTRACT(EPOCH FROM events.time)', 'repo.signed'] + aliases = ['id', 'state', 'create_event', 'creation_time', 'create_ts', 'signed'] joins = ['events ON repo.create_event = events.id'] clauses = ['repo.tag_id = %(id)i'] + if signed: + clauses.append('repo.signed is true') + else: + clauses.append('repo.signed is false') if event: # the repo table doesn't have all the fields of a _config table, just create_event clauses.append('create_event <= %(event)i') diff --git a/util/kojira b/util/kojira index e07f0b8..ed919cb 100755 --- a/util/kojira +++ b/util/kojira @@ -134,7 +134,11 @@ class ManagedRepo(object): (self.tag_id, self.repo_id)) return False tag_name = tag_info['name'] - path = pathinfo.repo(self.repo_id, tag_name) + rinfo = self.session.repoInfo(self.repo_id, strict=True) + if rinfo['signed']: + path = pathinfo.signedrepo(self.repo_id, tag_name) + else: + path = pathinfo.repo(self.repo_id, tag_name) try: #also check dir age. We do this because a repo can be created from an older event #and should not be removed based solely on that event's timestamp. From 43ead9c9d0a07b969b0a90443c41cde22520df41 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:13 +0000 Subject: [PATCH 60/70] fix newRepo in webui --- diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index d668f53..03393f5 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -218,6 +218,11 @@ $value #if $len($params) > 2 $printOpts($params[2]) #end if + #elif $task.method == 'newRepo' + Tag: $tag.name
+ #if $len($params) > 1 + $printOpts($params[1]) + #end if #elif $task.method == 'signedRepo' Tag: $tag.name
Repo ID: $params[1]
From ce333e81a0e699c5f3c75e70eec6356d688be59e Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:14 +0000 Subject: [PATCH 61/70] Make the signedRepo tasks happen in the createrepo channe --- diff --git a/hub/kojihub.py b/hub/kojihub.py index fec162a..643a542 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9660,7 +9660,7 @@ class RootExports(object): context.session.assertPerm('signed-repo') repo_id, event_id = signed_repo_init(tag, keys, task_opts) task_opts['event'] = event_id - return make_task('signedRepo', [tag, repo_id, keys, task_opts], priority=15) + return make_task('signedRepo', [tag, repo_id, keys, task_opts], priority=15, channel='createrepo') def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" From 1c967d59c04f9ba5db12e2a0f51f87f39406d138 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:14 +0000 Subject: [PATCH 62/70] fall back to a copy if we cannot hardlink --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 643a542..1f93297 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -11820,13 +11820,21 @@ class HostExports(object): os.link(src, dst) if fn.endswith('pkglist'): # hardlink the found rpms into the final repodir + # TODO: properly consider split-volume functionality with open(src) as pkgfile: for pkg in pkgfile: pkg = os.path.basename(pkg.strip()) rpmpath = fullpaths[pkg] bnp = os.path.basename(rpmpath) koji.ensuredir(os.path.join(archdir, bnp[0])) - os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) + try: + os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) + except OSError, ose: + if ose.error == 18: + shutil.copy2( + rpmpath, os.path.join(archdir, bnp[0], bnp)) + else: + raise ose os.unlink(src) def isEnabled(self): From ca4d5e89543b28a0c60ccda0644a73bac419ea98 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:14 +0000 Subject: [PATCH 63/70] errno not error --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 1f93297..ed111aa 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -11830,7 +11830,7 @@ class HostExports(object): try: os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) except OSError, ose: - if ose.error == 18: + if ose.errno == 18: shutil.copy2( rpmpath, os.path.join(archdir, bnp[0], bnp)) else: From c82923749201501312fc9fe4d53027cdb62e25cf Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:14 +0000 Subject: [PATCH 64/70] add builder plugins to the spec file --- diff --git a/koji.spec b/koji.spec index a1b3f28..ef97c30 100644 --- a/koji.spec +++ b/koji.spec @@ -113,6 +113,16 @@ Requires: createrepo >= 0.9.2 koji-builder is the daemon that runs on build machines and executes tasks that come through the Koji system. +%package builder-plugins +Summary: Koji builder plugins +Group: Applications/System +License: LGPLv2 +Requires: %{name} = %{version}-%{release} +Requires: %{name}-builder = %{version}-%{release} + +%description builder-plugins +Plugins for a Koji builder + %package vm Summary: Koji virtual machine management daemon Group: Applications/System From fdde5b23eb666eeeda5a41fe8e541583fa127291 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:14 +0000 Subject: [PATCH 65/70] add builder plugins to the spec file again --- diff --git a/koji.spec b/koji.spec index ef97c30..ecef177 100644 --- a/koji.spec +++ b/koji.spec @@ -262,6 +262,13 @@ rm -rf $RPM_BUILD_ROOT %config(noreplace) %{_sysconfdir}/kojid/kojid.conf %attr(-,kojibuilder,kojibuilder) %{_sysconfdir}/mock/koji +%files builder-plugins +%defattr(-,root,root) +%dir %{_sysconfdir}/kojid/plugins +%config(noreplace) %{_sysconfdir}/kojid/plugins/*.conf +%dir %{_prefix}/lib/koji-builder-plugins +%{_prefix}/lib/koji-builder-plugins/*.py* + %pre builder /usr/sbin/useradd -r -s /bin/bash -G mock -d /builddir -M kojibuilder 2>/dev/null ||: From f425d503371f88bd2a6ff495445c2712cee95d2c Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:14 +0000 Subject: [PATCH 66/70] fix builder-plugins spec file and add --non-latest --- diff --git a/builder/kojid b/builder/kojid index a698428..f7ae987 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5073,7 +5073,7 @@ enabled=1 builddirs = {} for a in (arch, 'noarch'): rpm_iter, builds = self.session.listTaggedRPMS(tag_id, - event=opts['event'], arch=a, + event=opts['event'], arch=a, latest=opts['latest'], inherit=opts['inherit'], rpmsigs=True) for build in builds: builddirs[build['id']] = self.pathinfo.build(build) diff --git a/cli/koji b/cli/koji index a9c4ec3..b489a76 100755 --- a/cli/koji +++ b/cli/koji @@ -6794,6 +6794,8 @@ def handle_signed_repo(options, session, args): help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times. These have to be reachable by the builder too, so the path needs to reach shared storage.')) parser.add_option('--event', type='int', help=_('create a signed repository based on a Brew event')) + parser.add_option('--non-latest', dest='latest', default=True, + action='store_false', help='Include older builds, not just the latest') parser.add_option('--multilib', default=None, help=_('Include multilib packages in the repository using a config')) parser.add_option("--noinherit", action='store_true', default=False, @@ -6858,10 +6860,11 @@ def handle_signed_repo(options, session, args): opts = { 'arch': task_opts.arch, 'comps': task_opts.comps, - 'event': task_opts.event, 'delta': task_opts.delta_rpms, - 'multilib': task_opts.multilib, + 'event': task_opts.event, 'inherit': not task_opts.noinherit, + 'latest': task_opts.latest, + 'multilib': task_opts.multilib, 'skip': task_opts.skip_unsigned, 'unsigned': task_opts.allow_unsigned } diff --git a/koji.spec b/koji.spec index ecef177..711d982 100644 --- a/koji.spec +++ b/koji.spec @@ -113,16 +113,6 @@ Requires: createrepo >= 0.9.2 koji-builder is the daemon that runs on build machines and executes tasks that come through the Koji system. -%package builder-plugins -Summary: Koji builder plugins -Group: Applications/System -License: LGPLv2 -Requires: %{name} = %{version}-%{release} -Requires: %{name}-builder = %{version}-%{release} - -%description builder-plugins -Plugins for a Koji builder - %package vm Summary: Koji virtual machine management daemon Group: Applications/System @@ -214,13 +204,6 @@ rm -rf $RPM_BUILD_ROOT %dir %{_sysconfdir}/koji-hub/plugins %config(noreplace) %{_sysconfdir}/koji-hub/plugins/*.conf -%files builder-plugins -%defattr(-,root,root) -%dir %{_sysconfdir}/kojid/plugins -%config(noreplace) %{_sysconfdir}/kojid/plugins/*.conf -%dir %{_prefix}/lib/koji-builder-plugins -%{_prefix}/lib/koji-builder-plugins/*.py* - %files utils %defattr(-,root,root) %{_sbindir}/kojira From c6c1774a928d44fc65828f40765e8367fe4976c1 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:14 +0000 Subject: [PATCH 67/70] fixes from testing and upstream comments --- diff --git a/builder/kojid b/builder/kojid index f7ae987..ce3ff22 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4831,8 +4831,12 @@ class NewSignedRepoTask(BaseTaskHandler): if len(task_opts['arch']) == 0: raise koji.GenericError('No arches specified nor for the tag!') subtasks = {} - arch32s = set() + # weed out subarchitectures + canonArches = set() for arch in task_opts['arch']: + canonArches.add(koji.canonArch(arch)) + arch32s = set() + for arch in canonArches: if not rpmUtils.arch.isMultiLibArch(arch): arch32s.add(arch) for arch in arch32s: @@ -4846,11 +4850,11 @@ class NewSignedRepoTask(BaseTaskHandler): results = self.wait(subtasks.values(), all=True, failany=True) for arch in arch32s: # move the 32-bit task output to the final resting place - # so the 64-bit arches can use it + # so the 64-bit arches can use it for multilib upload, files, keypaths = results[subtasks[arch]] self.session.host.signedRepoMove( repo_id, upload, files, arch, keypaths) - for arch in task_opts['arch']: + for arch in canonArches: # do the other arches if arch not in arch32s: arglist = [tag, repo_id, arch, keys, task_opts] @@ -4863,8 +4867,13 @@ class NewSignedRepoTask(BaseTaskHandler): for (arch, task_id) in subtasks.iteritems(): data[arch] = results[task_id] self.logger.debug("DEBUG: %r : %r " % (arch, data[arch])) - if arch not in arch32s: + if task_opts['multilib']: # we moved the 32-bit results before, do the 64-bit + if arch not in arch32s: + upload, files, keypaths = results[subtasks[arch]] + self.session.host.signedRepoMove( + repo_id, upload, files, arch, keypaths) + else: upload, files, keypaths = results[subtasks[arch]] self.session.host.signedRepoMove( repo_id, upload, files, arch, keypaths) @@ -4967,6 +4976,7 @@ class createSignedRepoTask(CreaterepoTask): if mlm.select(po) and self.archmap.has_key(arch): # we need a multilib package to be included # we assume the same signature level is available + # XXX: what is a subarchitecture is the right answer? pl_path = pkg.replace(arch, self.archmap[arch]).strip() # assume this exists in the task results for the ml arch real_path = os.path.join(mldir, pl_path) @@ -5059,9 +5069,9 @@ enabled=1 pkgwriter = open(self.pkglist, 'a') for ml_pkg in ml_needed: bnp = os.path.basename(ml_pkg) - pkgwriter.write(bnp[0] + '/' + bnp + '\n') - koji.ensuredir(os.path.join(self.repodir, bnp[0])) - os.symlink(ml_pkg, os.path.join(self.repodir, bnp[0], bnp)) + pkgwriter.write(bnp[0].lower() + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnp[0].lower())) + os.symlink(ml_pkg, os.path.join(self.repodir, bnp[0].lower(), bnp)) self.keypaths[bnp] = ml_pkg @@ -5071,7 +5081,7 @@ enabled=1 # it is possible to see the results of other committed transactions rpms = [] builddirs = {} - for a in (arch, 'noarch'): + for a in self.compat[arch] + ('noarch',): rpm_iter, builds = self.session.listTaggedRPMS(tag_id, event=opts['event'], arch=a, latest=opts['latest'], inherit=opts['inherit'], rpmsigs=True) @@ -5114,10 +5124,11 @@ enabled=1 fs_missing.add(pkgpath) else: bnp = os.path.basename(pkgpath) - pkglist.write(bnp[0] + '/' + bnp + '\n') - koji.ensuredir(os.path.join(self.repodir, bnp[0])) + pkglist.write(bnp[0].lower() + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnp[0].lower())) self.keypaths[bnp] = pkgpath - os.symlink(pkgpath, os.path.join(self.repodir, bnp[0], bnp)) + os.symlink(pkgpath, os.path.join(self.repodir, bnp[0].lower(), + bnp)) pkglist.close() if len(fs_missing) > 0: raise koji.GenericError('Packages missing from the filesystem:\n' + From 710276c048ed386dcc5b9b53956d38ff76310d95 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:14 +0000 Subject: [PATCH 68/70] make the src arch work in signed repos --- diff --git a/builder/kojid b/builder/kojid index ce3ff22..ce53832 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4900,6 +4900,7 @@ class createSignedRepoTask(CreaterepoTask): "arm": ("arm", "armv4l", "armv4tl", "armv5tel", "armv5tejl", "armv6l", "armv7l", "noarch"), "armhfp": ("armv7hl", "armv7hnl", "noarch"), "aarch64": ("aarch64", "noarch"), + "src": ("src",) } biarch = {"ppc": "ppc64", "x86_64": "i386", "sparc": From 1265a74d8a8dee23879c2d0e8a89c889f2bee645 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:12:14 +0000 Subject: [PATCH 69/70] lowercase directories in signed repos --- diff --git a/builder/kojid b/builder/kojid index ce53832..3757604 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5052,7 +5052,7 @@ enabled=1 ml_needed = set() for f in yumbase.tsInfo.getMembers(): bnp = os.path.basename(f.po.localPkg()) - dep_path = os.path.join(mldir, bnp[0], bnp) + dep_path = os.path.join(mldir, bnp[0].lower(), bnp) ml_needed.add(dep_path) self.logger.debug("added %s" % dep_path) if not os.path.exists(dep_path): @@ -5070,9 +5070,10 @@ enabled=1 pkgwriter = open(self.pkglist, 'a') for ml_pkg in ml_needed: bnp = os.path.basename(ml_pkg) - pkgwriter.write(bnp[0].lower() + '/' + bnp + '\n') - koji.ensuredir(os.path.join(self.repodir, bnp[0].lower())) - os.symlink(ml_pkg, os.path.join(self.repodir, bnp[0].lower(), bnp)) + bnplet = bnp[0].lower() + pkgwriter.write(bnplet + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnplet)) + os.symlink(ml_pkg, os.path.join(self.repodir, bnplet, bnp)) self.keypaths[bnp] = ml_pkg @@ -5125,11 +5126,11 @@ enabled=1 fs_missing.add(pkgpath) else: bnp = os.path.basename(pkgpath) - pkglist.write(bnp[0].lower() + '/' + bnp + '\n') - koji.ensuredir(os.path.join(self.repodir, bnp[0].lower())) + bnplet = bnp[0].lower() + pkglist.write(bnplet + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnplet)) self.keypaths[bnp] = pkgpath - os.symlink(pkgpath, os.path.join(self.repodir, bnp[0].lower(), - bnp)) + os.symlink(pkgpath, os.path.join(self.repodir, bnplet, bnp)) pkglist.close() if len(fs_missing) > 0: raise koji.GenericError('Packages missing from the filesystem:\n' + diff --git a/hub/kojihub.py b/hub/kojihub.py index ed111aa..00bd03e 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -11826,13 +11826,14 @@ class HostExports(object): pkg = os.path.basename(pkg.strip()) rpmpath = fullpaths[pkg] bnp = os.path.basename(rpmpath) - koji.ensuredir(os.path.join(archdir, bnp[0])) + bnplet = bnp[0].lower() + koji.ensuredir(os.path.join(archdir, bnplet)) try: - os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) + os.link(rpmpath, os.path.join(archdir, bnplet, bnp)) except OSError, ose: if ose.errno == 18: shutil.copy2( - rpmpath, os.path.join(archdir, bnp[0], bnp)) + rpmpath, os.path.join(archdir, bnplet, bnp)) else: raise ose os.unlink(src) From 23bec7763a1573ea27f353e22d2c30d66895adb7 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Jul 20 2016 20:39:44 +0000 Subject: [PATCH 70/70] accidentally wiped out changes in conflict resolution --- diff --git a/Makefile b/Makefile index 709400d..9a0bd34 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,10 @@ git-clean: @git clean -d -q -x test: - PYTHONPATH=hub/. nosetests --with-coverage --cover-package . + coverage erase + PYTHONPATH=hub/.:plugins/hub/. nosetests --with-coverage --cover-package . + coverage html + @echo Coverage report in htmlcov/index.html subdirs: for d in $(SUBDIRS); do make -C $$d; [ $$? = 0 ] || exit 1; done