From a174e5236a6cc93bf1a7080adea40085778eed10 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jun 07 2017 09:20:03 +0000 Subject: push build_tag, session, scratch into callbacks --- diff --git a/builder/kojid b/builder/kojid index 43fd1d9..047cdaf 100755 --- a/builder/kojid +++ b/builder/kojid @@ -899,7 +899,7 @@ class BuildTask(BaseTaskHandler): def getSRPMFromSCM(self, url, build_tag, repo_id): #TODO - allow different ways to get the srpm task_id = self.session.host.subtask(method='buildSRPMFromSCM', - arglist=[url, build_tag, {'repo_id': repo_id}], + arglist=[url, build_tag, {'repo_id': repo_id, 'scratch': self.opts.get('scratch')}], label='srpm', parent=self.id) # wait for subtask to finish @@ -1264,7 +1264,7 @@ class MavenTask(MultiPlatformTask): raise koji.BuildError('no repo for tag %s' % build_tag['name']) build_opts = dslice(opts, ['goals', 'profiles', 'properties', 'envs', 'patches', - 'packages', 'jvm_options', 'maven_options', 'deps'], + 'packages', 'jvm_options', 'maven_options', 'deps', 'scratch'], strict=False) build_opts['repo_id'] = repo_id @@ -1415,11 +1415,12 @@ class BuildMavenTask(BaseBuildTask): uploadpath = self.getUploadDir() - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) # Check out sources from the SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=opts.get('scratch'), srcdir=sourcedir) # zip up pristine sources for auditing purposes @@ -1430,12 +1431,13 @@ class BuildMavenTask(BaseBuildTask): patchlog = self.workdir + '/patches.log' patch_scm = SCM(self.opts.get('patches')) patch_scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=patch_scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=patch_scm.get_info(), build_tag=build_tag) # never try to check out a common/ dir when checking out patches patch_scm.use_common = False patchcheckoutdir = patch_scm.checkout(patchdir, self.session, uploadpath, patchlog) self.run_plugin("postSCMCheckout", scminfo=patch_scm.get_info(), + scratch=opts.get('scratch'), srcdir=patchcheckoutdir) self._zip_dir(patchcheckoutdir, os.path.join(outputdir, 'patches.zip')) @@ -1717,10 +1719,11 @@ class WrapperRPMTask(BaseBuildTask): logfile = os.path.join(self.workdir, 'checkout.log') scmdir = buildroot.rootdir() + '/tmp/scmroot' koji.ensuredir(scmdir) - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) specdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=opts.get('scratch'), srcdir=specdir) spec_template = None @@ -2631,7 +2634,7 @@ class ImageTask(BaseTaskHandler): self.logger.debug("Image buildroot ready: " + broot.rootdir()) return broot - def fetchKickstart(self, broot, ksfile): + def fetchKickstart(self, broot, ksfile, build_tag): """ Retrieve the kickstart file we were given (locally or remotely) and upload it. @@ -2645,6 +2648,7 @@ class ImageTask(BaseTaskHandler): @args: broot: a buildroot object ksfile: path to a kickstart file + build_tag: build tag name @returns: absolute path to the retrieved kickstart file """ scmdir = os.path.join(broot.rootdir(), 'tmp') @@ -2654,10 +2658,11 @@ class ImageTask(BaseTaskHandler): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) logfile = os.path.join(self.workdir, 'checkout.log') - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) scmsrcdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), srcdir=scmsrcdir) kspath = os.path.join(scmsrcdir, ksfile) else: @@ -2832,7 +2837,7 @@ class ApplianceTask(ImageTask): self.opts = opts broot = self.makeImgBuildRoot(build_tag, repo_info, arch, 'appliance-build') - kspath = self.fetchKickstart(broot, ksfile) + kspath = self.fetchKickstart(broot, ksfile, target_info['build_tag_name']) self.readKickstart(kspath, opts) kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts) # Figure out appliance-creator arguments, let it fail if something @@ -2981,7 +2986,7 @@ class LiveCDTask(ImageTask): broot = self.makeImgBuildRoot(build_tag, repo_info, arch, 'livecd-build') - kspath = self.fetchKickstart(broot, ksfile) + kspath = self.fetchKickstart(broot, ksfile, target_info['build_tag_name']) self.readKickstart(kspath, opts) kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts) @@ -3131,7 +3136,7 @@ class LiveMediaTask(ImageTask): broot = self.makeImgBuildRoot(build_tag, repo_info, arch, 'livemedia-build') - kspath = self.fetchKickstart(broot, ksfile) + kspath = self.fetchKickstart(broot, ksfile, target_info['build_tag_name']) self.readKickstart(kspath, opts) kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts) @@ -3249,7 +3254,7 @@ class LiveMediaTask(ImageTask): class OzImageTask(BaseTaskHandler): Methods = [] - def fetchKickstart(self): + def fetchKickstart(self, build_tag): """ Retrieve the kickstart file we were given (locally or remotely) and upload it to the hub. @@ -3260,7 +3265,8 @@ class OzImageTask(BaseTaskHandler): relative path in a remote scm. The user should have passed in an scm url with --ksurl. - @args: None, use self.opts for options + @args: build_tag: build tag name + use self.opts for options @returns: absolute path to the retrieved kickstart file """ @@ -3270,11 +3276,12 @@ class OzImageTask(BaseTaskHandler): scm = SCM(self.opts['ksurl']) scm.assert_allowed(self.options.allowed_scms) logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch) - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), srcdir=scmsrcdir) kspath = os.path.join(scmsrcdir, os.path.basename(ksfile)) else: @@ -3929,7 +3936,7 @@ class BaseImageTask(OzImageTask): self.formats = self._format_deps(opts.get('format')) # First, prepare the kickstart to use the repos we tell it - kspath = self.fetchKickstart() + kspath = self.fetchKickstart(build_tag=target_info['build_tag_name']) ks = self.prepareKickstart(kspath, inst_tree) kskoji = self.writeKickstart(ks, os.path.join(self.workdir, 'koji-%s-%i-base.ks' % @@ -4060,7 +4067,7 @@ class BuildIndirectionImageTask(OzImageTask): # END inefficient base image task method copies - def fetchHubOrSCM(self, filepath, fileurl): + def fetchHubOrSCM(self, filepath, fileurl, build_tag): """ Retrieve a file either from the hub or a remote scm @@ -4080,12 +4087,13 @@ class BuildIndirectionImageTask(OzImageTask): if fileurl: scm = SCM(fileurl) scm.assert_allowed(self.options.allowed_scms) - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) logfile = os.path.join(self.workdir, 'checkout.log') scmsrcdir = scm.checkout(self.workdir, self.session, self.getUploadDir(), logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), srcdir=scmsrcdir) final_path = os.path.join(scmsrcdir, os.path.basename(filepath)) else: @@ -4236,7 +4244,8 @@ class BuildIndirectionImageTask(OzImageTask): raise koji.ApplianceError('The Release may not have a hyphen') indirection_template = self.fetchHubOrSCM(opts.get('indirection_template'), - opts.get('indirection_template_url')) + opts.get('indirection_template_url'), + target_info['build_tag_name']) self.logger.debug('Got indirection template %s' % (indirection_template)) @@ -4427,11 +4436,12 @@ class BuildSRPMFromSCMTask(BaseBuildTask): logfile = self.workdir + '/checkout.log' uploadpath = self.getUploadDir() - self.run_plugin('preSCMCheckout', scminfo=scm.get_info()) + self.run_plugin('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag) # Check out spec file, etc. from SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) self.run_plugin("postSCMCheckout", scminfo=scm.get_info(), + scratch=self.opts.get('scratch'), srcdir=sourcedir) # chown the sourcedir and everything under it to the mockuser # so we can build the srpm as non-root diff --git a/koji/tasks.py b/koji/tasks.py index 3960901..5d7ac98 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -391,6 +391,7 @@ class BaseTaskHandler(object): self.taskinfo = self.session.getTaskInfo(self.id, request=True) taskinfo = self.taskinfo kwargs['taskinfo'] = taskinfo + kwargs['session'] = self.session koji.plugin.run_callbacks(plugin, *args, **kwargs) diff --git a/vm/kojikamid.py b/vm/kojikamid.py index ff38b41..5882c16 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -47,19 +47,20 @@ MANAGER_PORT = 7000 KOJIKAMID = True + ## INSERT kojikamid dup class fakemodule(object): pass -#make parts of the above insert accessible as koji.X + +# make parts of the above insert accessible as koji.X koji = fakemodule() koji.GenericError = GenericError koji.BuildError = BuildError class WindowsBuild(object): - LEADING_CHAR = re.compile('^[^A-Za-z_]') VAR_CHARS = re.compile('[^A-Za-z0-9_]') @@ -67,9 +68,9 @@ class WindowsBuild(object): """Get task info and setup build directory""" self.logger = logging.getLogger('koji.vm') self.server = server - self.task_info = server.getTaskInfo() - self.source_url = self.task_info[0] - self.build_tag = self.task_info[1] + self.info = server.getTaskInfo() + self.source_url = self.info[0] + self.build_tag = self.info[1] if len(info) > 2: self.task_opts = info[2] else: @@ -136,7 +137,7 @@ class WindowsBuild(object): self.logger.info('file %s exists', entry) if errors: raise BuildError('error validating build environment: %s' % \ - ', '.join(errors)) + ', '.join(errors)) def updateClam(self): """update ClamAV virus definitions""" @@ -166,17 +167,17 @@ class WindowsBuild(object): def checkout(self): """Checkout sources, winspec, and patches, and apply patches""" src_scm = SCM(self.source_url) - koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.task_info, scminfo=src_scm.get_info()) + self.server.runCallBacks('preSCMCheckout', scminfo=src_scm.get_info()) self.source_dir = src_scm.checkout(ensuredir(os.path.join(self.workdir, 'source'))) - koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.task_info, + self.server.runCallBacks('postSCMCheckout', scminfo=src_scm.get_info(), srcdir=self.source_dir) self.zipDir(self.source_dir, os.path.join(self.workdir, 'sources.zip')) if 'winspec' in self.task_opts: spec_scm = SCM(self.task_opts['winspec']) - koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.task_info, scminfo=spec_scm.get_info()) + self.server.runCallBacks('preSCMCheckout', scminfo=spec_scm.get_info()) self.spec_dir = spec_scm.checkout(ensuredir(os.path.join(self.workdir, 'spec'))) - koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.task_info, + self.server.runCallBacks('postSCMCheckout', scminfo=spec_scm.get_info(), srcdir=self.spec_dir) self.zipDir(self.spec_dir, os.path.join(self.workdir, 'spec.zip')) @@ -184,9 +185,9 @@ class WindowsBuild(object): self.spec_dir = self.source_dir if 'patches' in self.task_opts: patch_scm = SCM(self.task_opts['patches']) - koji.plugin.run_callbacks('preSCMCheckout', taskinfo=self.task_info, scminfo=patch_scm.get_info()) + self.server.runCallBacks('preSCMCheckout', scminfo=patch_scm.get_info()) self.patches_dir = patch_scm.checkout(ensuredir(os.path.join(self.workdir, 'patches'))) - koji.plugin.run_callbacks('postSCMCheckout', taskinfo=self.task_info, + self.server.runCallBacks('postSCMCheckout', scminfo=patch_scm.get_info(), srcdir=self.patch_dir) self.zipDir(self.patches_dir, os.path.join(self.workdir, 'patches.zip')) @@ -330,7 +331,7 @@ class WindowsBuild(object): # rpms don't have a md5sum in the fileinfo, but check it for everything else if ('md5sum' in fileinfo) and (digest != fileinfo['md5sum']): raise BuildError('md5 checksum validation failed for %s, %s (computed) != %s (provided)' % \ - (destpath, digest, fileinfo['md5sum'])) + (destpath, digest, fileinfo['md5sum'])) self.logger.info('Retrieved %s (%s bytes, md5: %s)', destpath, offset, digest) def fetchBuildReqs(self): @@ -480,7 +481,7 @@ class WindowsBuild(object): self.virusCheck(self.workdir) if errors: raise BuildError('error validating build output: %s' % \ - ', '.join(errors)) + ', '.join(errors)) def virusCheck(self, path): """ensure a path is virus free with ClamAV. path should be absolute""" @@ -511,6 +512,7 @@ class WindowsBuild(object): self.expireBuildroot() return self.gatherResults() + def run(cmd, chdir=None, fatal=False, log=True): global logfd output = '' @@ -540,6 +542,7 @@ def run(cmd, chdir=None, fatal=False, log=True): raise BuildError(msg) return ret, output + def find_net_info(): """ Find the network gateway configured for this VM. @@ -568,6 +571,7 @@ def find_net_info(): gateway = None return macaddr, gateway + def upload_file(server, prefix, path): """upload a single file to the vmd""" logger = logging.getLogger('koji.vm') @@ -588,6 +592,7 @@ def upload_file(server, prefix, path): server.verifyChecksum(path, digest, 'md5') logger.info('Uploaded %s (%s bytes, md5: %s)', destpath, offset, digest) + def get_mgmt_server(): """Get a ServerProxy object we can use to retrieve task info""" logger = logging.getLogger('koji.vm') @@ -606,6 +611,7 @@ def get_mgmt_server(): logger.debug('found task-specific port %s', task_port) return xmlrpclib.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True) + def get_options(): """handle usage and parse options""" usage = """%prog [options] @@ -615,10 +621,12 @@ def get_options(): parser = OptionParser(usage=usage) parser.add_option('-d', '--debug', action='store_true', help='Log debug statements') parser.add_option('-i', '--install', action='store_true', help='Install this daemon as a service', default=False) - parser.add_option('-u', '--uninstall', action='store_true', help='Uninstall this daemon if it was installed previously as a service', default=False) + parser.add_option('-u', '--uninstall', action='store_true', + help='Uninstall this daemon if it was installed previously as a service', default=False) (options, args) = parser.parse_args() return options + def setup_logging(opts): global logfile, logfd logger = logging.getLogger('koji.vm') @@ -633,11 +641,13 @@ def setup_logging(opts): logger.addHandler(handler) return handler + def log_local(msg): tb = ''.join(traceback.format_exception(*sys.exc_info())) sys.stderr.write('%s: %s\n' % (time.ctime(), msg)) sys.stderr.write(tb) + def stream_logs(server, handler, builds): """Stream logs incrementally to the server. The global logfile will always be streamed. @@ -675,6 +685,7 @@ def stream_logs(server, handler, builds): log_local('error uploading %s' % relpath) time.sleep(1) + def fail(server, handler): """do the right thing when a build fails""" global logfile, logfd @@ -701,6 +712,7 @@ def fail(server, handler): logfile = '/tmp/build.log' logfd = None + def main(): prog = os.path.basename(sys.argv[0]) opts = get_options() diff --git a/vm/kojivmd b/vm/kojivmd index 75e3eb9..7609253 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -298,7 +298,7 @@ class WinBuildTask(MultiPlatformTask): if not opts: opts = {} - subopts = koji.util.dslice(opts, ['winspec', 'patches'], + subopts = koji.util.dslice(opts, ['winspec', 'patches', 'scratch'], strict=False) # winspec and patches options are urls # verify the urls before passing them to the VM @@ -531,6 +531,20 @@ class VMExecTask(BaseTaskHandler): """ return self.task_info + def runCallbacks(self, method, **opts): + """ + Run callback plugins. + """ + taskinfo = self.session.getTaskInfo(self.id, request=True) + opts['taskinfo'] = taskinfo + opts['session'] = self.session + if method == 'preSCMCheckout': + opts['build_tag'] = self.task_info.get('build_tag') + elif method == 'postSCMCheckout': + opts['scratch'] = self.task_info.get('scratch') + koji.plugin.run_callbacks(method, **opts) + + def initBuildroot(self, repo_id, platform): """ Create the buildroot object on the hub.