#4045 Plugin taskrepos
Opened a year ago by jcupova. Modified 6 months ago
jcupova/koji issue-3871  into  master

Fix review
Jana Cupova • 6 months ago  
Fix review
Jana Cupova • 6 months ago  
Fix review
Jana Cupova • 6 months ago  
Plugin taskrepos
Jana Cupova • 6 months ago  
@@ -0,0 +1,29 @@ 

+ [taskrepos]

+ # expire_repos = 14

+ # ticketlink = 

+ # sourcecodelink = https://pagure.io/koji/blob/master/f/plugins/builder/taskrepos.py

+ email_template : From: %%(from_addr)s

+     Subject: repo for %%(build)s of %%(nvr)s is available

+     To: %%(recipient)s

+     A yum repository for the %%(build)s of %%(nvr)s (task %%(task_id)s) is available at:

+     %%(repodir)s/

+     %%(old_repodir_msg)s

+ 

+     You can install the rpms locally by putting this .repo file in your /etc/yum.repos.d/ directory:

+     %%(repofile)s

+ 

+     It is necessary to have internal Red Hat CA certificates installed on the system, if

+     you want to use yum repository. You can install required certificates from:

+     * http://hdn.corp.redhat.com/rhel7-csb-stage/repoview/redhat-internal-cert-install.html (rpm)

+     * https://certs.corp.redhat.com/ (cert files)

+ 

+     The full list of repos is:

+     %%(repo_urls)s

+ 

+     The repository will be available for the next %%(repolifetime)s days.

+     Scratch build output will be deleted earlier, based on the Brew scratch build retention policy.

+ 

+     If you found a bug or you wish to stop these emails, please create a ticket:

+     %%(ticketlink)s

+ 

+     Source code at %%(sourcecodelink)s 

\ No newline at end of file

@@ -0,0 +1,289 @@ 

+ # kojid plugin

+ 

+ import os

+ import smtplib

+ import subprocess

+ 

+ import six

+ 

+ import koji

+ from koji.tasks import BaseTaskHandler

+ from koji.util import joinpath

+ 

+ 

+ __all__ = ('TaskReposTask', 'taskRepoNotifications')

+ 

+ CONFIG_FILE = '/etc/kojid/plugins/taskrepos.conf'

+ config = None

+ 

+ 

+ def read_config():

+     global config

+     cp = koji.read_config_files(CONFIG_FILE)

+     config = {

+         'repo_lifetime': 14,  # days

+         'ticketlink': '',

+         'sourcecodelink': '',

+         'email_template': '',

+     }

+ 

+     # expire repos in days

+     if cp.has_option('taskrepos', 'expire_repos'):

+         config['repo_lifetime'] = cp.getint('taskrepos', 'expire_repos')

+ 

+     if cp.has_option('taskrepos', 'ticketlink'):

+         config['ticketlink'] = cp.get('taskrepos', 'ticketlink')

+ 

+     if cp.has_option('taskrepos', 'sourcecodelink'):

+         config['sourcecodelink'] = cp.get('taskrepos', 'sourcecodelink')

+ 

+     if cp.has_option('taskrepos', 'email_template'):

+         config['email_template'] = cp.get('taskrepos', 'email_template')

+ 

+ 

+ class TaskReposTask(BaseTaskHandler):

+ 

+     Methods = ['taskrepos']

+     _taskWeight = 2.0

+ 

+     def gen_repodata(self, rpmdir):

+         koji.ensuredir(rpmdir)

+         cmd = [

+             '/usr/bin/createrepo_c',

+             '--database',

+             '--outputdir=%s' % rpmdir,

+             '--checksum=sha256',

+             '--general-compress-type=gz',

+             '--pkglist=pkglist',

+             '--location-prefix=""',

+             '.',

+         ]

+         self.logger.debug('Running: %s' % " ".join(cmd))

+ 

+         proc = subprocess.run(cmd, cwd=rpmdir, capture_output=True)

+         self.logger.debug(proc.stdout.decode())

+         if proc.stderr:

+             self.logger.debug(proc.stderr.decode())

+         proc.check_returncode()

+ 

+     def merge_arch_repo(self, arch, repodir, srcdir):

+         archdir = joinpath(repodir, arch)

+         self.logger.debug('Creating %s repo under %s' % (arch, archdir))

+         koji.ensuredir(archdir)

+         cmd = [

+             '/usr/bin/mergerepo_c',

+             '--koji',

+             '--database',

+             '--outputdir=%s' % archdir,

+             '--archlist=%s,noarch,src' % arch,

+             '--compress-type=gz',

+         ]

+         for srcrepo in os.listdir(srcdir):

+             cmd.append('--repo=%s' % srcrepo)

+         self.logger.debug('Running: %s' " ".join(cmd))

+         proc = subprocess.run(cmd, cwd=srcdir, capture_output=True)

+         self.logger.debug(proc.stdout.decode())

+         if proc.stderr:

+             self.logger.debug(proc.stderr.decode())

+         proc.check_returncode()

+ 

+     def build_arches(self, rpms):

+         arches = set()

+         for rpm, arch in rpms:

+             if arch == 'src':

+                 continue

+             if arch not in arches:

+                 arches.add(arch)

+         return arches

+ 

+     def create_pkg_list(self, rpms, repodir):

+         remote_pi = koji.PathInfo(topdir='')

+         dest_dirs = []

+         pkglist = []

+         for rpm, arch in rpms:

+             url = joinpath(remote_pi.work(), rpm)

+             url = f'toplink{url}'

+             dest = joinpath(repodir, arch)

+             if dest not in dest_dirs:

+                 dest_dirs.append(dest)

+             pkglist.append(url)

+         for dest_dir in dest_dirs:

+             os.makedirs(dest_dir, exist_ok=True)

+             os.symlink(koji.pathinfo.topdir, joinpath(dest_dir, 'toplink'))

+             with open(joinpath(dest_dir, 'pkglist'), 'w') as dd:

+                 for pkg in pkglist:

+                     dd.write(f'{pkg}\n')

+                     self.logger.info('PkgLINE %s' % pkg)

+         return dest_dirs

+ 

+     def create_repos(self, tasksinfo):

+         if len(tasksinfo) != 0:

+             builds = self.session.listBuilds(taskID=tasksinfo[0]['parent'])

+             if builds:

+                 repotype = 'official'

+             else:

+                 repotype = 'scratch'

+         info = None

+         rpms = []

+         srpms = []

+         results = []

+         for taskinfo in tasksinfo:

+             task_result = self.session.getTaskResult(taskinfo['id'])

+             for tr_rpm in task_result['rpms']:

+                 results.append([tr_rpm, taskinfo['arch']])

+             if task_result['srpms'] != []:

+                 srpms.extend([task_result['srpms'][0], taskinfo['arch']])

+                 srpmname = os.path.basename(task_result['srpms'][0])

+                 info = koji.parse_NVRA(srpmname)

+             if info is None:

+                 raise koji.GenericError("SRPM is missing.")

+             if results != []:

+                 rpms.extend(results)

+         repodir = joinpath(self.workdir, info['name'], info['version'], info['release'])

+         repodir_nvr = joinpath(repotype, info['name'], info['version'], info['release'])

+         nvr = '%s-%s-%s' % (info['name'], info['version'], info['release'])

+ 

+         koji.ensuredir(repodir)

+         self.create_pkg_list(rpms, repodir)

+         rpms.append(srpms)

+         for arch in self.build_arches(rpms):

+             self.gen_repodata(joinpath(repodir, arch))

+         for arch in self.build_arches(rpms):

+             self.merge_arch_repo(arch, repodir, repodir)

+         return repodir, repodir_nvr, nvr

+ 

+     def write_repo_file(self, taskinfo, buildarch_subtasks, repodir, nvr):

+         if len(taskinfo['request']) > 2:

+             for tr in taskinfo['request']:

+                 if isinstance(tr, dict):

+                     scratch = tr.get('scratch', False)

+         else:

+             scratch = False

+         baseurl = joinpath(self.options.topurl, 'repos-tasks', 'tasks', str(self.id % 10000),

+                            str(self.id))

+         baseurls = []

+         for subtaskinfo in buildarch_subtasks:

+             if 'noarch' == subtaskinfo["label"]:

+                 baseurls.append('%s/noarch/' % baseurl)

+             else:

+                 baseurls.append('%s/%s/' % (baseurl, '$basearch'))

+         repo_file_name = nvr

+         repo_name = 'brew-task-repo-%s' % nvr.replace('+', '_')

+         build = 'build'

+         if scratch:

+             repo_file_name += '-scratch'

+             repo_name += '-scratch'

+             build = 'scratch build'

+         repo_file_name += '.repo'

+         repo_file = '%s/%s' % (repodir, repo_file_name)

+ 

+         with koji._open_text_file(repo_file, 'w') as repo_fd:

+             for baseurl in baseurls:

+                 repo_fd.write(

+                     """[%s]

+ name=Repo for Brew %s of %s

+ enabled=1

+ gpgcheck=0

+ baseurl=%s

+ module_hotfixes=1

+ """ % (repo_name, build, nvr, baseurl)

+                 )

+         self.logger.debug('Wrote repo file to %s' % repo_file)

+         return baseurl

+ 

+     def upload_repo(self, repodir):

+         repo_files = []

+         repos = []

+         repopath_part = '/'.join(repodir.split('/')[-4:])

+         for dirpath, dirs, files in os.walk(repodir):

+             relrepodir = os.path.relpath(dirpath, repodir)

+             for filename in files:

+                 path = "%s/%s" % (dirpath, filename)

+                 if os.path.islink(path):

+                     continue

+                 relpath = "%s/%s" % (relrepodir, filename)

+                 localpath = '%s/%s' % (repodir, relpath)

+                 reldir = os.path.dirname(relpath)

+                 if reldir:

+                     uploadpath = "%s/%s" % (repopath_part, reldir)

+                     fn = os.path.basename(relpath)

+                 else:

+                     uploadpath = repopath_part

+                     fn = relpath

+                 self.session.uploadWrapper(localpath, uploadpath, fn)

+                 repos.append([uploadpath, fn])

+                 repo_files.append(relpath)

+         return repo_files, repos, repopath_part

+         #return repopath_part

+ 

+     def handler(self, task_id, task_link=False):

+         read_config()

+         data = {}

+         if not isinstance(task_id, int):

+             task_id = int(task_id)

+         taskinfo = self.session.getTaskInfo(task_id, request=True)

+         if not taskinfo:

+             raise koji.BuildError('Invalid task ID: %s' % task_id)

+         if taskinfo['method'] != 'build':

+             raise koji.BuildError('%s is not a build task' % task_id)

+         if taskinfo['state'] != 2:

+             raise koji.BuildError('task %s has not completed successfully' % task_id)

+         policy_data = {

+             'user_id': taskinfo['owner'],

+         }

+         self.session.host.assertPolicy('taskrepos', policy_data)

+         buildarch_subtasks = [subtask for subtask in self.session.getTaskChildren(task_id)

+                               if subtask['method'] == 'buildArch']

+         repodir, repodir_nvr, data['nvr'] = self.create_repos(buildarch_subtasks)

+         self.logger.debug('Repos for task %s created under %s' % (task_id, repodir))

+         data['baseurl'] = self.write_repo_file(taskinfo, buildarch_subtasks, repodir, data['nvr'])

+         repo_files, repos, repopath_part = self.upload_repo(repodir)

+         data['owner'] = self.session.getUser(taskinfo['owner'])['name']

+         data['repo_urls'], uploadpath, data['old_repodir'] = self.session.host.taskRepoDone(

+             self.id, repos, repodir_nvr, self.options.topurl, task_link)

+         data['repodir'] = joinpath(koji.pathinfo.topdir, 'repos-tasks', repodir_nvr)

+         self.session.host.taskRepoNotifications(self.id, data)

+         return [uploadpath, repo_files]

+ 

+ 

+ class TaskRepoNotifications(BaseTaskHandler):

+     Methods = ['taskRepoNotifications']

+     _taskWeight = 0.1

+ 

+     def handler(self, recipient, data):

+         read_config()

+         data['from_addr'] = self.options.from_addr

+         repo_urls = data['repo_urls'].copy()

+         data['repo_urls'] = ''

+         for rf in repo_urls:

+             if '.repo' in rf:

+                 data['repofile'] = rf

+             else:

+                 data['repo_urls'] += '%s\n' % rf

+ 

+         if data['old_repodir']:

+             data['old_repodir_msg'] = (f"\nThis is latest version of repo. When you would like"

+                                        f" to use original version of repo, please use "

+                                        f"{data['old_repodir']}\n")

+         else:

+             data['old_repodir_msg'] = ''

+         data['ticketlink'] = config['ticketlink']

+         data['sourcecodelink'] = config['sourcecodelink']

+         data['repolifetime'] = config['repo_lifetime']

+         data['build'] = '-'.join(data['nvr'].split('-')[:-2])

+         data['recipient'] = recipient

+         message = config['email_template'] % data

+ 

+         # ensure message is in UTF-8

+         message = koji.fixEncoding(message)

+         # binary for python3

+         if six.PY3:

+             message = message.encode('utf8')

+         server = smtplib.SMTP(self.options.smtphost)

+         if self.options.smtp_user is not None and self.options.smtp_pass is not None:

+             server.login(self.options.smtp_user, self.options.smtp_pass)

+         # server.set_debuglevel(True)

+         server.sendmail(data['from_addr'], recipient, message.decode())

+         server.quit()

+ 

+         return 'sent notification of taskrepo %s to: %s' % (self.id, recipient)

@@ -0,0 +1,91 @@ 

+ # koji hub plugin

+ # There is a kojid plugin that goes with this hub plugin. The kojid builder

+ # plugin has a config file.  This hub plugin has no config file.

+ 

+ 

+ import logging

+ import os

+ 

+ import koji

+ import kojihub

+ from koji.context import context

+ from koji.plugin import export_in, callback

+ from koji.util import joinpath, safer_move, rmtree

+ 

+ __all__ = ('taskrepos', 'taskRepoNotifications', 'taskRepoDone')

+ 

+ 

+ logger = logging.getLogger('koji.hub')

+ 

+ 

+ @export_in('host')

+ def taskRepoDone(task_id, repos, repodir_nvr, topurl, task_link):

+     repo_urls = []

+     old_repodir = None

+     nvr_dir = '/'.join(repodir_nvr.split('/')[1:]) + '/'

+     for rp, fn in repos:

+         src = joinpath(koji.pathinfo.work(), rp, fn).replace('/.', '/')

+         task_dir = joinpath(koji.pathinfo.topdir, 'repos-tasks', 'tasks', str(task_id % 10000),

+                             str(task_id), rp.split(nvr_dir)[1]).replace('/.', '/')

+         taskfile = joinpath(task_dir, fn).replace('/.', '/')

+         koji.ensuredir(task_dir)

+         if not os.path.exists(src):

+             raise koji.GenericError("uploaded file missing: %s" % src)

+         safer_move(src, taskfile)

+         repo_urls.append(taskfile)

+ 

+     if task_link:

+         task_dir_with_id = joinpath(

+             koji.pathinfo.topdir, 'repos-tasks', 'tasks', str(task_id % 10000), str(task_id))

+         reponvr_dir = joinpath(koji.pathinfo.topdir, 'repos-tasks', repodir_nvr)

+         koji.ensuredir(reponvr_dir)

+         if os.path.islink(reponvr_dir) or os.path.isfile(reponvr_dir):

+             old_repodir = os.readlink(reponvr_dir)

+             os.remove(reponvr_dir)

+         elif os.path.isdir(reponvr_dir):

+             rmtree(reponvr_dir)

+         os.symlink(task_dir_with_id, reponvr_dir)

+ 

+     for i in range(0, len(repo_urls)):

+         if task_link:

+             repo_urls[i] = repo_urls[i].replace(

+                 joinpath(koji.pathinfo.topdir, 'repos-tasks', 'tasks', str(task_id % 10000),

+                          str(task_id)), reponvr_dir)

+         repo_urls[i] = repo_urls[i].replace(koji.pathinfo.topdir, topurl)

+ 

+     if task_link:

+         files_dir = reponvr_dir

+     else:

+         files_dir = joinpath(

+             koji.pathinfo.topdir, 'repos-tasks', 'tasks', str(task_id % 10000), str(task_id))

+ 

+     koji.util.rmtree(joinpath(

+         koji.pathinfo.work(), '/'.join(rp.split(str(task_id))[:1]), str(task_id)))

+     return repo_urls, files_dir, old_repodir

+     #return files_dir, old_repodir

+     #return repo_urls

+ 

+ 

+ @callback('postTaskStateChange')

+ def taskrepos(cbtype, *args, **kws):

+     if kws['attribute'] != 'state':

+         return

+     task_id = kws['info']['id']

+     task = kojihub.Task(task_id)

+     taskinfo = task.getInfo()

+     if taskinfo['method'] != 'build' or taskinfo['state'] != koji.TASK_STATES['CLOSED']:

+         return

+     task_link = True

+ 

+     kojihub.make_task('taskrepos', [task_id, task_link])

+ 

+ 

+ @export_in('host')

+ def taskRepoNotifications(task_id, data):

+     if context.opts.get('DisableNotifications'):

+         return

+     email_domain = context.opts['EmailDomain']

+     email = f"{data['owner']}@{email_domain}"

+ 

+     data['task_id'] = task_id

+     kojihub.make_task('taskRepoNotifications', [email, data])

@@ -0,0 +1,202 @@ 

+ import shutil

+ import tempfile

+ import unittest

+ 

+ import mock

+ from plugins.builder import taskrepos

+ 

+ import koji

+ 

+ 

+ class TestTaskReposTask(unittest.TestCase):

+ 

+     def setUp(self):

+         self.maxDiff = None

+         self.session = mock.MagicMock()

+         self.options = mock.MagicMock()

+         koji.ensuredir = mock.MagicMock()

+         self.workdir = tempfile.mkdtemp()

+         self.task = taskrepos.TaskReposTask(

+             123, 'taskrepos', {}, self.session, self.options, self.workdir)

+         self.os_symlink = mock.patch('os.symlink').start()

+         self.os_makedir = mock.patch('os.makedirs').start()

+ 

+     def tearDown(self):

+         shutil.rmtree(self.workdir)

+         mock.patch.stopall()

+ 

+     def test_build_arches(self):

+         rpms = [['rpm1', 'arch1'], ['rpm2', 'arch2'], ['rpm3', 'arch1']]

+         rv = self.task.build_arches(rpms)

+         self.assertEqual(rv, {'arch1', 'arch2'})

+ 

+     @mock.patch("builtins.open", new_callable=mock.mock_open, read_data="data")

+     def test_create_repo(self, builtins_open):

+         taskinfo = [{'parent': 2, 'id': 5, 'arch': 'arch1'},

+                     {'parent': 2, 'id': 6, 'arch': 'arch2'}]

+         self.session.getTaskResult.side_effect = [{'srpms': ['tasks/5/5/repo_-1.1-11.src.rpm'],

+                                                    'rpms': ['tasks/5/5/rpm-1.1-11.arch1.rpm',

+                                                             'tasks/5/5/rpm-1.2-11.arch1.rpm']},

+                                                   {'srpms': ['tasks/6/6/repo-1.1-11.src.rpm'],

+                                                    'rpms': ['tasks/6/6/rpm-1.1-11.arch2.rpm',

+                                                             'tasks/6/6/rpm-1.2-11.arch2.rpm']}]

+         self.task.build_arches = mock.MagicMock()

+         self.task.build_arches.side_effect = [['arch1', 'arch2'], ['arch1', 'arch2']]

+         self.task.gen_repodata = mock.MagicMock()

+         self.task.gen_repodata.return_value = None

+         self.task.merge_arch_repo = mock.MagicMock()

+         self.task.merge_arch_repo.return_value = None

+         rv = self.task.create_repos(taskinfo)

+         self.assertEqual(rv,

+                          (f'{self.workdir}/repo/1.1/11', 'official/repo/1.1/11', 'repo-1.1-11'))

+ 

+     @mock.patch("builtins.open", new_callable=mock.mock_open, read_data="data")

+     def test_create_pkg_list(self, builtins_open):

+         rpms = [['rpm1', 'arch1'], ['rpm2', 'arch2'], ['rpm3', 'arch1']]

+         rv = self.task.create_pkg_list(rpms, 'path/to/repodir')

+         self.assertEqual(rv, ['path/to/repodir/arch1', 'path/to/repodir/arch2'])

+ 

+     @mock.patch("builtins.open", new_callable=mock.mock_open, read_data="data")

+     def test_write_repo_file_non_scratch_noarch(self, builtins_open):

+         taskinfo = {'request': [{}, 'build-trg-test'], 'id': 1, 'arch': 'arch1'}

+         buildarch_subtasks = [{'parent': 2, 'id': 5, 'arch': 'arch1', 'label': 'noarch'},

+                               {'parent': 2, 'id': 6, 'arch': 'arch2', 'label': 'noarch'}]

+         repodir = 'path/to/repodir/'

+         nvr = 'rpm-1.1-11'

+         self.options.topurl = 'https://topurl.com'

+         rv = self.task.write_repo_file(taskinfo, buildarch_subtasks, repodir, nvr)

+         self.assertEqual(rv, 'https://topurl.com/repos-tasks/tasks/123/123/noarch/')

+ 

+     @mock.patch("builtins.open", new_callable=mock.mock_open, read_data="data")

+     def test_write_repo_file_scratch_basearch(self, builtins_open):

+         taskinfo = {'request': [{'scratch': True}, 'build-trg-test', 'giturl'],

+                     'id': 1, 'arch': 'arch1'}

+         buildarch_subtasks = [{'parent': 2, 'id': 5, 'arch': 'arch1', 'label': 'arch1'},

+                               {'parent': 2, 'id': 6, 'arch': 'arch1', 'label': 'arch1'}]

+         repodir = 'path/to/repodir/'

+         nvr = 'rpm-1.1-11'

+         self.options.topurl = 'https://topurl.com'

+         rv = self.task.write_repo_file(taskinfo, buildarch_subtasks, repodir, nvr)

+         self.assertEqual(rv, 'https://topurl.com/repos-tasks/tasks/123/123/$basearch/')

+ 

+     @mock.patch('subprocess.run')

+     def test_gen_repodata(self, subprocess_run):

+         mock_stdout = mock.MagicMock()

+         mock_stdout.configure_mock(

+             **{

+                 "stdout.decode.return_value": '{"A": 3}'

+             }

+         )

+         subprocess_run.return_value = mock_stdout

+         self.task.gen_repodata('path/to/repodir/arch')

+ 

+     @mock.patch('subprocess.run')

+     @mock.patch("os.listdir", return_value=['rpm-1.1-11.arch1.rpm', 'rpm-1.2-11.arch1.rpm'])

+     def test_merge_arch_repo_without_srcdir(self, subprocess_run, listdir):

+         arch = 'arch1'

+         repodir = 'path/to/repodir/'

+         srcdir = 'path/to/srcdir/'

+         mock_stdout = mock.MagicMock()

+         mock_stdout.configure_mock(

+             **{

+                 "stdout.decode.return_value": '{"A": 3}'

+             }

+         )

+         subprocess_run.return_value = mock_stdout

+         self.task.merge_arch_repo(arch, repodir, srcdir)

+ 

+     def test_handler_invalid_task_id(self):

+         taskrepos.read_config.return_value = {}

+         self.session.host.assertPolicy = mock.MagicMock()

+         self.session.host.assertPolicy.return_value = True

+         self.session.getTaskInfo.return_value = None

+         with self.assertRaises(koji.GenericError) as ex:

+             self.task.handler(111, True)

+         self.assertEqual(str(ex.exception), "Invalid task ID: 111")

+ 

+     def test_handler_not_build_task(self):

+         taskrepos.read_config.return_value = {}

+         self.session.host.assertPolicy = mock.MagicMock()

+         self.session.host.assertPolicy.return_value = True

+         self.session.getTaskInfo.return_value = {

+             'request': [{}, 'build-trg-test'], 'id': 111, 'arch': 'arch1', 'owner': 'testuser',

+             'method': 'buildArch', 'state': 2}

+         with self.assertRaises(koji.GenericError) as ex:

+             self.task.handler(111, True)

+         self.assertEqual(str(ex.exception), "111 is not a build task")

+ 

+     def test_handler_not_closed_task(self):

+         taskrepos.read_config.return_value = {}

+         self.session.host.assertPolicy = mock.MagicMock()

+         self.session.host.assertPolicy.return_value = True

+         self.session.getTaskInfo.return_value = {

+             'request': [{}, 'build-trg-test'], 'id': 111, 'arch': 'arch1', 'owner': 'testuser',

+             'method': 'build', 'state': 1}

+         with self.assertRaises(koji.GenericError) as ex:

+             self.task.handler(111, True)

+         self.assertEqual(str(ex.exception), "task 111 has not completed successfully")

+ 

+     def test_handler_valid(self):

+         taskrepos.read_config.return_value = {}

+         self.session.host.assertPolicy = mock.MagicMock()

+         self.session.host.assertPolicy.return_value = True

+         self.session.getTaskInfo.return_value = {

+             'request': [{}, 'build-trg-test'], 'id': 111, 'arch': 'arch1', 'owner': 'testuser',

+             'method': 'build', 'state': 2}

+         self.session.getTaskChildren.return_value = [

+             {'id': 112, 'arch': 'arch1', 'owner': 'testuser', 'method': 'buildArch', 'state': 2},

+             {'id': 113, 'arch': 'arch1', 'owner': 'testuser', 'method': 'tagBuild', 'state': 2}]

+         self.task.create_repos = mock.MagicMock()

+         self.task.create_repos.return_value = [

+             'tmp/dir//repo/1.1/11', 'official/repo/1.1/11', 'repo-1.1-11']

+         self.task.write_repo_file = mock.MagicMock()

+         self.task.write_repo_file.return_value = \

+             'https://topurl.com/repos-tasks/tasks/123/123/noarch/'

+         self.task.upload_repo = mock.MagicMock()

+         self.task.upload_repo.return_value = (['../../../dir/file'],

+                                               [['path/to/repodir//../../../dir', 'file']],

+                                               'path/to/repodir/')

+         self.session.host.taskRepoDone.return_value = (

+             ['https://topurl.com/repos-tasks/tasks/123/123/upload_path_1/fn_1'],

+             'topdir/repos-tasks/tasks/123/123', None)

+         rv = self.task.handler(111, True)

+         self.assertEqual(rv, (['topdir/repos-tasks/tasks/123/123', ['../../../dir/file']]))

+ 

+     @mock.patch('os.path.islink')

+     @mock.patch('os.walk')

+     def test_upload_repo(self, os_walk, islink):

+         repodir = 'path/to/repodir/'

+         os_walk.return_value = (('dir', [], ['file']),)

+         islink.return_value = False

+         rv = self.task.upload_repo(repodir)

+         self.assertEqual(rv, (['../../../dir/file'],

+                               [['path/to/repodir//../../../dir', 'file']],

+                               'path/to/repodir/'))

+ 

+ 

+ class TestTaskRepoNotifications(unittest.TestCase):

+ 

+     def setUp(self):

+         self.maxDiff = None

+         self.session = mock.MagicMock()

+         self.options = mock.MagicMock()

+         self.options.from_addr = 'notifuser@domain.com'

+         self.task_notif = taskrepos.TaskRepoNotifications(

+             123, 'taskRepoNotifications', {}, self.session, self.options)

+         self.SMTP = mock.patch('smtplib.SMTP').start()

+ 

+     def tearDown(self):

+         mock.patch.stopall()

+ 

+     def test_notif(self):

+         recipient = 'testuser@domain.com'

+         server = mock.MagicMock()

+         self.SMTP.return_value = server

+         taskrepos.read_config.return_value = {'ticketlink': 'https://ticketlink.com',

+                                               'sourcecodelink': 'https://sourcecodelink.com',

+                                               'repo_lifetime': 14,

+                                               'email_template': 'email_template'}

+         rv = self.task_notif.handler(recipient, {

+             'repo_urls': ['rpm-1.1-11.arch.rpm', 'rpm-1.2-11.arch.rpm', 'url/repo/repofile.repo'],

+             'nvr': 'rpm-1.2-11', 'old_repodir': None})

+         self.assertEqual(rv, 'sent notification of taskrepo 123 to: %s' % recipient)

@@ -0,0 +1,141 @@ 

+ import unittest

+ 

+ import mock

+ from plugins.hub import taskrepos_hub

+ 

+ import koji

+ import kojihub

+ 

+ QP = kojihub.QueryProcessor

+ 

+ 

+ class TestTaskrepos(unittest.TestCase):

+     def getQuery(self, *args, **kwargs):

+         query = QP(*args, **kwargs)

+         query.executeOne = self.query_executeone

+         self.queries.append(query)

+         return query

+ 

+     def setUp(self):

+         self._dml = mock.patch('kojihub.db._dml').start()

+         self.context = mock.patch('plugins.hub.taskrepos_hub.context').start()

+         self.context.session.assertPerm = mock.MagicMock()

+         kojihub.make_task = mock.MagicMock()

+         kojihub.make_task.return_value = 1

+         self.QueryProcessor = mock.patch('kojihub.kojihub.QueryProcessor',

+                                          side_effect=self.getQuery).start()

+         self.queries = []

+         self.query_executeone = mock.MagicMock()

+ 

+     def tearDown(self):

+         mock.patch.stopall()

+ 

+     def test_valid(self):

+         self.query_executeone.return_value = {'method': 'build', 'state': 2}

+         taskrepos_hub.taskrepos('cbtype', 'test', attribute='state', info={'id': 111})

+         kojihub.make_task.assert_called_once_with('taskrepos', [111, True])

+         self.assertEqual(len(self.queries), 1)

+ 

+     def test_attribute_not_state(self):

+         self.query_executeone.return_value = {'method': 'build', 'state': 2}

+         taskrepos_hub.taskrepos('cbtype', 'test', attribute='test', info={'id': 111})

+         kojihub.make_task.assert_not_called()

+         self.assertEqual(len(self.queries), 0)

+ 

+     def test_task_not_build(self):

+         self.query_executeone.return_value = {'method': 'buildArch', 'state': 2}

+         taskrepos_hub.taskrepos('cbtype', 'test', attribute='state', info={'id': 111})

+         kojihub.make_task.assert_not_called()

+         self.assertEqual(len(self.queries), 1)

+ 

+     def test_task_not_closed(self):

+         self.query_executeone.return_value = {'method': 'build', 'state': 1}

+         taskrepos_hub.taskrepos('cbtype', 'test', attribute='state', info={'id': 111})

+         kojihub.make_task.assert_not_called()

+         self.assertEqual(len(self.queries), 1)

+ 

+ 

+ class TestTaskRepoDone(unittest.TestCase):

+     def setUp(self):

+         self.maxDiff = None

+         self.context = mock.patch('plugins.hub.taskrepos_hub.context').start()

+         self.context.session.assertPerm = mock.MagicMock()

+         self.pathinfo_work = mock.patch('koji.pathinfo.work').start()

+         koji.pathinfo.topdir = 'topdir'

+         self.os_symlink = mock.patch('os.symlink').start()

+ 

+     def tearDown(self):

+         mock.patch.stopall()

+ 

+     @mock.patch("koji.util.rmtree")

+     @mock.patch('shutil.move')

+     @mock.patch('koji.ensuredir')

+     @mock.patch('os.mkdir')

+     @mock.patch('os.path.exists')

+     @mock.patch('os.path.isdir')

+     def test_valid_with_task_link(self, mock_isdir, mock_exists, mock_mkdir, ensuredir,

+                                   shutil_move, rmtree):

+         mock_exists.side_effect = [True, False, False]

+         mock_isdir.return_value = True

+         self.pathinfo_work.return_value = '/'

+         rv = taskrepos_hub.taskRepoDone(123, [['repo/dir/nvr/1.1/upload_path_1', 'fn_1']],

+                                         'repo/dir/nvr/1.1', 'https://topurl.com', True)

+         self.assertEqual(rv, (

+             ['https://topurl.com/repos-tasks/repo/dir/nvr/1.1/upload_path_1/fn_1'],

+             'topdir/repos-tasks/repo/dir/nvr/1.1', None))

+ 

+     @mock.patch("koji.util.rmtree")

+     @mock.patch('shutil.move')

+     @mock.patch('koji.ensuredir')

+     @mock.patch('os.mkdir')

+     @mock.patch('os.path.exists')

+     @mock.patch('os.path.isdir')

+     def test_valid_without_task_link(self, mock_isdir, mock_exists, mock_mkdir, ensuredir,

+                                      shutil_move, rmtree):

+         mock_exists.side_effect = [True, False, False]

+         mock_isdir.return_value = True

+         self.pathinfo_work.return_value = '/'

+         rv = taskrepos_hub.taskRepoDone(123, [['repo/dir/nvr/1.1/upload_path_1', 'fn_1']],

+                                         'repo/dir/nvr/1.1', 'https://topurl.com', False)

+         self.assertEqual(rv, (

+             ['https://topurl.com/repos-tasks/tasks/123/123/upload_path_1/fn_1'],

+             'topdir/repos-tasks/tasks/123/123', None))

+ 

+     @mock.patch('koji.ensuredir')

+     @mock.patch('os.path.exists')

+     def test_uploaded_file_missing(self, mock_exists, ensuredir):

+         mock_exists.return_value = False

+         self.pathinfo_work.return_value = '/'

+         with self.assertRaises(koji.GenericError) as ex:

+             taskrepos_hub.taskRepoDone(123, [['repo/dir/nvr/1.1/upload_path_1', 'fn_1']],

+                                        'repo/dir/nvr/1.1', 'https://topurl.com', True)

+         self.assertEqual(str(ex.exception),

+                          "uploaded file missing: /repo/dir/nvr/1.1/upload_path_1/fn_1")

+ 

+ 

+ class TestTaskReposNotifications(unittest.TestCase):

+     def setUp(self):

+         self.context = mock.patch('plugins.hub.taskrepos_hub.context').start()

+         self.context.session.assertPerm = mock.MagicMock()

+         kojihub.make_task = mock.MagicMock()

+         kojihub.make_task.return_value = 1

+         self.context.opts = {'EmailDomain': 'testdomain.com'}

+ 

+     def tearDown(self):

+         mock.patch.stopall()

+ 

+     def test_taskrepoNotification(self):

+         self.context.opts['DisableNotifications'] = False

+         taskrepos_hub.taskRepoNotifications(123, {'owner': 'testuser', 'task_id': 112})

+         kojihub.make_task.assert_called_once_with(

+             'taskRepoNotifications',

+             [

+                 'testuser@testdomain.com',

+                 {'owner': 'testuser', 'task_id': 123}

+             ],

+         )

+ 

+     def test_taskrepoNotification_disabled(self):

+         self.context.opts['DisableNotifications'] = True

+         taskrepos_hub.taskRepoNotifications(123, {'owner': 'testuser', 'task_id': 112})

+         kojihub.make_task.assert_not_called()

Hub:

  • why is in __all__ taskrepos and not taskRepoNotifications?
    Testing?

  • This is not triggered for scratch builds. postTaskStateChange is correct one.
    Only tasks moved to CLOSED state makes sense here. Not every call. In the
    current code, it is triggered even e.g. if build is moved between volumes.
    Further, it needs to be triggered for every buildArch task.

  • repos have to end in /mnt/koji/repos-tasks (s/taskrepos/repos-tasks/
    just for better listing), not in /mnt/work/taskrepos (or maybe in
    /mnt/scratch/repos-tasks - it would be a) easier to automatically
    cleanup, b) more visible that it is just a transient repo) Anyway, thinking
    more about it, /mnt/koji/scratch doesn't look like a good place as it
    would be misleading for non-scratch builds. Check layout in the end of the
    text.

  • baseurl in repo file doesn't point to correct location. a) wrong location b)
    $basearch shouldn't be used if there is only one architecture (it would be
    typically noarch and no user has set it as $basearch)

  • repofile contains lines which starts with whitespaces

  • notification task should receive only .repo files. There is a lost of unused
    arguments now.

  • taskRepoNotifications should be builder-only call (host. namespace,
    probably best via export_in decorator)

Builder:

  • syntax errors: usage of py2.7 incompatible f-strings over all builder plugin.

  • config could be read only once (see save_failed_tree) and should be used
    for both tasks. Use lowercase_names for config items to be consistent
    across all configs.

  • don't use requests directly. There is an koji.util.downloadFile function
    which handles some error types.

  • mirror_rpms is not a good way to do this. Expect (and put is a default
    channel) that task runs in createrepo channel. In such case you'll get
    access to all those files without downloading them. In such case construct
    pi = PathInfo(topdif=self.options.topdir)

  • using mergerepo_c is not safe as only mergerepo could exist at the
    builder. Anyway, it is not that important now, we can revisit it later (or
    document that this plugin needs mergerepo_c installed)

  • create_repos can be simplified to if builds: else part. Scratch
    builds will be found by that and non-scratch rpms can be discovered in the
    same way.

  • using assert is dangerous as it could be turned off by optimization.
    Better to use logging/exceptions.

  • flock part is not needed - no one else touch our work directory. Probably
    an artifact from original code.

  • not sure what rm_contents is trying to solve. What is the issue with
    rmtree usage? Another leftover maybe?

  • baseurl construction is flawed.

  • content_dir is not needed on createrepo builder.

  • link_task_repos muse use relative symlinks, so it will work after
    uploading to the hub (or solved some other way)

  • E-mail template should refer to koji and should be placed in config.

  • recipient check is weird. a) email is always passed through hub creation
    call b) Its length will be always at least 1 "@".

  • Don't feed templates with locals() as it will be part of config and could
    leak some information nobody wanted. Be explicit what is propagated there.

  • NVR is constructed as N.V.R (in create_repos)- it should be always N-V-R.
    In the other direction (taskRepoNotifications) use koji.parse_NVR or
    koji.parse_NVRA if needed. There could be problems with epoch.

Structure of /mnt/koji/repos-tasks should be:

  • official/<name>/<version>/<release>
  • scratch/<user>/<name>/<version>/<release>
  • tasks/<task_id % 10000>/<task_id> here is a suggested difference to original code from <tasks>/<task_id>

1 new commit added

  • Fix review
a year ago

rebased onto 56f3ce855e0cb9d0173ea11dbdec7607456cee4d

a year ago

2 new commits added

  • Fix review
  • Plugin taskrepos
a year ago

It is not checking the version, so it is not needed at all. If command is not present, call will fail anyway with "missing program"-like error.

1 new commit added

  • Fix review
11 months ago

1 new commit added

  • Fix review
10 months ago

missing EOLs - results in one long line and createrepo_c will fail on this.

  • There is a problem with original plugin - if I run scratch build twice with different sources, but same NVR there will be a conflict when linking to scratch repo which doesn't differentiate. We either have to add task_id into the path or update symlink only to the latest version. In such case, notification must contain also the original task repos, not only symlinked ones, so user can switch accordingly.
  • Scratch repo name is missing -scratch suffix (inside .repo file)
  • baseurl inside of .repo is not working in noarch case. ($basearch will be e.g. x86_64)

1 new commit added

  • Fix review
8 months ago

5 new commits added

  • Fix review
  • Fix review
  • Fix review
  • Fix review
  • Plugin taskrepos
8 months ago

5 new commits added

  • Fix review
  • Fix review
  • Fix review
  • Fix review
  • Plugin taskrepos
8 months ago

5 new commits added

  • Fix review
  • Fix review
  • Fix review
  • Fix review
  • Plugin taskrepos
8 months ago

1 new commit added

  • Fix code review
8 months ago

6 new commits added

  • Fix code review
  • Fix review
  • Fix review
  • Fix review
  • Fix review
  • Plugin taskrepos
8 months ago

1 new commit added

  • Fix code review
7 months ago

rebased onto 0823802

7 months ago

There's a lot here, but let me start with what jumps out.

The hub calls should have access controls. Both are clearly intended to be host-only. Both include a task arg, so they should further be limited to the host running that task.

Also, they are inconsistently exported. Both should be under host.*

The naming is inconsistent. Is it "taskRepo" or "tasksRepo".

The recursive_walk function in the hub code needs work, or possibly to be removed entirely. It shouldn't need to recurse on itself since os.walk already traverses the directory tree. This is the point of using os.walk. Furthermore, I am suspicious of this feature needing a full tree traversal like this. It seems like we should have the information to generate these repo urls more decisively.

The configuration management in the builder plugin seems overly complex. Do we really need two separate config files for this plugin?

Inconsistent config naming: sourcecode vs sourcecodelink

Why are we running mergerepo_c? It seems like we should just generate the repo we want and be done with it.

The policy data seems sparse. Granted, it's not clear what we're trying to accomplish with a policy hook here. Users cannot submit these tasks directly, they are automatically generated by (every) completed buildArch task. I suppose we might want to allow filtering this with policy, but I'm not sure if this is the right place to do that.

A fair chunk of this is the notification portion, which mimics existing notifications in Koji. This isn't the best model to emulate, and I wonder if these notifications are actually required.

Also, it would be a good idea to include unit tests

The builder has a parameter namedtask_id_child, but the task passed in is definitely not a child task. They do not have a parent/child relationship.

@mikem I pinged @veruu about notifications and result is, that we should have enabled notifications as default and add some option to config file for disable it. Are you ok with this?

2 new commits added

  • Fix review
  • Fix code review
6 months ago

rebased onto a204491

6 months ago