From 9bcce9dc3e678ecb485fab565d1512ad2f1f2ecb Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Oct 06 2017 21:16:03 +0000 Subject: pull in core volume policy functions from earlier branches --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 2391488..91d18f6 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -51,6 +51,7 @@ import subprocess import sys import tarfile import tempfile +import traceback import time import types import xmlrpclib @@ -1528,6 +1529,7 @@ def _direct_tag_build(tag, build, user, force=False): insert.execute() koji.plugin.run_callbacks('postTag', tag=tag, build=build, user=user, force=force) + def _untag_build(tag, build, user_id=None, strict=True, force=False): """Untag a build @@ -4709,6 +4711,11 @@ def change_build_volume(build, volume, strict=True): context.session.assertPerm('admin') volinfo = lookup_name('volume', volume, strict=True) binfo = get_build(build, strict=True) + _set_build_volume(binfo, volinfo, strict) + + +def _set_build_volume(binfo, volinfo, strict=True): + """Move a build to a different storage volume""" if binfo['volume_id'] == volinfo['id']: if strict: raise koji.GenericError("Build %(nvr)s already on volume %(volume_name)s" % binfo) @@ -4785,6 +4792,57 @@ def change_build_volume(build, volume, strict=True): koji.plugin.run_callbacks('postBuildStateChange', attribute='volume_id', old=old_binfo['volume_id'], new=volinfo['id'], info=binfo) +def check_volume_policy(data, strict=False): + """Check volume policy for the given data + + If strict is True, raises exception on bad policies or no matches + Returns volume info, or None if no match + """ + result = None + try: + ruleset = context.policy.get('volume') + result = ruleset.apply(data) + except Exception: + logger.error('Volume policy error') + if strict: + raise + tb_str = ''.join(traceback.format_exception(*sys.exc_info())) + logger.debug(tb_str) + if result is None: + if strict: + raise koji.GenericError('No volume policy match') + logger.warn('No volume policy match') + return None + logger.debug('Volume policy returned %s', result) + vol = lookup_name('volume', result) + if not vol: + if strict: + raise koji.GenericError("Policy returned invalid volume: %s" % result) + logger.error('Volume policy returned unknown volume %s', result) + return None + return vol + + +def apply_volume_policy(build, strict=False): + """Apply volume policy, moving build as needed + + build should be the buildinfo returned by get_build() + + The strict options determines what happens in the case of a bad policy. + If strict is True, and exception will be raised. Otherwise, the existing + volume we be retained (or DEFAULT will be used if the build has no volume) + """ + policy_data = {'build': build} + volume = check_volume_policy(policy_data, strict=strict) + if volume is None: + # just leave the build where it is + return + if build['volume_id'] == volume['id']: + # nothing to do + return + _set_build_volume(build, volume, strict=True) + + def new_build(data): """insert a new build entry""" @@ -5135,6 +5193,7 @@ class CG_Importer(object): self.prep_outputs() self.assert_policy() + self.set_volume() koji.plugin.run_callbacks('preImport', type='cg', metadata=metadata, directory=directory) @@ -5206,6 +5265,23 @@ class CG_Importer(object): assert_policy('cg_import', policy_data) + def set_volume(self): + """Use policy to determine what the volume should be""" + # we have to be careful and provide sufficient data + policy_data = { + 'build': self.buildinfo, + 'package': self.buildinfo['name'], + 'source': self.buildinfo['source'], + 'cgs': self.cgs, + 'volume': 'DEFAULT', # ??? + 'cg_import': True, + } + vol = check_volume_policy(policy_data, strict=False) + if vol: + self.buildinfo['volume_id'] = vol['id'] + self.buildinfo['volume_name'] = vol['name'] + + def prep_build(self): metadata = self.metadata buildinfo = get_build(metadata['build'], strict=False) @@ -7859,29 +7935,51 @@ def policy_get_pkg(data): raise koji.GenericError("policy requires package data") -def policy_get_cgs(data): +def policy_get_brs(data): """Determine content generators from policy data""" - if 'build' not in data: - raise koji.GenericError("policy requires build data") - binfo = get_build(data['build'], strict=True) + if 'buildroots' in data: + return set(data['buildroots']) + elif 'build' in data: + binfo = get_build(data['build'], strict=True) + rpm_brs = [r['buildroot_id'] for r in list_rpms(buildID=binfo['id'])] + archive_brs = [a['buildroot_id'] for a in list_archives(buildID=binfo['id'])] + return set(rpm_brs + archive_brs) + else: + return set() - # first get buildroots used - rpm_brs = [r['buildroot_id'] for r in list_rpms(buildID=binfo['id'])] - archive_brs = [a['buildroot_id'] for a in list_archives(buildID=binfo['id'])] +def policy_get_cgs(data): # pull cg info out # note that br_id will be None if a component had no buildroot cgs = set() - for br_id in set(rpm_brs + archive_brs): + for br_id in policy_get_brs(data): if br_id is None: cgs.add(None) else: cgs.add(get_buildroot(br_id, strict=True)['cg_name']) - return cgs +def policy_get_build_tags(data): + # pull cg info out + # note that br_id will be None if a component had no buildroot + if 'build_tag' in data: + return [get_tag(data['build_tag'], strict=True)['name']] + elif 'build_tags' in data: + return [get_tag(t, strict=True)['name'] for t in data['build_tags']] + elif 'build' in data: + tags = set() + for br_id in policy_get_brs(data): + if br_id is None: + tags.add(None) + else: + tags.add(get_buildroot(br_id, strict=True)['tag_name']) + return tags + else: + return [] + + class NewPackageTest(koji.policy.BaseSimpleTest): """Checks to see if a package exists yet""" name = 'is_new_package' @@ -7996,6 +8094,8 @@ class HasTagTest(koji.policy.BaseSimpleTest): """Check to see if build (currently) has a given tag""" name = 'hastag' def run(self, data): + if 'build' not in data: + return False tags = list_tags(build=data['build']) #True if any of these tags match any of the patterns args = self.str.split()[1:] @@ -8015,8 +8115,9 @@ class SkipTagTest(koji.policy.BaseSimpleTest): def run(self, data): return bool(data.get('skip_tag')) + class BuildTagTest(koji.policy.BaseSimpleTest): - """Check the build tag of the build + """Check the build tag(s) of the build If build_tag is not provided in policy data, it is determined by the buildroots of the component rpms @@ -8024,37 +8125,15 @@ class BuildTagTest(koji.policy.BaseSimpleTest): name = 'buildtag' def run(self, data): args = self.str.split()[1:] - if 'build_tag' in data: - tagname = get_tag(data['build_tag'], strict=True)['name'] - for pattern in args: - if fnmatch.fnmatch(tagname, pattern): - return True - #else - return False - elif 'build' in data: - #determine build tag from buildroots - #in theory, we should find only one unique build tag - #it is possible that some rpms could have been imported later and hence - #not have a buildroot. - #or if the entire build was imported, there will be no buildroots - rpms = context.handlers.call('listRPMs', buildID=data['build']) - archives = list_archives(buildID=data['build']) - br_list = [r['buildroot_id'] for r in rpms] - br_list.extend([a['buildroot_id'] for a in archives]) - for br_id in br_list: - if br_id is None: - continue - tagname = get_buildroot(br_id)['tag_name'] - if tagname is None: - # content generator buildroots might not have tag info - continue - for pattern in args: - if fnmatch.fnmatch(tagname, pattern): - return True - #otherwise... - return False - else: - return False + for tagname in policy_get_build_tags(data): + if tagname is None: + # content generator buildroots might not have tag info + continue + if multi_fnmatch(tagname, args): + return True + #otherwise... + return False + class ImportedTest(koji.policy.BaseSimpleTest): """Check if any part of a build was imported diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index 056680a..7809c4c 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -526,6 +526,9 @@ _default_policies = { 'cg_import': ''' all :: allow ''', + 'volume': ''' + all :: DEFAULT + ''', } def get_policy(opts, plugins): diff --git a/koji/policy.py b/koji/policy.py index ec2e119..0842d84 100644 --- a/koji/policy.py +++ b/koji/policy.py @@ -18,6 +18,8 @@ # Mike McLean import fnmatch +import logging + import koji @@ -180,6 +182,7 @@ class SimpleRuleSet(object): self.rules = self.parse_rules(rules) self.lastrule = None self.lastaction = None + self.logger = logging.getLogger('koji.policy') def parse_rules(self, lines): """Parse rules into a ruleset data structure @@ -297,7 +300,9 @@ class SimpleRuleSet(object): self.lastrule = [] value = False for test in tests: - if not test.run(data): + check = test.run(data) + self.logger.debug("%s -> %s", test, check) + if not check: break else: #all tests in current rule passed @@ -307,6 +312,7 @@ class SimpleRuleSet(object): if value: self.lastrule.append([tests, negate]) if isinstance(action, list): + self.logger.debug("matched: entering subrule") # action is a list of subrules ret = self._apply(action, data) if ret is not None: @@ -314,12 +320,15 @@ class SimpleRuleSet(object): # if ret is None, then none of the subrules matched, # so we keep going else: + self.logger.debug("matched: action=%s", action) return action return None def apply(self, data): + self.logger.debug("policy start") self.lastrule = [] self.lastaction = self._apply(self.ruleset, data, top=True) + self.logger.debug("policy done") return self.lastaction def last_rule(self): diff --git a/tests/test_hub/test_policy_tests.py b/tests/test_hub/test_policy_tests.py new file mode 100644 index 0000000..64309bc --- /dev/null +++ b/tests/test_hub/test_policy_tests.py @@ -0,0 +1,223 @@ +import mock +import unittest + +import koji +import kojihub + + +class TestBasicTests(unittest.TestCase): + + def test_operation_test(self): + obj = kojihub.OperationTest('operation foo*') + self.assertFalse(obj.run({'operation': 'FOOBAR'})) + self.assertTrue(obj.run({'operation': 'foobar'})) + + @mock.patch('kojihub.policy_get_pkg') + def test_package_test(self, policy_get_pkg): + obj = kojihub.PackageTest('package foo*') + policy_get_pkg.return_value = {'name': 'mypackage'} + self.assertFalse(obj.run({})) + policy_get_pkg.return_value = {'name': 'foobar'} + self.assertTrue(obj.run({})) + + @mock.patch('kojihub.policy_get_pkg') + def test_new_package_test(self, policy_get_pkg): + obj = kojihub.NewPackageTest('is_new_package') + policy_get_pkg.return_value = {'name': 'mypackage', 'id': 42} + self.assertFalse(obj.run({})) + policy_get_pkg.return_value = {'name': 'foobar', 'id': None} + self.assertTrue(obj.run({})) + + def test_skip_tag_test(self): + obj = kojihub.SkipTagTest('skip_tag') + data = {'skip_tag': True} + self.assertTrue(obj.run(data)) + data = {'skip_tag': False} + self.assertFalse(obj.run(data)) + data = {'skip_tag': None} + self.assertFalse(obj.run(data)) + data = {} + self.assertFalse(obj.run(data)) + + +class TestPolicyGetUser(unittest.TestCase): + + def setUp(self): + self.get_user = mock.patch('kojihub.get_user').start() + self.context = mock.patch('kojihub.context').start() + + def tearDown(self): + mock.patch.stopall() + + def test_get_user_specified(self): + self.get_user.return_value = 'USER' + result = kojihub.policy_get_user({'user_id': 42}) + self.assertEqual(result, 'USER') + self.get_user.assert_called_once_with(42) + + def test_get_no_user(self): + self.context.session.logged_in = False + result = kojihub.policy_get_user({}) + self.assertEqual(result, None) + self.get_user.assert_not_called() + + def test_get_logged_in_user(self): + self.context.session.logged_in = True + self.context.session.user_id = 99 + self.get_user.return_value = 'USER' + result = kojihub.policy_get_user({}) + self.assertEqual(result, 'USER') + self.get_user.assert_called_once_with(99) + + def test_get_user_specified_with_login(self): + self.get_user.return_value = 'USER' + self.context.session.logged_in = True + self.context.session.user_id = 99 + result = kojihub.policy_get_user({'user_id': 42}) + self.assertEqual(result, 'USER') + self.get_user.assert_called_once_with(42) + + +class TestPolicyGetCGs(unittest.TestCase): + + def setUp(self): + self.get_build = mock.patch('kojihub.get_build').start() + self.list_rpms = mock.patch('kojihub.list_rpms').start() + self.list_archives = mock.patch('kojihub.list_archives').start() + self.get_buildroot = mock.patch('kojihub.get_buildroot').start() + + def tearDown(self): + mock.patch.stopall() + + def _fakebr(self, br_id, strict): + self.assertEqual(strict, True) + return {'cg_name': self._cgname(br_id)} + + def _cgname(self, br_id): + if br_id is None: + return None + return 'cg for br %s'% br_id + + def test_policy_get_cg_basic(self): + self.get_build.return_value = {'id': 42} + br1 = [1,1,1,2,3,4,5,5] + br2 = [2,2,7,7,8,8,9,9,None] + self.list_rpms.return_value = [{'buildroot_id': n} for n in br1] + self.list_archives.return_value = [{'buildroot_id': n} for n in br2] + self.get_buildroot.side_effect = self._fakebr + # let's see... + result = kojihub.policy_get_cgs({'build': 'NVR'}) + expect = set([self._cgname(n) for n in br1 + br2]) + self.assertEqual(result, expect) + self.list_rpms.called_once_with(buildID=42) + self.list_archives.called_once_with(buildID=42) + self.get_build.called_once_with('NVR', strict=True) + + def test_policy_get_cg_nobuild(self): + result = kojihub.policy_get_cgs({'package': 'foobar'}) + self.get_build.assert_not_called() + self.assertEqual(result, set()) + + +class TestBuildTagTest(unittest.TestCase): + + def setUp(self): + self.get_build = mock.patch('kojihub.get_build').start() + self.get_tag = mock.patch('kojihub.get_tag').start() + self.list_rpms = mock.patch('kojihub.list_rpms').start() + self.list_archives = mock.patch('kojihub.list_archives').start() + self.get_buildroot = mock.patch('kojihub.get_buildroot').start() + + def tearDown(self): + mock.patch.stopall() + + def test_build_tag_given(self): + obj = kojihub.BuildTagTest('buildtag foo*') + data = {'build_tag': 'TAGINFO'} + self.get_tag.return_value = {'name': 'foo-3.0-build'} + self.assertTrue(obj.run(data)) + self.list_rpms.assert_not_called() + self.list_archives.assert_not_called() + self.get_buildroot.assert_not_called() + self.get_tag.assert_called_once_with('TAGINFO', strict=True) + + def test_build_tag_given_alt(self): + obj = kojihub.BuildTagTest('buildtag foo*') + data = {'build_tag': 'TAGINFO'} + self.get_tag.return_value = {'name': 'foo-3.0-build'} + self.assertTrue(obj.run(data)) + self.get_tag.return_value = {'name': 'bar-1.2-build'} + self.assertFalse(obj.run(data)) + + obj = kojihub.BuildTagTest('buildtag foo-3* foo-4* fake-*') + data = {'build_tag': 'TAGINFO', 'build': 'BUILDINFO'} + self.get_tag.return_value = {'name': 'foo-4.0-build'} + self.assertTrue(obj.run(data)) + self.get_tag.return_value = {'name': 'foo-3.0.1-build'} + self.assertTrue(obj.run(data)) + self.get_tag.return_value = {'name': 'fake-0.99-build'} + self.assertTrue(obj.run(data)) + self.get_tag.return_value = {'name': 'foo-2.1'} + self.assertFalse(obj.run(data)) + self.get_tag.return_value = {'name': 'foo-5.5-alt'} + self.assertFalse(obj.run(data)) + self.get_tag.return_value = {'name': 'baz-2-candidate'} + self.assertFalse(obj.run(data)) + + self.list_rpms.assert_not_called() + self.list_archives.assert_not_called() + self.get_buildroot.assert_not_called() + + def _fakebr(self, br_id, strict=None): + return {'tag_name': self._brtag(br_id)} + + def _brtag(self, br_id): + if br_id == '': + return None + return br_id + + def test_build_tag_from_build(self): + # Note: the match is for any buildroot tag + brtags = [None, '', 'a', 'b', 'c', 'd', 'not-foo-5', 'foo-3-build'] + self.list_rpms.return_value = [{'buildroot_id': x} for x in brtags] + self.list_archives.return_value = [{'buildroot_id': x} for x in brtags] + self.get_buildroot.side_effect = self._fakebr + + obj = kojihub.BuildTagTest('buildtag foo-*') + data = {'build': 'BUILDINFO'} + self.assertTrue(obj.run(data)) + + obj = kojihub.BuildTagTest('buildtag bar-*') + data = {'build': 'BUILDINFO'} + self.assertFalse(obj.run(data)) + + self.get_tag.assert_not_called() + + def test_build_tag_no_info(self): + obj = kojihub.BuildTagTest('buildtag foo*') + data = {} + self.assertFalse(obj.run(data)) + self.list_rpms.assert_not_called() + self.list_archives.assert_not_called() + self.get_buildroot.assert_not_called() + + +class TestHasTagTest(unittest.TestCase): + + def setUp(self): + self.list_tags = mock.patch('kojihub.list_tags').start() + + def tearDown(self): + mock.patch.stopall() + + def test_has_tag_simple(self): + obj = kojihub.HasTagTest('hastag *-candidate') + tags = ['foo-1.0', 'foo-2.0', 'foo-3.0-candidate'] + self.list_tags.return_value = [{'name': t} for t in tags] + data = {'build': 'NVR'} + self.assertTrue(obj.run(data)) + self.list_tags.assert_called_once_with(build='NVR') + + # check no match case + self.list_tags.return_value = [] + self.assertFalse(obj.run(data)) diff --git a/www/kojiweb/buildinfo.chtml b/www/kojiweb/buildinfo.chtml index 690883b..cfff038 100644 --- a/www/kojiweb/buildinfo.chtml +++ b/www/kojiweb/buildinfo.chtml @@ -65,6 +65,10 @@ + Volume + $build.volume_name + + Started$util.formatTimeLong($start_time) #if $build.state == $koji.BUILD_STATES.BUILDING