PR#495: osbs: Enable specifying extra repos - pungi

pungi

#495 osbs: Enable specifying extra repos

Merged 8 years ago by ausil. Opened 8 years ago by lsedlar.

lsedlar/pungi osbs-extra-repo into master

osbs: Validate config in tests

Lubomír Sedlář • 8 years ago

e7c8b2a

osbs: Verify the .repo files contain correct URL

Lubomír Sedlář • 8 years ago

61a4c43

osbs: Enable specifying extra repos

Lubomír Sedlář • 8 years ago

814bf44

doc/configuration.rst

file modified

+3 -1

		`@@ -1246,7 +1246,9 @@`
		This includes ``name``, ``version``, ``scratch`` and ``priority``.

		A value for ``yum_repourls`` will be created automatically and point at a
		`- repository in the current compose.`
		`+ repository in the current compose. You can add extra repositories with`
		+ ``repo`` key having a list of urls pointing to ``.repo`` files or
		+ ``repo_from`` as a list of variants in current compose.


		`Example config`

pungi/checks.py

file modified

		`@@ -828,6 +828,8 @@`
		`"version": {"type": "string"},`
		`"scratch": {"type": "boolean"},`
		`"priority": {"type": "number"},`
		`+ "repo": {"$ref": "#/definitions/strings"},`
		`+ "repo_from": {"$ref": "#/definitions/strings"},`
		`},`
		`"required": ["url", "target"]`
		`}`

pungi/phases/osbs.py

file modified

+15 -8

		`@@ -3,6 +3,7 @@`
		`import json`
		`import os`
		`from kobo.threads import ThreadPool, WorkerThread`
		`+ from kobo import shortcuts`

		`from .base import ConfigGuardedPhase, PhaseLoggerMixin`
		`from .. import util`
		`@@ -50,15 +51,15 @@`
		`koji.login()`

		`# Start task`
		`- try:`
		`- source = util.resolve_git_url(config.pop('url'))`
		`- target = config.pop('target')`
		`- except KeyError as exc:`
		`- raise RuntimeError('OSBS: missing config key %s for %s'`
		`- % (exc, variant.uid))`
		`+ source = util.resolve_git_url(config.pop('url'))`
		`+ target = config.pop('target')`
		`priority = config.pop('priority', None)`
		`+ repos = shortcuts.force_list(config.pop('repo', []))`
		`+ compose_repos = [self._get_repo(compose, v)`
		`+ for v in [variant.uid] + shortcuts.force_list(`
		`+ config.pop('repo_from', []))]`

		`- config['yum_repourls'] = [self._get_repo(compose, variant)]`
		`+ config['yum_repourls'] = compose_repos + repos`

		`task_id = koji.koji_proxy.buildContainer(source, target, config,`
		`priority=priority)`
		`@@ -106,11 +107,17 @@`
		`self.pool.metadata.setdefault(`
		`variant.uid, {}).setdefault(arch, []).append(data)`

		`- def _get_repo(self, compose, variant):`
		`+ def _get_repo(self, compose, variant_uid):`
		`"""`
		`Write a .repo file pointing to current variant and return URL to the`
		`file.`
		`"""`
		`+ try:`
		`+ variant = compose.all_variants[variant_uid]`
		`+ except KeyError:`
		`+ raise RuntimeError(`
		`+ 'There is no variant %s to get repo from to pass to OSBS.'`
		`+ % (variant_uid))`
		`os_tree = compose.paths.compose.os_tree('$basearch', variant,`
		`create_dir=False)`
		`repo_file = os.path.join(compose.paths.work.tmp_dir(None, variant),`

tests/test_osbs_phase.py

file modified

+102 -10

		`@@ -8,12 +8,14 @@`
		`import mock`
		`import json`

		`+ import copy`
		`import os`
		`import sys`

		`sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))`

		`from tests import helpers`
		`+ from pungi import checks`
		`from pungi.phases import osbs`


		`@@ -192,10 +194,28 @@`
		`mock.call.koji_proxy.getBuild(54321),`
		`mock.call.koji_proxy.listArchives(54321)])`

		`- def _assertRepoFile(self):`
		`- with open(self.topdir + '/work/global/tmp-Server/compose-rpms-1.repo') as f:`
		`- lines = f.read().split('\n')`
		`- self.assertIn('baseurl=http://root/compose/Server/$baseurl/os', lines)`
		`+ def _assertRepoFile(self, variants=None):`
		`+ variants = variants or ['Server']`
		`+ for variant in variants:`
		`+ with open(self.topdir + '/work/global/tmp-%s/compose-rpms-1.repo' % variant) as f:`
		`+ lines = f.read().split('\n')`
		`+ self.assertIn('baseurl=http://root/compose/%s/$basearch/os' % variant, lines)`
		`+`
		`+ def _assertConfigCorrect(self, cfg):`
		`+ config = copy.deepcopy(self.compose.conf)`
		`+ config['osbs'] = {`
		`+ '^Server$': cfg`
		`+ }`
		`+ self.assertEqual(([], []), checks.validate(config))`
		`+`
		`+ def _assertConfigMissing(self, cfg, key):`
		`+ config = copy.deepcopy(self.compose.conf)`
		`+ config['osbs'] = {`
		`+ '^Server$': cfg`
		`+ }`
		`+ self.assertEqual(`
		`+ (['Failed validation in osbs.^Server$: \'%s\' is a required property' % key], []),`
		`+ checks.validate(config))`

		`@mock.patch('pungi.util.resolve_git_url')`
		`@mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')`
		`@@ -205,11 +225,13 @@`
		`'target': 'f24-docker-candidate',`
		`}`
		`self._setupMock(KojiWrapper, resolve_git_url)`
		`+ self._assertConfigCorrect(cfg)`

		`self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)`

		`self._assertCorrectCalls({})`
		`self._assertCorrectMetadata()`
		`+ self._assertRepoFile()`

		`@mock.patch('pungi.util.resolve_git_url')`
		`@mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')`
		`@@ -220,11 +242,13 @@`
		`'failable': ['*']`
		`}`
		`self._setupMock(KojiWrapper, resolve_git_url)`
		`+ self._assertConfigCorrect(cfg)`

		`self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)`

		`self._assertCorrectCalls({})`
		`self._assertCorrectMetadata()`
		`+ self._assertRepoFile()`

		`@mock.patch('pungi.util.resolve_git_url')`
		`@mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')`
		`@@ -236,39 +260,104 @@`
		`'version': '1.0',`
		`}`
		`self._setupMock(KojiWrapper, resolve_git_url)`
		`+ self._assertConfigCorrect(cfg)`

		`self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)`

		`self._assertCorrectCalls({'name': 'my-name', 'version': '1.0'})`
		`self._assertCorrectMetadata()`
		`+ self._assertRepoFile()`

		`@mock.patch('pungi.util.resolve_git_url')`
		`@mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')`
		`- def test_run_with_missing_url(self, KojiWrapper, resolve_git_url):`
		`+ def test_run_with_extra_repos(self, KojiWrapper, resolve_git_url):`
		`cfg = {`
		`+ 'url': 'git://example.com/repo?#HEAD',`
		`'target': 'f24-docker-candidate',`
		`'name': 'my-name',`
		`+ 'version': '1.0',`
		`+ 'repo': 'http://pkgs.example.com/my.repo',`
		`+ 'repo_from': 'Everything',`
		`}`
		`self._setupMock(KojiWrapper, resolve_git_url)`
		`+ self._assertConfigCorrect(cfg)`

		`- with self.assertRaises(RuntimeError) as ctx:`
		`- self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)`
		`+ self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)`
		`+`
		`+ options = {`
		`+ 'name': 'my-name',`
		`+ 'version': '1.0',`
		`+ 'yum_repourls': [`
		`+ 'http://root/work/global/tmp-Server/compose-rpms-1.repo',`
		`+ 'http://root/work/global/tmp-Everything/compose-rpms-1.repo',`
		`+ 'http://pkgs.example.com/my.repo',`
		`+ ]`
		`+ }`
		`+ self._assertCorrectCalls(options)`
		`+ self._assertCorrectMetadata()`
		`+ self._assertRepoFile(['Server', 'Everything'])`
		`+`
		`+ @mock.patch('pungi.util.resolve_git_url')`
		`+ @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')`
		`+ def test_run_with_extra_repos_in_list(self, KojiWrapper, resolve_git_url):`
		`+ cfg = {`
		`+ 'url': 'git://example.com/repo?#HEAD',`
		`+ 'target': 'f24-docker-candidate',`
		`+ 'name': 'my-name',`
		`+ 'version': '1.0',`
		`+ 'repo': ['http://pkgs.example.com/my.repo'],`
		`+ 'repo_from': ['Everything', 'Client'],`
		`+ }`
		`+ self._assertConfigCorrect(cfg)`
		`+ self._setupMock(KojiWrapper, resolve_git_url)`

		`- self.assertIn("missing config key 'url' for Server", str(ctx.exception))`
		`+ self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)`
		`+`
		`+ options = {`
		`+ 'name': 'my-name',`
		`+ 'version': '1.0',`
		`+ 'yum_repourls': [`
		`+ 'http://root/work/global/tmp-Server/compose-rpms-1.repo',`
		`+ 'http://root/work/global/tmp-Everything/compose-rpms-1.repo',`
		`+ 'http://root/work/global/tmp-Client/compose-rpms-1.repo',`
		`+ 'http://pkgs.example.com/my.repo',`
		`+ ]`
		`+ }`
		`+ self._assertCorrectCalls(options)`
		`+ self._assertCorrectMetadata()`
		`+ self._assertRepoFile(['Server', 'Everything', 'Client'])`

		`@mock.patch('pungi.util.resolve_git_url')`
		`@mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')`
		`- def test_run_with_missing_target(self, KojiWrapper, resolve_git_url):`
		`+ def test_run_with_extra_repos_missing_variant(self, KojiWrapper, resolve_git_url):`
		`cfg = {`
		`'url': 'git://example.com/repo?#HEAD',`
		`+ 'target': 'f24-docker-candidate',`
		`'name': 'my-name',`
		`+ 'version': '1.0',`
		`+ 'repo_from': 'Gold',`
		`}`
		`+ self._assertConfigCorrect(cfg)`
		`self._setupMock(KojiWrapper, resolve_git_url)`

		`with self.assertRaises(RuntimeError) as ctx:`
		`self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)`

		`- self.assertIn("missing config key 'target' for Server", str(ctx.exception))`
		`+ self.assertIn('no variant Gold', str(ctx.exception))`
		`+`
		`+ def test_run_with_missing_url(self):`
		`+ cfg = {`
		`+ 'target': 'f24-docker-candidate',`
		`+ 'name': 'my-name',`
		`+ }`
		`+ self._assertConfigMissing(cfg, 'url')`
		`+`
		`+ def test_run_with_missing_target(self):`
		`+ cfg = {`
		`+ 'url': 'git://example.com/repo?#HEAD',`
		`+ 'name': 'my-name',`
		`+ }`
		`+ self._assertConfigMissing(cfg, 'target')`

		`@mock.patch('pungi.util.resolve_git_url')`
		`@mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')`
		`@@ -277,6 +366,7 @@`
		`'url': 'git://example.com/repo?#HEAD',`
		`'target': 'fedora-24-docker-candidate',`
		`}`
		`+ self._assertConfigCorrect(cfg)`
		`self._setupMock(KojiWrapper, resolve_git_url)`
		`self.wrapper.watch_task.return_value = 1`

		`@@ -293,6 +383,7 @@`
		`'target': 'fedora-24-docker-candidate',`
		`'failable': ['*']`
		`}`
		`+ self._assertConfigCorrect(cfg)`
		`self._setupMock(KojiWrapper, resolve_git_url)`
		`self.wrapper.watch_task.return_value = 1`

		`@@ -306,6 +397,7 @@`
		`'target': 'fedora-24-docker-candidate',`
		`'scratch': True,`
		`}`
		`+ self._assertConfigCorrect(cfg)`
		`self._setupMock(KojiWrapper, resolve_git_url)`

		`self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)`

lsedlar commented 8 years ago

The same way live_media and image_build accept additional external repos or variants list, there is now a repo and repo_from configuration key to add these.

Additional commits clean up and improve the unit tests a little by validating the configuration and checking the generated .repo files.

Fixes: #486