PR#1344: Allow specifying a specific module build to reuse components from - fm-orchestrator

fm-orchestrator

#1344 Allow specifying a specific module build to reuse components from

Merged 4 years ago by mprahl. Opened 4 years ago by mprahl.

reuse-specific-module-build into master

Allow specifying a specific module build to reuse components from

mprahl • 4 years ago

1bcb40d

Alphabetize the build parameters in the documentation

mprahl • 4 years ago

6c4d6ee

README.rst

file modified

+13 -9

		`@@ -119,23 +119,25 @@`
		- ``buildrequire_overrides`` - the buildrequires to override the modulemd with. The overrides must
		`be to existing buildrequires on the modulemd. The expected format is`
		``{'platform': ['f28', 'f29']}``.
		+ - ``modulemd`` - a string for submitting a YAML modulemd file as a parameter in the JSON data as
		+ an alternative to sending it in a ``multipart/form-data`` request. Only allowed if
		+ ``scratch`` is ``True`` or if the MBS setting ``YAML_SUBMIT_ALLOWED`` is ``True``.
		+ - ``module_name`` - a string to use as the module name if a scratch build is requested and the
		+ YAML modulemd is submitted using the ``modulemd`` parameter.
		+ - ``rebuild_strategy`` - a string of the desired rebuild strategy (affects what components get
		`+ rebuilt). For the available options, please look at the "Rebuild Strategies" section below.`
		- ``require_overrides`` - the requires to override the modulemd with. The overrides must be to
		existing requires on the modulemd. The expected format is ``{'platform': ['f28', 'f29']}``.
		+ - ``reuse_components_from`` - the ID or NSVC of the module build to reuse components from. If it's
		`+ not set, MBS will try to find a compatible module build to reuse components from.`
		- ``scratch`` - a boolean indicating if a scratch module build should be performed.
		Only allowed to be ``True`` if the MBS setting ``MODULES_ALLOW_SCRATCH`` is ``True``.
		+ - ``srpms`` - an optional list of Koji upload URLs of SRPMs to include in a module scratch build.
		+ Only allowed if ``scratch`` is ``True``.
		- ``yaml`` - a string of the input file when submitting a YAML modulemd file directly in a
		``multipart/form-data`` request. Only allowed if ``scratch`` is ``True`` or if the MBS
		setting ``YAML_SUBMIT_ALLOWED`` is ``True``. The basename of the file will be used as
		`the module name.`
		- - ``modulemd`` - a string for submitting a YAML modulemd file as a parameter in the JSON data as
		- an alternative to sending it in a ``multipart/form-data`` request. Only allowed if
		- ``scratch`` is ``True`` or if the MBS setting ``YAML_SUBMIT_ALLOWED`` is ``True``.
		- - ``module_name`` - a string to use as the module name if a scratch build is requested and the
		- YAML modulemd is submitted using the ``modulemd`` parameter.
		- - ``srpms`` - an optional list of Koji upload URLs of SRPMs to include in a module scratch build.
		- Only allowed if ``scratch`` is ``True``.
		- - ``rebuild_strategy`` - a string of the desired rebuild strategy (affects what components get
		`- rebuilt). For the available options, please look at the "Rebuild Strategies" section below.`


		`Module build state query`
		`@@ -357,6 +359,7 @@`
		`"modulemd": "...."`
		`"name": "testmodule",`
		`"owner": "mprahl",`
		`+ "reused_module_id": 121,`
		`"scmurl": "https://src.fedoraproject.org/modules/testmodule.git?#86d9cfe53d20118d863ae051641fc3784d91d981",`
		`"state": 5,`
		`"state_name": "ready",`
		`@@ -479,6 +482,7 @@`
		- ``new_repo_task_id``
		- ``owner``
		- ``rebuild_strategy``
		+ - ``reuse_components_from`` - the compatible module that was used for component reuse
		- ``scmurl``
		- ``state`` - Can be the state name or the state ID e.g. ``state=done``. This
		`parameter can be given multiple times, in which case or-ing will be used.`

docs/REBUILD_STRATEGIES.rst

file modified

		`@@ -75,6 +75,9 @@`
		Additionally, if the rebuild strategy for the module being built is ``changed-and-after``, then the
		module to reuse components from will have a rebuild strategy of ``changed-and-after`` or ``all``.

		+ If the user wants to specify the compatible module, they can use the ``reuse_components_from``
		`+ parameter.`
		`+`

		`How the Rebuild Strategies Work`
		`===============================`

module_build_service/models.py

file modified

		`@@ -877,6 +877,7 @@`
		`"build_context": self.build_context,`
		`"modulemd": self.modulemd,`
		`"ref_build_context": self.ref_build_context,`
		`+ "reused_module_id": self.reused_module_id,`
		`"runtime_context": self.runtime_context,`
		`"state_trace": [`
		`{`

module_build_service/utils/submit.py

file modified

		`@@ -1014,6 +1014,7 @@`
		`scmurl=params.get("scmurl"),`
		`username=username,`
		`rebuild_strategy=params.get("rebuild_strategy"),`
		`+ reused_module_id=params.get("reuse_components_from"),`
		`scratch=params.get("scratch"),`
		`srpms=params.get("srpms"),`
		`)`

module_build_service/views.py

file modified

+35

		`@@ -305,6 +305,7 @@`
		`"module_name",`
		`"owner",`
		`"rebuild_strategy",`
		`+ "reuse_components_from",`
		`"require_overrides",`
		`"scmurl",`
		`"scratch",`
		`@@ -382,6 +383,40 @@`
		`self._validate_dep_overrides_format("buildrequire_overrides")`
		`self._validate_dep_overrides_format("require_overrides")`

		`+ if "reuse_components_from" in self.data:`
		`+ if "rebuild_strategy" in self.data and self.data["rebuild_strategy"] == "all":`
		`+ raise ValidationError(`
		`+ 'You cannot specify the parameter "reuse_components_from" when the '`
		`+ '"rebuild_strategy" parameter is set to "all"'`
		`+ )`
		`+`
		`+ invalid_identifier_msg = (`
		`+ 'The parameter "reuse_components_from" contains an invalid module identifier')`
		`+`
		`+ if isinstance(self.data["reuse_components_from"], int):`
		`+ reuse_module = models.ModuleBuild.get_by_id(`
		`+ db.session, self.data["reuse_components_from"])`
		`+ elif isinstance(self.data["reuse_components_from"], string_types):`
		`+ try:`
		`+ n, s, v, c = self.data["reuse_components_from"].split(":")`
		`+ except ValueError:`
		`+ raise ValidationError(invalid_identifier_msg)`
		`+ reuse_module = models.ModuleBuild.get_build_from_nsvc(db.session, n, s, v, c)`
		`+ else:`
		`+ raise ValidationError(invalid_identifier_msg)`
		`+`
		`+ if not reuse_module:`
		`+ raise ValidationError(`
		`+ 'The module in the parameter "reuse_components_from" could not be found')`
		`+`
		`+ if reuse_module.state != models.BUILD_STATES["ready"]:`
		`+ raise ValidationError(`
		`+ 'The module in the parameter "reuse_components_from" must be in the ready state'`
		`+ )`
		`+`
		`+ # Normalize the value so that it simplifies any code that uses this value`
		`+ self.data["reuse_components_from"] = reuse_module.id`
		`+`

		`class SCMHandler(BaseHandler):`
		`def validate(self, skip_branch=False, skip_optional_params=False):`

tests/test_views/test_views.py

file modified

+112 -1

		`@@ -31,6 +31,7 @@`
		`from os.path import basename, dirname, splitext`
		`from requests.utils import quote`
		`import hashlib`
		`+ import koji`
		`import pytest`
		`import re`
		`import sqlalchemy`
		`@@ -39,7 +40,7 @@`
		`from tests import read_staged_data`
		`from tests.test_scm import base_dir as scm_base_dir`
		`from module_build_service.errors import UnprocessableEntity`
		`- from module_build_service.models import ModuleBuild`
		`+ from module_build_service.models import ModuleBuild, BUILD_STATES`
		`from module_build_service import db, version`
		`import module_build_service.config as mbs_config`
		`import module_build_service.scheduler.handlers.modules`
		`@@ -207,6 +208,7 @@`
		`assert data["name"] == "nginx"`
		`assert data["owner"] == "Moe Szyslak"`
		`assert data["rebuild_strategy"] == "changed-and-after"`
		`+ assert data["reused_module_id"] is None`
		`assert data["scmurl"] == \`
		`"git://pkgs.domain.local/modules/nginx?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9"`
		`assert data["scratch"] is False`
		`@@ -2592,3 +2594,112 @@`
		`mock_get.assert_called_once_with(expected_url, timeout=15)`
		`else:`
		`mock_get.assert_not_called()`
		`+`
		`+ @pytest.mark.parametrize("reuse_components_from", (7, "testmodule:4.3.43:7:00000000"))`
		`+ @patch("module_build_service.auth.get_user", return_value=user)`
		`+ @patch("module_build_service.scm.SCM")`
		`+ def test_submit_build_reuse_components_from(`
		`+ self, mocked_scm, mocked_get_user, reuse_components_from,`
		`+ ):`
		`+ """Test a build submission using the reuse_components_from parameter."""`
		`+ module_to_reuse = db.session.query(ModuleBuild).get(7)`
		`+ module_to_reuse.state = BUILD_STATES["ready"]`
		`+ for c in module_to_reuse.component_builds:`
		`+ c.state = koji.BUILD_STATES["COMPLETE"]`
		`+ db.session.commit()`
		`+`
		`+ FakeSCM(`
		`+ mocked_scm, "testmodule", "testmodule.yaml", "620ec77321b2ea7b0d67d82992dda3e1d67055b4")`
		`+ rv = self.client.post(`
		`+ "/module-build-service/1/module-builds/",`
		`+ data=json.dumps({`
		`+ "branch": "master",`
		`+ "reuse_components_from": reuse_components_from,`
		`+ "scmurl": (`
		`+ "https://src.stg.fedoraproject.org/modules/testmodule.git?"`
		`+ "#68931c90de214d9d13feefbd35246a81b6cb8d49"`
		`+ ),`
		`+ }),`
		`+ )`
		`+ data = json.loads(rv.data)`
		`+ assert data["reused_module_id"] == 7`
		`+`
		`+ @pytest.mark.parametrize(`
		`+ "reuse_components_from, expected_error",`
		`+ (`
		`+ (`
		`+ "testmodule:4.3.43:7",`
		`+ 'The parameter "reuse_components_from" contains an invalid module identifier',`
		`+ ),`
		`+ (`
		`+ {},`
		`+ 'The parameter "reuse_components_from" contains an invalid module identifier',`
		`+ ),`
		`+ (`
		`+ 912312312,`
		`+ 'The module in the parameter "reuse_components_from" could not be found',`
		`+ ),`
		`+ (`
		`+ 7,`
		`+ 'The module in the parameter "reuse_components_from" must be in the ready state',`
		`+ )`
		`+ )`
		`+ )`
		`+ @patch("module_build_service.auth.get_user", return_value=user)`
		`+ @patch("module_build_service.scm.SCM")`
		`+ def test_submit_build_reuse_components_from_errors(`
		`+ self, mocked_scm, mocked_get_user, reuse_components_from, expected_error,`
		`+ ):`
		`+ """`
		`+ Test a build submission using an invalid value for the reuse_components_from parameter.`
		`+ """`
		`+ FakeSCM(`
		`+ mocked_scm, "testmodule", "testmodule.yaml", "620ec77321b2ea7b0d67d82992dda3e1d67055b4")`
		`+ rv = self.client.post(`
		`+ "/module-build-service/1/module-builds/",`
		`+ data=json.dumps({`
		`+ "branch": "master",`
		`+ "reuse_components_from": reuse_components_from,`
		`+ "scmurl": (`
		`+ "https://src.stg.fedoraproject.org/modules/testmodule.git?"`
		`+ "#68931c90de214d9d13feefbd35246a81b6cb8d49"`
		`+ ),`
		`+ }),`
		`+ )`
		`+ data = json.loads(rv.data)`
		`+ assert rv.status_code == 400`
		`+ assert data["message"] == expected_error`
		`+`
		`+ @patch("module_build_service.auth.get_user", return_value=user)`
		`+ @patch("module_build_service.scm.SCM")`
		`+ @patch(`
		`+ "module_build_service.config.Config.rebuild_strategy_allow_override",`
		`+ new_callable=PropertyMock,`
		`+ return_value=True,`
		`+ )`
		`+ def test_submit_build_reuse_components_rebuild_strategy_all(`
		`+ self, mock_rsao, mocked_scm, mocked_get_user,`
		`+ ):`
		`+ """`
		`+ Test a build submission using reuse_components_from and the rebuild_strategy of all.`
		`+ """`
		`+ FakeSCM(`
		`+ mocked_scm, "testmodule", "testmodule.yaml", "620ec77321b2ea7b0d67d82992dda3e1d67055b4")`
		`+ rv = self.client.post(`
		`+ "/module-build-service/1/module-builds/",`
		`+ data=json.dumps({`
		`+ "branch": "master",`
		`+ "rebuild_strategy": "all",`
		`+ "reuse_components_from": 7,`
		`+ "scmurl": (`
		`+ "https://src.stg.fedoraproject.org/modules/testmodule.git?"`
		`+ "#68931c90de214d9d13feefbd35246a81b6cb8d49"`
		`+ ),`
		`+ }),`
		`+ )`
		`+ data = json.loads(rv.data)`
		`+ assert rv.status_code == 400`
		`+ assert data["message"] == (`
		`+ 'You cannot specify the parameter "reuse_components_from" when the "rebuild_strategy" '`
		`+ 'parameter is set to "all"'`
		`+ )`