From 7a302725f423ae4fcb9b2bec76aae5c738dff469 Mon Sep 17 00:00:00 2001 From: Valerij Maljulin Date: Jan 28 2020 16:08:52 +0000 Subject: [PATCH 1/2] Allow usage of the side tags repo JIRA: FACTORY-5691 This also reverts commit be53a785d30527719db1f2bdb6859337eb1fb510. Signed-off-by: Valerij Maljulin --- diff --git a/Dockerfile b/Dockerfile index cbd4406..89157d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ ARG cacert_url=undefined WORKDIR /src RUN dnf -y install \ + git-core \ python3-dogpile-cache \ python3-fedmsg \ python3-flask \ diff --git a/Vagrantfile b/Vagrantfile index a8fc358..044524a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,6 +5,7 @@ $script = <<-'SCRIPT' set -e dnf -y install \ + git-core \ postgresql-server \ postgresql-contrib \ python3-gunicorn \ diff --git a/docs/package-specific-policies.rst b/docs/package-specific-policies.rst index 73e083e..e3ac892 100644 --- a/docs/package-specific-policies.rst +++ b/docs/package-specific-policies.rst @@ -52,7 +52,7 @@ Refer to :doc:`policies` for details about each of the keys in the YAML file. .. _tolerate-invalid-gating-yaml: -Tolerate an invalid gating.yaml file +Tolerate an invalid remote rule file ------------------------------------ A gating.yaml file is considered invalid if it has an invalid syntax (yaml @@ -64,21 +64,20 @@ gating.yaml file) and it will not be possible to ship the build. To skip this problem, it is possible to submit a waiver with the tool `waiverdb-cli `_. This waiver must have -``testcase`` equal to ``invalid-gating-yaml``. It is not necessary to have +``testcase`` equal to ``invalid-remote-rule-yaml``. It is not necessary to have a result in Resultsdb for this testcase. -The side effect is that all the policies defined in the gating.yaml +The side effect is that all the policies defined in the remote rule file will be completely ignored by Greenwave. -.. _missing-gating-yaml: +.. _missing-remote-rule-yaml: -Missing gating.yaml file +Missing remote rule file ------------------------ -Missing gating.yaml file (i.e. not present in dist-git repo of the tested -package in the required revision) is just skipped and not treated as -unsatisfied requirement by default. To change this, ``required`` boolean +Missing remote rule file (i.e. not present in the configured repo) is just skipped +and not treated as unsatisfied requirement by default. To change this, ``required`` boolean attribute of ``RemoteRule`` must be set to ``true``. .. code-block:: yaml @@ -91,8 +90,7 @@ attribute of ``RemoteRule`` must be set to ``true``. rules: - !RemoteRule {required: true} -For such policy, if gating.yaml is missing, could result in the following -decision. +For such policy, missing remote rule file could result in the following decision. .. code-block:: json @@ -104,8 +102,8 @@ decision. "unsatisfied_requirements": [{ "subject_identifier": "nethack-1.2.3-1.f31", "subject_type": "koji_build", - "testcase": "missing-gating-yaml", - "type": "missing-gating-yaml" + "testcase": "missing-remote-rule-yaml", + "type": "missing-remote-rule-yaml" }], } @@ -118,7 +116,7 @@ Tutorial - How to configure the RemoteRule If you want to add some additional policies, you can follow this tutorial. -We need to write the gating.yaml file. The one for this example will +We need to write a remote rule file. The one for this example will be this one: :: @@ -131,7 +129,7 @@ be this one: rules: - !PassingTestCaseRule {test_case_name: dist.depcheck} -*NB*. It is not possible to insert a RemoteRule inside a gating.yaml file. +*NB*. It is not possible to insert a RemoteRule inside a remote rule file. This will provoke an error. You need now to push the new file (or the changes) in your dist-git @@ -145,8 +143,8 @@ source code of your project): Now you can find in the link of the build in Koji the nvr of the build. Example: ``python-ansi2html-1.1.1-114.fc28`` -In case of a misconfigured gating.yaml you would need to repeate the -build. To avoid this it is possible to validate the gating.yaml file +In case of a misconfigured remote rule you would need to repeate the +build. To avoid this it is possible to validate the remote rule file before starting the build. To do that you can use this command (in this example we are using the Fedora Greenwave instance in production): @@ -195,28 +193,26 @@ Once you create a result in ResultsDB for that testcase (with decision will change and all the requirements will be satisfied (if everything was configured in the correct way). -If your gating.yaml file will be misconfigured, Greenwave will reply -that the gating.yaml file is wrong. If you just want to skip this check +If your remote rule file is misconfigured, Greenwave will reply +that the remote rule file is wrong. If you just want to skip this check without build again, just look at the previous section in this page. .. _fetching-gating-yaml: -How is gating.yaml file retrieved? ----------------------------------- +How is the remote rule file being retrieved? +-------------------------------------------- -The "gating.yaml" file is downloaded from a dist-git repository based on the -source URL of a specific build in Koji. - -The file is fetched from specific git commit (the revision is part of the -build's source URL). +The remote rule file (usually called ``gating.yaml``) is downloaded +from a repository based on the source URL of a specific build in Koji. +Different URLs can be set for different subject types. More specifically, Greenwave first gets the build data ``koji call getBuild $NVR``. Then it parses URL in "source" field to get namespace ("rpms" or "containers" etc.), the git commit and package name (or rather the git repository name). -The "gating.yaml" URL is constructed based on ``DIST_GIT_URL_TEMPLATE`` +For HTTP method, the remote rule URL is constructed based on ``HTTP_URL_TEMPLATE`` specified in Greenwave configuration. The URL template is something like:: - {DIST_GIT_BASE_URL}/{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml + {BASE_URL}/{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml diff --git a/docs/policies.rst b/docs/policies.rst index 6db6fa8..c155e7b 100644 --- a/docs/policies.rst +++ b/docs/policies.rst @@ -249,19 +249,39 @@ Here's an example of a RemoteRule: Once the code is pushed, Greenwave will start to check if there is a -gating.yaml file in your dist-git repo. If you didn't configure any -gating.yaml file nothing will change. +remote rule file in your repo. If you didn't configure any remote rule file +nothing will change. -Greenwave will check if a gating.yaml exists, if it does, it pulls it +Greenwave will check if a remote rule file exists, if it does, it pulls it down, loads it, and uses it to additionally evaluate the subject of the decision. -Greenwave requires these configuration parameters ``KOJI_BASE_URL``, -``DIST_GIT_BASE_URL`` and ``DIST_GIT_URL_TEMPLATE``. Here's the default -for the Fedora instance: +Greenwave requires these configuration parameters ``KOJI_BASE_URL`` and +``REMOTE_RULE_POLICIES``. + +``REMOTE_RULE_POLICIES`` is a map, where the key is the subject type pattern +(file matching wildcards (* and ?) are allowed). There should be a default +pattern "*". Each pattern should contain a map of two parameters +depending on a retrieval mechanism. + +Greenwave has two mechanisms to retrieve the remote rule file: ``git archive`` is +using for side tags rules and using a git front-end. ``GIT_URL`` and ``GIT_PATH_TEMPLATE`` +should be set for ``git archive`` mechanism, ``HTTP_BASE_URL`` and ``HTTP_URL_TEMPLATE`` +should be set if you are going to use a git front-end. + +Below is an example configuration where ``git archive`` is being used for "brew-build-group" +subject type and HTTP is being used for other: .. code-block:: console - DIST_GIT_BASE_URL = 'https://src.fedoraproject.org/' - DIST_GIT_URL_TEMPLATE = '{DIST_GIT_BASE_URL}{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml' - KOJI_BASE_URL = 'https://koji.fedoraproject.org/kojihub' + REMOTE_RULE_POLICIES = { + 'brew-build-group': { + 'GIT_URL': 'git@gitlab.cee.redhat.com:devops/greenwave-policies/side-tags.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + }, + '*': { + 'HTTP_BASE_URL': 'https://src.fedoraproject.org/', + 'HTTP_URL_TEMPLATE': '{BASE_URL}{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml' + } + } + KOJI_BASE_URL = 'https://koji.fedoraproject.org/kojihub' diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 3a0f252..68c6bfa 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -60,7 +60,7 @@ Released 27 June 2019 * ``RemoteRule`` has a new optional attribute ``required`` which allows to treat a missing ``gating.yaml`` file as a failed requirement. See - :ref:`missing-gating-yaml`. + :ref:`missing-remote-rule-yaml`. * Status code 500 is no longer returned if a ``gating.yaml`` file cannot be retrieved. Instead, status code 502 is returned with a specific error. * Documentation now contains recommendation for the maximum number of subjects diff --git a/greenwave/app_factory.py b/greenwave/app_factory.py index 8bbe8b5..28bf27c 100644 --- a/greenwave/app_factory.py +++ b/greenwave/app_factory.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0+ import logging + from flask import Flask from greenwave.api_v1 import api from greenwave.monitor import monitor_api @@ -16,12 +17,17 @@ log = logging.getLogger(__name__) def _can_use_remote_rule(config): - # Ensure that the required config settings are set - return (bool( - config.get('DIST_GIT_BASE_URL') and - config.get('DIST_GIT_URL_TEMPLATE') and - config.get('KOJI_BASE_URL')) - ) + if not config.get('KOJI_BASE_URL'): + return False + + if config.get('REMOTE_RULE_POLICIES'): + return all( + (conf_item.get('GIT_URL') and conf_item.get('GIT_PATH_TEMPLATE')) or + conf_item.get('HTTP_URL_TEMPLATE') + for conf_item in config.get('REMOTE_RULE_POLICIES').values() + ) and '*' in config.get('REMOTE_RULE_POLICIES') + + return False def _has_remote_rule(policies): @@ -50,10 +56,11 @@ def create_app(config_obj=None): if not _can_use_remote_rule(app.config) and _has_remote_rule(app.config['policies']): raise RuntimeError( - "If you want to apply a RemoteRule" - " you need to configure 'DIST_GIT_BASE_URL', " - "'DIST_GIT_URL_TEMPLATE' and KOJI_BASE_URL in " - "your configuration." + 'If you want to apply a RemoteRule, you must have "KOJI_BASE_URL" and ' + '"REMOTE_RULE_POLICIES" or both set in your configuration. ' + 'Each field in "REMOTE_RULE_POLICIES" map have to contain either ' + '"GIT_URL"/"GIT_PATH_TEMPLATE" or "HTTP_URL_TEMPLATE". ' + 'Setting the default configuration by using "*" in map is also mandatory' ) # register error handlers @@ -82,4 +89,4 @@ def healthcheck(): Returns a 200 response if the application is alive and able to serve requests. """ - return ('Health check OK', 200, [('Content-Type', 'text/plain')]) + return 'Health check OK', 200, [('Content-Type', 'text/plain')] diff --git a/greenwave/config.py b/greenwave/config.py index aa372df..d5f009a 100644 --- a/greenwave/config.py +++ b/greenwave/config.py @@ -22,9 +22,18 @@ class Config(object): RESULTSDB_API_URL = 'https://taskotron.fedoraproject.org/resultsdb_api/api/v2.0' WAIVERDB_API_URL = 'https://waiverdb.fedoraproject.org/api/v1.0' - # Remote rule configuration - DIST_GIT_BASE_URL = 'https://src.fedoraproject.org/' - DIST_GIT_URL_TEMPLATE = '{DIST_GIT_BASE_URL}{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml' + REMOTE_RULE_POLICIES = { + 'brew-build-group': { + 'GIT_URL': 'git@gitlab.cee.redhat.com:devops/greenwave-policies/side-tags.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + }, + '*': { + 'HTTP_URL_TEMPLATE': + 'https://src.fedoraproject.org/{pkg_namespace}{pkg_name}/raw/{rev}/f/gating.yaml' + } + } + REMOTE_RULE_GIT_TIMEOUT = 30 + REMOTE_RULE_GIT_MAX_RETRY = 3 KOJI_BASE_URL = 'https://koji.fedoraproject.org/kojihub' # Options for outbound HTTP requests made by python-requests REQUESTS_TIMEOUT = (6.1, 15) diff --git a/greenwave/policies.py b/greenwave/policies.py index 8165d56..e3e0e9e 100644 --- a/greenwave/policies.py +++ b/greenwave/policies.py @@ -213,7 +213,7 @@ class TestResultErrored(RuleNotSatisfied): return TestResultPassed(self.test_case_name, self.result_id) -class InvalidGatingYaml(RuleNotSatisfied): +class InvalidRemoteRuleYaml(RuleNotSatisfied): """ Remote policy parsing failed. """ @@ -225,7 +225,7 @@ class InvalidGatingYaml(RuleNotSatisfied): def to_json(self): return { - 'type': 'invalid-gating-yaml', + 'type': 'invalid-remote-rule-yaml', 'testcase': self.test_case_name, 'subject_type': self.subject.type, 'subject_identifier': self.subject.identifier, @@ -236,12 +236,12 @@ class InvalidGatingYaml(RuleNotSatisfied): return None -class MissingGatingYaml(RuleNotSatisfied): +class MissingRemoteRuleYaml(RuleNotSatisfied): """ Remote policy not found in remote repository. """ - test_case_name = 'missing-gating-yaml' + test_case_name = 'missing-remote-rule-yaml' def __init__(self, subject): self.subject = subject @@ -433,6 +433,14 @@ class RemoteRule(Rule): 'required': SafeYAMLBool(optional=True, default=False), } + def _get_config_urls(self, rr_config, subject): + for subj, cnf in rr_config.items(): + if subj == '*': + continue + if fnmatch(subject, subj): + return cnf + return rr_config['*'] + def _get_sub_policies(self, policy, subject): if not subject.supports_remote_rule: return [] @@ -447,10 +455,14 @@ class RemoteRule(Rule): # if the element is actually a container and not a pkg there will be a "-container" # string at the end of the "pkg_name" and it will not match with the one in the - # gating.yaml URL + # remote rule file URL if pkg_namespace == 'containers': pkg_name = re.sub('-container$', '', pkg_name) - response = greenwave.resources.retrieve_yaml_remote_rule(rev, pkg_name, pkg_namespace) + rr_policies_conf = current_app.config.get('REMOTE_RULE_POLICIES') + cur_subject_config = self._get_config_urls(rr_policies_conf, policy.subject_type) + response = greenwave.resources.retrieve_yaml_remote_rule( + rev, pkg_name, pkg_namespace, cur_subject_config + ) if response is None: # greenwave extension file not found @@ -479,12 +491,12 @@ class RemoteRule(Rule): policies = self._get_sub_policies(policy, subject) except SafeYAMLError as e: return [ - InvalidGatingYaml(subject, 'invalid-gating-yaml', str(e)) + InvalidRemoteRuleYaml(subject, 'invalid-gating-yaml', str(e)) ] if policies is None: if self.required: - return [MissingGatingYaml(subject)] + return [MissingRemoteRuleYaml(subject)] return [] answers = [] diff --git a/greenwave/resources.py b/greenwave/resources.py index 8242d13..67bb862 100644 --- a/greenwave/resources.py +++ b/greenwave/resources.py @@ -8,6 +8,9 @@ waiverdb, etc..). import logging import re +from io import BytesIO +import tarfile +import subprocess import socket from urllib.parse import urlparse @@ -47,6 +50,7 @@ class ResultsRetriever(BaseRetriever): """ Retrieves results from cache or ResultsDB. """ + def __init__(self, **args): super().__init__(**args) self.cache = {} @@ -92,6 +96,7 @@ class WaiversRetriever(BaseRetriever): """ Retrieves waivers from WaiverDB. """ + def _retrieve_all(self, filters): if self.since: for filter_ in filters: @@ -138,8 +143,8 @@ def retrieve_scm_from_koji_build(nvr, build, koji_url): if not source: raise NoSourceException( 'Failed to retrieve SCM URL from Koji build "{}" at "{}" ' - '(expected SCM URL in "source" attribute)' - .format(nvr, koji_url)) + '(expected SCM URL in "source" attribute)'.format(nvr, koji_url) + ) url = urlparse(source) @@ -153,8 +158,8 @@ def retrieve_scm_from_koji_build(nvr, build, koji_url): if not rev: raise BadGateway( 'Failed to parse SCM URL "{}" from Koji build "{}" at "{}" ' - '(missing URL fragment with SCM revision information)' - .format(source, nvr, koji_url)) + '(missing URL fragment with SCM revision information)'.format(source, nvr, koji_url) + ) pkg_name = url.path.split('/')[-1] pkg_name = re.sub(r'\.git$', '', pkg_name) @@ -162,29 +167,77 @@ def retrieve_scm_from_koji_build(nvr, build, koji_url): @cached -def retrieve_yaml_remote_rule(rev, pkg_name, pkg_namespace): - """ Retrieve cached gating.yaml content for a given rev from the dist-git web UI. """ +def retrieve_yaml_remote_rule(rev, pkg_name, pkg_namespace, rr_config): + """ Retrieve a remote rule file content from the given repo""" + if rr_config.get('GIT_URL') and rr_config.get('GIT_PATH_TEMPLATE'): + return _retrieve_yaml_remote_rule_git_archive( + pkg_name, pkg_namespace, rr_config['GIT_URL'], rr_config['GIT_PATH_TEMPLATE'] + ) + else: + return _retrieve_yaml_remote_rule_web( + rev, pkg_name, pkg_namespace, rr_config['HTTP_URL_TEMPLATE'] + ) + + +_retrieve_remote_rule_error = 'Error occurred while retrieving a remote rule file from the repo.' + + +def _retrieve_yaml_remote_rule_web(rev, pkg_name, pkg_namespace, url_template): + """ Retrieve a remote rule file content from the git web UI. """ data = { - "DIST_GIT_BASE_URL": (current_app.config['DIST_GIT_BASE_URL'].rstrip('/') + - ('/' if pkg_namespace else '')), - "pkg_namespace": pkg_namespace, + "pkg_namespace": pkg_namespace + ('/' if pkg_namespace else ''), "pkg_name": pkg_name, "rev": rev } - url = current_app.config['DIST_GIT_URL_TEMPLATE'].format(**data) + url = url_template.format(**data) response = requests_session.request('HEAD', url) if response.status_code == 404: return None if response.status_code != 200: - raise BadGateway('Error occurred looking for gating.yaml file in the dist-git repo.') + raise BadGateway(_retrieve_remote_rule_error) - # gating.yaml found... + # remote rule file found... response = requests_session.request('GET', url) response.raise_for_status() return response.content +def _retrieve_yaml_remote_rule_git_archive(pkg_name, pkg_namespace, git_url, path_template): + """ Retrieve a remote rule file content from a git repo using git archive. """ + git_path = path_template.format(pkg_name=pkg_name, pkg_namespace=pkg_namespace) + cmd = ['git', 'archive', f'--remote={git_url}', 'master', git_path] + # Retry thrice if TimeoutExpired exception is raised + MAX_RETRY = current_app.config.get('REMOTE_RULE_GIT_MAX_RETRY', 3) + git_archive = None + for _ in range(MAX_RETRY): + try: + git_archive = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error_output = git_archive.communicate( + timeout=current_app.config.get('REMOTE_RULE_GIT_TIMEOUT', 30) + ) + break + except subprocess.TimeoutExpired: + if git_archive: + git_archive.kill() + continue + + if git_archive.returncode != 0: + error_output = error_output.decode('utf-8') + if 'path not found' in error_output: + return None + + cmd_str = ' '.join(cmd) + log.error('The following exception occurred while running "%s": %s', cmd_str, error_output) + raise BadGateway(_retrieve_remote_rule_error) + + # Convert the output to a file-like object with BytesIO, then tar can read it + # in memory rather than writing it to a file first + remote_rule_archive = tarfile.open(fileobj=BytesIO(output)) + remote_rule_content = remote_rule_archive.extractfile(git_path).read().decode('utf-8') + return remote_rule_content + + # NOTE - not cached. def retrieve_decision(greenwave_url, data): response = requests_session.post(greenwave_url, json=data) diff --git a/greenwave/tests/test_app_factory.py b/greenwave/tests/test_app_factory.py index 1915a1d..805fbe8 100644 --- a/greenwave/tests/test_app_factory.py +++ b/greenwave/tests/test_app_factory.py @@ -29,7 +29,7 @@ def test_remote_rules_misconfigured(mock_load_policies): mock_load_policies.return_value = policies config = TestingConfig() - config.DIST_GIT_BASE_URL = '' + config.REMOTE_RULE_POLICIES = {} expected_error = 'If you want to apply a RemoteRule' @@ -40,20 +40,36 @@ def test_remote_rules_misconfigured(mock_load_policies): def test_can_use_remote_rule_http(): """ Test that _can_use_remote_rule verifies the configuration properly if HTTP is used. """ config = { - 'DIST_GIT_BASE_URL': 'https://dist-git.domain.local', 'KOJI_BASE_URL': 'https://koji.domain.local/kojihub', - 'DIST_GIT_URL_TEMPLATE': ('{DIST_GIT_BASE_URL}{pkg_namespace}/{pkg_name}/raw/{rev}/f/' - 'gating.yaml') + 'REMOTE_RULE_POLICIES': { + '*': { + 'HTTP_URL_TEMPLATE': + 'https://src.fedoraproject.org/{pkg_namespace}/{pkg_name}/{rev}/f/gating.yaml' + } + } } assert _can_use_remote_rule(config) is True @pytest.mark.parametrize('config', ( { - 'DIST_GIT_BASE_URL': 'https://dist-git.domain.local', + 'REMOTE_RULE_POLICIES': { + '*': { + 'GIT_URL': 'git@gitlab.cee.redhat.com:devops/greenwave-policies/side-tags.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + } + }, }, { - 'DIST_GIT_BASE_URL': 'https://dist-git.domain.local', + 'KOJI_BASE_URL': 'https://koji.domain.local/kojihub' + }, + { + 'REMOTE_RULE_POLICIES': { + 'brew-build-group': { + 'GIT_URL': 'git@gitlab.cee.redhat.com:devops/greenwave-policies/side-tags.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + } + }, 'KOJI_BASE_URL': 'https://koji.domain.local/kojihub' } )) diff --git a/greenwave/tests/test_policies.py b/greenwave/tests/test_policies.py index 11845d4..83032fb 100644 --- a/greenwave/tests/test_policies.py +++ b/greenwave/tests/test_policies.py @@ -17,8 +17,8 @@ from greenwave.policies import ( TestResultMissing, TestResultFailed, TestResultPassed, - InvalidGatingYaml, - MissingGatingYaml, + InvalidRemoteRuleYaml, + MissingRemoteRuleYaml, OnDemandPolicy ) from greenwave.resources import ResultsRetriever @@ -550,7 +550,7 @@ def test_remote_rule_malformed_yaml(tmpdir): results = DummyResultsRetriever() decision = policy.check('fedora-26', subject, results) assert len(decision) == 1 - assert isinstance(decision[0], InvalidGatingYaml) + assert isinstance(decision[0], InvalidRemoteRuleYaml) assert decision[0].is_satisfied is False @@ -640,7 +640,7 @@ def test_remote_rule_required(): results = DummyResultsRetriever() decision = policy.check('fedora-rawhide', subject, results) assert len(decision) == 1 - assert isinstance(decision[0], MissingGatingYaml) + assert isinstance(decision[0], MissingRemoteRuleYaml) assert not decision[0].is_satisfied assert decision[0].subject.identifier == subject.identifier @@ -1109,7 +1109,7 @@ def test_remote_rule_policy_on_demand_policy_required(): results = DummyResultsRetriever() decision = policy.check('fedora-26', subject, results) assert len(decision) == 1 - assert isinstance(decision[0], MissingGatingYaml) + assert isinstance(decision[0], MissingRemoteRuleYaml) assert not decision[0].is_satisfied assert decision[0].subject.identifier == subject.identifier diff --git a/greenwave/tests/test_retrieve_gating_yaml.py b/greenwave/tests/test_retrieve_gating_yaml.py index 222d335..75a68ea 100644 --- a/greenwave/tests/test_retrieve_gating_yaml.py +++ b/greenwave/tests/test_retrieve_gating_yaml.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0+ +import subprocess +import io import socket from requests.exceptions import ConnectionError, HTTPError @@ -120,7 +122,9 @@ def test_retrieve_yaml_remote_rule_no_namespace(): response = mock.MagicMock() response.status_code = 404 session.request.return_value = response - retrieve_yaml_remote_rule("deadbeaf", "pkg", "") + retrieve_yaml_remote_rule( + "deadbeaf", "pkg", "", app.config['REMOTE_RULE_POLICIES']['*'] + ) expected_call = mock.call( 'HEAD', 'https://src.fedoraproject.org/pkg/raw/deadbeaf/f/gating.yaml') @@ -138,7 +142,9 @@ def test_retrieve_yaml_remote_rule_connection_error(): ] with pytest.raises(HTTPError) as excinfo: - retrieve_yaml_remote_rule("deadbeaf", "pkg", "") + retrieve_yaml_remote_rule( + "deadbeaf", "pkg", "", app.config['REMOTE_RULE_POLICIES']['*'] + ) assert str(excinfo.value) == ( '502 Server Error: Something went terribly wrong... for url: ' @@ -146,6 +152,69 @@ def test_retrieve_yaml_remote_rule_connection_error(): ) +@mock.patch('tarfile.open') +@mock.patch('subprocess.Popen') +def test_retrieve_yaml_remote_rule_git_archive(mock_subp, mock_tar): + # Make the git archive call return bytes + mock_subp.return_value.communicate.return_value = (b'tar file', '') + mock_subp.return_value.returncode = 0 + # Make the tar archive, based on the return value of git archive, return a file-like + # object representing the gating.yaml file + mock_tar.return_value.extractfile.return_value = io.BytesIO(b'some gating yaml file') + + app = greenwave.app_factory.create_app() + rr_config = { + 'GIT_URL': 'git://dist-git.domain.local/abc/abc.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + } + with app.app_context(): + gating_yaml = retrieve_yaml_remote_rule('abcdef', 'python-requests', 'rpms', rr_config) + + assert gating_yaml == 'some gating yaml file' + expected_cmd = [ + 'git', 'archive', '--remote=git://dist-git.domain.local/abc/abc.git', + 'master', 'rpms/python-requests.yaml'] + mock_subp.assert_called_once_with(expected_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + tar_file = mock_tar.call_args[1]['fileobj'].read() + assert tar_file == b'tar file' + mock_tar.return_value.extractfile.assert_called_once_with('rpms/python-requests.yaml') + + +@mock.patch('subprocess.Popen') +def test_retrieve_yaml_remote_rule_git_archive_no_file(mock_subp): + # Make the git archive command return an error saying the file isn't in the repo + mock_subp.return_value.communicate.return_value = \ + (None, b'remote: fatal: path not found: xxx.yaml') + mock_subp.return_value.returncode = 1 + + app = greenwave.app_factory.create_app() + rr_config = { + 'GIT_URL': 'git://dist-git.domain.local/abc/abc.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + } + with app.app_context(): + gating_yaml = retrieve_yaml_remote_rule('master', 'python-requests', 'rpms', rr_config) + + assert gating_yaml is None + + +@mock.patch('subprocess.Popen') +def test_retrieve_yaml_remote_rule_git_archive_error(mock_subp): + # Make the git archive command return an error + mock_subp.return_value.communicate.return_value = (None, b'remote: fatal: some error') + mock_subp.return_value.returncode = 1 + + app = greenwave.app_factory.create_app() + rr_config = { + 'GIT_URL': 'git://dist-git.domain.local/abc/abc.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + } + expected_error = 'Error occurred while retrieving a remote rule file from the repo.' + with pytest.raises(BadGateway, match=expected_error): + with app.app_context(): + retrieve_yaml_remote_rule('master', 'python-requests', 'rpms', rr_config) + + @mock.patch('greenwave.resources.xmlrpc.client.ServerProxy') def test_retrieve_scm_from_koji_build_socket_error(mock_xmlrpc_client): mock_auth_server = mock_xmlrpc_client.return_value diff --git a/greenwave/tests/test_summary.py b/greenwave/tests/test_summary.py index ccda425..202f5bd 100644 --- a/greenwave/tests/test_summary.py +++ b/greenwave/tests/test_summary.py @@ -6,7 +6,7 @@ from greenwave.policies import ( TestResultFailed, TestResultMissing, TestResultMissingWaived, - InvalidGatingYaml, + InvalidRemoteRuleYaml, ) from greenwave.subjects.subject import Subject @@ -23,7 +23,7 @@ testResultMissing = TestResultMissing( testSubject, 'test', None) testResultMissingWaived = TestResultMissingWaived( testSubject, 'test', None) -testInvalidGatingYaml = InvalidGatingYaml( +testInvalidGatingYaml = InvalidRemoteRuleYaml( testSubject, 'test', 'Missing !Policy tag') diff --git a/greenwave/tests/test_waive.py b/greenwave/tests/test_waive.py index 10b8615..054dbea 100644 --- a/greenwave/tests/test_waive.py +++ b/greenwave/tests/test_waive.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0+ from greenwave.policies import ( - InvalidGatingYaml, + InvalidRemoteRuleYaml, TestResultMissing, TestResultFailed, ) @@ -79,7 +79,7 @@ def test_waive_missing_result(): def test_waive_invalid_gatin_yaml(): answers = [ - InvalidGatingYaml( + InvalidRemoteRuleYaml( subject=test_subject(), test_case_name='invalid-gating-yaml', details='', From 626ed0c871bd543f908793e310061ef7f4f542a6 Mon Sep 17 00:00:00 2001 From: Valerij Maljulin Date: Jan 29 2020 15:53:12 +0000 Subject: [PATCH 2/2] fixup! Allow usage of the side tags repo --- diff --git a/docs/policies.rst b/docs/policies.rst index c155e7b..d956608 100644 --- a/docs/policies.rst +++ b/docs/policies.rst @@ -260,13 +260,13 @@ Greenwave requires these configuration parameters ``KOJI_BASE_URL`` and ``REMOTE_RULE_POLICIES``. ``REMOTE_RULE_POLICIES`` is a map, where the key is the subject type pattern -(file matching wildcards (* and ?) are allowed). There should be a default +(file matching wildcards (* and ?) are allowed). There could be a default pattern "*". Each pattern should contain a map of two parameters depending on a retrieval mechanism. Greenwave has two mechanisms to retrieve the remote rule file: ``git archive`` is using for side tags rules and using a git front-end. ``GIT_URL`` and ``GIT_PATH_TEMPLATE`` -should be set for ``git archive`` mechanism, ``HTTP_BASE_URL`` and ``HTTP_URL_TEMPLATE`` +should be set for ``git archive`` mechanism, ``HTTP_URL_TEMPLATE`` should be set if you are going to use a git front-end. Below is an example configuration where ``git archive`` is being used for "brew-build-group" @@ -280,8 +280,7 @@ subject type and HTTP is being used for other: 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' }, '*': { - 'HTTP_BASE_URL': 'https://src.fedoraproject.org/', - 'HTTP_URL_TEMPLATE': '{BASE_URL}{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml' + 'HTTP_URL_TEMPLATE': 'https://src.fedoraproject.org/{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml' } } KOJI_BASE_URL = 'https://koji.fedoraproject.org/kojihub' diff --git a/greenwave/app_factory.py b/greenwave/app_factory.py index 28bf27c..42c3abf 100644 --- a/greenwave/app_factory.py +++ b/greenwave/app_factory.py @@ -25,7 +25,7 @@ def _can_use_remote_rule(config): (conf_item.get('GIT_URL') and conf_item.get('GIT_PATH_TEMPLATE')) or conf_item.get('HTTP_URL_TEMPLATE') for conf_item in config.get('REMOTE_RULE_POLICIES').values() - ) and '*' in config.get('REMOTE_RULE_POLICIES') + ) return False @@ -59,8 +59,7 @@ def create_app(config_obj=None): 'If you want to apply a RemoteRule, you must have "KOJI_BASE_URL" and ' '"REMOTE_RULE_POLICIES" or both set in your configuration. ' 'Each field in "REMOTE_RULE_POLICIES" map have to contain either ' - '"GIT_URL"/"GIT_PATH_TEMPLATE" or "HTTP_URL_TEMPLATE". ' - 'Setting the default configuration by using "*" in map is also mandatory' + '"GIT_URL"/"GIT_PATH_TEMPLATE" or "HTTP_URL_TEMPLATE".' ) # register error handlers diff --git a/greenwave/policies.py b/greenwave/policies.py index e3e0e9e..85642e7 100644 --- a/greenwave/policies.py +++ b/greenwave/policies.py @@ -439,6 +439,9 @@ class RemoteRule(Rule): continue if fnmatch(subject, subj): return cnf + if '*' not in rr_config: + raise RuntimeError(f'Cannot use a remote rule for {subject} subject ' + f'as it has not been configured') return rr_config['*'] def _get_sub_policies(self, policy, subject): diff --git a/greenwave/tests/test_app_factory.py b/greenwave/tests/test_app_factory.py index 805fbe8..8597294 100644 --- a/greenwave/tests/test_app_factory.py +++ b/greenwave/tests/test_app_factory.py @@ -67,7 +67,6 @@ def test_can_use_remote_rule_http(): 'REMOTE_RULE_POLICIES': { 'brew-build-group': { 'GIT_URL': 'git@gitlab.cee.redhat.com:devops/greenwave-policies/side-tags.git', - 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' } }, 'KOJI_BASE_URL': 'https://koji.domain.local/kojihub'