From b00b6094fd65e557b62898450bf0c6b514e6779e Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Aug 21 2017 23:59:47 +0000 Subject: Add a new "Any" rule providing logical disjunction. This enables a atomic_ci_pipeline_results policy. We wouldn't need the 'Any' rule, except that the ci pipeline oddly reports results for complete or ignored as different testcase names. Really - it's the same testcase, not two different ones, so we should work with them to consolidate that into a single testcase with PASSED or INFO(skipped) result status. Anyways, this is a short-term fix to be able to provide gating decisions based on their results. Fixes #61. --- diff --git a/conf/policies/fedora.yaml b/conf/policies/fedora.yaml index 53b5e28..59309eb 100644 --- a/conf/policies/fedora.yaml +++ b/conf/policies/fedora.yaml @@ -8,4 +8,13 @@ decision_context: bodhi_update_push_stable rules: - !PassingTestCaseRule {test_case_name: dist.abicheck} - !PassingTestCaseRule {test_case_name: dist.rpmdeplint} - - !PassingTestCaseRule {test_case_name: dist.upgradepath} \ No newline at end of file + - !PassingTestCaseRule {test_case_name: dist.upgradepath} +# Fedora Atomic CI pipeline +# http://fedoraproject.org/wiki/CI +--- !Policy +id: "atomic_ci_pipeline_results" +product_versions: + - fedora-26 +decision_context: bodhi_update_push_stable +rules: + - !Any { test_case_names: [org.centos.prod.ci.pipeline.package.complete, org.centos.prod.ci.pipeline.package.ignored] } diff --git a/functional-tests/test_api_v1.py b/functional-tests/test_api_v1.py index d21a4ae..5cbec4f 100644 --- a/functional-tests/test_api_v1.py +++ b/functional-tests/test_api_v1.py @@ -83,6 +83,11 @@ TASKTRON_RELEASE_CRITICAL_TASKS = [ 'dist.upgradepath', ] +ATOMIC_CI_TASKS = [ + 'org.centos.prod.ci.pipeline.package.ignored', + 'org.centos.prod.ci.pipeline.package.complete', +] + def test_cannot_make_decision_without_product_version(requests_session, greenwave_server): data = { @@ -286,6 +291,10 @@ def test_bodhi_push_update_stable_policy( testdatabuilder.create_result(item=nvr, testcase_name=testcase_name, outcome='PASSED') + # mark atomic ci as ignored, but leave the 'complete' result absent. + testdatabuilder.create_result(item=nvr, + testcase_name=ATOMIC_CI_TASKS[0], + outcome='PASSED') data = { 'decision_context': 'bodhi_update_push_stable', 'product_version': 'fedora-26', @@ -297,7 +306,10 @@ def test_bodhi_push_update_stable_policy( assert r.status_code == 200 res_data = r.json() assert res_data['policies_satisified'] is True - assert res_data['applicable_policies'] == ['taskotron_release_critical_tasks'] + assert res_data['applicable_policies'] == [ + 'taskotron_release_critical_tasks', + 'atomic_ci_pipeline_results', + ] expected_summary = 'all required tests passed' assert res_data['summary'] == expected_summary assert res_data['unsatisfied_requirements'] == [] @@ -322,6 +334,10 @@ def test_multiple_results_in_a_subject( testdatabuilder.create_result(item=nvr, testcase_name=testcase_name, outcome='PASSED') + # mark atomic ci as ignored, but leave the 'complete' result absent. + testdatabuilder.create_result(item=nvr, + testcase_name=ATOMIC_CI_TASKS[0], + outcome='PASSED') data = { 'decision_context': 'bodhi_update_push_stable', 'product_version': 'fedora-26', @@ -334,8 +350,11 @@ def test_multiple_results_in_a_subject( res_data = r.json() # The failed result should be taken into account. assert res_data['policies_satisified'] is False - assert res_data['applicable_policies'] == ['taskotron_release_critical_tasks'] - assert res_data['summary'] == '1 of 3 required tests failed' + assert res_data['applicable_policies'] == [ + 'taskotron_release_critical_tasks', + 'atomic_ci_pipeline_results', + ] + assert res_data['summary'] == '1 of 4 required tests failed' expected_unsatisfied_requirements = [ { 'item': {'item': nvr, 'type': 'koji_build'}, @@ -345,3 +364,42 @@ def test_multiple_results_in_a_subject( }, ] assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements + + +def test_failing_any_rule( + requests_session, greenwave_server, testdatabuilder): + """ + This make sure that Greenwave returns an intelligent response if the Any + rule fails. + """ + nvr = testdatabuilder.unique_nvr() + # All passing here.. but no atomic_ci results. + for testcase_name in TASKTRON_RELEASE_CRITICAL_TASKS: + testdatabuilder.create_result(item=nvr, + testcase_name=testcase_name, + outcome='PASSED') + data = { + 'decision_context': 'bodhi_update_push_stable', + 'product_version': 'fedora-26', + 'subject': [{'item': nvr, 'type': 'koji_build'}] + } + r = requests_session.post(greenwave_server.url + 'api/v1.0/decision', + headers={'Content-Type': 'application/json'}, + data=json.dumps(data)) + assert r.status_code == 200 + res_data = r.json() + # The failed result should be taken into account. + assert res_data['policies_satisified'] is False + assert res_data['applicable_policies'] == [ + 'taskotron_release_critical_tasks', + 'atomic_ci_pipeline_results', + ] + assert res_data['summary'] == '1 of 4 required tests not found' + expected_unsatisfied_requirements = [ + { + 'item': {'item': nvr, 'type': 'koji_build'}, + 'testcase': 'org.centos.prod.ci.pipeline.package.complete', + 'type': 'test-result-missing' + }, + ] + assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements diff --git a/greenwave/policies.py b/greenwave/policies.py index b95fe03..55bc4ea 100644 --- a/greenwave/policies.py +++ b/greenwave/policies.py @@ -159,6 +159,37 @@ class PassingTestCaseRule(Rule): return "%s(test_case_name=%r)" % (self.__class__.__name__, self.test_case_name) +class Any(Rule): + """ + This rule requires at least one of the given testcases to either pass or be + waived. + """ + yaml_tag = u'!Any' + yaml_loader = yaml.SafeLoader + + def __init__(self, test_case_names): + self.test_case_names = test_case_names + + def check(self, item, results, waivers): + exemplar = None + sub_rules = [ + PassingTestCaseRule(test_case_name=name) + for name in self.test_case_names + ] + for rule in sub_rules: + result = rule.check(item, results, waivers) + if isinstance(result, RuleSatisfied): + return result + if not exemplar and isinstance(result, TestResultMissing): + exemplar = result + if isinstance(result, TestResultFailed): + exemplar = result + return exemplar + + def __repr__(self): + return "%s(test_case_names=%r)" % (self.__class__.__name__, self.test_case_names) + + class Policy(yaml.YAMLObject): yaml_tag = u'!Policy' yaml_loader = yaml.SafeLoader diff --git a/greenwave/tests/test_policies.py b/greenwave/tests/test_policies.py index 7af3b48..1209436 100644 --- a/greenwave/tests/test_policies.py +++ b/greenwave/tests/test_policies.py @@ -2,7 +2,13 @@ # SPDX-License-Identifier: GPL-2.0+ from greenwave.app_factory import create_app -from greenwave.policies import summarize_answers, RuleSatisfied, TestResultMissing, TestResultFailed +from greenwave.policies import ( + summarize_answers, + RuleSatisfied, + TestResultMissing, + TestResultFailed, + Any, +) def test_summarize_answers(): @@ -23,7 +29,9 @@ def test_load_policies(): app = create_app('greenwave.config.TestingConfig') assert len(app.config['policies']) > 0 assert any(policy.id == '1' for policy in app.config['policies']) - assert any(policy.decision_context == 'errata_newfile_to_qe' for policy in - app.config['policies']) - assert any(rule.test_case_name == 'dist.rpmdiff.analysis.abi_symbols' for policy in - app.config['policies'] for rule in policy.rules) + assert any(policy.decision_context == 'errata_newfile_to_qe' + for policy in app.config['policies']) + assert any(getattr(rule, 'test_case_name', None) == 'dist.rpmdiff.analysis.abi_symbols' + for policy in app.config['policies'] for rule in policy.rules) + assert any(isinstance(rule, Any) + for policy in app.config['policies'] for rule in policy.rules)