From 0bdaae679bb994ff15c6978e1ef8004e4895c5fe Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Oct 20 2017 14:38:09 +0000 Subject: Get "scenarios" working for greenwave. I spent some time this afternoon with @mohanboddu and @adamwill investigating use of greenwave for gating rawhide composes. https://infrastructure.fedoraproject.org/cgit/ansible.git/commit/?id=f693b6de0d6566881adc189404176edb0bef6121 One thing we ran into was that the *requirements* for a rawhide compose can't be expressed only in terms of testcase names. They must be specified in terms of both the testcase name *and* the scenario. This patch gives greenwave the ability to do that. A passing testcaserule now optionally allows also specifying a required scenario for that testcase. I don't yet have a full list of testcase X scenario combinations, but when I have one, I'll commit it to ansible. --- diff --git a/conf/policies/fedora.yaml b/conf/policies/fedora.yaml index 3f5410e..05d9917 100644 --- a/conf/policies/fedora.yaml +++ b/conf/policies/fedora.yaml @@ -31,3 +31,12 @@ blacklist: [] rules: - !PassingTestCaseRule {test_case_name: dist.rpmdeplint} - !PassingTestCaseRule {test_case_name: dist.upgradepath} +--- !Policy +id: "openqa_important_stuff_for_rawhide" +product_versions: + - fedora-rawhide +decision_context: rawhide_compose_sync_to_mirrors +blacklist: [] +rules: + - !PassingTestCaseRule {test_case_name: compose.install_no_user, scenario: scenario1} + - !PassingTestCaseRule {test_case_name: compose.install_no_user, scenario: scenario2} diff --git a/functional-tests/conftest.py b/functional-tests/conftest.py index bc400b6..f3be25a 100644 --- a/functional-tests/conftest.py +++ b/functional-tests/conftest.py @@ -123,12 +123,17 @@ class TestDataBuilder(object): def unique_nvr(self): return 'glibc-1.0-{}.el7'.format(self._counter.next()) - def create_result(self, item, testcase_name, outcome): + def unique_compose_id(self): + return 'Fedora-9000-19700101.n.{}'.format(self._counter.next()) + + def create_result(self, item, testcase_name, outcome, scenario=None): data = { 'testcase': {'name': testcase_name}, 'data': {'item': item, 'type': 'koji_build'}, 'outcome': outcome, } + if scenario: + data['data']['scenario'] = scenario response = self.requests_session.post( self.resultsdb_url + 'api/v2.0/results', headers={'Content-Type': 'application/json'}, diff --git a/functional-tests/test_api_v1.py b/functional-tests/test_api_v1.py index a3878ed..10e422a 100644 --- a/functional-tests/test_api_v1.py +++ b/functional-tests/test_api_v1.py @@ -83,6 +83,14 @@ TASKTRON_RELEASE_CRITICAL_TASKS = [ 'dist.upgradepath', ] +OPENQA_TASKS = [ + 'compose.install_no_user', +] +OPENQA_SCENARIOS = [ + 'scenario1', + 'scenario2', +] + def test_inspect_policies(requests_session, greenwave_server): r = requests_session.get(greenwave_server.url + 'api/v1.0/policies', @@ -90,20 +98,23 @@ def test_inspect_policies(requests_session, greenwave_server): assert r.status_code == 200 body = r.json() policies = body['policies'] - assert len(policies) == 4 + assert len(policies) == 5 assert any(p['id'] == 'taskotron_release_critical_tasks' for p in policies) assert any(p['decision_context'] == 'bodhi_update_push_stable' for p in policies) assert any(p['product_versions'] == ['fedora-26'] for p in policies) expected_rules = [ {'rule': 'PassingTestCaseRule', - 'test_case_name': 'dist.abicheck'}, + 'test_case_name': 'dist.abicheck', + 'scenario': None}, ] assert any(p['rules'] == expected_rules for p in policies) expected_rules = [ {'rule': 'PassingTestCaseRule', - 'test_case_name': 'dist.rpmdeplint'}, + 'test_case_name': 'dist.rpmdeplint', + 'scenario': None}, {'rule': 'PassingTestCaseRule', - 'test_case_name': 'dist.upgradepath'}] + 'test_case_name': 'dist.upgradepath', + 'scenario': None}] assert any(p['rules'] == expected_rules for p in policies) @@ -418,6 +429,66 @@ def test_ignore_result(requests_session, greenwave_server, testdatabuilder): assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements +def test_make_a_decison_on_passed_result_with_scenario(requests_session, greenwave_server, testdatabuilder): + """ + If we require two scenarios to pass, and both pass, then we pass. + """ + compose_id = testdatabuilder.unique_compose_id() + for testcase_name in OPENQA_TASKS: + for scenario in OPENQA_SCENARIOS: + testdatabuilder.create_result(item=compose_id, + testcase_name=testcase_name, + scenario=scenario, + outcome='PASSED') + data = { + 'decision_context': 'rawhide_compose_sync_to_mirrors', + 'product_version': 'fedora-rawhide', + 'subject': [{'item': compose_id}], + } + r = requests_session.post(greenwave_server.url + 'api/v1.0/decision', + headers={'Content-Type': 'application/json'}, + data=json.dumps(data)) + assert r.status_code == 200 + res_data = r.json() + assert res_data['policies_satisified'] is True + assert res_data['applicable_policies'] == ['openqa_important_stuff_for_rawhide'] + expected_summary = 'all required tests passed' + assert res_data['summary'] == expected_summary + + +def test_make_a_decison_on_failing_result_with_scenario(requests_session, greenwave_server, testdatabuilder): + """ + If we require two scenarios to pass, and one is failing, then we fail. + """ + + compose_id = testdatabuilder.unique_compose_id() + for testcase_name in OPENQA_TASKS: + # Scenario 1 passes.. + testdatabuilder.create_result(item=compose_id, + testcase_name=testcase_name, + scenario='scenario1', + outcome='PASSED') + # But scenario 2 fails! + testdatabuilder.create_result(item=compose_id, + testcase_name=testcase_name, + scenario='scenario2', + outcome='FAILED') + data = { + 'decision_context': 'rawhide_compose_sync_to_mirrors', + 'product_version': 'fedora-rawhide', + 'subject': [{'item': compose_id}], + } + r = requests_session.post(greenwave_server.url + 'api/v1.0/decision', + headers={'Content-Type': 'application/json'}, + data=json.dumps(data)) + assert r.status_code == 200 + res_data = r.json() + assert res_data['policies_satisified'] is False + assert res_data['applicable_policies'] == ['openqa_important_stuff_for_rawhide'] + expected_summary = '1 of 2 required tests failed' + assert res_data['summary'] == expected_summary + + def test_ignore_waiver(requests_session, greenwave_server, testdatabuilder): """ This tests that a waiver can be ignored when making the decision. diff --git a/greenwave/policies.py b/greenwave/policies.py index eb02db6..55ec540 100644 --- a/greenwave/policies.py +++ b/greenwave/policies.py @@ -147,6 +147,11 @@ class PassingTestCaseRule(Rule): def check(self, item, results, waivers): matching_results = [r for r in results if r['testcase']['name'] == self.test_case_name] + + # Rules may optionally specify a scenario to limit applicability. + if self._scenario: + matching_results = [r for r in matching_results if self.scenario in r['data'].get('scenario', [])] + if not matching_results: return TestResultMissing(item, self.test_case_name) # If we find multiple matching results, we always use the first one which @@ -160,13 +165,18 @@ class PassingTestCaseRule(Rule): return RuleSatisfied() return TestResultFailed(item, self.test_case_name, matching_result['id']) + @property + def _scenario(self): + return getattr(self, 'scenario', None) + def __repr__(self): - return "%s(test_case_name=%r)" % (self.__class__.__name__, self.test_case_name) + return "%s(test_case_name=%r, scenario=%r)" % (self.__class__.__name__, self.test_case_name, self._scenario) def to_json(self): return { 'rule': self.__class__.__name__, 'test_case_name': self.test_case_name, + 'scenario': self._scenario, } diff --git a/greenwave/tests/test_policies.py b/greenwave/tests/test_policies.py index 7241ac9..c2d8d7e 100644 --- a/greenwave/tests/test_policies.py +++ b/greenwave/tests/test_policies.py @@ -190,6 +190,20 @@ rules: assert 'Policies are not configured properly' in str(excinfo.value) +def test_passing_testcasename_with_scenario(tmpdir): + p = tmpdir.join('fedora.yaml') + p.write(""" +--- !Policy +id: "rawhide_compose_sync_to_mirrors" +product_versions: + - fedora-rawhide +decision_context: rawhide_compose_sync_to_mirrors +rules: + - !PassingTestCaseRule {test_case_name: compose.install_default_upload, scenario: somescenario} + """) + load_policies(tmpdir.strpath) + + def test_version_endpoint(): app = create_app('greenwave.config.TestingConfig') test_app = app.test_client()