PR#200: Koschei: Pull latest build metadata from koschei - fedora-qa/oraculum

fedora-qa / oraculum

#200 Koschei: Pull latest build metadata from koschei

Opened 2 years ago by frantisekz. Modified 3 months ago

issue_55 into master

Koschei: Pull latest build metadata from koschei

František Zatloukal • 2 years ago

250c32b

oraculum/utils/koschei.py

file modified

+6 -43

		`@@ -23,45 +23,9 @@`

		`from collections import defaultdict`

		`- import koji`
		`-`
		`from oraculum import app, CACHE`
		`from oraculum.utils.dashboard_helpers import get_json, release_from_dist, release_is_active`

		`-`
		`- def process_koji_queue(session, data, releases=None):`
		`- if releases is None:`
		`- releases = CACHE.get("fedora_releases")`
		`-`
		`- koji_results = session.multiCall()`
		`- for element in koji_results:`
		`- try:`
		`- # koji multicall returns results packed inside lists of one element, redefine element for better readability later`
		`- element = element[0][0]`
		`- except IndexError:`
		`- # Sometimes, koji returns incomplete results`
		`- continue`
		`- for entry in data[element["name"]]:`
		`- """`
		`- We need to iterate throughout all members of data["package_name"] to find and alter only the architecture we`
		`- now have in the element.`
		`- Example of element (only keys we use are listed below):`
		`- {'tag_name': 'fXX',`
		`- 'build_id': XYZ,`
		`- 'completion_time': 'YYYY-MM-DD HH:MM:SS.FFFFFF',`
		`- 'name': 'package_name',`
		`- ...}`
		`- """`
		`- # Since dict doesn't guarantee order, we must pair koji result and data entry`
		`- if release_from_dist(element["tag_name"], releases) == entry["release"]:`
		`- data[element["name"]][data[element["name"]].index(entry)]["last_success"] = {`
		`- "time": element["completion_time"],`
		`- "url": app.config["KOJI_URL"] + "buildinfo?buildID=" + str(element["build_id"])}`
		`- # And we can jump to another element early if we found what we needed, yay!`
		`- continue`
		`- return data`
		`-`
		`-`
		`def parse_koschei_data():`
		`"""`
		`Prepares data from koschei for easier parsing in get_user_koschei_data()`
		`@@ -74,9 +38,6 @@`
		`data = defaultdict(list)`
		`koschei_resp = get_json(app.config['KOSCHEI_API_URL'])`
		`releases = CACHE.get("fedora_releases")`
		`- # Set up koji API`
		`- koji_session = koji.ClientSession(app.config['KOJIHUB_URL'])`
		`- koji_session.multicall = True`

		`for item in koschei_resp:`
		`if "playground" in item["collection"]:`
		`@@ -92,11 +53,13 @@`
		`"release": release_from_dist(item["collection"], releases),`
		`"status": item["state"],`
		`"url": "https://koschei.fedoraproject.org/package/%s?collection=%s" % (item["name"], item["collection"]),`
		`- "last_success": {"time": None, "url": None}})`
		`- if item["state"] == "failing":`
		`- koji_session.getLatestBuilds(item["collection"], package=item["name"])`
		`+ "last_success": {`
		`+ "time": item["last_complete_build"]["time_finished"],`
		`+ "url": app.config["KOJI_URL"] + "taskinfo?taskID=" + str(item["last_complete_build"]["task_id"])`
		`+ } if item["last_complete_build"] is not None else {"time": None, "url": None}`
		`+ })`

		`- return process_koji_queue(koji_session, data)`
		`+ return data`


		`def get_user_koschei_data(packages):`

tests/test_packager_dashboard_koschei.py

file modified

+7 -44

		`@@ -3,16 +3,6 @@`
		`from oraculum.utils import koschei`
		`from oraculum import app, CACHE`

		`- class MockedKoji():`
		`-`
		`- def multiCall():`
		`- # Koji returns only one build per release`
		- stub_koji_element_a = [[{'tag_id': 2, 'tag_name': 'f98', 'id': 1, 'build_id': 1, 'version': '1.2.2', 'release': '2.fc98', 'epoch': None, 'state': 1, 'completion_time': '2020-01-28 11:08:00.940846+00:00', 'start_time': '2020-01-28 11:06:16.563830+00:00', 'task_id': 1, 'creation_event_id': 1, 'creation_time': '2020-01-28 11:06:16.563830+00:00', 'volume_id': 0, 'volume_name': 'DEFAULT', 'package_id': 1, 'package_name': 'package_a', 'name': 'package_a', 'nvr': 'package_a-1.2.2-2.fc99', 'owner_id': 1, 'owner_name': 'releng'}]]
		`-`
		- stub_koji_element_b = [[{'tag_id': 3, 'tag_name': 'f99', 'id': 2, 'build_id': 2, 'version': '1.2.2', 'release': '1.fc99', 'epoch': None, 'state': 1, 'completion_time': '2020-01-01 11:08:00.940846+00:00', 'start_time': '2020-01-01 11:06:16.563830+00:00', 'task_id': 1, 'creation_event_id': 1, 'creation_time': '2020-01-01 11:06:16.563830+00:00', 'volume_id': 0, 'volume_name': 'DEFAULT', 'package_id': 1, 'package_name': 'package_a', 'name': 'package_a', 'nvr': 'package_a-1.2.2-1.fc99', 'owner_id': 1, 'owner_name': 'releng'}]]
		`-`
		`- return [stub_koji_element_a, stub_koji_element_b]`
		`-`

		`class TestKoschei(object):`

		`@@ -34,7 +24,7 @@`
		`'url': 'https://koschei.fedoraproject.org/package/package_a?collection=f98'`
		`},`
		`{`
		`- 'last_success': {'time': None, 'url': None},`
		`+ 'last_success': {'time': "2020-01-01T11:08:00.940846", 'url': "https://koji.fedoraproject.org/koji/taskinfo?taskID=2"},`
		`'release': 'Fedora Rawhide',`
		`'status': 'failing',`
		`'url': 'https://koschei.fedoraproject.org/package/package_a?collection=f99'`
		`@@ -61,10 +51,7 @@`
		`"name":"package_a",`
		`"collection":"f98",`
		`"state":"ok",`
		`- "last_complete_build":`
		`- {`
		`- "task_id":1`
		`- }`
		`+ "last_complete_build": None`
		`},`
		`{`
		`"name":"package_a",`
		`@@ -72,35 +59,27 @@`
		`"state":"failing",`
		`"last_complete_build":`
		`{`
		`- "task_id":2`
		`+ "task_id": 2,`
		`+ "time_finished": "2020-01-01T11:08:00.940846"`
		`}`
		`},`
		`{`
		`"name":"package_a",`
		`"collection":"epel1",`
		`"state":"ok",`
		`- "last_complete_build":`
		`- {`
		`- "task_id":3`
		`- }`
		`+ "last_complete_build": None`
		`},`
		`{`
		`"name":"package_a",`
		`"collection":"epel2",`
		`"state":"blocked",`
		`- "last_complete_build":`
		`- {`
		`- "task_id":4`
		`- }`
		`+ "last_complete_build": None`
		`},`
		`{`
		`"name":"package_a",`
		`"collection":"eln",`
		`"state":"blocked",`
		`- "last_complete_build":`
		`- {`
		`- "task_id":5`
		`- }`
		`+ "last_complete_build": None`
		`}`
		`]`

		`@@ -111,20 +90,4 @@`

		`monkeypatch.setattr(CACHE, 'get', mock.MagicMock(return_value=self.stub_fedora_releases))`
		`monkeypatch.setattr(koschei, 'get_json', mock.MagicMock(return_value=stub_koschei_data))`
		`- monkeypatch.setattr(koschei, 'koji', mock.MagicMock(return_value=True))`
		`assert koschei.parse_koschei_data() == self.koschei_result`
		`-`
		`- def test_koji_data(self, monkeypatch):`
		`-`
		`- expected_result = {`
		`- 'last_success': {'time': '2020-01-01 11:08:00.940846+00:00', 'url': 'https://koji.fedoraproject.org/koji/buildinfo?buildID=2'},`
		`- 'release': 'Fedora Rawhide',`
		`- 'status': 'failing',`
		`- 'url': 'https://koschei.fedoraproject.org/package/package_a?collection=f99'`
		`- }`
		`-`
		`- monkeypatch.setattr(CACHE, 'get', mock.MagicMock(return_value=self.stub_fedora_releases))`
		`- result = koschei.process_koji_queue(MockedKoji, self.koschei_result)`
		`-`
		`- assert len(result) == 1`
		`- assert expected_result in result['package_a']`

frantisekz commented 2 years ago

Fixes https://pagure.io/fedora-qa/packager_dashboard/issue/55

zuul commented 2 years ago

Build succeeded.

oraculum-pytest : SUCCESS in 1m 59s

frantisekz commented 2 years ago

@mizdebsk is my assumption that it is safe to assume that koschei would always have the latest metadata for any failing build, correct, so I don't need to bother with asking koji?

Current dataset looks like it's true, just wanted to be sure...

rebased onto 093a6c7

2 years ago

zuul commented 2 years ago

Build succeeded.

oraculum-pytest : SUCCESS in 2m 02s

rebased onto bfd8bfc

2 years ago

zuul commented 2 years ago

Build succeeded.

oraculum-pytest : SUCCESS in 1m 57s

rebased onto 250c32b

2 years ago

zuul commented 2 years ago

Build succeeded.

oraculum-pytest : SUCCESS in 2m 00s

imcinerney commented 3 months ago

I came here from the linked issue, and it would be great if this could be finally merged to get the actual last build time displayed. What is the current blocker for this?