PR#129: Handle pagure pagination - fedora-qa/oraculum

fedora-qa / oraculum

#129 Handle pagure pagination

Merged 4 years ago by frantisekz. Opened 4 years ago by frantisekz.

pagination into master

Handle pagure pagination

František Zatloukal • 4 years ago

9f99896

oraculum/utils/pagure.py

file modified

+19 -3

		`@@ -90,15 +90,21 @@`
		`}`
		`"""`
		`groups_users = {}`
		`- resp = get_json("https://src.fedoraproject.org/api/0/groups?per_page=100") # TODO: Handle pagination properly`
		`+ resp = get_json("https://src.fedoraproject.org/api/0/groups?per_page=100")`
		`if not resp:`
		`return None`
		`+ groups = resp["groups"]`
		`+ while resp["pagination"]["next"]:`
		`+ resp = get_json(resp["pagination"]["next"])`
		`+ if not resp:`
		`+ return None`
		`+ groups.extend(resp["groups"])`
		`allowed_groups = [re.compile(a) for a in app.config["ALLOWED_PACKAGER_GROUPS"]]`
		`- for group in resp["groups"]:`
		`+ for group in groups:`
		`if not any(regex.match(group) for regex in allowed_groups):`
		`continue`
		`app.logger.debug("Checking out Pagure group %s" % group)`
		`- group_resp = get_json("https://src.fedoraproject.org/api/0/group/%s?projects=1&acl=commit" % group)`
		`+ group_resp = get_json("https://src.fedoraproject.org/api/0/group/%s?projects=1&acl=commit&per_page=100" % group)`
		`if not group_resp:`
		`continue`
		`try:`
		`@@ -109,6 +115,16 @@`
		`except TypeError:`
		`app.logger.error("Skipped Pagure group %s because of an error" % group)`
		`continue`
		`+ try:`
		`+ while group_resp["pagination"]["next"]:`
		`+ group_resp = get_json(group_resp["pagination"]["next"])`
		`+ if not group_resp:`
		`+ return None`
		`+ groups_users[group]["packages"].extend([project["name"] for project in group_resp["projects"]])`
		`+ except TypeError:`
		`+ app.logger.error("Skipped Pagure group %s because of an error" % group)`
		`+ continue`
		`+`
		`return groups_users`

tests/test_packager_dashboard_pagure.py

file modified

+78

		`@@ -7,6 +7,11 @@`
		`def stub_get_json_for_pagure_groups(url):`

		`stub_groups = {`
		`+ "pagination": {`
		`+ "page": 1,`
		`+ "next": None,`
		`+ "pages": 1`
		`+ },`
		`"groups": [`
		`"group_not_to_be_included",`
		`"included-cause-sig",`
		`@@ -16,6 +21,11 @@`
		`}`

		`stub_group = {`
		`+ "pagination": {`
		`+ "page": 1,`
		`+ "next": None,`
		`+ "pages": 1`
		`+ },`
		`"members": ["jdoe", "mr_packager"],`
		`"projects": [`
		`{`
		`@@ -31,6 +41,69 @@`
		`return stub_group`
		`return stub_groups`

		`+ def stub_get_json_for_pagure_groups_paged(url):`
		`+`
		`+ stub_groups = [`
		`+ {`
		`+ "pagination": {`
		`+ "page": 1,`
		`+ "next": url + "&page=2",`
		`+ "pages": 2`
		`+ },`
		`+ "groups": [`
		`+ "group_not_to_be_included",`
		`+ "included-cause-sig"`
		`+ ]`
		`+ },`
		`+ {`
		`+ "pagination": {`
		`+ "page": 2,`
		`+ "next": None,`
		`+ "pages": 2`
		`+ },`
		`+ "groups": [`
		`+ "sig-but-not-really",`
		`+ "sssd-maintainers"`
		`+ ]`
		`+ },`
		`+ ]`
		`+`
		`+ stub_group = [`
		`+ {`
		`+ "pagination": {`
		`+ "page": 1,`
		`+ "next": url + "&page=2",`
		`+ "pages": 2`
		`+ },`
		`+ "members": ["jdoe", "mr_packager"],`
		`+ "projects": [`
		`+ {`
		`+ "name": "glorious_package"`
		`+ }`
		`+ ]`
		`+ },`
		`+ {`
		`+ "pagination": {`
		`+ "page": 2,`
		`+ "next": None,`
		`+ "pages": 2`
		`+ },`
		`+ "members": ["jdoe", "mr_packager"],`
		`+ "projects": [`
		`+ {`
		`+ "name": "some_other_package"`
		`+ }`
		`+ ]`
		`+ },`
		`+ ]`
		`+ if "projects=1&acl=commit" in url:`
		`+ if url.endswith("page=2"):`
		`+ return stub_group[1]`
		`+ return stub_group[0]`
		`+`
		`+ if url.endswith("page=2"):`
		`+ return stub_groups[1]`
		`+ return stub_groups[0]`

		`def stub_get_json_for_pagure_versions(url, *args):`

		`@@ -180,6 +253,11 @@`

		`assert expected_user_group_result == result_complete["group"]`

		`+ # and finally try pagination`
		`+ monkeypatch.setattr(pagure, 'get_json', stub_get_json_for_pagure_groups_paged)`
		`+ result_groups = pagure.get_pagure_groups()`
		`+ assert result_groups == expected_groups_result`
		`+`

		`def test_pagure_versions(self, monkeypatch):`
		`pagure_versions_result = {`