From 9f99896a7ccf4d1609388c73a953eb48d66a8bf2 Mon Sep 17 00:00:00 2001
From: František Zatloukal <fzatlouk@redhat.com>
Date: Feb 05 2021 21:30:48 +0000
Subject: Handle pagure pagination


Fixes https://pagure.io/fedora-qa/oraculum/issue/93

---

diff --git a/oraculum/utils/pagure.py b/oraculum/utils/pagure.py
index 5403c4a..fb38470 100644
--- a/oraculum/utils/pagure.py
+++ b/oraculum/utils/pagure.py
@@ -90,15 +90,21 @@ def get_pagure_groups():
     }
     """
     groups_users = {}
-    resp = get_json("https://src.fedoraproject.org/api/0/groups?per_page=100")  # TODO: Handle pagination properly
+    resp = get_json("https://src.fedoraproject.org/api/0/groups?per_page=100")
     if not resp:
         return None
+    groups = resp["groups"]
+    while resp["pagination"]["next"]:
+        resp = get_json(resp["pagination"]["next"])
+        if not resp:
+            return None
+        groups.extend(resp["groups"])
     allowed_groups = [re.compile(a) for a in app.config["ALLOWED_PACKAGER_GROUPS"]]
-    for group in resp["groups"]:
+    for group in groups:
         if not any(regex.match(group) for regex in allowed_groups):
             continue
         app.logger.debug("Checking out Pagure group %s" % group)
-        group_resp = get_json("https://src.fedoraproject.org/api/0/group/%s?projects=1&acl=commit" % group)
+        group_resp = get_json("https://src.fedoraproject.org/api/0/group/%s?projects=1&acl=commit&per_page=100" % group)
         if not group_resp:
             continue
         try:
@@ -109,6 +115,16 @@ def get_pagure_groups():
         except TypeError:
             app.logger.error("Skipped Pagure group %s because of an error" % group)
             continue
+        try:
+            while group_resp["pagination"]["next"]:
+                group_resp = get_json(group_resp["pagination"]["next"])
+                if not group_resp:
+                    return None
+                groups_users[group]["packages"].extend([project["name"] for project in group_resp["projects"]])
+        except TypeError:
+            app.logger.error("Skipped Pagure group %s because of an error" % group)
+            continue
+
     return groups_users
 
 
diff --git a/tests/test_packager_dashboard_pagure.py b/tests/test_packager_dashboard_pagure.py
index 69ccbad..71b819a 100644
--- a/tests/test_packager_dashboard_pagure.py
+++ b/tests/test_packager_dashboard_pagure.py
@@ -7,6 +7,11 @@ from oraculum import app, CACHE
 def stub_get_json_for_pagure_groups(url):
 
     stub_groups = {
+        "pagination": {
+            "page": 1,
+            "next": None,
+            "pages": 1
+        },
         "groups": [
             "group_not_to_be_included",
             "included-cause-sig",
@@ -16,6 +21,11 @@ def stub_get_json_for_pagure_groups(url):
     }
 
     stub_group = {
+        "pagination": {
+            "page": 1,
+            "next": None,
+            "pages": 1
+        },
         "members": ["jdoe", "mr_packager"],
         "projects": [
             {
@@ -31,6 +41,69 @@ def stub_get_json_for_pagure_groups(url):
         return stub_group
     return stub_groups
 
+def stub_get_json_for_pagure_groups_paged(url):
+
+    stub_groups = [
+        {
+        "pagination": {
+            "page": 1,
+            "next": url + "&page=2",
+            "pages": 2
+        },
+        "groups": [
+            "group_not_to_be_included",
+            "included-cause-sig"
+        ]
+        },
+        {
+        "pagination": {
+            "page": 2,
+            "next": None,
+            "pages": 2
+        },
+        "groups": [
+            "sig-but-not-really",
+            "sssd-maintainers"
+        ]
+        },
+    ]
+
+    stub_group = [
+        {
+        "pagination": {
+            "page": 1,
+            "next": url + "&page=2",
+            "pages": 2
+        },
+        "members": ["jdoe", "mr_packager"],
+        "projects": [
+            {
+                "name": "glorious_package"
+            }
+        ]
+    },
+        {
+        "pagination": {
+            "page": 2,
+            "next": None,
+            "pages": 2
+        },
+        "members": ["jdoe", "mr_packager"],
+        "projects": [
+            {
+                "name": "some_other_package"
+            }
+        ]
+    },
+    ]
+    if "projects=1&acl=commit" in url:
+        if url.endswith("page=2"):
+            return stub_group[1]
+        return stub_group[0]
+
+    if url.endswith("page=2"):
+        return stub_groups[1]
+    return stub_groups[0]
 
 def stub_get_json_for_pagure_versions(url, *args):
 
@@ -180,6 +253,11 @@ class TestPagure(object):
 
         assert expected_user_group_result == result_complete["group"]
 
+        # and finally try pagination
+        monkeypatch.setattr(pagure, 'get_json', stub_get_json_for_pagure_groups_paged)
+        result_groups = pagure.get_pagure_groups()
+        assert result_groups == expected_groups_result
+
 
     def test_pagure_versions(self, monkeypatch):
         pagure_versions_result = {