#134 Fetch package versions from oraculum
Merged 3 years ago by pingou. Opened 4 years ago by frantisekz.
frantisekz/pagure-dist-git oraculum-versions  into  master

file modified
+1
@@ -53,6 +53,7 @@ 

                                  'refs/heads/epel[0-9]+',

                                  'refs/heads/el[0-9]+',

                                  'refs/heads/olpc[0-9]+']

+    ORACULUM_URL = 'https://packager-dashboard.fedoraproject.org/api/'

  

  CentOS

  ------

file modified
+106 -15
@@ -395,24 +395,82 @@ 

      return flask.jsonify(output)

  

  

- @DISTGIT_NS.route("/bodhi_updates/<namespace>/<repo>", methods=["GET"])

- @api_method

- def bodhi_updates_endpoint(namespace, repo):

-     """Retrieves the current updates in bodhi for the specified package."""

-     if namespace not in ["rpms"]:

+ def query_oraculum_versions(package):

+     """

+     Receives package versions from oraculum

+     Returns triplet: fedora-releases, updates-for-package, F%number-of-current-rawhide

+     """

+ 

+     oraculum_base_url = "https://packager-dashboard.fedoraproject.org/api/"

+     # easier debugging when running directly outside of flask context

+     try:

+         oraculum_url = flask.current_app.config.get("ORACULUM_URL", oraculum_base_url)

+     except RuntimeError:

+         oraculum_url = oraculum_base_url

+ 

+     if not oraculum_url:

          raise pagure.exceptions.APIError(

-             400,

-             error_code=APIERROR.EINVALIDREQ,

-             errors=["Namespace not supported"],

+             500,

+             error_code=APIERROR.ENOCODE,

+             error="This pagure instance has no ORACULUM_URL configured, please "

+             "inform your pagure administrators",

          )

  

-     _log.info(

-         "Received request for the bodhi updates of: %s/%s", namespace, repo

-     )

+     _log.debug("Based ORACULUM url: %s", oraculum_url)

+     req_versions = requests.get("%sv1/package_versions/%s" % (oraculum_url, package))

+     req_releases = requests.get("%sv1/releases" % oraculum_url)

  

-     repo = _get_repo(repo, namespace=namespace)

-     html = pagure.utils.is_true(flask.request.args.get("html", False))

+     if not req_releases.ok:

+         raise pagure.exceptions.APIError(

+                 400,

+                 error_code=APIERROR.EINVALIDREQ,

+                 errors=["Could not call oraculum to query current Fedora Releases."],

+             )

+ 

+     if not req_versions.ok:

+         raise pagure.exceptions.APIError(

+                 400,

+                 error_code=APIERROR.EINVALIDREQ,

+                 errors=["Could not call oraculum to query current package versions."],

+             )

+ 

+     versions_resp = req_versions.json()

+     releases_resp = req_releases.json()

+ 

+     # Releases mapping ({"FXX": "Fedora XX", "EPEL X": "Fedora EPEL X"})

+     releases = {}

+     releases["ELN"] = "Fedora ELN"

+     for epel_release in releases_resp["epel"]:

+         releases["EPEL-%d" % epel_release] = "Fedora EPEL %d" % epel_release

+     for release in releases_resp["fedora"]["values"]:

+         releases["F%d" % release] = "Fedora %d" % release

+ 

+     rawhide_num = max(releases_resp["fedora"]["values"])

+     rawhide = "F" + str(rawhide_num)

  

+     # Rename fedora releases to what pagure's dist-git api uses now

+     # (Fedora XX => FXX; Fedora Rawhide => FXX, EPEL X => EPEL-X)

+     versions = {}

+ 

+     for release in list(versions_resp.keys()):

+         if release in ["Fedora ELN", "Fedora Rawhide"]:

+             continue

+         if release.startswith("Fedora"):

+             versions[release.replace("Fedora ", "F")] = versions_resp[release]

+         elif release.startswith("EPEL"):

+             versions[release.replace(" ", "-")] = versions_resp[release]

+ 

+     # And finally Rawhide and ELN

+     versions[rawhide] = versions_resp["Fedora Rawhide"]

+     versions["ELN"] = versions_resp["Fedora ELN"]

+ 

+     return releases, versions, rawhide

+ 

+ def query_bodhi_versions(repo):

+     """

+     Receives package versions from bodhi

+     Returns triplet: fedora-releases, updates-for-package, F%number-of-current-rawhide

+     """

      bodhi_base_url = "https://bodhi.fedoraproject.org/"

  

      # Retrieves the active releases from bodhi
@@ -465,7 +523,7 @@ 

      # Retrieves the updates of that package in bodhi

      update_url = "%s/updates?packages=%s" % (

          bodhi_base_url.rstrip("/"),

-         repo.name,

+         repo,

      )

      updates = collections.defaultdict(dict)

  
@@ -479,18 +537,51 @@ 

          )

  

      data = req.json()

+ 

+     # To be consistent with oraculum

+     for release in releases:

+         updates[release] = {"pending": None, "stable": None, "testing": None}

+ 

      for update in data["updates"]:

          if update["release"]["name"] not in releases:

              continue

+         if update["status"] == "obsolete":

+             # To be consistent with oraculum

+             continue

          name = update["title"]

          for build in update["builds"]:

-             if repo.name in build["nvr"]:

+             if repo in build["nvr"]:

                  name = build["nvr"]

                  break

  

          if not updates[update["release"]["name"]].get(update["status"]):

              updates[update["release"]["name"]][update["status"]] = name

  

+     return releases, updates, rawhide

+ 

+ @DISTGIT_NS.route("/bodhi_updates/<namespace>/<repo>", methods=["GET"])

+ @api_method

+ def bodhi_updates_endpoint(namespace, repo):

+     """Retrieves the current updates in bodhi for the specified package."""

+     if namespace not in ["rpms"]:

+         raise pagure.exceptions.APIError(

+             400,

+             error_code=APIERROR.EINVALIDREQ,

+             errors=["Namespace not supported"],

+         )

+ 

+     _log.info(

+         "Received request for the bodhi updates of: %s/%s", namespace, repo

+     )

+ 

+     repo = _get_repo(repo, namespace=namespace)

+     html = pagure.utils.is_true(flask.request.args.get("html", False))

+ 

+     try:

+         releases, updates, rawhide = query_oraculum_versions(repo.name)

+     except pagure.exceptions.APIError:

+         releases, updates, rawhide = query_bodhi_versions(repo.name)

+ 

      if html:

          html_output = (

              '<table class="table table-bordered table-striped">\n'

It depends on not yet deployed https://pagure.io/fedora-qa/oraculum/pull-request/142 . - DONE

Obsoleted updates are missing (do we need them though?).

Reviewed the code in-person with @frantisekz, there will be some cleanup done to this :D

rebased onto 0e0b56d98c55b300ab0f3c193baec8c9da46c3f8

4 years ago

rebased onto 7d2aa04fc2aabb807c044ab338e3c8fc38ce094d

4 years ago

@pingou I've added ELN Support here, so this is complete minus obsolete updates. They are not shown on pagure, do we need them?

Apart from that, how would you want to handle the configuration of oraculum url here?

Apart from that, how would you want to handle the configuration of oraculum url here?

Take it from pagure's config?

1 new commit added

  • Use oraculum url from pagure config
4 years ago

Apart from that, how would you want to handle the configuration of oraculum url here?

Take it from pagure's config?

Yeah, I should have figured that out, thanks! Filled https://pagure.io/pagure/pull-request/5149# for this and updated this PR.

I'll be deploying new oraculum build to production tomorrow, but if you really wanted to test this out, using ORACULUM_URL = "https://packager-dashboard.stg.fedoraproject.org/api/" should work already.

frantisekz commented on the pull-request: Fetch package versions from oraculum that you are following:
``

Apart from that, how would you want to handle the configuration of oraculum url here?

Take it from pagure's config?

Yeah, I should have figured that out, thanks! Filled https://pagure.io/pagure/pull-request/5149# for this and updated this PR.

It shouldn't be upstream, you can have it in the config without it being in the
default config.

IIRC we document some pagure-dist-git's specific configuration keys in the
README.

I'll be deploying new oraculum build to production tomorrow, but if you really
wanted to test this out, using ORACULUM_URL =
"https://packager-dashboard.stg.fedoraproject.org/api/" should work already.

roger, thanks!

1 new commit added

  • Config: don't rely on it being in pagure
4 years ago

It shouldn't be upstream, you can have it in the config without it being in the
default config.

IIRC we document some pagure-dist-git's specific configuration keys in the
README.

Oh, okay, would something like this be okay? Or shall I import pagure_config as it is in dist_git_auth.py?

Thanks!

FYI: changes in the backend are now deployed in production.

Just tested it and it works fine :)

My worry is introducing a dependency between oraculum and dist-git.

@kevin any thoughts on this?

Just tested it and it works fine :)

My worry is introducing a dependency between oraculum and dist-git.

@kevin any thoughts on this?

Thinking about this, if you're worried too much, I can leave the bodhi code in place too (or rather, move it into a separate function) and call it if/when oraculum fails to return meaningful data.

EDIT: I mean, it shouldn't happen, unless repo metadata format or bodhi api changes, relevant code around versions parsing is here, if you want to take a look: https://pagure.io/fedora-qa/oraculum/blob/master/f/oraculum/utils/versions.py and https://pagure.io/fedora-qa/oraculum/blob/master/f/oraculum/utils/bodhi.py .

Just tested it and it works fine :)

My worry is introducing a dependency between oraculum and dist-git.

@kevin any thoughts on this?

Thinking about this, if you're worried too much, I can leave the bodhi code in
place too (or rather, move it into a separate function) and call it if/when
oraculum fails to return meaningful data.

I have been thinking about this a bit as well. That'd be +1 for me!

I missed whats the advantage(s) here? Just faster/less load on bodhi? or ?

Faster and I think more correct data (and less load on bodhi and mdapi with the caching that oraculum has)

I missed whats the advantage(s) here? Just faster/less load on bodhi? or ?

Yeah, we're parsing version information from repo metadata and bodhi anyway and I thought pagure might make use of cached pakage-version pairs.

1 new commit added

  • Add bodhi code back as a fallback
3 years ago

@pingou Added back bodhi as a fallback code (while few making adjustments so the returned data are consistent with what oraculum path returns - no obsolete updates and existing keys with None values for no data). This PR also includes https://pagure.io/pagure-dist-git/pull-request/137# for now (I expect you'll merge that one first, I'll rebase and squash this then).

If it's just that info sure... I wonder if the new packager-static couldn't be the source, but thats just adding more confusion/apps. :)

@frantisekz #137 has been merged, you can rebase this one :)

rebased onto e60b548

3 years ago

@frantisekz #137 has been merged, you can rebase this one :)

Rebased and squashed :)

Pull-Request has been merged by pingou

3 years ago
Metadata