PR#2274: frontend: add page for exploring projects - copr/copr

copr / copr

#2274 frontend: add page for exploring projects

Merged 2 years ago by msuchy. Opened 2 years ago by frostyx.

copr/ frostyx/copr explore into main

frontend: add page for exploring projects

Jakub Kadlcik • 2 years ago

a32b52b

frontend/coprs_frontend/coprs/__init__.py

file modified

		`@@ -143,6 +143,7 @@`
		`NonAdminCannotCreatePersistentProject,`
		`NonAdminCannotDisableAutoPrunning,`
		`)`
		`+ from coprs.views.explore_ns import explore_ns`
		`from coprs.error_handlers import get_error_handler`
		`import coprs.context_processors`

		`@@ -164,6 +165,7 @@`
		`app.register_blueprint(user_ns)`
		`app.register_blueprint(webhooks_ns)`
		`app.register_blueprint(rss_ns)`
		`+ app.register_blueprint(explore_ns)`

		`if app.config["MEMORY_ANALYZER"]:`
		`from coprs.views.memory_analyzer import memory_analyzer`

frontend/coprs_frontend/coprs/logic/coprs_logic.py

file modified

+16

		`@@ -10,6 +10,7 @@`

		`from sqlalchemy import not_`
		`from sqlalchemy import desc`
		`+ from sqlalchemy import func`
		`from sqlalchemy.event import listens_for`
		`from sqlalchemy.orm.attributes import NEVER_SET, NO_VALUE`
		`from sqlalchemy.orm.exc import NoResultFound`
		`@@ -1234,6 +1235,21 @@`
		`def reset(cls, copr):`
		`cls.get(copr, flask.g.user).delete()`

		`+ @classmethod`
		`+ def get_popular_projects(cls, limit=10):`
		`+ """`
		`+ Get projects with the highest score (upvotes/downvotes feature).`
		`+ The result is returned as tuples Copr.id and its score, but may be`
		`+ changed to return tuples of Copr object and its score in the future`
		`+ """`
		`+ query = db.session.query(`
		`+ models.CoprScore.copr_id,`
		`+ func.sum(models.CoprScore.score).label("score_sum")`
		`+ )`
		`+ return (query.group_by(models.CoprScore.copr_id)`
		`+ .order_by(desc("score_sum"))`
		`+ .limit(limit))`
		`+`

		`class MockChrootsLogic(object):`
		`@classmethod`

frontend/coprs_frontend/coprs/logic/stat_logic.py

file modified

+26 -1

		`@@ -5,7 +5,7 @@`
		`from coprs import app`
		`from coprs import db`
		`from coprs.models import CounterStat`
		`- from coprs import helpers`
		`+ from coprs import helpers, models`


		`class CounterStatLogic(object):`
		`@@ -70,6 +70,31 @@`

		`return repo_dl_stats`

		`+ @classmethod`
		`+ def get_popular_projects(cls):`
		`+ """`
		`+ Return CounterStat results for projects with the most downloaded RPMs`
		`+ """`
		`+ return cls.get_popular(helpers.CounterStatType.PROJECT_RPMS_DL)`
		`+`
		`+ @classmethod`
		`+ def get_popular_chroots(cls):`
		`+ """`
		`+ Return CounterStat results for chroots with the most downloaded RPMs`
		`+ """`
		`+ return cls.get_popular(helpers.CounterStatType.CHROOT_RPMS_DL)`
		`+`
		`+ @classmethod`
		`+ def get_popular(cls, counter_type, limit=10):`
		`+ """`
		`+ Return CounterStat results with the highest counter for a given`
		`+ CounterStatType.`
		`+ """`
		`+ return (CounterStat.query`
		`+ .filter(CounterStat.counter_type == counter_type)`
		`+ .order_by(models.CounterStat.counter.desc())`
		`+ .limit(limit))`
		`+`

		`def handle_be_stat_message(stat_data):`
		`"""`

frontend/coprs_frontend/coprs/models.py

file modified

+16

		`@@ -2149,6 +2149,22 @@`

		`counter = db.Column(db.Integer, default=0, server_default="0")`

		`+ @property`
		`+ def pretty_name(self):`
		`+ """`
		`+ Return owner/project, or owner/project/chroot value`
		`+ """`
		`+`
		`+ # TODO Once we add relationships to Copr and CoprChroot, use them here`
		`+ suffix = self.name.rsplit("::", 1)[-1]`
		`+ owner, project = suffix.rsplit("@", 1)`
		`+`
		`+ # Chrootname after colon the separator`
		`+ if ":" in project:`
		`+ project = project.replace(":", "/")`
		`+`
		`+ return "{0}/{1}".format(owner, project)`
		`+`

		`class Group(db.Model, helpers.Serializer):`

frontend/coprs_frontend/coprs/templates/explore.html

file added

+50

		`@@ -0,0 +1,50 @@`
		`+ {% extends "coprs/show.html" %}`
		`+ {% from "_helpers.html" import alert %}`
		`+ {% block title %}Explore projects{% endblock %}`
		`+ {% block header %}Explore projects{% endblock %}`
		`+`
		`+`
		`+ {% block content %}`
		`+ {{ alert("This page is a feature preview", type="warning") }}`
		`+`
		`+`
		`+ <h1>Explore projects</h1>`
		`+ <p>Explore popular, featured, and interesting projects</p>`
		`+`
		`+`
		`+ <h2>Most downloaded projects</h2>`
		`+`
		`+ <ul>`
		`+ {% for stat in stats_projects %}`
		`+ <li>`
		`+ {{ stat.counter }} downloads -`
		`+ <strong>{{ stat.pretty_name }}</strong>`
		`+ </li>`
		`+ {% endfor %}`
		`+ </ul>`
		`+`
		`+`
		`+ <h2>Most downloaded chroots</h2>`
		`+`
		`+ <ul>`
		`+ {% for stat in stats_chroots %}`
		`+ <li>`
		`+ {{ stat.counter }} downloads -`
		`+ <strong>{{ stat.pretty_name }}</strong>`
		`+ </li>`
		`+ {% endfor %}`
		`+ </ul>`
		`+`
		`+`
		`+ <h2>Most upvoted projects</h2>`
		`+`
		`+ <ul>`
		`+ {% for project in upvoted_projects \|sort(attribute='score', reverse=True) %}`
		`+ <li>`
		`+ {{ project.score }} upvotes -`
		`+ <strong>{{ project.full_name }}</strong>`
		`+ </li>`
		`+ {% endfor %}`
		`+ </ul>`
		`+`
		`+ {% endblock %}`

frontend/coprs_frontend/coprs/views/explore_ns.py

file added

+31

		`@@ -0,0 +1,31 @@`
		`+ import flask`
		`+ from coprs import models`
		`+ from coprs.logic.stat_logic import CounterStatLogic`
		`+ from coprs.logic.coprs_logic import CoprScoreLogic, CoprsLogic`
		`+ from coprs.logic.builds_logic import BuildsLogic`
		`+ from coprs.logic.complex_logic import ComplexLogic`
		`+`
		`+`
		`+ explore_ns = flask.Blueprint("explore_ns", __name__, url_prefix="/explore")`
		`+`
		`+`
		`+ @explore_ns.route("/")`
		`+ def explore_home():`
		`+ stats_projects = CounterStatLogic.get_popular_projects()`
		`+ stats_chroots = CounterStatLogic.get_popular_chroots()`
		`+`
		`+ upvoted_projects_ids = [x.copr_id for x in`
		`+ CoprScoreLogic.get_popular_projects()]`
		`+ upvoted_projects = (CoprsLogic.get_multiple()`
		`+ .filter(models.Copr.id.in_(upvoted_projects_ids)))`
		`+`
		`+ return flask.render_template(`
		`+ "explore.html",`
		`+ stats_projects=stats_projects,`
		`+ stats_chroots=stats_chroots,`
		`+ upvoted_projects=upvoted_projects,`
		`+`
		`+ # Meh this should be done automatically`
		`+ tasks_info=ComplexLogic.get_queue_sizes(),`
		`+ graph=BuildsLogic.get_small_graph_data('30min'),`
		`+ )`

frostyx commented 2 years ago

See #1890
(Popular projects)

See #1919
(Editor's choice tags)

For the long time we have been discussing a page where users could
discover interesting projects (whatever that means). Maybe most
downloaded projects, most upvoted projects, manually picked projects
featured as interesting, projects that has been picked for Fedora
Magazine article, etc.

It is quite a lot of work and nobody will probably have enough time
to implement it all at once. So I am starting the implementation, and
leaving it unfinished, so we can continue to work on it next time.

I wasn't sure what terminology to use, "Explore Projects", "Discover
Projects", or simply "Popular projects", feel free to change as we go
on.

zuul commented 2 years ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

csdiff-pylint : NODE_FAILURE in 0s

frostyx commented 2 years ago

At this moment the page looks like this

Of course, it is ugly and not meant to be consumed by users yet :-)

rebased onto f6be96560e2e4ffd29407eeb84ec9e5b1f86aeca

2 years ago

zuul commented 2 years ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

csdiff-pylint : NODE_FAILURE in 0s

praiskup commented 2 years ago

Brainstorm dump from the meeting:
- not sure about the srpm-builds in the output
- %2540 in group name

frostyx commented 2 years ago

%2540 in group name

coprdb=# select name from counter_stat where counter_type='chroot_rpms_dl' order by counter DESC LIMIT 10;
┌──────────────────────────────────────────────────────────────────────────────────┐
│                                       name                                       │
├──────────────────────────────────────────────────────────────────────────────────┤
│ chroot_rpms_dl_stat:hset::jsynacek@systemd-backports-for-centos-7:epel-7-x86_64  │
│ chroot_rpms_dl_stat:hset::%40spacewalkproject@spacewalk-2.9-client:epel-7-x86_64 │
│ chroot_rpms_dl_stat:hset::stenstorp@MATE:epel-8-x86_64                           │
│ chroot_rpms_dl_stat:hset::zawertun@kde:fedora-35-x86_64                          │
│ chroot_rpms_dl_stat:hset::zawertun@kde:fedora-36-x86_64                          │
│ chroot_rpms_dl_stat:hset::ovirt@ovirt-master-snapshot:centos-stream-8-x86_64     │
│ chroot_rpms_dl_stat:hset::@pki@master:fedora-35-x86_64                           │
│ chroot_rpms_dl_stat:hset::@copr@PyPI:srpm-builds                                 │
│ chroot_rpms_dl_stat:hset::%2540oamg@leapp:epel-7-x86_64                          │
│ chroot_rpms_dl_stat:hset::ibotty@prometheus-exporters:epel-7-x86_64              │
└──────────────────────────────────────────────────────────────────────────────────┘

It's wrong in the database, we will have to add a migration to fix it. I want to add more changes that require migration (adding relationships to Copr and CoprChroot tables) so I will do it together.

frostyx commented 2 years ago

not sure about the srpm-builds in the output

I think those srpm-builds downloads are caused by copr-dist-git. When we build SRPM on a builder, it is then downloaded via SSH to the backend. Then it is downloaded from backend to distgit using HTTP. The hitcounter script ignores accesses that have "Mock" in the user-agent, or the user-agent is a known bot. But we download the SRPM using the SafeRequest using a default user-agent, e.g. "python-requests/2.26.0".

We could either ignore all downloaded SRPMs like this:

diff --git a/backend/copr_backend/hitcounter.py b/backend/copr_backend/hitcounter.py
index 2a2e03276..0ec00d1a9 100644
--- a/backend/copr_backend/hitcounter.py
+++ b/backend/copr_backend/hitcounter.py
@@ -125,6 +125,12 @@ def get_hit_data(accesses, log):
             log.debug("Skipping: %s", url)
             continue

+        if any(x for x in key_strings
+               if x.startswith("chroot_rpms_dl_stat|")
+               and x.endswith("|srpm-builds")):
+            log.debug("Skipping %s (SRPM build)", url)
+            continue
+
         log.debug("Processing: %s", url)

         # When counting RPM access, we want to iterate both project hits and

Or we can specify the user-agent for SafeRequest from our copr-dist-git and ignore just those.

praiskup commented 2 years ago

%2540 in group name

Ack, we need migration.

not sure about the srpm-builds in the output
The solution you propose seems good. +1
We could in theory initiate some IP denylist, too, starting with DistGit's IP (long-term that will be needed anyway, I think).