From df9b15c896dd702b70ce1602e792a32cb3269daf Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 03 2020 14:30:54 +0000 Subject: [PATCH 1/2] Make pagure work with recent sqlalchemy versions Fixes https://pagure.io/pagure/issue/4415 Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/query.py b/pagure/lib/query.py index bf12bb0..ac7d5ba 100644 --- a/pagure/lib/query.py +++ b/pagure/lib/query.py @@ -2439,7 +2439,7 @@ def search_projects( model.User.user == private, ) ) - sub_q2 = session.query(model.Project.id).filter( + sub_q2 = session.query(sqlalchemy.distinct(model.Project.id)).filter( # User got admin or commit right sqlalchemy.and_( model.Project.private == True, # noqa: E712 @@ -2452,7 +2452,7 @@ def search_projects( ), ) ) - sub_q3 = session.query(model.Project.id).filter( + sub_q3 = session.query(sqlalchemy.distinct(model.Project.id)).filter( # User created a group that has admin or commit right sqlalchemy.and_( model.Project.private == True, # noqa: E712 @@ -2467,7 +2467,7 @@ def search_projects( ), ) ) - sub_q4 = session.query(model.Project.id).filter( + sub_q4 = session.query(sqlalchemy.distinct(model.Project.id)).filter( # User is part of a group that has admin or commit right sqlalchemy.and_( model.Project.private == True, # noqa: E712 @@ -2494,14 +2494,25 @@ def search_projects( model.PagureGroup.group_name.notin_(exclude_groups) ) - projects = projects.filter( - model.Project.id.in_( - subquery0.union(sub_q1) - .union(sub_q2) - .union(sub_q3) - .union(sub_q4) + private_repo_subq = ( + subquery0.union(sub_q1).union(sub_q2).union(sub_q3).union(sub_q4) + ) + + # There is something going on here, we shouldn't have to invoke/call + # the sub-query, it should work fine with: + # model.Project.id.in_(private_repo_subq.subquery()) + # however, it does not. Either something gets really confused with + # sqlite or the generated SQL is broken + # The issues seems to be with the unions in the subquery just above. + # Since we can't quite get this to work, let's bite the bullet and go + # with this approach, but damn I don't like it! + # This issue appeared with sqlalchemy 1.3.0 and is still present in + # 1.3.13 tested today. + private_repos = private_repo_subq.all() + if private_repos: + projects = projects.filter( + model.Project.id.in_(list(set(zip(*private_repos)))[0]) ) - ) if fork is not None: if fork is True: @@ -2532,7 +2543,7 @@ def search_projects( projects = projects.filter(model.Project.namespace == namespace) query = session.query(model.Project).filter( - model.Project.id.in_(projects.subquery()) + model.Project.id.in_(projects.as_scalar()) ) if sort == "latest": diff --git a/requirements.txt b/requirements.txt index c877d94..5d88237 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,11 +27,7 @@ redis requests six # sqlalchemy minimum 0.8 -# sqlalchemy 1.3.0 is causing issues on the pip container leading -# test_pagure_lib.py to raise a: -# "(sqlite3.OperationalError) no such column: users.user" -# in test_search_projects_private line 319 -sqlalchemy < 1.3.0 +sqlalchemy >= 0.8 # 1.4.0 is broken, 1.4.0-post-1 works but gives odd results on newer setuptools # the latest version 1.5.0 is also known to work straight.plugin From 5c556b8f050597489aa416fe6d159677da6e81f7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 04 2020 11:39:44 +0000 Subject: [PATCH 2/2] Stop setting a backref on relations that are viewonly From sqlalachemy 1.3.14 relations that are specifying a backref when viewonly=True will issue a warning and this will be disallowed in future releases of sqlalchemy. So, after checking that these backref were not used in the project, we have simply removed them. Source: https://github.com/sqlalchemy/sqlalchemy/releases/tag/rel_1_3_14 which also explains why this behavior is not desired. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/model.py b/pagure/lib/model.py index 103bfc9..ea6413e 100644 --- a/pagure/lib/model.py +++ b/pagure/lib/model.py @@ -417,7 +417,6 @@ class Project(BASE): primaryjoin="projects.c.id==user_projects.c.project_id", secondaryjoin="and_(users.c.id==user_projects.c.user_id,\ user_projects.c.access=='admin')", - backref="co_projects_admins", viewonly=True, ) @@ -428,7 +427,6 @@ class Project(BASE): secondaryjoin="and_(users.c.id==user_projects.c.user_id,\ or_(user_projects.c.access=='commit',\ user_projects.c.access=='admin'))", - backref="co_projects_committers", viewonly=True, ) @@ -453,7 +451,6 @@ class Project(BASE): primaryjoin="projects.c.id==projects_groups.c.project_id", secondaryjoin="and_(pagure_group.c.id==projects_groups.c.group_id,\ projects_groups.c.access=='admin')", - backref="projects_admin_groups", order_by="PagureGroup.group_name.asc()", viewonly=True, ) @@ -465,7 +462,6 @@ class Project(BASE): secondaryjoin="and_(pagure_group.c.id==projects_groups.c.group_id,\ or_(projects_groups.c.access=='admin',\ projects_groups.c.access=='commit'))", - backref="projects_committer_groups", order_by="PagureGroup.group_name.asc()", viewonly=True, )