#4460 Make pagure work with recent sqlalchemy versions
Merged 4 years ago by pingou. Opened 5 years ago by pingou.

file modified
-4
@@ -417,7 +417,6 @@ 

          primaryjoin="projects.c.id==user_projects.c.project_id",

          secondaryjoin="and_(users.c.id==user_projects.c.user_id,\

                  user_projects.c.access=='admin')",

-         backref="co_projects_admins",

          viewonly=True,

      )

  
@@ -428,7 +427,6 @@ 

          secondaryjoin="and_(users.c.id==user_projects.c.user_id,\

                  or_(user_projects.c.access=='commit',\

                      user_projects.c.access=='admin'))",

-         backref="co_projects_committers",

          viewonly=True,

      )

  
@@ -453,7 +451,6 @@ 

          primaryjoin="projects.c.id==projects_groups.c.project_id",

          secondaryjoin="and_(pagure_group.c.id==projects_groups.c.group_id,\

                  projects_groups.c.access=='admin')",

-         backref="projects_admin_groups",

          order_by="PagureGroup.group_name.asc()",

          viewonly=True,

      )
@@ -465,7 +462,6 @@ 

          secondaryjoin="and_(pagure_group.c.id==projects_groups.c.group_id,\

                  or_(projects_groups.c.access=='admin',\

                      projects_groups.c.access=='commit'))",

-         backref="projects_committer_groups",

          order_by="PagureGroup.group_name.asc()",

          viewonly=True,

      )

file modified
+22 -11
@@ -2439,7 +2439,7 @@ 

                  model.User.user == private,

              )

          )

-         sub_q2 = session.query(model.Project.id).filter(

+         sub_q2 = session.query(sqlalchemy.distinct(model.Project.id)).filter(

              # User got admin or commit right

              sqlalchemy.and_(

                  model.Project.private == True,  # noqa: E712
@@ -2452,7 +2452,7 @@ 

                  ),

              )

          )

-         sub_q3 = session.query(model.Project.id).filter(

+         sub_q3 = session.query(sqlalchemy.distinct(model.Project.id)).filter(

              # User created a group that has admin or commit right

              sqlalchemy.and_(

                  model.Project.private == True,  # noqa: E712
@@ -2467,7 +2467,7 @@ 

                  ),

              )

          )

-         sub_q4 = session.query(model.Project.id).filter(

+         sub_q4 = session.query(sqlalchemy.distinct(model.Project.id)).filter(

              # User is part of a group that has admin or commit right

              sqlalchemy.and_(

                  model.Project.private == True,  # noqa: E712
@@ -2494,14 +2494,25 @@ 

                  model.PagureGroup.group_name.notin_(exclude_groups)

              )

  

-         projects = projects.filter(

-             model.Project.id.in_(

-                 subquery0.union(sub_q1)

-                 .union(sub_q2)

-                 .union(sub_q3)

-                 .union(sub_q4)

+         private_repo_subq = (

+             subquery0.union(sub_q1).union(sub_q2).union(sub_q3).union(sub_q4)

+         )

+ 

+         # There is something going on here, we shouldn't have to invoke/call

+         # the sub-query, it should work fine with:

+         #    model.Project.id.in_(private_repo_subq.subquery())

+         # however, it does not. Either something gets really confused with

+         # sqlite or the generated SQL is broken

+         # The issues seems to be with the unions in the subquery just above.

+         # Since we can't quite get this to work, let's bite the bullet and go

+         # with this approach, but damn I don't like it!

+         # This issue appeared with sqlalchemy 1.3.0 and is still present in

+         # 1.3.13 tested today.

+         private_repos = private_repo_subq.all()

+         if private_repos:

+             projects = projects.filter(

+                 model.Project.id.in_(list(set(zip(*private_repos)))[0])

              )

-         )

  

      if fork is not None:

          if fork is True:
@@ -2532,7 +2543,7 @@ 

          projects = projects.filter(model.Project.namespace == namespace)

  

      query = session.query(model.Project).filter(

-         model.Project.id.in_(projects.subquery())

+         model.Project.id.in_(projects.as_scalar())

      )

  

      if sort == "latest":

file modified
+1 -5
@@ -27,11 +27,7 @@ 

  requests

  six

  # sqlalchemy minimum 0.8

- # sqlalchemy 1.3.0 is causing issues on the pip container leading

- # test_pagure_lib.py to raise a:

- # "(sqlite3.OperationalError) no such column: users.user"

- # in test_search_projects_private line 319

- sqlalchemy < 1.3.0

+ sqlalchemy >= 0.8

  # 1.4.0 is broken, 1.4.0-post-1 works but gives odd results on newer setuptools

  # the latest version 1.5.0 is also known to work

  straight.plugin

What's the impact of this change, performance wise?

Unknown at this time.

Tbh, I really dislike this solution :(

rebased onto 2bc92e7df80481d228f7ca104f595ae446f8a41f

5 years ago

Have you forwarded the breakage to SQLAlchemy upstream?

Have you forwarded the breakage to SQLAlchemy upstream?

Discussing with @puiterwijk he believes it's not a SQLAlchemy issue but a pagure one.

Our minimum version is still 0.8

rebased onto 0653ca2d65ff7a874ef149b34649cedfa5a0f4dd

4 years ago

This comment is not valid anymore with this change.

rebased onto 87c458e1c5bc591d0663fe27ce9171c8b8b2163a

4 years ago

rebased onto fefe88909a0f849891add3910a56f2dd6ff545c7

4 years ago

rebased onto df9b15c

4 years ago

Only one test failing...

11:44:34  Failed tests:
11:44:34  FAILED test: py-test_pagure_lib_git_auth

1 new commit added

  • Stop setting a backref on relations that are viewonly
4 years ago

As much as it pains me, let's get this in

Pull-Request has been merged by pingou

4 years ago