From 652727dbd41599fef588c535821707bdef7fdeee Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: May 26 2017 14:55:07 +0000 Subject: [frontend][dist-git][backend] arbitrary dist-git branching This stays 100% compatible with existing copr instances, though the branch <-> chroots mapping is now stored in database to allow more flexibility. --- diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py index 04cbc20..0d04826 100644 --- a/backend/backend/helpers.py +++ b/backend/backend/helpers.py @@ -144,16 +144,6 @@ def _get_conf(cp, section, option, default, mode=None): return path return default -def chroot_to_branch(chroot): - """ - Get a git branch name from chroot. Follow the fedora naming standard. - """ - os_name, version, _ = chroot.split("-") - if os_name == "fedora": - os_name = "f" - elif os_name == "epel" and int(version) <= 6: - os_name = "el" - return "{}{}".format(os_name, version) class BackendConfigReader(object): def __init__(self, config_file=None, ext_opts=None): diff --git a/dist-git/dist_git/dist_git_importer.py b/dist-git/dist_git/dist_git_importer.py index e458357..c208c24 100755 --- a/dist-git/dist_git/dist_git_importer.py +++ b/dist-git/dist_git/dist_git_importer.py @@ -672,7 +672,14 @@ class DistGitImporter(object): self.teardown_per_task_logging(per_task_log_handler) def setup_per_task_logging(self, task): - handler = logging.FileHandler(os.path.join(self.opts.per_task_log_dir, "{0}.log".format(task.task_id))) + # Avoid putting logs into subdirectories when dist git branch name + # contains slashes. + task_id = str(task.task_id).replace('/', '_') + + handler = logging.FileHandler( + os.path.join(self.opts.per_task_log_dir, + "{0}.log".format(task_id)) + ) handler.setLevel(logging.DEBUG) logging.getLogger('').addHandler(handler) return handler @@ -796,6 +803,8 @@ class VM(object): sandbox(dst_dir) dcmd = ["docker", "run"] full_name = "-".join([name or "copr", hashlib.md5().hexdigest()]) + # For dist-git branches containing slashes. + full_name = full_name.replace('/', '_') if sys_admin: dcmd.extend(["--cap-add=SYS_ADMIN"]) diff --git a/frontend/coprs_frontend/alembic/schema/versions/bf4b5dc74740_map_mock_croots_to_dits_git_branch.py b/frontend/coprs_frontend/alembic/schema/versions/bf4b5dc74740_map_mock_croots_to_dits_git_branch.py new file mode 100644 index 0000000..7099536 --- /dev/null +++ b/frontend/coprs_frontend/alembic/schema/versions/bf4b5dc74740_map_mock_croots_to_dits_git_branch.py @@ -0,0 +1,52 @@ +"""map mock croots to dits-git branch + +Revision ID: bf4b5dc74740 +Revises: 38ea34def9a +Create Date: 2017-05-19 07:55:05.743045 + +""" + +# revision identifiers, used by Alembic. +revision = 'bf4b5dc74740' +down_revision = '38ea34def9a' + +from alembic import op +import sqlalchemy as sa + +from sqlalchemy.orm import sessionmaker +from coprs.models import MockChroot, DistGitBranch +from coprs.helpers import chroot_to_branch +from coprs.logic.coprs_logic import BranchesLogic + +def upgrade(): + bind = op.get_bind() + Session = sessionmaker() + session = Session(bind=bind) + + op.create_table('dist_git_branch', + sa.Column('name', sa.String(length=50), nullable=False), + sa.PrimaryKeyConstraint('name') + ) + + # Nullable at this point. + op.add_column(u'mock_chroot', sa.Column('distgit_branch_name', sa.String(length=50), nullable=True)) + op.create_foreign_key(None, 'mock_chroot', 'dist_git_branch', ['distgit_branch_name'], ['name']) + + for chroot in session.query(MockChroot).all(): + # Pick the predefined default. + branch = chroot_to_branch(chroot.name) + chroot.distgit_branch = BranchesLogic.get_or_create(branch, session) + session.add(chroot.distgit_branch) + session.add(chroot) + + session.commit() + + # not nulllable since now.. + op.alter_column('mock_chroot', 'distgit_branch_name', + existing_type=sa.VARCHAR(length=50), + nullable=False) + + +def downgrade(): + op.drop_column(u'mock_chroot', 'distgit_branch_name') + op.drop_table('dist_git_branch') diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py index 9770fe8..f194e36 100644 --- a/frontend/coprs_frontend/coprs/helpers.py +++ b/frontend/coprs_frontend/coprs/helpers.py @@ -232,30 +232,6 @@ def chroot_to_branch(chroot): return "{}{}".format(os, version) -def branch_to_os_version(branch): - os = None - version = None - if branch == "master": - os = "fedora" - version = "rawhide" - elif branch[0] == "f": - os = "fedora" - version = branch[1:] - elif branch[:4] == "epel" or branch[:2] == "el": - os = "epel" - version = branch[-1:] - elif branch[:6] == "custom": - os = "custom" - version = branch[-1:] - elif branch[:3] == "mga": - os = "mageia" - version = branch[3:] - elif branch[:8] == "cauldron": - os = "mageia" - version = "cauldron" - return os, version - - def splitFilename(filename): """ Pass in a standard style rpm fullname diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 5fe8132..2942f4f 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -638,13 +638,15 @@ GROUP BY task_id consists of a name of git branch + build id Example: 42-f22 -> build id 42, chroots fedora-22-* """ - build_id, branch = task_id.split("-") + build_id, branch = task_id.split("-", 1) build = cls.get_by_id(build_id).one() build_chroots = build.build_chroots - os, version = helpers.branch_to_os_version(branch) - chroot_halfname = "{}-{}".format(os, version) - matching = [ch for ch in build_chroots if chroot_halfname in ch.name] - return matching + + # What chroots this branch is for? + branch_chroots = models.DistGitBranch.query.get(branch).chroots + branch_chroots = [x.name for x in branch_chroots] + + return [ch for ch in build_chroots if ch.name in branch_chroots] @classmethod diff --git a/frontend/coprs_frontend/coprs/logic/coprs_logic.py b/frontend/coprs_frontend/coprs/logic/coprs_logic.py index 971fa91..02cdb5d 100644 --- a/frontend/coprs_frontend/coprs/logic/coprs_logic.py +++ b/frontend/coprs_frontend/coprs/logic/coprs_logic.py @@ -430,6 +430,19 @@ listen(models.Copr.auto_createrepo, 'set', on_auto_createrepo_change, active_history=True, retval=False) +class BranchesLogic(object): + @classmethod + def get_or_create(cls, name, session=db.session): + item = session.query(models.DistGitBranch).filter_by(name=name).first() + if item: + return item + + branch = models.DistGitBranch() + branch.name = name + session.add(branch) + return branch + + class CoprChrootsLogic(object): @classmethod def mock_chroots_from_names(cls, names): diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index 2b770ba..39d4dc3 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -764,6 +764,15 @@ class Build(db.Model, helpers.Serializer): return result +class DistGitBranch(db.Model, helpers.Serializer): + """ + 1:N mapping: branch -> chroots + """ + + # Name of the branch used on dist-git machine. + name = db.Column(db.String(50), primary_key=True) + + class MockChroot(db.Model, helpers.Serializer): """ @@ -782,6 +791,14 @@ class MockChroot(db.Model, helpers.Serializer): arch = db.Column(db.String(50), nullable=False) is_active = db.Column(db.Boolean, default=True) + # Reference branch name + distgit_branch_name = db.Column(db.String(50), + db.ForeignKey("dist_git_branch.name"), + nullable=False) + + distgit_branch = db.relationship("DistGitBranch", + backref=db.backref("chroots")) + @property def name(self): """ @@ -942,7 +959,7 @@ class BuildChroot(db.Model, helpers.Serializer): @property def import_task_id(self): - return "{}-{}".format(self.build_id, helpers.chroot_to_branch(self.name)) + return "{}-{}".format(self.build_id, self.mock_chroot.distgit_branch_name) @property def dist_git_url(self): @@ -961,7 +978,7 @@ class BuildChroot(db.Model, helpers.Serializer): def import_log_url(self): if app.config["COPR_DIST_GIT_LOGS_URL"]: return "{}/{}.log".format(app.config["COPR_DIST_GIT_LOGS_URL"], - self.import_task_id) + self.import_task_id.replace('/', '_')) return None @property diff --git a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py index 9165783..1b105e7 100644 --- a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py +++ b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py @@ -40,7 +40,7 @@ def dist_git_importing_queue(): "task_id": task.import_task_id, "user": copr.owner_name, # TODO: user -> owner "project": task.build.copr.name, - "branch": helpers.chroot_to_branch(task.mock_chroot.name), + "branch": task.mock_chroot.distgit_branch_name, "source_type": task.build.source_type, "source_json": task.build.source_json, } @@ -152,7 +152,7 @@ def waiting(): "enable_net": task.build.enable_net, "git_repo": task.build.package.dist_git_repo, "git_hash": task.git_hash, - "git_branch": helpers.chroot_to_branch(task.mock_chroot.name), + "git_branch": task.mock_chroot.distgit_branch_name, "package_name": task.build.package.name, "package_version": task.build.pkg_version } diff --git a/frontend/coprs_frontend/manage.py b/frontend/coprs_frontend/manage.py index 5f04391..cde58a1 100755 --- a/frontend/coprs_frontend/manage.py +++ b/frontend/coprs_frontend/manage.py @@ -20,6 +20,7 @@ from coprs.views.misc import create_user_wrapper from coprs.whoosheers import CoprWhoosheer from run import generate_repo_packages from sqlalchemy import or_ +from coprs.helpers import chroot_to_branch class TestCommand(Command): @@ -118,10 +119,21 @@ class CreateChrootCommand(ChrootCommand): "Creates a mock chroot in DB" - def run(self, chroot_names): + def __init__(self): + self.option_list += Option( + "--dist-git-branch", + "-b", + dest="branch", + help="Branch name for this set of new chroots"), + + def run(self, chroot_names, branch=None): for chroot_name in chroot_names: + if not branch: + branch = chroot_to_branch(chroot_name) + branch_object = coprs_logic.BranchesLogic.get_or_create(branch) try: - coprs_logic.MockChrootsLogic.add(chroot_name) + chroot = coprs_logic.MockChrootsLogic.add(chroot_name) + chroot.distgit_branch = branch_object db.session.commit() except exceptions.MalformedArgumentException: self.print_invalid_format(chroot_name) diff --git a/frontend/coprs_frontend/tests/coprs_test_case.py b/frontend/coprs_frontend/tests/coprs_test_case.py index 0189f7b..f220f34 100644 --- a/frontend/coprs_frontend/tests/coprs_test_case.py +++ b/frontend/coprs_frontend/tests/coprs_test_case.py @@ -15,6 +15,7 @@ import coprs from coprs import helpers from coprs import models +from coprs.logic.coprs_logic import BranchesLogic import six from coprs.helpers import StatusEnum @@ -148,12 +149,19 @@ class CoprsTestCase(object): def f_mock_chroots(self): self.mc1 = models.MockChroot( os_release="fedora", os_version="18", arch="x86_64", is_active=True) + self.mc1.distgit_branch = models.DistGitBranch(name='f18') + self.mc2 = models.MockChroot( os_release="fedora", os_version="17", arch="x86_64", is_active=True) + self.mc2.distgit_branch = models.DistGitBranch(name='fedora-17') + self.mc3 = models.MockChroot( os_release="fedora", os_version="17", arch="i386", is_active=True) + self.mc3.distgit_branch = self.mc2.distgit_branch + self.mc4 = models.MockChroot( os_release="fedora", os_version="rawhide", arch="i386", is_active=True) + self.mc4.distgit_branch = models.DistGitBranch(name='master') self.mc_basic_list = [self.mc1, self.mc2, self.mc3, self.mc4] # only bind to coprs if the test has used the f_coprs fixture @@ -191,12 +199,20 @@ class CoprsTestCase(object): mc = models.MockChroot( os_release="fedora", os_version=os_version, arch=arch, is_active=True) + # Let's try slashes now. for example. Some copr instances use + # this pattern. + mc.distgit_branch = BranchesLogic.get_or_create( + 'fedora/{0}'.format(os_version), + session=self.db.session) self.mc_list.append(mc) for os_version in [5, 6, 7]: mc = models.MockChroot( os_release="epel", os_version=os_version, arch=arch, is_active=True) + mc.distgit_branch = BranchesLogic.get_or_create( + 'el{0}'.format(os_version), + session=self.db.session) self.mc_list.append(mc) self.mc_list[-1].is_active = False @@ -206,7 +222,8 @@ class CoprsTestCase(object): for mc in self.mc_list: cc = models.CoprChroot() cc.mock_chroot = mc - self.c1.copr_chroots.append(cc) + # TODO: why 'self.c1.copr_chroots.append(cc)' doesn't work here? + cc.copr = self.c1 self.db.session.add_all(self.mc_list)