From 605eaaa8ad75b1733e4a56ad1f0506b9055e5f80 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 05 2018 19:50:47 +0000 Subject: Make the mirroring feature work with older git The GIT_SSH_COMMAND environment variable was added in 2.10 but we want pagure to work with older git version, so we are switching to GIT_SSH with a helper script generated on the fly. In addition, this commit adds the logic to pull all the branches stored in pagure so when we push --mirror they are all synced. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/ssh_script.sh b/pagure/lib/ssh_script.sh new file mode 100755 index 0000000..246d30d --- /dev/null +++ b/pagure/lib/ssh_script.sh @@ -0,0 +1,2 @@ +#!/bin/sh +ssh -i $SSHKEY -oStrictHostKeyChecking=no "$@" diff --git a/pagure/lib/tasks_mirror.py b/pagure/lib/tasks_mirror.py index f9aa457..22f66d0 100644 --- a/pagure/lib/tasks_mirror.py +++ b/pagure/lib/tasks_mirror.py @@ -13,12 +13,9 @@ from __future__ import unicode_literals import base64 import logging import os -import shutil import stat import struct -import tempfile -import pygit2 import six import werkzeug @@ -209,13 +206,14 @@ def mirror_project(self, session, username, namespace, name): _log.info("Git folder not found at: %s, bailing", repopath) return - newpath = tempfile.mkdtemp(prefix="pagure-mirror-") - pygit2.clone_repository(repopath, newpath) - ssh_folder = pagure_config["MIRROR_SSHKEYS_FOLDER"] public_key_name = werkzeug.secure_filename(project.fullname) private_key_file = os.path.join(ssh_folder, public_key_name) + # Add the utility script allowing this feature to work on old(er) git. + here = os.path.join(os.path.dirname(os.path.abspath(__file__))) + script_file = os.path.join(here, "ssh_script.sh") + # Get the list of remotes remotes = [ remote.strip() @@ -225,44 +223,26 @@ def mirror_project(self, session, username, namespace, name): and ssh_urlpattern.match(remote.strip()) ] - # Add the remotes - for idx, remote in enumerate(remotes): - remote_name = "%s_%s" % (public_key_name, idx) - _log.info("Adding remote %s as %s", remote, remote_name) - (stdout, stderr) = pagure.lib.git.read_git_lines( - ["remote", "add", remote_name, remote, "--mirror=push"], - abspath=newpath, - error=True, - ) - _log.info( - "Output from git remote add:\n stdout: %s\n stderr: %s", - stdout, - stderr, - ) - # Push logs = [] - for idx, remote in enumerate(remotes): - remote_name = "%s_%s" % (public_key_name, idx) + for remote in remotes: _log.info( - "Pushing to remote %s using key: %s", remote_name, private_key_file + "Pushing to remote %s using key: %s", remote, private_key_file ) (stdout, stderr) = pagure.lib.git.read_git_lines( - ["push", remote_name], - abspath=newpath, + ["push", "--mirror", remote], + abspath=repopath, error=True, - env={"GIT_SSH_COMMAND": "ssh -i %s" % private_key_file}, + env={"SSHKEY": private_key_file, "GIT_SSH": script_file}, ) log = "Output from the push:\n stdout: %s\n stderr: %s" % ( stdout, stderr, ) logs.append(log) + if logs: project.mirror_hook.last_log = "\n".join(logs) session.add(project.mirror_hook) session.commit() _log.info("\n".join(logs)) - - # Remove the clone - shutil.rmtree(newpath) diff --git a/tests/test_pagure_lib_task_mirror.py b/tests/test_pagure_lib_task_mirror.py index f4752df..fa20664 100644 --- a/tests/test_pagure_lib_task_mirror.py +++ b/tests/test_pagure_lib_task_mirror.py @@ -273,9 +273,6 @@ class PagureLibTaskMirrorSetuptests(tests.Modeltests): project = pagure.lib.get_authorized_project(self.session, 'test') self.assertIsNone(project.mirror_hook.public_key) - @patch( - 'tempfile.mkdtemp', - MagicMock(return_value='/tmp/pagure-mirror-fdgqcF')) @patch('pagure.lib.git.read_git_lines') def test_mirror_project(self,rgl): """ Test the mirror_project method. """ @@ -307,27 +304,23 @@ class PagureLibTaskMirrorSetuptests(tests.Modeltests): self.assertTrue( project.mirror_hook.public_key.startswith('ssh-rsa ')) + ssh_script = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), + '..', 'pagure','lib', 'ssh_script.sh')) + calls = [ call( - [ - u'remote', u'add', u'test_0', - u'ssh://user@localhost.localdomain/foobar.git', - u'--mirror=push' - ], - abspath=u'/tmp/pagure-mirror-fdgqcF', - error=True - ), - call( - [u'push', u'test_0'], - abspath=u'/tmp/pagure-mirror-fdgqcF', + [u'push', u'--mirror', u'ssh://user@localhost.localdomain/foobar.git'], + abspath=os.path.join(self.path, 'repos', 'test.git'), env={ - u'GIT_SSH_COMMAND': u'ssh -i %s/sshkeys/test' % self.path + u'GIT_SSH': ssh_script, + u'SSHKEY': u'%s/sshkeys/test' % self.path }, error=True ) ] - self.assertEqual(rgl.call_count, 2) + self.assertEqual(rgl.call_count, 1) self.assertEqual( calls, rgl.mock_calls