From 91a98bdfda63370d6c0170cd4896816bc6fcb03f Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Dec 18 2016 21:29:52 +0000 Subject: [PATCH 1/4] Created new class to manage the cleanup of the cloned repo Signed-off-by: Clement Verna --- diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 13fa426..a4cf15c 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -5,6 +5,7 @@ import json import click import pygit2 import re +import shutil from urllib.parse import urlparse from configparser import ConfigParser from github import Github @@ -237,3 +238,19 @@ def is_image(filename): return True else: return False + + +class Importer: + ''' Common Class for Github and Feodrahosted importer''' + + def __init__(self, username, password, repo_name, repo_folder): + self.username = username + self.password = password + self.repo_name = repo_name + self.repo_folder = repo_folder + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + shutil.rmtree(os.path.join(self.repo_folder, 'clone-' + self.repo_name)) diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index af3e6ef..7cf8f51 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -6,22 +6,20 @@ import requests from base64 import b64decode from datetime import datetime from pagure_importer.utils import ( - get_pagure_namespace, get_close_status, is_image) + get_pagure_namespace, get_close_status, is_image, Importer) from pagure_importer.utils.git import ( clone_repo, get_secure_filename, push_delete_repo, update_git) from pagure_importer.utils.models import User, Issue, IssueComment -class TracImporter(): +class TracImporter(Importer): ''' Pagure importer for trac instance ''' - def __init__(self, project_url, username, password, offset, - fasclient=None, tags=False, private=False): + def __init__(self, project_url, username, password, offset, repo_name, + repo_folder, fasclient=None, tags=False, private=False): ''' Instantiate a TracImporter object ''' - + Importer.__init__(self, username, password, repo_name, repo_folder) self.url = project_url - self.username = username - self.password = password self.fas = fasclient self.tags = tags self.private = private @@ -80,11 +78,10 @@ class TracImporter(): custom_fields.append(current_field) return custom_fields - def import_issues(self, repo_name, repo_folder, - trac_query='max=0&order=id'): + def import_issues(self, trac_query='max=0&order=id'): ''' Import issues from trac instance using jsonrpc API ''' - newpath, new_repo = clone_repo(repo_name, repo_folder) + newpath, new_repo = clone_repo(self.repo_name, self.repo_folder) tickets_id = self.request('ticket.query', trac_query) for ticket_id in tickets_id: @@ -97,7 +94,7 @@ class TracImporter(): if comments[key].attachment is not None and \ any(attachment in comments[key].attachment for attachment in pagure_issue.attachment): - project = get_pagure_namespace(repo_folder, repo_name) + project = get_pagure_namespace(self.repo_folder, self.repo_name) for attach_name in comments[key].attachment: filename = get_secure_filename( @@ -112,7 +109,7 @@ class TracImporter(): pagure_issue.comments.append(comments[key].to_json()) # update the local git repo new_repo = update_git(pagure_issue, newpath, new_repo) - click.echo('Updated ' + repo_name + ' with issue :' + + click.echo('Updated ' + self.repo_name + ' with issue :' + str(ticket_id) + '/' + str(tickets_id[-1])) push_delete_repo(newpath, new_repo) From 6563dd4604a5acca204d60c9c0f401de7f2d4256 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Dec 18 2016 21:29:52 +0000 Subject: [PATCH 2/4] Using new class for fedorahosted import Signed-off-by: Clement Verna --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 811161c..ee87533 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -20,15 +20,22 @@ def fedorahosted(project_url, tags, private, username, password, offset): project_url = project_url.rstrip('/') fasclient = FASclient(username, password, 'https://admin.fedoraproject.org/accounts') - project_url = project_url + '/login/jsonrpc' + project_url += '/login/jsonrpc' repos = pagure_importer.utils.display_repo() if repos: repo_index = click.prompt('Choose the import destination repo ', default=1) repo_name = repos[int(repo_index)-1] - trac_importer = importer_trac.TracImporter(project_url, username, - password, offset, fasclient, - tags, private) - trac_importer.import_issues(repo_name=repo_name, repo_folder=REPO_PATH) + with importer_trac.TracImporter(project_url=project_url, + username=username, + password=password, + offset=offset, + repo_name=repo_name, + repo_folder=REPO_PATH, + fasclient=fasclient, + tags=tags, + private=private) as trac_importer: + + trac_importer.import_issues() else: click.echo('No ticket repository found. Use pgimport clone command') From 8f07e11ac71b1afe6815e19cef264aeac538bd15 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Dec 18 2016 21:29:52 +0000 Subject: [PATCH 3/4] Added new class to github import Signed-off-by: Clement Verna --- diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py index befbbdd..0b91541 100644 --- a/pagure_importer/commands/github.py +++ b/pagure_importer/commands/github.py @@ -30,13 +30,13 @@ def github(username, password, project): 'Choose the import destination repo', default=1) repo_name = repos[int(repo_index)-1] - github_importer = GithubImporter( - username=username, - password=password, - project=project) + with GithubImporter(username=username, + password=password, + project=project, + repo_name=repo_name, + repo_folder=REPO_PATH) as github_importer: - github_importer.import_issues( - repo_path=repo_name, repo_folder=REPO_PATH) + github_importer.import_issues() else: click.echo( 'No ticket repository found. Use pgimport clone command') diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index 2ef86b9..8a3b4c9 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -2,7 +2,7 @@ import click from github import Github from pagure_importer.utils import ( - models, gh_get_user_email, get_auth_token) + models, gh_get_user_email, get_auth_token, Importer) from pagure_importer.utils.git import ( clone_repo, push_delete_repo, update_git) from pagure_importer.utils.exceptions import ( @@ -10,14 +10,13 @@ from pagure_importer.utils.exceptions import ( ) -class GithubImporter(): +class GithubImporter(Importer): ''' Imports from Github using PyGithub and libpagure ''' - def __init__(self, username, password, project): + def __init__(self, username, password, project, repo_name, repo_folder): ''' Instantiate GithubImporter object ''' - self.github_username = username - self.github_password = password + Importer.__init__(self, username, password, repo_name, repo_folder) self.github_project_name = project self.github = Github(username, password) @@ -39,7 +38,7 @@ class GithubImporter(): if assignee is not None: return assignee.to_json() - def import_issues(self, repo_path, repo_folder, status='all'): + def import_issues(self, status='all'): ''' Imports the issues on github for the given project ''' @@ -50,7 +49,7 @@ class GithubImporter(): raise GithubRepoNotFound( 'Repo not found, project name wrong') - newpath, new_repo = clone_repo(repo_path, repo_folder) + newpath, new_repo = clone_repo(self.repo_name, self.repo_folder) repo_issues = repo.get_issues(state=status) issues_length = 0 for issue in repo_issues: From e7d276fbf79adbf341b3d4bdb771257ef2181556 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Dec 18 2016 21:29:52 +0000 Subject: [PATCH 4/4] Always delete the repo when the object exit Signed-off-by: Clement Verna --- diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index a4cf15c..b38935b 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -253,4 +253,5 @@ class Importer: return self def __exit__(self, exc_type, exc_val, exc_tb): + ''' Delete the cloned repo where the commits were going ''' shutil.rmtree(os.path.join(self.repo_folder, 'clone-' + self.repo_name)) diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py index 6b8ea13..ff51324 100644 --- a/pagure_importer/utils/git.py +++ b/pagure_importer/utils/git.py @@ -35,9 +35,8 @@ def clone_repo(repo_name, repo_folder): return (newpath, new_repo) -def push_delete_repo(newpath, new_repo): - ''' Push the changes to the originally cloned repo from pagure and delete - the cloned repo where the commits were going ''' +def push_repo(newpath, new_repo): + ''' Push the changes to the originally cloned repo from pagure ''' # Push to origin ori_remote = new_repo.remotes[0] @@ -46,9 +45,6 @@ def push_delete_repo(newpath, new_repo): ori_remote.push([refname]) - # Remove the clone - shutil.rmtree(newpath) - def update_git(obj, newpath, new_repo): """ Update the given issue in its git. diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index 8a3b4c9..c87525d 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -4,7 +4,7 @@ from github import Github from pagure_importer.utils import ( models, gh_get_user_email, get_auth_token, Importer) from pagure_importer.utils.git import ( - clone_repo, push_delete_repo, update_git) + clone_repo, push_repo, update_git) from pagure_importer.utils.exceptions import ( GithubRepoNotFound ) @@ -156,4 +156,4 @@ class GithubImporter(Importer): # update the local git repo new_repo = update_git(pagure_issue, newpath, new_repo) click.echo('Updated issue %s out of %s' % (idx+1, issues_length)) - push_delete_repo(newpath, new_repo) + push_repo(newpath, new_repo) diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 7cf8f51..63ae0a6 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -8,7 +8,7 @@ from datetime import datetime from pagure_importer.utils import ( get_pagure_namespace, get_close_status, is_image, Importer) from pagure_importer.utils.git import ( - clone_repo, get_secure_filename, push_delete_repo, update_git) + clone_repo, get_secure_filename, push_repo, update_git) from pagure_importer.utils.models import User, Issue, IssueComment @@ -111,7 +111,7 @@ class TracImporter(Importer): new_repo = update_git(pagure_issue, newpath, new_repo) click.echo('Updated ' + self.repo_name + ' with issue :' + str(ticket_id) + '/' + str(tickets_id[-1])) - push_delete_repo(newpath, new_repo) + push_repo(newpath, new_repo) def get_custom_fields_of_ticket(self, trac_ticket): ''' Given the trac ticket, it will return all the