From c8b1c10c0f0ed81894394b885ecfd7feb49799c1 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 09 2017 23:41:02 +0000 Subject: [PATCH 1/9] github: Attachment support Signed-off-by: Vivek Anand --- diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py index f66aea8..1032d45 100644 --- a/pagure_importer/commands/github.py +++ b/pagure_importer/commands/github.py @@ -24,9 +24,17 @@ import pagure_importer.utils.git as gitutils help="Status of issue/PR to be imported(open/closed/all)") @click.option('--nopush', is_flag=True, help="Do not push the result of pagure-importer back") -def github(username, project, nopush, status, gencsv): - ''' Command to import from github ''' +@click.option('--pagure_project', + prompt='Enter name of pagure project: \n' + ' 1. it is a fork then like: fork//\n' + ' 2. it has a namespace: /\n' + ' 3. it is a fork of namespaced project:' + ' fork///') +def github(username, project, nopush, pagure_project, status, gencsv): + ''' For imports from github ''' + password = click.prompt("Github Password", hide_input=True) + pagure_project = pagure_project.rstrip('/').lstrip('/') if gencsv: gh_get_contributors(username, password, project) gh_get_issue_users(username, password, project) @@ -45,6 +53,7 @@ def github(username, project, nopush, status, gencsv): project=project, repo_name=repo_name, repo_folder=REPO_PATH, + pagure_project=pagure_project, nopush=nopush) as github_importer: repo = github_importer.github.get_repo( diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 8ea6db6..dff2f02 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -3,10 +3,10 @@ import os import sys import json import re +import hashlib import shutil import click -import hashlib import pygit2 import werkzeug diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index 866c750..faa5965 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -1,17 +1,24 @@ -import click +import re +import base64 import os import shutil + +import click +import requests from github import Github from pagure_importer.utils import ( - models, gh_get_user_email, get_auth_token, issue_to_json + models, gh_get_user_email, get_auth_token, issue_to_json, get_secure_filename ) +from pagure_importer.utils.git import clone_repo +from pagure_importer.utils.exceptions import GithubRepoNotFound class GithubImporter(object): ''' Imports from Github using PyGithub and libpagure ''' - def __init__(self, username, password, project, repo_name, repo_folder, nopush): + def __init__(self, username, password, + project, repo_name, repo_folder, nopush, pagure_project): ''' Instantiate GithubImporter object ''' self.username = username @@ -22,6 +29,7 @@ class GithubImporter(object): repo_folder, 'clone-' + repo_name) self.nopush = nopush self.github_project_name = project + self.pagure_project = pagure_project self.github = Github(username, password) otp_auth = get_auth_token(self.github) @@ -51,9 +59,34 @@ class GithubImporter(object): if assignee is not None: return assignee.to_json() + def get_comment_body(self, comment, pagure_attachments): + ''' Return the comment body. Check if there is + an attachment, if so return the attachment as well ''' + + whole_body = comment.body + attach_regex = '!\[.*\]\((.*)\)' + attach_url_list = re.findall(attach_regex, whole_body) + print (attach_url_list) + format_list = [] + for attach_url in attach_url_list: + attach_name = attach_url.strip().rstrip('/').rsplit('/')[-1] + format_list.append(attach_name) + response = requests.get(attach_url) + if response.status_code == 200: + pagure_attachments[attach_name] = response.content + filename = get_secure_filename( + pagure_attachments[attach_name], attach_name) + url = '/%s/issue/raw/files/%s' % (self.pagure_project, filename) + else: + url = '#Attachment Unavailable' + format_list.append(url) + format_list.append(url) + unformatted_body = re.sub(attach_regex, '\n[![%s](%s)](%s)', whole_body) + whole_body = unformatted_body % tuple(format_list) + return whole_body, pagure_attachments + def import_issues(self, repo, status='all'): - ''' Imports the issues on github for the given project - ''' + ''' Imports the issues on github for the given project ''' repo_issues = repo.get_issues(state=status) issues_length = sum(1 for issue in repo_issues) @@ -120,10 +153,14 @@ class GithubImporter(object): # comments on the issue comments = [] - for comment in github_issue.get_comments(): + # only github comments can have attachments + pagure_attachments = {} + for comment in github_issue.get_comments(): + comment_attachments = {} comment_user = comment.user - pagure_issue_comment_body = comment.body + pagure_issue_comment_body, comment_attachments = self.get_comment_body( + comment, comment_attachments) pagure_issue_comment_created_at = comment.created_at.strftime('%s') # No idea what to do with this right now @@ -152,10 +189,13 @@ class GithubImporter(object): editor=pagure_issue_comment_editor, attachment=None) + pagure_attachments.update(comment_attachments) comments.append(pagure_issue_comment.to_json()) - # add all the comments to the issue object + # add all the comments to the issue object and the attachments pagure_issue.comments = comments + pagure_issue.attachment = pagure_attachments + print (pagure_attachments) click.echo('Updated %s with issue : %s/%s' % (self.repo_name, idx + 1, issues_length)) issue_to_json(pagure_issue, self.clone_repo_location) From 13d99817053928d6ca419a37c34fd5d9ae864c01 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 09 2017 23:41:02 +0000 Subject: [PATCH 2/9] remove debugging code Signed-off-by: Vivek Anand --- diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index faa5965..b28f862 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -66,7 +66,6 @@ class GithubImporter(object): whole_body = comment.body attach_regex = '!\[.*\]\((.*)\)' attach_url_list = re.findall(attach_regex, whole_body) - print (attach_url_list) format_list = [] for attach_url in attach_url_list: attach_name = attach_url.strip().rstrip('/').rsplit('/')[-1] @@ -195,7 +194,6 @@ class GithubImporter(object): # add all the comments to the issue object and the attachments pagure_issue.comments = comments pagure_issue.attachment = pagure_attachments - print (pagure_attachments) click.echo('Updated %s with issue : %s/%s' % (self.repo_name, idx + 1, issues_length)) issue_to_json(pagure_issue, self.clone_repo_location) From 613141390eb3e3ba0cffad181b0fd12b644d2e85 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 09 2017 23:41:02 +0000 Subject: [PATCH 3/9] Remove unused import Signed-off-by: Vivek Anand --- diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index b28f862..78ad2c6 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -10,8 +10,6 @@ from github import Github from pagure_importer.utils import ( models, gh_get_user_email, get_auth_token, issue_to_json, get_secure_filename ) -from pagure_importer.utils.git import clone_repo -from pagure_importer.utils.exceptions import GithubRepoNotFound class GithubImporter(object): From b48240bced309a5d1fe370876978e0f80da8e40f Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 09 2017 23:41:02 +0000 Subject: [PATCH 4/9] replace .rstrip and .strip with .strip Signed-off-by: Vivek Anand --- diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py index 1032d45..444940a 100644 --- a/pagure_importer/commands/github.py +++ b/pagure_importer/commands/github.py @@ -34,7 +34,7 @@ def github(username, project, nopush, pagure_project, status, gencsv): ''' For imports from github ''' password = click.prompt("Github Password", hide_input=True) - pagure_project = pagure_project.rstrip('/').lstrip('/') + pagure_project = pagure_project.strip().strip('/') if gencsv: gh_get_contributors(username, password, project) gh_get_issue_users(username, password, project) From 79e71f65044f805f4b34b35a4d86e9203d95992a Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 09 2017 23:42:00 +0000 Subject: [PATCH 5/9] github: introduce --namespace and --is_fork Signed-off-by: Vivek Anand --- diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py index 444940a..6cd9460 100644 --- a/pagure_importer/commands/github.py +++ b/pagure_importer/commands/github.py @@ -4,7 +4,7 @@ import pagure_importer from pagure_importer.app import app, REPO_PATH from pagure_importer.utils.importer_github import GithubImporter from pagure_importer.utils import ( - gh_get_contributors, gh_get_issue_users, + gh_get_contributors, gh_get_issue_users, prepare_pagure_project, gh_assemble_users, validate_gh_project, ) @@ -24,17 +24,21 @@ import pagure_importer.utils.git as gitutils help="Status of issue/PR to be imported(open/closed/all)") @click.option('--nopush', is_flag=True, help="Do not push the result of pagure-importer back") -@click.option('--pagure_project', - prompt='Enter name of pagure project: \n' - ' 1. it is a fork then like: fork//\n' - ' 2. it has a namespace: /\n' - ' 3. it is a fork of namespaced project:' - ' fork///') -def github(username, project, nopush, pagure_project, status, gencsv): +@click.option('--pagure_project', prompt='Name of pagure project', + help="Name of the pagure project without namespace or 'fork'") +@click.option('--namespace', help="Name of the namespace of the pagure project") +@click.option('--is_fork', is_flag=True, default=False) +def github(username, project, nopush, pagure_project, + status, gencsv, namespace, is_fork): ''' For imports from github ''' password = click.prompt("Github Password", hide_input=True) - pagure_project = pagure_project.strip().strip('/') + pagure_project = prepare_pagure_project( + name=pagure_project, + namespace=namespace, + is_fork=is_fork + ) + if gencsv: gh_get_contributors(username, password, project) gh_get_issue_users(username, password, project) diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index dff2f02..4bba5f5 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -312,3 +312,16 @@ def validate_gh_project(ctx, param, value): click.echo("Repo doesn't exist or is private") ctx.exit() return value + + +def prepare_pagure_project(name, namespace, is_fork): + ''' Prepare pagure project name in a way that can be used to view + the attachments associated with the project's ticket attachments ''' + + pagure_project = name.rstrip('/').lstrip('/') + if namespace: + pagure_project = '/'.join([namespace, pagure_project]) + if is_fork: + username = click.prompt('Pagure username') + pagure_project = '/'.join(['fork', username, pagure_project]) + return pagure_project From 4a998c1c87983978c62a40919d6c95ee36d6f932 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 09 2017 23:42:00 +0000 Subject: [PATCH 6/9] Update readme for --namespace and --is_fork --- diff --git a/README.md b/README.md index 8582225..62142eb 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,16 @@ To add some new close status just edit the config file as follow. Where ```Foo`` This command will assume that you have the csv ready for use and will begin importing issues from github to your local git repository. + If the project is under some namespace or is a fork, use the below commands instead: + $ pgimport github --namespace + + If the project is a fork: + $ pgimport github --is_fork + + The namespace and is_fork option can be joined, if the project is a fork of a namespaced project: + $ pgimport github --namespace --is_fork + + The issues will be imported to /tmp/foobar.git repository. 3) The push command can be used to push a clone pagure ticket repo back to pagure. From 4a366ce95eadb94109e2ee9591302ca438b29591 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 09 2017 23:42:00 +0000 Subject: [PATCH 7/9] Remove unused import base64 Signed-off-by: Vivek Anand --- diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index 78ad2c6..6d74720 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -1,5 +1,4 @@ import re -import base64 import os import shutil From 538a595f02ae291ce0223aec7a529d327b4942f4 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 09 2017 23:42:00 +0000 Subject: [PATCH 8/9] Support for non-image file attachments Signed-off-by: Vivek Anand --- diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index 6d74720..2e281e3 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -56,12 +56,10 @@ class GithubImporter(object): if assignee is not None: return assignee.to_json() - def get_comment_body(self, comment, pagure_attachments): - ''' Return the comment body. Check if there is - an attachment, if so return the attachment as well ''' + def _get_attachments(self, attach_regex, whole_body, + pagure_attachments, is_image=True): + ''' Get the Attachment ''' - whole_body = comment.body - attach_regex = '!\[.*\]\((.*)\)' attach_url_list = re.findall(attach_regex, whole_body) format_list = [] for attach_url in attach_url_list: @@ -71,16 +69,62 @@ class GithubImporter(object): if response.status_code == 200: pagure_attachments[attach_name] = response.content filename = get_secure_filename( - pagure_attachments[attach_name], attach_name) + pagure_attachments[attach_name], attach_name) url = '/%s/issue/raw/files/%s' % (self.pagure_project, filename) else: url = '#Attachment Unavailable' format_list.append(url) - format_list.append(url) + + # the difference is because of md + if is_image: + format_list.append(url) + return (format_list, whole_body, pagure_attachments) + + def _get_image_attachments(self, whole_body, pagure_attachments): + ''' Get the image attachments from github comment body ''' + + attach_regex = r'!\[.*\]\((.*)\)' + format_list, whole_body, pagure_attachments = self._get_attachments( + attach_regex=attach_regex, + whole_body=whole_body, + pagure_attachments=pagure_attachments, + ) + unformatted_body = re.sub(attach_regex, '\n[![%s](%s)](%s)', whole_body) whole_body = unformatted_body % tuple(format_list) return whole_body, pagure_attachments + def _get_file_attachments(self, whole_body, pagure_attachments): + ''' Get the file from github comment body ''' + + attach_regex = r'\[.*\]\((.*%s\/.*files.*)\)' \ + % self.github_project_name.replace('/', r'\/') + format_list, whole_body, pagure_attachments = self._get_attachments( + attach_regex=attach_regex, + whole_body=whole_body, + pagure_attachments=pagure_attachments, + is_image=False, + ) + unformatted_body = re.sub(attach_regex, '\n[%s](%s)', whole_body) + whole_body = unformatted_body % tuple(format_list) + return whole_body, pagure_attachments + + def get_comment_body(self, comment, pagure_attachments): + ''' Return the comment body. Check if there is + an attachment, if so return the attachment as well ''' + + whole_body = comment.body + whole_body, pagure_attachments = self._get_file_attachments( + whole_body=whole_body, + pagure_attachments=pagure_attachments + ) + whole_body, pagure_attachments = self._get_image_attachments( + whole_body=whole_body, + pagure_attachments=pagure_attachments + ) + + return whole_body, pagure_attachments + def import_issues(self, repo, status='all'): ''' Imports the issues on github for the given project ''' From 6feaa4b4aa1b2e65636a201f8f06bc5cef5e1008 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 09 2017 23:42:00 +0000 Subject: [PATCH 9/9] Add comments explaining regexes for image and non-image attachments Signed-off-by: Vivek Anand --- diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index 2e281e3..98528ca 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -83,6 +83,7 @@ class GithubImporter(object): def _get_image_attachments(self, whole_body, pagure_attachments): ''' Get the image attachments from github comment body ''' + # Markdown for image: ![]() attach_regex = r'!\[.*\]\((.*)\)' format_list, whole_body, pagure_attachments = self._get_attachments( attach_regex=attach_regex, @@ -97,6 +98,10 @@ class GithubImporter(object): def _get_file_attachments(self, whole_body, pagure_attachments): ''' Get the file from github comment body ''' + # Markdown for a non-image file is same as any normal link ([]()) + # in markdown. To differentiate i have used the url at which + # github stores these attachments: + # https://github.com///// attach_regex = r'\[.*\]\((.*%s\/.*files.*)\)' \ % self.github_project_name.replace('/', r'\/') format_list, whole_body, pagure_attachments = self._get_attachments(