From 341d8a19d0bf6b74dfdc471421bbcebdadc42f62 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Nov 16 2016 09:03:34 +0000 Subject: [PATCH 1/6] small pep8 fixes --- diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 6aa3e4d..3efa188 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -133,12 +133,12 @@ def generate_json_for_github_issue_commentors(github_username, issue_commentors = [] for issue in project.get_issues(state='all'): - if not issue.user.login in issue_commentors: + if issue.user.login not in issue_commentors: issue_commentors.append(issue.user.login) click.echo('commentor added: ' + issue.user.login) for comment in project.get_issues_comments(): - if not comment.user.login in issue_commentors: + if comment.user.login not in issue_commentors: issue_commentors.append(comment.user.login) click.echo('commentor added: ' + comment.user.login) diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index dde829e..73a90cd 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -48,7 +48,7 @@ class GithubImporter(): # Some details of a issue if github_issue.state != 'closed': pagure_issue_status = 'Open' - close_status='' + close_status = '' else: pagure_issue_status = 'Closed' close_status = 'Fixed' From a99db1913d6cbe21b0d111a64c421b0442a924c5 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Nov 16 2016 10:00:34 +0000 Subject: [PATCH 2/6] Add assignee support for github importer --- diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 3efa188..099b793 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -124,26 +124,32 @@ def generate_json_for_github_issue_commentors(github_username, github_project_name): ''' Will create a json file containing details of all the user who have commented on or filed any issue in the given project + This also contains users who are assignee of an issue ''' github_obj = Github(github_username, github_password) otp_auth = get_auth_token(github_obj) github_obj = Github(otp_auth) project = github_obj.get_repo(github_project_name) - issue_commentors = [] + issue_commentors_assignees = [] for issue in project.get_issues(state='all'): - if issue.user.login not in issue_commentors: - issue_commentors.append(issue.user.login) + if issue.user.login not in issue_commentors_assignees: + issue_commentors_assignees.append(issue.user.login) click.echo('commentor added: ' + issue.user.login) + if issue.assignee and \ + issue.assignee.login not in issue_commentors_assignees: + issue_commentors_assignees.append(issue.assignee.login) + click.echo('assignee added: ' + issue.assignee.login) + for comment in project.get_issues_comments(): - if comment.user.login not in issue_commentors: - issue_commentors.append(comment.user.login) + if comment.user.login not in issue_commentors_assignees: + issue_commentors_assignees.append(comment.user.login) click.echo('commentor added: ' + comment.user.login) with open('issue_commentors.json', 'w') as f: - f.write(json.dumps(issue_commentors)) + f.write(json.dumps(issue_commentors_assignees)) return diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index 73a90cd..023f02e 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -23,6 +23,21 @@ class GithubImporter(): otp_auth = get_auth_token(self.github) self.github = Github(otp_auth) + def get_issue_assignee(self, github_issue): + ''' From the github issue object, return the + assignee of the issue if any ''' + + assignee = None + if github_issue.assignee: + assignee = models.User( + name=github_issue.assignee.login, + fullname=github_issue.assignee.name, + emails=[github_get_commentor_email( + github_issue.assignee.login)] + ) + + return assignee + def import_issues(self, repo_path, repo_folder, status='all'): ''' Imports the issues on github for the given project @@ -54,8 +69,9 @@ class GithubImporter(): close_status = 'Fixed' pagure_issue_created_at = github_issue.created_at.strftime('%s') - # Not sure how to deal with this atm - pagure_issue_assignee = None + + # Get the assignee of the issue + pagure_issue_assignee = self.get_issue_assignee(github_issue) if github_issue.labels: pagure_issue_tags = [i.name for i in github_issue.labels] From 5a59c0cd287b9587307af23618342943e75eeb78 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Nov 16 2016 10:25:26 +0000 Subject: [PATCH 3/6] The user object of assignee should be json serializable --- diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index 023f02e..d4a16e9 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -36,7 +36,8 @@ class GithubImporter(): github_issue.assignee.login)] ) - return assignee + if assignee: + return assignee.to_json() def import_issues(self, repo_path, repo_folder, status='all'): ''' Imports the issues on github for From fc2192b7c022861eeccfba0aa12a7449903aaa31 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Nov 17 2016 09:10:29 +0000 Subject: [PATCH 4/6] Tell the user when one issue is created locally Otherwise, it looks like that the program has stopped responding --- diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index d4a16e9..46ad8cf 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -1,3 +1,4 @@ +import click from github import Github from pagure_importer.utils import ( @@ -49,8 +50,11 @@ class GithubImporter(): except: raise GithubRepoNotFound( 'Repo not found, project name wrong') + newpath, new_repo = clone_repo(repo_path, repo_folder) - for github_issue in repo.get_issues(state=status): + repo_issues = repo.get_issues(state=status) + issues_length = len(repo_issues) + for idx, github_issue in enumerate(repo_issues): # title of the issue pagure_issue_title = github_issue.title @@ -151,4 +155,5 @@ class GithubImporter(): # update the local git repo new_repo = update_git(pagure_issue, newpath, new_repo) + click.echo('Updated issue %s out of %s' % idx+1, issues_length) push_delete_repo(newpath, new_repo) From e1152691b78cb2d00fe24273052558a955b0c78d Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Nov 17 2016 09:10:34 +0000 Subject: [PATCH 5/6] Change few function and file names, they were big unnecessarily --- diff --git a/README.md b/README.md index 675bba6..dacb9a4 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ contributors and issue commentors. If you are running the script for github for the first time, the answer is 'y'. 3. The above step will create 3 different files: ```contributors.json``` -```issue_commentors.json``` and ```assembled_commentors.csv```. The last file +```issue_users.json``` and ```assembled_users.csv```. The last file is where all the edit has to go. All the missing entries in the assembled commentors file has to be filled for the running of the script. diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py index 37c4fb6..befbbdd 100644 --- a/pagure_importer/commands/github.py +++ b/pagure_importer/commands/github.py @@ -3,9 +3,7 @@ import pagure_importer from pagure_importer.app import app, REPO_PATH from pagure_importer.utils.importer_github import GithubImporter from pagure_importer.utils import ( - generate_json_for_github_contributors, - generate_json_for_github_issue_commentors, - assemble_github_contributors_commentors + gh_get_contributors, gh_get_issue_users, gh_assemble_users, ) @@ -22,15 +20,9 @@ def github(username, password, project): "Do you want to generate jsons for project's contributers" " and issue commentors?") if gen_json: - generate_json_for_github_contributors( - username, - password, - project) - generate_json_for_github_issue_commentors( - username, - password, - project) - assemble_github_contributors_commentors() + gh_get_contributors(username, password, project) + gh_get_issue_users(username, password, project) + gh_assemble_users() else: repos = pagure_importer.utils.display_repo() if repos: diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 099b793..6875441 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -61,9 +61,7 @@ def display_repo(): return repo -def generate_json_for_github_contributors(github_username, - github_password, - github_project_name): +def gh_get_contributors(github_username, github_password, github_project_name): ''' Creates a file containing a list of dicts containing the username and emails of the contributors in the given github project ''' @@ -119,9 +117,7 @@ def generate_json_for_github_contributors(github_username, return -def generate_json_for_github_issue_commentors(github_username, - github_password, - github_project_name): +def gh_get_issue_users(github_username, github_password, github_project_name): ''' Will create a json file containing details of all the user who have commented on or filed any issue in the given project This also contains users who are assignee of an issue @@ -148,17 +144,17 @@ def generate_json_for_github_issue_commentors(github_username, issue_commentors_assignees.append(comment.user.login) click.echo('commentor added: ' + comment.user.login) - with open('issue_commentors.json', 'w') as f: + with open('issue_users.json', 'w') as f: f.write(json.dumps(issue_commentors_assignees)) return -def assemble_github_contributors_commentors(): +def gh_assemble_users(): ''' It uses the files: issue_commentors.json and contributors.json Assembles and creates a file: assembled_commentors.csv To use: just fill the empty blocks under emails column''' - with open('issue_commentors.json', 'r') as ic: + with open('issue_users.json', 'r') as ic: issue_names = json.load(ic) with open('contributors.json', 'r') as c: @@ -177,7 +173,7 @@ def assemble_github_contributors_commentors(): d = {'name': i, 'fullname': None, 'emails': None} names.append(d) - with open('assembled_commentors.csv', 'w') as ac: + with open('assembled_users.csv', 'w') as ac: field_names = ['name', 'fullname', 'emails'] writer = csv.DictWriter(ac, fieldnames=field_names) @@ -186,18 +182,18 @@ def assemble_github_contributors_commentors(): writer.writerow(name) -def github_get_commentor_email(name): +def gh_get_user_email(name): ''' Will return the issue commentor email as given in the - assembled_commentors.csv file + assembled_users.csv file ''' - if not os.path.exists('assembled_commentors.csv'): + if not os.path.exists('assembled_users.csv'): raise FileNotFound('The assembled_commentors.json file must be present' ' Rerun the program and choose to generate the json' ' files') data = [] - with open('assembled_commentors.csv') as ac: + with open('assembled_users.csv') as ac: reader = csv.DictReader(ac) for row in reader: data.append(dict( diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index 46ad8cf..23127e9 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -2,7 +2,7 @@ import click from github import Github from pagure_importer.utils import ( - models, github_get_commentor_email, get_auth_token) + models, gh_get_user_email, get_auth_token) from pagure_importer.utils.git import ( clone_repo, push_delete_repo, update_git) from pagure_importer.utils.exceptions import ( @@ -33,8 +33,7 @@ class GithubImporter(): assignee = models.User( name=github_issue.assignee.login, fullname=github_issue.assignee.name, - emails=[github_get_commentor_email( - github_issue.assignee.login)] + emails=[gh_get_user_email(github_issue.assignee.login)] ) if assignee: @@ -96,7 +95,7 @@ class GithubImporter(): name=github_issue.user.login, fullname=github_issue.user.name, emails=[github_issue.user.email] if github_issue.user.email - else [github_get_commentor_email(github_issue.user.login)]) + else [gh_get_user_email(github_issue.user.login)]) pagure_issue = models.Issue( id=None, @@ -135,7 +134,7 @@ class GithubImporter(): name=comment_user.login, fullname=comment_user.name, emails=[comment_user.email] if comment_user.email - else [github_get_commentor_email(comment_user.login)]) + else [gh_get_user_email(comment_user.login)]) # Object to represent comment on an issue pagure_issue_comment = models.IssueComment( From 0fa1f4e27c9c1970c88d426f962158d38c86926c Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Nov 17 2016 09:10:34 +0000 Subject: [PATCH 6/6] Use 'is not None' instead of anything resulting to True while checking against None --- diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 6875441..9dcdad4 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -134,7 +134,7 @@ def gh_get_issue_users(github_username, github_password, github_project_name): issue_commentors_assignees.append(issue.user.login) click.echo('commentor added: ' + issue.user.login) - if issue.assignee and \ + if issue.assignee is not None and \ issue.assignee.login not in issue_commentors_assignees: issue_commentors_assignees.append(issue.assignee.login) click.echo('assignee added: ' + issue.assignee.login) diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index 23127e9..fc829c0 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -29,14 +29,14 @@ class GithubImporter(): assignee of the issue if any ''' assignee = None - if github_issue.assignee: + if github_issue.assignee is not None: assignee = models.User( name=github_issue.assignee.login, fullname=github_issue.assignee.name, emails=[gh_get_user_email(github_issue.assignee.login)] ) - if assignee: + if assignee is not None: return assignee.to_json() def import_issues(self, repo_path, repo_folder, status='all'):