From 75398708bf577654a18beb3508ffc3df770613d6 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Feb 12 2017 11:05:54 +0000 Subject: [PATCH 1/4] Beginning of refactor to remove TracImporter object Signed-off-by: Clement Verna --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 5e22a64..0f3004d 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -2,7 +2,6 @@ import click import pagure_importer from pagure_importer.app import app, REPO_PATH from pagure_importer.utils import importer_trac, get_pagure_namespace -from pagure_importer.utils.fas import FASclient import pagure_importer.utils.git as gitutils diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index ffaee9d..004a7bd 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -1,15 +1,20 @@ +import os import sys import re import time import click import requests -import shutil -import os +from random import randint from base64 import b64decode from datetime import datetime from pagure_importer.utils import ( get_close_status, is_image, issue_to_json, get_secure_filename) from pagure_importer.utils.models import User, Issue, IssueComment +from pagure_importer.utils.fas import FASclient +from pagure_importer.app import REPO_PATH + +SOMEBODY = User(name='somebody', fullname='somebody', + emails=['some@body.com']) def to_timestamp(tm): @@ -21,298 +26,276 @@ def to_timestamp(tm): return ts -class TracImporter(object): - ''' Pagure importer for trac instance ''' - - def __init__(self, project_url, username, password, offset, repo_name, - repo_folder, nopush, fasclient=None, tags=False, private=False): - ''' Instantiate a TracImporter object ''' - self.username = username - self.password = password - self.repo_name = repo_name - self.repo_folder = repo_folder - self.clone_repo_location = os.path.join( - repo_folder, 'clone-' + repo_name) - self.nopush = nopush - self.url = project_url - self.fas = fasclient - self.tags = tags - self.private = private - self.offset = offset - self.somebody = User(name='somebody', fullname='somebody', - emails=['some@body.com']) - self.reqid = 0 - self.custom_fields = [] - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - ''' Delete the cloned repo where the commits were going ''' - if os.path.exists(self.clone_repo_location): - if not self.nopush: - shutil.rmtree(self.clone_repo_location) - - def request(self, method, *args): - ''' Common method for querying trac ''' - - self.reqid += 1 - req = {'params': args, - 'method': method, - 'id': self.reqid} - resp = requests.post(self.url, json=req, - auth=(self.username, self.password)) - resp = resp.json() - if resp['id'] != self.reqid: - click.echo('ERROR: Invalid response for request! ' + - 'ID does not match') - sys.exit(1) - if resp['error'] is not None: - # Ignore missing attachment errors - if 'Attachment ' not in resp['error']['message'] and \ - ' not found' not in resp['error']['message']: - click.echo("ERROR: Error in response: %s" % resp['error']) - sys.exit(1) - - return resp['result'] - - def get_custom_fields(self): - ''' Queries the fedorahosted api to get all ticket fields - and filters all the custom fields, returns - a list of dicts - dict with keys 'name' and 'key_type' ''' - - all_ticket_fields = self.request('ticket.getTicketFields') - custom_fields = [] - for field in all_ticket_fields: - if field.get('custom') is True: - current_field = {} - current_field['name'] = field['name'] - key_type = 'text' - if field['type'] == 'checkbox': - key_type = 'boolean' - current_field['key_type'] = key_type - custom_fields.append(current_field) - return custom_fields - - def import_issues(self, repo_name, trac_query='max=0&order=id'): - ''' Queries the trac instance via its jsonrpc API and convert the - tickets into JSON blob to be imported into pagure's ticket git repo. - - :arg repo_name: the name of the repository - :arg repo_folder: the folder in which is the repository - :kwarg trac_query: the query to call trac with in order to retrieve - all the tickets. - Defaults to ``max=0&order=id`` - - ''' - - tickets_id = self.request('ticket.query', trac_query) - self.custom_fields = self.get_custom_fields() - - for ticket_id in tickets_id: - pagure_issue = self.create_issue(ticket_id) - pagure_issue.comments = [] - pagure_issue_comments = self.request('ticket.changeLog', ticket_id) - comments = self.create_comments(pagure_issue_comments) - # add all the comments to the issue object - for key in comments: - if comments[key].attachment is not None and \ - any(attachment in comments[key].attachment for attachment in - pagure_issue.attachment): - - for attach_name in comments[key].attachment: - filename = get_secure_filename( - pagure_issue.attachment[attach_name], attach_name) - url = '/%s/issue/raw/files/%s' % (repo_name, filename) - if is_image(attach_name): - comments[key].comment += ('\n[![%s](%s)](%s)' % - (attach_name, url, url)) - else: - comments[key].comment += ('\n[%s](%s)' % - (attach_name, url)) - pagure_issue.comments.append(comments[key].to_json()) - click.echo('Updated ' + repo_name + ' with issue :' + - str(ticket_id) + '/' + str(tickets_id[-1])) - issue_to_json(pagure_issue, self.clone_repo_location) - - def get_custom_fields_of_ticket(self, trac_ticket): - ''' Given the trac ticket, it will return all the - custom fields of the ticket, in a form that it can - be used for pagure Issue ''' - - pagure_fields = [] - for field in self.custom_fields: - if field['name'] in trac_ticket: - pagure_field = {} - pagure_field['name'] = field.get('name') - pagure_field['key_type'] = field.get('key_type') - pagure_field['value'] = trac_ticket.get( - pagure_field['name'], "").strip() - if pagure_field['value']: - pagure_fields.append(pagure_field) - return pagure_fields - - def create_issue(self, ticket_id): - ''' Create Issue object from track ticket ''' - - trac_ticket_info = self.request('ticket.get', ticket_id) - trac_ticket = trac_ticket_info[3] - trac_attachments = self.request('ticket.listAttachments', ticket_id) - - pagure_attachment = {} - for attachment in trac_attachments: - filename = attachment[0] - attachment_resp = self.request( - 'ticket.getAttachment', - ticket_id, filename) - if attachment_resp: - if is_image(filename): - content = b64decode(attachment_resp['__jsonclass__'][1]) - pagure_attachment[filename] = content - else: - content = b64decode( - attachment_resp['__jsonclass__'][1].replace('\n', '')) - pagure_attachment[filename] = content - - pagure_custom_fields = self.get_custom_fields_of_ticket(trac_ticket) - pagure_issue_title = trac_ticket['summary'] - - pagure_issue_content = trac_ticket['description'] - if pagure_issue_content == '': - pagure_issue_content = '#No Description Provided' - - issue_status, close_status = self.get_ticket_status(trac_ticket) - - pagure_issue_created_at = to_timestamp( - trac_ticket_info[1]['__jsonclass__'][1]) - - if self.fas: - pagure_issue_assignee = self.fas.find_fas_user( - trac_ticket['owner']) - pagure_issue_user = self.fas.find_fas_user(trac_ticket['reporter']) - if not pagure_issue_user.name: - pagure_issue_user = User( - name=trac_ticket['reporter'], - fullname=trac_ticket['reporter'], - emails=[trac_ticket['reporter'] + '@fedoraproject.org']) - else: - pagure_issue_assignee = User(name='', fullname='', emails=[]) - pagure_issue_user = User( - name=trac_ticket['reporter'], - fullname=trac_ticket['reporter'], - emails=[trac_ticket['reporter'] + '@fedoraproject.org']) - - # The milestone of the issue - pagure_milestone = None - if 'milestone' in trac_ticket and trac_ticket['milestone'] != '': - pagure_milestone = trac_ticket['milestone'] - - # Issue tags - pagure_issue_tags = [] - if self.tags: - pagure_issue_tags = filter( - lambda x: x != '', trac_ticket['keywords'].split(' ')) - - pagure_issue_tags = self.pre_process_tags(pagure_issue_tags) - - pagure_issue_depends = [] - pagure_issue_blocks = [] - if self.private: - pagure_issue_is_private = True - else: - pagure_issue_is_private = False - - pagure_issue = Issue( - id=ticket_id + self.offset, - title=pagure_issue_title, - content=pagure_issue_content, - status=issue_status, - close_status=close_status, - date_created=pagure_issue_created_at, - user=pagure_issue_user.to_json(), - private=pagure_issue_is_private, - attachment=pagure_attachment, - tags=pagure_issue_tags, - milestone=pagure_milestone, - depends=pagure_issue_depends, - blocks=pagure_issue_blocks, - assignee=pagure_issue_assignee.to_json(), - custom_fields=pagure_custom_fields,) - return pagure_issue - - def pre_process_tags(self, tags): - ''' Pre process the tags before sending it to pagure ''' - - delims_str = ',|/|;|:|-|\+|\*|\'|\"' - - # Remove the delims between the tags and convert all tags to lower case - pagure_issue_tags = list(set([j for k in [ - re.split(delims_str, i.lower()) for i in tags] for j in k])) - - return pagure_issue_tags - - def get_ticket_status(self, trac_ticket): - ''' Returns the corresponding status of ticket on pagure ''' - close_status = get_close_status() - if close_status is not None: - if trac_ticket['status'] != 'closed': - return ('Open', '') - else: - for status in close_status: - if trac_ticket['resolution'] in close_status[status]: - return ('Closed', status) - return ('Closed', 'Fixed') - else: - click.echo('ERROR: Close Status not read from config file') - sys.exit(1) +def pre_process_tags(tags): + ''' Pre process the tags before sending it to pagure ''' - def get_comment_user(self, comment): - ''' Returns the user who commented on the ticket ''' + delims_str = ',|/|;|:|-|\+|\*|\'|\"' - # The User who commented - if self.fas and comment[1]: - pagure_issue_comment_user = self.fas.find_fas_user(comment[1]) - if not pagure_issue_comment_user.name: - pagure_issue_comment_user = self.somebody - else: - pagure_issue_comment_user = self.somebody - return pagure_issue_comment_user + # Remove the delims between the tags and convert all tags to lower case + pagure_issue_tags = list(set([j for k in [ + re.split(delims_str, i.lower()) for i in tags] for j in k])) - def create_comments(self, trac_comments): - ''' Create IssueComment objects from the trac comments ''' + return pagure_issue_tags - comments = {} - for comment in trac_comments: - ts = to_timestamp(comment[0]['__jsonclass__'][1]) - if comment[2] == 'comment' and comment[4] != '': - if ts in comments: - attachment = comments[ts].attachment - else: - attachment = [] +def request(method, rest, *args): + ''' Common method for querying trac ''' - pagure_issue_comment_body = comment[4] - pagure_issue_comment_created_at = ts + reqid = randint(0, 100) + req = {'params': args, + 'method': method, + 'id': reqid} + resp = requests.post(rest.url, json=req, + auth=(rest.username, rest.password)) + resp = resp.json() + if resp['id'] != reqid: + click.echo('ERROR: Invalid response for request! ' + + 'ID does not match') + sys.exit(1) + if resp['error'] is not None: + # Ignore missing attachment errors + if 'Attachment ' not in resp['error']['message'] and \ + ' not found' not in resp['error']['message']: + click.echo("ERROR: Error in response: %s" % resp['error']) + sys.exit(1) + return resp['result'] + + +def import_issues(repo_name, rest, trac_query='max=0&order=id'): + ''' Queries the trac instance via its jsonrpc API and convert the + tickets into JSON blob to be imported into pagure's ticket git repo. + + :arg repo_name: the name of the repository + :arg repo_folder: the folder in which is the repository + :kwarg trac_query: the query to call trac with in order to retrieve + all the tickets. + Defaults to ``max=0&order=id`` + + ''' + + tickets_id = request('ticket.query', rest, trac_query) + ticket_fields = request('ticket.getTicketFields', rest) + custom_fields = get_custom_fields(ticket_fields) + clone_repo_location = os.path.join(REPO_PATH, 'clone-' + repo_name) + + for ticket_id in tickets_id: + pagure_issue = create_issue(ticket_id=ticket_id, + custom_fields=custom_fields, + rest=rest) + pagure_issue.comments = [] + pagure_issue_comments = request('ticket.changeLog', rest, ticket_id) + comments = create_comments(pagure_issue_comments) + # add all the comments to the issue object + pagure_issue = add_comment_to_issue(comments, pagure_issue) + + click.echo('Updated ' + repo_name + ' with issue :' + + str(ticket_id) + '/' + str(tickets_id[-1])) + issue_to_json(pagure_issue, clone_repo_location) + + +def get_custom_fields(ticket_fields): + ''' Queries the fedorahosted api to get all ticket fields + and filters all the custom fields, returns + a list of dicts - dict with keys 'name' and 'key_type' ''' + + custom_fields = [] + for field in ticket_fields: + if field.get('custom') is True: + current_field = {} + current_field['name'] = field['name'] + key_type = 'text' + if field['type'] == 'checkbox': + key_type = 'boolean' + current_field['key_type'] = key_type + custom_fields.append(current_field) + return custom_fields + + +def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags): + ''' Create Issue object from track ticket ''' + + trac_ticket_info = request('ticket.get', rest, ticket_id) + trac_ticket = trac_ticket_info[3] + trac_attachments = request('ticket.listAttachments', rest, ticket_id) + + pagure_attachment = {} + for attachment in trac_attachments: + filename = attachment[0] + attachment_resp = request('ticket.getAttachment', + rest, ticket_id, filename) + if attachment_resp: + if is_image(filename): + content = b64decode(attachment_resp['__jsonclass__'][1]) + pagure_attachment[filename] = content + else: + content = b64decode( + attachment_resp['__jsonclass__'][1].replace('\n', '')) + pagure_attachment[filename] = content + + pagure_custom_fields = get_custom_fields_of_ticket(trac_ticket=trac_ticket, + custom_fields=custom_fields) + pagure_issue_title = trac_ticket['summary'] + + pagure_issue_content = trac_ticket['description'] + if pagure_issue_content == '': + pagure_issue_content = '#No Description Provided' + + issue_status, close_status = get_ticket_status(trac_ticket) + + pagure_issue_created_at = to_timestamp( + trac_ticket_info[1]['__jsonclass__'][1]) + + fas = FASclient(rest.username, rest.password, + 'https://admin.fedoraproject.org/accounts') + pagure_issue_assignee = fas.find_fas_user(trac_ticket['owner']) + pagure_issue_user = fas.find_fas_user(trac_ticket['reporter']) + if not pagure_issue_user.name: + pagure_issue_user = User( + name=trac_ticket['reporter'], + fullname=trac_ticket['reporter'], + emails=[trac_ticket['reporter'] + '@fedoraproject.org']) + + # The milestone of the issue + pagure_milestone = None + if 'milestone' in trac_ticket and trac_ticket['milestone'] != '': + pagure_milestone = trac_ticket['milestone'] + + # Issue tags + pagure_issue_tags = [] + if tags: + pagure_issue_tags = filter( + lambda x: x != '', trac_ticket['keywords'].split(' ')) + + pagure_issue_tags = pre_process_tags(pagure_issue_tags) + + pagure_issue_depends = [] + pagure_issue_blocks = [] + if private: + pagure_issue_is_private = True + else: + pagure_issue_is_private = False + + pagure_issue = Issue( + id=ticket_id + offset, + title=pagure_issue_title, + content=pagure_issue_content, + status=issue_status, + close_status=close_status, + date_created=pagure_issue_created_at, + user=pagure_issue_user.to_json(), + private=pagure_issue_is_private, + attachment=pagure_attachment, + tags=pagure_issue_tags, + milestone=pagure_milestone, + depends=pagure_issue_depends, + blocks=pagure_issue_blocks, + assignee=pagure_issue_assignee.to_json(), + custom_fields=pagure_custom_fields,) + return pagure_issue + + +def get_custom_fields_of_ticket(trac_ticket, custom_fields): + ''' Given the trac ticket, it will return all the + custom fields of the ticket, in a form that it can + be used for pagure Issue ''' + + pagure_fields = [] + for field in custom_fields: + if field['name'] in trac_ticket: + pagure_field = {} + pagure_field['name'] = field.get('name') + pagure_field['key_type'] = field.get('key_type') + pagure_field['value'] = trac_ticket.get(pagure_field['name']) + pagure_fields.append(pagure_field) + return pagure_fields + + +def get_ticket_status(trac_ticket): + ''' Returns the corresponding status of ticket on pagure ''' + close_status = get_close_status() + if close_status is not None: + if trac_ticket['status'] != 'closed': + return ('Open', '') + else: + for status in close_status: + if trac_ticket['resolution'] in close_status[status]: + return ('Closed', status) + return ('Closed', 'Fixed') + else: + click.echo('ERROR: Close Status not read from config file') + sys.exit(1) + + +def create_comments(trac_comments): + ''' Create IssueComment objects from the trac comments ''' + + comments = {} + for comment in trac_comments: + ts = to_timestamp(comment[0]['__jsonclass__'][1]) + + if comment[2] == 'comment' and comment[4] != '': + if ts in comments: + attachment = comments[ts].attachment + else: + attachment = [] + + pagure_issue_comment_body = comment[4] + pagure_issue_comment_created_at = ts + + pagure_issue_comment_user = get_comment_user(comment) + # Object to represent comment on an issue + comments[ts] = IssueComment( + id=None, + comment=pagure_issue_comment_body, + date_created=pagure_issue_comment_created_at, + attachment=attachment, + user=pagure_issue_comment_user.to_json()) + + elif comment[2] == 'attachment': + if ts in comments: + comments[ts].attachment.append(comment[4]) + else: pagure_issue_comment_user = self.get_comment_user(comment) - # Object to represent comment on an issue comments[ts] = IssueComment( id=None, - comment=pagure_issue_comment_body, - date_created=pagure_issue_comment_created_at, - attachment=attachment, + comment='attachment', + date_created=ts, + attachment=[comment[4]], user=pagure_issue_comment_user.to_json()) - elif comment[2] == 'attachment': - if ts in comments: - comments[ts].attachment.append(comment[4]) + return comments + + +def get_comment_user(comment, fas): + ''' Returns the user who commented on the ticket ''' + + # The User who commented + if fas and comment[1]: + pagure_issue_comment_user = fas.find_fas_user(comment[1]) + if not pagure_issue_comment_user.name: + pagure_issue_comment_user = SOMEBODY + else: + pagure_issue_comment_user = SOMEBODY + return pagure_issue_comment_user + + +def add_comment_to_issue(comments, pagure_issue, repo_name): + '''Adding comment to an issue''' + for key in comments: + if comments[key].attachment is not None and \ + any(attachment in comments[key].attachment for attachment in + pagure_issue.attachment): + + for attach_name in comments[key].attachment: + filename = get_secure_filename( + pagure_issue.attachment[attach_name], attach_name) + url = '/%s/issue/raw/files/%s' % (repo_name, filename) + if is_image(attach_name): + comments[key].comment += ('\n[![%s](%s)](%s)' % + (attach_name, url, url)) else: - pagure_issue_comment_user = self.get_comment_user(comment) - comments[ts] = IssueComment( - id=None, - comment='attachment', - date_created=ts, - attachment=[comment[4]], - user=pagure_issue_comment_user.to_json()) - - return comments + comments[key].comment += ('\n[%s](%s)' % + (attach_name, url)) + pagure_issue.comments.append(comments[key].to_json()) + return pagure_issue From cb8e1f576cee7efd168f0c24a65b9962258727f7 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Feb 12 2017 11:05:54 +0000 Subject: [PATCH 2/4] Move logic to import issues into fedorahosted command code Signed-off-by: Clement Verna --- diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index 0f3004d..cc4d822 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -1,9 +1,13 @@ +import os +from collections import namedtuple import click import pagure_importer from pagure_importer.app import app, REPO_PATH -from pagure_importer.utils import importer_trac, get_pagure_namespace +from pagure_importer.utils import get_pagure_namespace, issue_to_json +from pagure_importer.utils.fas import FASclient import pagure_importer.utils.git as gitutils +import pagure_importer.utils.importer_trac as libtrac @app.command() @@ -19,36 +23,60 @@ import pagure_importer.utils.git as gitutils help='Number of issue in pagure before import') @click.option('--nopush', is_flag=True, help="Do not push the result of pagure-importer back") -def fedorahosted( - project_url, tags, private, username, password, offset, nopush): - fasclient = FASclient(username, password, - 'https://admin.fedoraproject.org/accounts') +def fedorahosted(project_url, tags, private, username, password, + offset, nopush): + """Import issues from fedorahosted""" + if project_url.endswith('.git'): project_url = project_url.replace('.git', '') project_url += '/login/jsonrpc' repos = pagure_importer.utils.display_repo() + if repos: repo_index = click.prompt('Choose the import destination repo ', default=1) repo_name = repos[int(repo_index)-1] - newpath, new_repo = gitutils.clone_repo(repo_name, REPO_PATH) + _, new_repo = gitutils.clone_repo(repo_name, REPO_PATH) project = get_pagure_namespace(REPO_PATH, repo_name) - with importer_trac.TracImporter(project_url=project_url, - username=username, - password=password, - offset=offset, - repo_name=repo_name, - repo_folder=REPO_PATH, - fasclient=fasclient, - tags=tags, - private=private, - nopush=nopush) as trac_importer: - - trac_importer.import_issues(project) + rest_param = namedtuple('rest_param', 'url username password') + rest = rest_param(project_url, username, password) + fas = FASclient(rest.username, rest.password, + 'https://admin.fedoraproject.org/accounts') + + # import issues + tickets_id = libtrac.get_project_tickets(rest=rest) + ticket_fields = libtrac.get_project_tickets_fields(rest=rest) + custom_fields = libtrac.get_custom_fields(ticket_fields=ticket_fields) + clone_repo_location = os.path.join(REPO_PATH, 'clone-' + repo_name) + + for ticket_id in tickets_id: + + pagure_issue = libtrac.create_issue(ticket_id=ticket_id, + custom_fields=custom_fields, + rest=rest, + fas=fas, + offset=offset, + private=private, + tags=tags) + pagure_issue.comments = [] + pagure_issue_comments = libtrac.get_ticket_comments(rest=rest, + ticket_id=ticket_id) + comments = libtrac.create_comments(trac_comments=pagure_issue_comments, + fas=fas) + # add all the comments to the issue object + pagure_issue = libtrac.add_comment_to_issue(comments=comments, + pagure_issue=pagure_issue, + repo_name=project) + + click.echo('Updated ' + repo_name + ' with issue :' + + str(ticket_id) + '/' + str(tickets_id[-1])) + issue_to_json(issue=pagure_issue, folder=clone_repo_location) + # update the local git repo new_repo = gitutils.update_git( new_repo, commit_message='Imported issues from fedorahosted project: %s' % repo_name) + if not nopush: gitutils.push_repo(new_repo) else: From ce3531f9fb16f50f88160a397766770d46da5bc4 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Feb 12 2017 11:05:54 +0000 Subject: [PATCH 3/4] Remove TracImporter object and use function instead Signed-off-by: Clement Verna --- diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 004a7bd..0457883 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -1,17 +1,14 @@ -import os import sys import re import time -import click -import requests from random import randint from base64 import b64decode from datetime import datetime +import click +import requests from pagure_importer.utils import ( - get_close_status, is_image, issue_to_json, get_secure_filename) + get_close_status, is_image, get_secure_filename) from pagure_importer.utils.models import User, Issue, IssueComment -from pagure_importer.utils.fas import FASclient -from pagure_importer.app import REPO_PATH SOMEBODY = User(name='somebody', fullname='somebody', emails=['some@body.com']) @@ -29,7 +26,7 @@ def to_timestamp(tm): def pre_process_tags(tags): ''' Pre process the tags before sending it to pagure ''' - delims_str = ',|/|;|:|-|\+|\*|\'|\"' + delims_str = r',|/|;|:|-|\+|\*|\'|\"' # Remove the delims between the tags and convert all tags to lower case pagure_issue_tags = list(set([j for k in [ @@ -62,36 +59,33 @@ def request(method, rest, *args): return resp['result'] -def import_issues(repo_name, rest, trac_query='max=0&order=id'): - ''' Queries the trac instance via its jsonrpc API and convert the - tickets into JSON blob to be imported into pagure's ticket git repo. - - :arg repo_name: the name of the repository - :arg repo_folder: the folder in which is the repository - :kwarg trac_query: the query to call trac with in order to retrieve - all the tickets. - Defaults to ``max=0&order=id`` +def get_project_tickets(rest, trac_query='max=0&order=id'): + """ Queries the trac instance via the jsonrpc API to get a list + tickets. By default returns all the project tickets + :arg rest: the rest connection parameters (url, username, password) + :arg trac_query: query use by trac to return tickets + """ + ticket_id = request('ticket.query', rest, trac_query) + return ticket_id - ''' - tickets_id = request('ticket.query', rest, trac_query) +def get_project_tickets_fields(rest): + """ Queries the trac instance via the jsonrpc API to get a list + of all the field used in the trac tickets + :arg rest: the rest connection parameters (url, username, password) + """ ticket_fields = request('ticket.getTicketFields', rest) - custom_fields = get_custom_fields(ticket_fields) - clone_repo_location = os.path.join(REPO_PATH, 'clone-' + repo_name) + return ticket_fields - for ticket_id in tickets_id: - pagure_issue = create_issue(ticket_id=ticket_id, - custom_fields=custom_fields, - rest=rest) - pagure_issue.comments = [] - pagure_issue_comments = request('ticket.changeLog', rest, ticket_id) - comments = create_comments(pagure_issue_comments) - # add all the comments to the issue object - pagure_issue = add_comment_to_issue(comments, pagure_issue) - click.echo('Updated ' + repo_name + ' with issue :' + - str(ticket_id) + '/' + str(tickets_id[-1])) - issue_to_json(pagure_issue, clone_repo_location) +def get_ticket_comments(rest, ticket_id): + """ Queries the trac instance via the jsonrpc API to get a list + of all the ticket's comments + :arg rest: the rest connection parameters (url, username, password) + :arg ticket_id: id of the ticket to query + """ + issue_comments = request('ticket.changeLog', rest, ticket_id) + return issue_comments def get_custom_fields(ticket_fields): @@ -112,18 +106,51 @@ def get_custom_fields(ticket_fields): return custom_fields +def get_ticket_info(rest, ticket_id): + """ Queries the trac instance via the jsonrpc API to get the + details of a ticket. + :arg rest: the rest connection parameters (url, username, password) + :arg ticket_id: id of the ticket to query + """ + ticket_info = request('ticket.get', rest, ticket_id) + return ticket_info + + +def get_ticket_attachments(rest, ticket_id): + """ Queries the trac instance via the jsonrpc API to get all + of a ticket's attachments. + :arg rest: the rest connection parameters (url, username, password) + :arg ticket_id: id of the ticket to query + """ + attachments = request('ticket.listAttachments', rest, ticket_id) + return attachments + + +def get_attachment(rest, ticket_id, attachment): + """ Queries the trac instance via the jsonrpc API to get a + ticket's attachment. + :arg rest: the rest connection parameters (url, username, password) + :arg ticket_id: id of the ticket to query + :arg attachment: name of the attachment + """ + attachment = request('ticket.getAttachment', + rest, ticket_id, attachment) + return attachment + + def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags): ''' Create Issue object from track ticket ''' - trac_ticket_info = request('ticket.get', rest, ticket_id) + trac_ticket_info = get_ticket_info(rest=rest, ticket_id=ticket_id) trac_ticket = trac_ticket_info[3] - trac_attachments = request('ticket.listAttachments', rest, ticket_id) + trac_attachments = get_ticket_attachments(rest=rest, ticket_id=ticket_id) pagure_attachment = {} for attachment in trac_attachments: filename = attachment[0] - attachment_resp = request('ticket.getAttachment', - rest, ticket_id, filename) + attachment_resp = get_attachment(rest=rest, + ticket_id=ticket_id, + attachment=filename) if attachment_resp: if is_image(filename): content = b64decode(attachment_resp['__jsonclass__'][1]) @@ -146,8 +173,6 @@ def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags): pagure_issue_created_at = to_timestamp( trac_ticket_info[1]['__jsonclass__'][1]) - fas = FASclient(rest.username, rest.password, - 'https://admin.fedoraproject.org/accounts') pagure_issue_assignee = fas.find_fas_user(trac_ticket['owner']) pagure_issue_user = fas.find_fas_user(trac_ticket['reporter']) if not pagure_issue_user.name: @@ -171,10 +196,6 @@ def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags): pagure_issue_depends = [] pagure_issue_blocks = [] - if private: - pagure_issue_is_private = True - else: - pagure_issue_is_private = False pagure_issue = Issue( id=ticket_id + offset, @@ -184,7 +205,7 @@ def create_issue(ticket_id, custom_fields, rest, fas, private, offset, tags): close_status=close_status, date_created=pagure_issue_created_at, user=pagure_issue_user.to_json(), - private=pagure_issue_is_private, + private=private, attachment=pagure_attachment, tags=pagure_issue_tags, milestone=pagure_milestone, @@ -227,7 +248,7 @@ def get_ticket_status(trac_ticket): sys.exit(1) -def create_comments(trac_comments): +def create_comments(trac_comments, fas): ''' Create IssueComment objects from the trac comments ''' comments = {} @@ -243,7 +264,7 @@ def create_comments(trac_comments): pagure_issue_comment_body = comment[4] pagure_issue_comment_created_at = ts - pagure_issue_comment_user = get_comment_user(comment) + pagure_issue_comment_user = get_comment_user(comment, fas) # Object to represent comment on an issue comments[ts] = IssueComment( id=None, @@ -256,7 +277,7 @@ def create_comments(trac_comments): if ts in comments: comments[ts].attachment.append(comment[4]) else: - pagure_issue_comment_user = self.get_comment_user(comment) + pagure_issue_comment_user = get_comment_user(comment, fas) comments[ts] = IssueComment( id=None, comment='attachment', From be87612049548e9395e98691bbcc81a7c24db619 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Feb 12 2017 11:05:54 +0000 Subject: [PATCH 4/4] Remove TracImporter object from test suite Signed-off-by: Clement Verna --- diff --git a/tests/test_pgimport_importer_trac.py b/tests/test_pgimport_importer_trac.py index fe3886b..a2c0539 100644 --- a/tests/test_pgimport_importer_trac.py +++ b/tests/test_pgimport_importer_trac.py @@ -1,28 +1,23 @@ import unittest from unittest.mock import MagicMock -from pagure_importer.utils.importer_trac import (TracImporter, to_timestamp) +from pagure_importer.utils.fas import FASclient +from pagure_importer.utils.models import User +import pagure_importer.utils.importer_trac as libtrac class PgimportImporterTrac (unittest.TestCase): def setUp(self): - self.trac = TracImporter(project_url="https://foobar.bar", - username="foo", - password="bar", - offset=0, - repo_folder="myfolder", - repo_name="myname", - nopush=False) + self.fas = FASclient('foo', 'bar', 'http://example.com') + self.fas_mock = MagicMock() + self.fas.find_fas_user = self.fas_mock + self.fas.find_fas_user.return_value = User(name='', + fullname='', + emails=['']) + self.somebody = {'name': 'somebody', 'fullname': 'somebody', 'emails': ['some@body.com']} - self.mock_request = MagicMock() - self.old_request = TracImporter.request - TracImporter.request = self.mock_request - - def tearDown(self): - TracImporter.request = self.old_request - def test_import_2_comments(self): # Case 1 - Import 2 comments @@ -32,9 +27,9 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2016-11-08T20:32:20']}, u'foobar', u'comment', u'2', u'Well this will improve the code base :).', 1]] - results = TracImporter.create_comments(self.trac, trac_comments) + results = libtrac.create_comments(trac_comments, self.fas) - ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) + ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) self.assertEqual("Yeah, let\'s have some test.", results[ts_1].comment) @@ -42,7 +37,7 @@ class PgimportImporterTrac (unittest.TestCase): self.assertEqual([], results[ts_1].attachment) self.assertEqual(self.somebody, results[ts_1].user) - ts_2 = to_timestamp(trac_comments[1][0]['__jsonclass__'][1]) + ts_2 = libtrac.to_timestamp(trac_comments[1][0]['__jsonclass__'][1]) self.assertEqual("Well this will improve the code base :).", results[ts_2].comment) @@ -56,7 +51,7 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2016-11-08T20:30:13']}, u'foobar', u'comment', u'1', u'', 1]] - results = TracImporter.create_comments(self.trac, trac_comments) + results = libtrac.create_comments(trac_comments, self.fas) self.assertEqual({}, results) def test_import_not_a_comments(self): @@ -65,7 +60,7 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2014-04-09T16:02:20']}, u'foobar', u'resolution', u'', u'wontfix', 1]] - results = TracImporter.create_comments(self.trac, trac_comments) + results = libtrac.create_comments(trac_comments, self.fas) self.assertEqual({}, results) def test_import_attachment(self): @@ -74,8 +69,8 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar', u'attachment', u'', u'mytest.png', 0]] - results = TracImporter.create_comments(self.trac, trac_comments) - ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) + results = libtrac.create_comments(trac_comments, self.fas) + ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) self.assertEqual("attachment", results[ts_1].comment) @@ -91,8 +86,8 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar', u'comment', u'', u'test attachment', 0]] - results = TracImporter.create_comments(self.trac, trac_comments) - ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) + results = libtrac.create_comments(trac_comments, self.fas) + ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) self.assertEqual("test attachment", results[ts_1].comment) @@ -108,8 +103,8 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar', u'attachment', u'', u'mytest.png', 0]] - results = TracImporter.create_comments(self.trac, trac_comments) - ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) + results = libtrac.create_comments(trac_comments, self.fas) + ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) self.assertEqual("test attachment", results[ts_1].comment) @@ -127,8 +122,8 @@ class PgimportImporterTrac (unittest.TestCase): [{u'__jsonclass__': [u'datetime', u'2014-10-31T12:32:18']}, u'foobar', u'comment', u'', u'test 2 attachments', 0]] - results = TracImporter.create_comments(self.trac, trac_comments) - ts_1 = to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) + results = libtrac.create_comments(trac_comments, self.fas) + ts_1 = libtrac.to_timestamp(trac_comments[0][0]['__jsonclass__'][1]) self.assertEqual("test 2 attachments", results[ts_1].comment) @@ -140,60 +135,67 @@ class PgimportImporterTrac (unittest.TestCase): trac_ticket = {'status': 'open'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Open', ''), results) def test_get_ticket_status_closedInvalid(self): trac_ticket = {'status': 'closed', 'resolution': 'invalid'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Invalid'), results) def test_get_ticket_status_closedWontFix(self): trac_ticket = {'status': 'closed', 'resolution': 'wontfix'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Invalid'), results) def test_get_ticket_status_closedWorksforme(self): trac_ticket = {'status': 'closed', 'resolution': 'worksforme'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Invalid'), results) def test_get_ticket_status_closedDuplicate(self): trac_ticket = {'status': 'closed', 'resolution': 'duplicate'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Duplicate'), results) def test_get_ticket_status_closedInsufficient(self): trac_ticket = {'status': 'closed', 'resolution': 'insufficient_info'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Insufficient data'), results) def test_get_ticket_status_closedFixed(self): trac_ticket = {'status': 'closed', 'resolution': 'fixed'} - results = TracImporter.get_ticket_status(self.trac, trac_ticket) + results = libtrac.get_ticket_status(trac_ticket) self.assertEqual(('Closed', 'Fixed'), results) def test_get_custom_fieldsText(self): - self.mock_request.return_value = [{'custom': True, 'name': 'foo', 'type': 'textarea'}] - results = TracImporter.get_custom_fields(self.trac) + ticket_fields = [{'custom': True, 'name': 'foo', 'type': 'textarea'}] + results = libtrac.get_custom_fields(ticket_fields) self.assertEqual([{'key_type': 'text', 'name': 'foo'}], results) def test_get_custom_fieldsBoolean(self): - self.mock_request.return_value = [{'custom': True, 'name': 'bar', 'type': 'checkbox'}] - results = TracImporter.get_custom_fields(self.trac) + ticket_fields = [{'custom': True, 'name': 'bar', 'type': 'checkbox'}] + results = libtrac.get_custom_fields(ticket_fields) self.assertEqual([{'key_type': 'boolean', 'name': 'bar'}], results) def test_get_tickets_custom_fiedls(self): - self.trac.custom_fields = [{'key_type': 'text', 'name': 'bar'}, - {'key_type': 'boolean', 'name': 'foo'}] + custom_fields = [{'key_type': 'text', 'name': 'bar'}, + {'key_type': 'boolean', 'name': 'foo'}] trac_ticket = {'bar': 'hello', 'foo': 'world', 'foobar': 'hello world'} - results = TracImporter.get_custom_fields_of_ticket(self.trac, trac_ticket) - self.assertDictEqual({'key_type': 'text', 'value': 'hello', 'name': 'bar'}, results[0]) - self.assertDictEqual({'key_type': 'boolean', 'value': 'world', 'name': 'foo'}, results[1]) + results = libtrac.get_custom_fields_of_ticket(trac_ticket, custom_fields) + self.assertDictEqual({'key_type': 'text', 'value': 'hello', 'name': 'bar'}, + results[0]) + self.assertDictEqual({'key_type': 'boolean', 'value': 'world', 'name': 'foo'}, + results[1]) + + +if __name__ == '__main__': + SUITE = unittest.TestLoader().loadTestsFromTestCase(PgimportImporterTrac) + unittest.TextTestRunner(verbosity=2).run(SUITE)