From 5770c132bbdc0cbbc2a11d88b566b3e04653ef75 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Jan 09 2017 20:23:21 +0000 Subject: [PATCH 1/2] Add Support for multi-threaded import of fedorahosted issues Signed-off-by: Clement Verna --- diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index eb1d20c..c6c3bfe 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -3,6 +3,9 @@ import re import time import click import requests +from threading import Thread +from random import randint +from queue import Queue from base64 import b64decode from datetime import datetime from pagure_importer.utils import ( @@ -34,20 +37,19 @@ class TracImporter(Importer): self.offset = offset self.somebody = User(name='somebody', fullname='somebody', emails=['some@body.com']) - self.reqid = 0 self.custom_fields = self.get_custom_fields() + self.queue = Queue() def request(self, method, *args): ''' Common method for querying trac ''' - - self.reqid += 1 + reqid = randint(0,100) req = {'params': args, 'method': method, - 'id': self.reqid} + 'id': reqid} resp = requests.post(self.url, json=req, auth=(self.username, self.password)) resp = resp.json() - if resp['id'] != self.reqid: + if resp['id'] != reqid: click.echo('ERROR: Invalid response for request! ' + 'ID does not match') sys.exit(1) @@ -91,8 +93,19 @@ class TracImporter(Importer): ''' tickets_id = self.request('ticket.query', trac_query) + for i in range(10): + t = Thread(target=self.import_thread, args=(self.queue, repo_name, tickets_id)) + t.setDaemon(True) + t.start() for ticket_id in tickets_id: + self.queue.put(ticket_id) + + self.queue.join() + + def import_thread(self, queue, repo_name, tickets_id): + while True: + ticket_id = queue.get() pagure_issue = self.create_issue(ticket_id) pagure_issue.comments = [] pagure_issue_comments = self.request('ticket.changeLog', ticket_id) @@ -117,6 +130,7 @@ class TracImporter(Importer): click.echo('Updated ' + repo_name + ' with issue :' + str(ticket_id) + '/' + str(tickets_id[-1])) issue_to_json(pagure_issue, self.clone_repo_location) + self.queue.task_done() def get_custom_fields_of_ticket(self, trac_ticket): ''' Given the trac ticket, it will return all the From 3dbc95a765ea49b3cadd478476348d11163637c7 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Jan 09 2017 20:46:30 +0000 Subject: [PATCH 2/2] Make the number of thread a parameter Signed-off-by: Clement Verna --- diff --git a/README.md b/README.md index 3ac953a..c86d876 100644 --- a/README.md +++ b/README.md @@ -84,9 +84,10 @@ To add some new close status just edit the config file as follow. Where ```Foo`` --private By default make all issues private. --username TEXT FAS username --password TEXT FAS password - --offset INTEGER Number of issue in pagure before import + --offset INTEGER Number of issue in pagure before import (Default = 0) --help Show this message and exit. --nopush Do not push the result of pagure-importer back + --threads Number of thread used during the issue import (Default = 5) $ pgimport fedorahosted https://fedorahosted.org/foobar --tags @@ -114,6 +115,10 @@ To add some new close status just edit the config file as follow. Where ```Foo`` representation of the issues to import. This can be used to process the issues using the json files before running the import. Default location of the cloned repository is under /tmp/clone-foobar.git + + $ pgimport fedorahosted https://fedorahosted.org/foobar --threads 50 + + This command will run the importer using 50 threads to import the issues in parallel. 3) The push command can be used to push a clone pagure ticket repo back to pagure. diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py index b3b1986..1c19c9d 100644 --- a/pagure_importer/commands/fedorahosted.py +++ b/pagure_importer/commands/fedorahosted.py @@ -20,8 +20,10 @@ import pagure_importer.utils.git as gitutils help='Number of issue in pagure before import') @click.option('--nopush', is_flag=True, help="Do not push the result of pagure-importer back") +@click.option('--threads', default=5, + help="Number of thread used during the issue import") def fedorahosted( - project_url, tags, private, username, password, offset, nopush): + project_url, tags, private, username, password, offset, nopush, threads): fasclient = FASclient(username, password, 'https://admin.fedoraproject.org/accounts') project_url += '/login/jsonrpc' @@ -41,7 +43,8 @@ def fedorahosted( fasclient=fasclient, tags=tags, private=private, - nopush=nopush) as trac_importer: + nopush=nopush, + threads=threads) as trac_importer: trac_importer.import_issues(project) # update the local git repo diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index c6c3bfe..a5f1a0d 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -27,7 +27,7 @@ class TracImporter(Importer): ''' Pagure importer for trac instance ''' def __init__(self, project_url, username, password, offset, repo_name, - repo_folder, nopush, fasclient=None, tags=False, private=False): + repo_folder, nopush, threads, fasclient=None, tags=False, private=False): ''' Instantiate a TracImporter object ''' Importer.__init__(self, username, password, repo_name, repo_folder, nopush) self.url = project_url @@ -35,6 +35,7 @@ class TracImporter(Importer): self.tags = tags self.private = private self.offset = offset + self.threads = threads self.somebody = User(name='somebody', fullname='somebody', emails=['some@body.com']) self.custom_fields = self.get_custom_fields() @@ -42,7 +43,7 @@ class TracImporter(Importer): def request(self, method, *args): ''' Common method for querying trac ''' - reqid = randint(0,100) + reqid = randint(0, self.threads) req = {'params': args, 'method': method, 'id': reqid} @@ -93,7 +94,7 @@ class TracImporter(Importer): ''' tickets_id = self.request('ticket.query', trac_query) - for i in range(10): + for i in range(self.threads): t = Thread(target=self.import_thread, args=(self.queue, repo_name, tickets_id)) t.setDaemon(True) t.start()