From 38bbf7cfa416384d429fc48e4dbf350c1d9c1866 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 1/12] Move the statistics on the issues list to their own (inner) section This will make re-using the JS in easier as it won't rely on something that is specific to that page. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/static/issues_stats.js b/pagure/static/issues_stats.js index 6ec0baf..4724c64 100644 --- a/pagure/static/issues_stats.js +++ b/pagure/static/issues_stats.js @@ -2,7 +2,7 @@ issues_history_stats_plot = function(url, _b, _s) { var svg = d3.select("svg"), margin = {top: 20, right: 20, bottom: 30, left: 50}, - width = $('#tags').width() - margin.left - margin.right, + width = $('#stats').width() - margin.left - margin.right, height = +svg.attr("height") - margin.top - margin.bottom, g = svg.append("g").attr("transform", "translate(" + margin.left + "," + margin.top + ")"); diff --git a/pagure/templates/issues.html b/pagure/templates/issues.html index b0392f0..9fb915b 100644 --- a/pagure/templates/issues.html +++ b/pagure/templates/issues.html @@ -127,8 +127,10 @@
- - +
+ + +
From 716362a982f4a2b5cd6c75d6c5fe30662508e2ad Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 2/12] Add a stats endpoint to project This endpoint provides two types of statistics at the moment: - number of commits per person - number of tickets open over time (same graph as on the issues list) Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/internal/__init__.py b/pagure/internal/__init__.py index 2df97c2..ee156b2 100644 --- a/pagure/internal/__init__.py +++ b/pagure/internal/__init__.py @@ -27,6 +27,7 @@ import pagure.exceptions # noqa: E402 import pagure.forms # noqa: E402 import pagure.lib # noqa: E402 import pagure.lib.git # noqa: E402 +import pagure.lib.tasks # noqa: E402 import pagure.ui.fork # noqa: E402 @@ -570,3 +571,59 @@ def get_branches_head(): 'heads': heads, } ) + + +@PV.route('/task/', methods=['GET']) +def task_info(taskid): + """ Return the results of the specified task or a 418 if the task is + still being processed. + """ + task = pagure.lib.tasks.get_result(taskid) + + if task.ready(): + result = task.get(timeout=0, propagate=False) + return flask.jsonify({'results': result}) + else: + flask.abort(418) + + +@PV.route('/stats/commits/authors', methods=['POST']) +def get_stats_commits(): + """ Return statistics about the commits made on the specified repo. + + """ + form = pagure.forms.ConfirmationForm() + if not form.validate_on_submit(): + response = flask.jsonify({ + 'code': 'ERROR', + 'message': 'Invalid input submitted', + }) + response.status_code = 400 + return response + + repo = pagure.get_authorized_project( + pagure.SESSION, + flask.request.form.get('repo', '').strip() or None, + namespace=flask.request.form.get('namespace', '').strip() or None, + user=flask.request.form.get('repouser', '').strip() or None) + + if not repo: + response = flask.jsonify({ + 'code': 'ERROR', + 'message': 'No repo found with the information provided', + }) + response.status_code = 404 + return response + + repopath = os.path.join(pagure.APP.config['GIT_FOLDER'], repo.path) + + task = pagure.lib.tasks.commits_author_stats.delay(repopath) + + return flask.jsonify( + { + 'code': 'OK', + 'message': 'Stats asked', + 'url': flask.url_for('internal_ns.task_info', taskid=task.id), + 'task_id': task.id, + } + ) diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 5fda99d..c70e0f5 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -8,6 +8,7 @@ """ +import collections import gc import hashlib import os @@ -722,3 +723,31 @@ def update_checksums_file(folder, filenames): for algo in algos: stream.write('%s (%s) = %s\n' % ( algo.upper(), filename, algos[algo].hexdigest())) + + +@conn.task +def commits_author_stats(repopath): + """ Returns some statistics about commits made against the specified + git repository. + """ + if not os.path.exists(repopath): + raise ValueError('Git repository not found.') + + repo_obj = pygit2.Repository(repopath) + + stats = collections.defaultdict(int) + cnt = 0 + authors_email = set() + for commit in repo_obj.walk( + repo_obj.head.get_object().oid.hex, pygit2.GIT_SORT_TIME): + cnt += 1 + email = commit.author.email + author = commit.author.name + stats[(author, email)] += 1 + authors_email.add(email) + + out_stats = collections.defaultdict(list) + for authors, val in stats.items(): + out_stats[val].append(authors) + + return (cnt, out_stats, len(authors_email), commit.commit_time) diff --git a/pagure/static/issues_stats.js b/pagure/static/issues_stats.js index 4724c64..847913e 100644 --- a/pagure/static/issues_stats.js +++ b/pagure/static/issues_stats.js @@ -58,3 +58,45 @@ issues_history_stats_plot = function(url, _b, _s) { }); }; + +wait_for_task = function(url, callback){ + $.get(url) + .done(function(data){ + callback(data); + }) + .fail(function(){ + window.setTimeout(wait_for_task(url, callback), 1000); + }) +} + +show_commits_authors = function(data) { + var _b = $("#data_stats"); + var _s = $("#data_stats_spinner"); + var html = '

Since ' + data.results[3] + ' there has been ' + + data.results[0] + ' commits found in this repo, from ' + + data.results[2] + ' contributors

\n' + + '
\n'; + for (key in data.results[1]){ + for (key2 in data.results[1][key]){ + entry = data.results[1][key][key2] + html += ' ' + + entry[0] + + '
' + key + ' commits
' + + '\n'; + } + } + html += '
'; + _b.html(html); + _b.show(); + _s.hide(); +} + +commits_authors = function(url, _data) { + $.post( url, _data ) + .done(function(data) { + wait_for_task(data.url, show_commits_authors); + }) + .fail(function(data) { + }) +}; diff --git a/pagure/templates/repo_stats.html b/pagure/templates/repo_stats.html new file mode 100644 index 0000000..f22e2ca --- /dev/null +++ b/pagure/templates/repo_stats.html @@ -0,0 +1,106 @@ +{% extends "repo_master.html" %} + +{% block title %}{{ select.capitalize() }} - {{ + g.repo.namespace + '/' if g.repo.namespace }}{{ g.repo.name }}{% endblock %} +{% set tag = "home" %} + + +{% block repo %} + +
+
+
+ + Commits + +
+
+ + Authors + +
+
+ + Issues + +
+
+
+
+ + +
+
+
+{% endblock %} + +{% block jscripts %} +{{ super() }} + + + + +{% endblock %} diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index ae6ab16..9b4b514 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -2806,3 +2806,23 @@ def project_dowait(repo, username=None, namespace=None): name=repo, namespace=namespace, user=username).id return pagure.wait_for_task(taskid) + + +@APP.route('//stats/') +@APP.route('//stats') +@APP.route('///stats/') +@APP.route('///stats') +@APP.route('/fork///stats/') +@APP.route('/fork///stats') +@APP.route('/fork////stats/') +@APP.route('/fork////stats') +def view_stats(repo, username=None, namespace=None): + """ Displays some statistics about the specified repo. + """ + return flask.render_template( + 'repo_stats.html', + select='stats', + username=username, + repo=flask.g.repo, + form=pagure.forms.ConfirmationForm(), + ) From c1607b0d488ea44ca32d7b161620e32776804ba7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 3/12] Fix the logic to find out the number of tickets open over time Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 16e6c42..47694da 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -4582,19 +4582,25 @@ def issues_history_stats(session, project): output = {} for week in range(53): start = tomorrow - datetime.timedelta(days=(week * 7)) - query = session.query( + closed_ticket = session.query( model.Issue ).filter( model.Issue.project_id == project.id ).filter( - sqlalchemy.or_( - model.Issue.closed_at == None, # noqa - model.Issue.closed_at <= start - ) + model.Issue.closed_at >= start + ).filter( + model.Issue.date_created <= start + ) + open_ticket = session.query( + model.Issue + ).filter( + model.Issue.project_id == project.id + ).filter( + model.Issue.status == 'Open' ).filter( model.Issue.date_created <= start ) - cnt = query.count() - to_ignore + cnt = open_ticket.count() + closed_ticket.count() - to_ignore if cnt < 0: cnt = 0 output[start.isoformat()] = cnt diff --git a/tests/test_pagure_flask_api_issue.py b/tests/test_pagure_flask_api_issue.py index 2ceac32..d7a354d 100644 --- a/tests/test_pagure_flask_api_issue.py +++ b/tests/test_pagure_flask_api_issue.py @@ -2992,7 +2992,7 @@ class PagureFlaskApiIssuetests(tests.Modeltests): self.assertEqual(len(data), 1) self.assertEqual(len(data['stats']), 53) last_key = sorted(data['stats'].keys())[-1] - self.assertEqual(data['stats'][last_key], 8) + self.assertEqual(data['stats'][last_key], 7) for k in sorted(data['stats'].keys())[:-1]: self.assertEqual(data['stats'][k], 0) From 1828a2bd3a6a0677035c16936910ee4df214415c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 4/12] Add a graph of the evolution of the commits on the repo over the last year Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/internal/__init__.py b/pagure/internal/__init__.py index ee156b2..c33cd24 100644 --- a/pagure/internal/__init__.py +++ b/pagure/internal/__init__.py @@ -627,3 +627,45 @@ def get_stats_commits(): 'task_id': task.id, } ) + + +@PV.route('/stats/commits/trend', methods=['POST']) +def get_stats_commits_trend(): + """ Return evolution of the commits made on the specified repo. + + """ + form = pagure.forms.ConfirmationForm() + if not form.validate_on_submit(): + response = flask.jsonify({ + 'code': 'ERROR', + 'message': 'Invalid input submitted', + }) + response.status_code = 400 + return response + + repo = pagure.get_authorized_project( + pagure.SESSION, + flask.request.form.get('repo', '').strip() or None, + namespace=flask.request.form.get('namespace', '').strip() or None, + user=flask.request.form.get('repouser', '').strip() or None) + + if not repo: + response = flask.jsonify({ + 'code': 'ERROR', + 'message': 'No repo found with the information provided', + }) + response.status_code = 404 + return response + + repopath = os.path.join(pagure.APP.config['GIT_FOLDER'], repo.path) + + task = pagure.lib.tasks.commits_history_stats.delay(repopath) + + return flask.jsonify( + { + 'code': 'OK', + 'message': 'Stats asked', + 'url': flask.url_for('internal_ns.task_info', taskid=task.id), + 'task_id': task.id, + } + ) diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index c70e0f5..c558c9c 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -9,6 +9,7 @@ """ import collections +import datetime import gc import hashlib import os @@ -19,6 +20,7 @@ import time from celery import Celery from celery.result import AsyncResult +import arrow import pygit2 import tempfile import six @@ -751,3 +753,25 @@ def commits_author_stats(repopath): out_stats[val].append(authors) return (cnt, out_stats, len(authors_email), commit.commit_time) + + +@conn.task +def commits_history_stats(repopath): + """ Returns the evolution of the commits made against the specified + git repository. + """ + if not os.path.exists(repopath): + raise ValueError('Git repository not found.') + + repo_obj = pygit2.Repository(repopath) + + dates = collections.defaultdict(int) + for commit in repo_obj.walk( + repo_obj.head.get_object().oid.hex, pygit2.GIT_SORT_TIME): + delta = datetime.datetime.utcnow() \ + - arrow.get(commit.commit_time).naive + if delta.days > 365: + break + dates[arrow.get(commit.commit_time).date().isoformat()] += 1 + + return [(key, dates[key]) for key in sorted(dates)] diff --git a/pagure/static/issues_stats.js b/pagure/static/issues_stats.js index 847913e..999fc24 100644 --- a/pagure/static/issues_stats.js +++ b/pagure/static/issues_stats.js @@ -100,3 +100,74 @@ commits_authors = function(url, _data) { .fail(function(data) { }) }; + + +show_commits_history = function(data) { + var _b = $("#data_stats"); + var _s = $("#data_stats_spinner"); + + var parseTime = d3.timeParse("%Y-%m-%d"); + + var _out = data.results.map(function(x){ + var t = {}; + t.date = parseTime(x[0]); + t.value = x[1]; + return t; + }) + + var svg = d3.select("svg"), + margin = {top: 20, right: 20, bottom: 30, left: 50}, + width = $('#stats').width() - margin.left - margin.right, + height = +svg.attr("height") - margin.top - margin.bottom, + g = svg.append("g").attr( + "transform", "translate(" + margin.left + "," + margin.top + ")"); + + var x = d3.scaleTime() + .rangeRound([0, width]); + + var y = d3.scaleLinear() + .rangeRound([height, 0]); + + var area = d3.area() + .x(function(d) { return x(d.date); }) + .y1(function(d) { return y(d.value); }); + + function draw_graph(data) { + + x.domain(d3.extent(data, function(d) { return d.date; })); + y.domain([0, d3.max(data, function(d) { return d.value; })]); + area.y0(y(0)); + + g.append("path") + .datum(data) + .attr("fill", "steelblue") + .attr("d", area); + + g.append("g") + .attr("transform", "translate(0," + height + ")") + .call(d3.axisBottom(x)); + + g.append("g") + .call(d3.axisLeft(y)) + .append("text") + .attr("fill", "#000") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", "0.71em") + .attr("text-anchor", "end") + .text("Number of commits"); + }; + + draw_graph(_out); + _b.show(); + _s.hide(); +} + +commits_history = function(url, _data) { + $.post( url, _data ) + .done(function(data) { + wait_for_task(data.url, show_commits_history); + }) + .fail(function(data) { + }) +}; diff --git a/pagure/templates/repo_stats.html b/pagure/templates/repo_stats.html index f22e2ca..4ae0b29 100644 --- a/pagure/templates/repo_stats.html +++ b/pagure/templates/repo_stats.html @@ -11,7 +11,7 @@
+ title="Evolution of the commits over time"> Commits
@@ -88,6 +88,27 @@ commits_authors_call = function(){ commits_authors(_stats_url, data); }; +commits_history_call = function(){ + var _stats_url = "{{ url_for('internal_ns.get_stats_commits_trend') }}"; + var _b = $("#data_stats"); + var _s = $("#data_stats_spinner"); + _s.html( + "" + ) + _s.show(); + _b.html( + "

Evolution of the number of commits over the last year

" + + "" + ); + var data = { + csrf_token: "{{ form.csrf_token.current_token }}", + repo: "{{ g.repo.name }}", + username: "{{ username or '' }}", + namespace: "{{ g.repo.namespace or '' }}", + } + commits_history(_stats_url, data); +}; + $(document).ready(function() { @@ -96,11 +117,15 @@ $(document).ready(function() { _b.hide(); if ($(this).attr('name') == 'issues') { issues_history_stats_plot_call(); + } else if ($(this).attr('name') == 'authors') { + commits_authors_call(); } else if ($(this).attr('name') == 'commits') { - commits_stats_call(); + commits_history_call(); } }); + commits_history_call(); + }); {% endblock %} From 019cbc7e474059a64b00b69c1159a01a453b1ccf Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 5/12] Add a link to the stats page in the navigation bar Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/templates/repo_master.html b/pagure/templates/repo_master.html index ea2f5af..27d3f27 100644 --- a/pagure/templates/repo_master.html +++ b/pagure/templates/repo_master.html @@ -287,6 +287,20 @@ {% endif %} +
  • + + + Stats  + +
    • + {% if repo and repo.forks %}
  • Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 6/12] Transform the exceptions in string so they can be sent back Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/internal/__init__.py b/pagure/internal/__init__.py index c33cd24..0e8be0f 100644 --- a/pagure/internal/__init__.py +++ b/pagure/internal/__init__.py @@ -582,6 +582,8 @@ def task_info(taskid): if task.ready(): result = task.get(timeout=0, propagate=False) + if isinstance(result, Exception): + result = str(result) return flask.jsonify({'results': result}) else: flask.abort(418) From e8a12798051c4d9c995846a8f23ec5ada8f9c8df Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 7/12] Add unit-tests for the internal endpoints added Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/test_pagure_flask_internal.py b/tests/test_pagure_flask_internal.py index fffb04e..b048910 100644 --- a/tests/test_pagure_flask_internal.py +++ b/tests/test_pagure_flask_internal.py @@ -11,6 +11,7 @@ __requires__ = ['SQLAlchemy >= 0.8'] import pkg_resources +import datetime import json import unittest import shutil @@ -1353,6 +1354,223 @@ class PagureFlaskInternaltests(tests.Modeltests): self.assertEqual(len(js_data['heads']), 3) self.assertEqual(len(js_data['branches']), 3) + def test_get_stats_commits_no_token(self): + ''' Test the get_stats_commits from the internal API. ''' + # No CSRF token + data = { + 'repo': 'fakerepo', + } + output = self.app.post('/pv/stats/commits/authors', data=data) + self.assertEqual(output.status_code, 400) + js_data = json.loads(output.data.decode('utf-8')) + self.assertDictEqual( + js_data, + {u'code': u'ERROR', u'message': u'Invalid input submitted'} + ) + + def test_get_stats_commits_invalid_repo(self): + ''' Test the get_stats_commits from the internal API. ''' + user = tests.FakeUser() + user.username = 'pingou' + with tests.user_set(pagure.APP, user): + csrf_token = self.get_csrf() + + # Invalid repo + data = { + 'repo': 'fakerepo', + 'csrf_token': csrf_token, + } + output = self.app.post('/pv/stats/commits/authors', data=data) + self.assertEqual(output.status_code, 404) + js_data = json.loads(output.data.decode('utf-8')) + self.assertDictEqual( + js_data, + {u'code': u'ERROR', + u'message': u'No repo found with the information provided'} + ) + + def test_get_stats_commits_empty_git(self): + ''' Test the get_stats_commits from the internal API. ''' + tests.create_projects(self.session) + tests.create_projects_git(os.path.join(self.path, 'repos')) + + user = tests.FakeUser() + user.username = 'pingou' + with tests.user_set(pagure.APP, user): + csrf_token = self.get_csrf() + + # No content in git + data = { + 'repo': 'test', + 'csrf_token': csrf_token, + } + output = self.app.post('/pv/stats/commits/authors', data=data) + self.assertEqual(output.status_code, 200) + js_data = json.loads(output.data.decode('utf-8')) + self.assertEqual( + sorted(js_data.keys()), + ['code', 'message', 'task_id', 'url'] + ) + self.assertEqual(js_data['code'], 'OK') + self.assertEqual(js_data['message'], 'Stats asked') + self.assertTrue(js_data['url'].startswith('/pv/task/')) + + output = self.app.get(js_data['url']) + js_data2 = json.loads(output.data.decode('utf-8')) + self.assertDictEqual( + js_data2, + {u'results': u"Reference 'refs/heads/master' not found"} + ) + + def test_get_stats_commits_git_populated(self): + ''' Test the get_stats_commits from the internal API. ''' + tests.create_projects(self.session) + tests.create_projects_git( + os.path.join(self.path, 'repos'), bare=True) + tests.add_content_git_repo( + os.path.join(self.path, 'repos', 'test.git')) + + user = tests.FakeUser() + user.username = 'pingou' + with tests.user_set(pagure.APP, user): + csrf_token = self.get_csrf() + + # Content in git + data = { + 'repo': 'test', + 'csrf_token': csrf_token, + } + output = self.app.post('/pv/stats/commits/authors', data=data) + self.assertEqual(output.status_code, 200) + js_data = json.loads(output.data.decode('utf-8')) + self.assertEqual( + sorted(js_data.keys()), + ['code', 'message', 'task_id', 'url'] + ) + self.assertEqual(js_data['code'], 'OK') + self.assertEqual(js_data['message'], 'Stats asked') + self.assertTrue(js_data['url'].startswith('/pv/task/')) + + output = self.app.get(js_data['url']) + js_data2 = json.loads(output.data.decode('utf-8')) + self.assertTrue(js_data2['results'][3] > 1509110062) + js_data2['results'][3] = 1509110062 + self.assertDictEqual( + js_data2, + {u'results': [ + 2, + {u'2': [[u'Alice Author', u'alice@authors.tld']]}, + 1, + 1509110062 + ] + } + ) + + def test_get_stats_commits_trend_no_token(self): + ''' Test the get_stats_commits_trend from the internal API. ''' + # No CSRF token + data = { + 'repo': 'fakerepo', + } + output = self.app.post('/pv/stats/commits/trend', data=data) + self.assertEqual(output.status_code, 400) + js_data = json.loads(output.data.decode('utf-8')) + self.assertDictEqual( + js_data, + {u'code': u'ERROR', u'message': u'Invalid input submitted'} + ) + + def test_get_stats_commits_trend_invalid_repo(self): + """ Test the get_stats_commits_trend from the internal API. """ + user = tests.FakeUser() + user.username = 'pingou' + with tests.user_set(pagure.APP, user): + csrf_token = self.get_csrf() + + # Invalid repo + data = { + 'repo': 'fakerepo', + 'csrf_token': csrf_token, + } + output = self.app.post('/pv/stats/commits/trend', data=data) + self.assertEqual(output.status_code, 404) + js_data = json.loads(output.data.decode('utf-8')) + self.assertDictEqual( + js_data, + {u'code': u'ERROR', + u'message': u'No repo found with the information provided'} + ) + + def test_get_stats_commits_trend_empty_git(self): + ''' Test the get_stats_commits_trend from the internal API. ''' + tests.create_projects(self.session) + tests.create_projects_git(os.path.join(self.path, 'repos')) + + user = tests.FakeUser() + user.username = 'pingou' + with tests.user_set(pagure.APP, user): + csrf_token = self.get_csrf() + + # No content in git + data = { + 'repo': 'test', + 'csrf_token': csrf_token, + } + output = self.app.post('/pv/stats/commits/trend', data=data) + self.assertEqual(output.status_code, 200) + js_data = json.loads(output.data.decode('utf-8')) + self.assertEqual( + sorted(js_data.keys()), + ['code', 'message', 'task_id', 'url'] + ) + self.assertEqual(js_data['code'], 'OK') + self.assertEqual(js_data['message'], 'Stats asked') + self.assertTrue(js_data['url'].startswith('/pv/task/')) + + output = self.app.get(js_data['url']) + js_data2 = json.loads(output.data.decode('utf-8')) + self.assertDictEqual( + js_data2, + {u'results': u"Reference 'refs/heads/master' not found"} + ) + + def test_get_stats_commits_trend_git_populated(self): + ''' Test the get_stats_commits_trend from the internal API. ''' + tests.create_projects(self.session) + tests.create_projects_git( + os.path.join(self.path, 'repos'), bare=True) + tests.add_content_git_repo( + os.path.join(self.path, 'repos', 'test.git')) + + user = tests.FakeUser() + user.username = 'pingou' + with tests.user_set(pagure.APP, user): + csrf_token = self.get_csrf() + + # Content in git + data = { + 'repo': 'test', + 'csrf_token': csrf_token, + } + output = self.app.post('/pv/stats/commits/trend', data=data) + self.assertEqual(output.status_code, 200) + js_data = json.loads(output.data.decode('utf-8')) + self.assertEqual( + sorted(js_data.keys()), + ['code', 'message', 'task_id', 'url'] + ) + self.assertEqual(js_data['code'], 'OK') + self.assertEqual(js_data['message'], 'Stats asked') + self.assertTrue(js_data['url'].startswith('/pv/task/')) + + output = self.app.get(js_data['url']) + js_data2 = json.loads(output.data.decode('utf-8')) + today = datetime.datetime.utcnow().date() + self.assertDictEqual( + js_data2, + {u'results': [[str(today), 2]]} + ) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/tests/test_pagure_flask_ui_repo.py b/tests/test_pagure_flask_ui_repo.py index 0d52365..0269d35 100644 --- a/tests/test_pagure_flask_ui_repo.py +++ b/tests/test_pagure_flask_ui_repo.py @@ -1337,6 +1337,9 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertIn( '
    \n' 'test project #1
    ', output.data) + self.assertIn( + 'Stats ', + output.data) self.perfMaxWalks(0, 0) self.perfReset() From 609c2b9947a9880a7bcd49722df2a01c2f1e6ffe Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 8/12] Change the output of the author stats so they appear ordered directly Moving from objects to list allows to do the sorting in python rather than fighting to do this in javascript. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index c558c9c..5d6632b 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -751,8 +751,12 @@ def commits_author_stats(repopath): out_stats = collections.defaultdict(list) for authors, val in stats.items(): out_stats[val].append(authors) + out_list = [ + (key, out_stats[key]) + for key in sorted(out_stats, reverse=True) + ] - return (cnt, out_stats, len(authors_email), commit.commit_time) + return (cnt, out_list, len(authors_email), commit.commit_time) @conn.task diff --git a/pagure/static/issues_stats.js b/pagure/static/issues_stats.js index 999fc24..32c1e11 100644 --- a/pagure/static/issues_stats.js +++ b/pagure/static/issues_stats.js @@ -77,12 +77,13 @@ show_commits_authors = function(data) { + data.results[2] + ' contributors

    \n' + '
    \n'; for (key in data.results[1]){ - for (key2 in data.results[1][key]){ - entry = data.results[1][key][key2] + cnt = data.results[1][key][0]; + for (entry in data.results[1][key][1]){ + entry = data.results[1][key][1][entry]; html += ' ' + entry[0] - + '
    ' + key + ' commits
    ' + + '
    ' + cnt + ' commits
    ' + '    \n'; } } diff --git a/tests/test_pagure_flask_internal.py b/tests/test_pagure_flask_internal.py index b048910..036b68d 100644 --- a/tests/test_pagure_flask_internal.py +++ b/tests/test_pagure_flask_internal.py @@ -1459,7 +1459,7 @@ class PagureFlaskInternaltests(tests.Modeltests): js_data2, {u'results': [ 2, - {u'2': [[u'Alice Author', u'alice@authors.tld']]}, + [[2, [[u'Alice Author', u'alice@authors.tld']]]], 1, 1509110062 ] From 5b4c94893180587192175fb3b1d441dff5beba05 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 9/12] Add titles to the different section, fix style and fix date in the authors stats Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/static/issues_stats.js b/pagure/static/issues_stats.js index 32c1e11..571f3f0 100644 --- a/pagure/static/issues_stats.js +++ b/pagure/static/issues_stats.js @@ -72,7 +72,8 @@ wait_for_task = function(url, callback){ show_commits_authors = function(data) { var _b = $("#data_stats"); var _s = $("#data_stats_spinner"); - var html = '

    Since ' + data.results[3] + ' there has been ' + var html = '

    Authors stats

    Since ' + + new Date(data.results[3]*1000) + ' there has been ' + data.results[0] + ' commits found in this repo, from ' + data.results[2] + ' contributors

    \n' + '
    \n'; diff --git a/pagure/templates/repo_stats.html b/pagure/templates/repo_stats.html index 4ae0b29..22b90cb 100644 --- a/pagure/templates/repo_stats.html +++ b/pagure/templates/repo_stats.html @@ -52,7 +52,7 @@ var view_commits_url = "{{ url_for('view_commits', namespace=repo.namespace, author='---') }}"; -issues_history_stats_plot_call = function(){ +issues_history_stats_plot_call = function() { var _stats_url = "{{ url_for( 'api_ns.api_view_issues_history_stats', repo=g.repo.name, @@ -65,13 +65,13 @@ issues_history_stats_plot_call = function(){ ) _s.show(); _b.html( - "

    Number of issues open on 7 days windows

    " + "

    Issues stats

    Number of issues open on 7 days windows

    " + "" ); issues_history_stats_plot(_stats_url, _b, _s); }; -commits_authors_call = function(){ +commits_authors_call = function() { var _stats_url = "{{ url_for('internal_ns.get_stats_commits') }}"; var _b = $("#data_stats"); var _s = $("#data_stats_spinner"); @@ -88,7 +88,7 @@ commits_authors_call = function(){ commits_authors(_stats_url, data); }; -commits_history_call = function(){ +commits_history_call = function() { var _stats_url = "{{ url_for('internal_ns.get_stats_commits_trend') }}"; var _b = $("#data_stats"); var _s = $("#data_stats_spinner"); @@ -97,7 +97,7 @@ commits_history_call = function(){ ) _s.show(); _b.html( - "

    Evolution of the number of commits over the last year

    " + "

    Commits stats

    Evolution of the number of commits over the last year

    " + "" ); var data = { @@ -110,8 +110,8 @@ commits_history_call = function(){ }; -$(document).ready(function() { +$(document).ready(function() { $('.stats_btn').click(function(ev){ var _b = $("#data_stats"); _b.hide(); From cb23f4a4ec0593993b6609155d7b492e428bf72b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 10/12] Make the different stats section by alphabetical order Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/templates/repo_stats.html b/pagure/templates/repo_stats.html index 22b90cb..a9c01c5 100644 --- a/pagure/templates/repo_stats.html +++ b/pagure/templates/repo_stats.html @@ -10,15 +10,18 @@
    + + Authors + +
    +
    Commits
    - - Authors
    From e13b7a36504cacefe0cbb34a225708781cd5a640 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 11/12] Move the forks into the stats tab Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/templates/repo_master.html b/pagure/templates/repo_master.html index 27d3f27..13c21fb 100644 --- a/pagure/templates/repo_master.html +++ b/pagure/templates/repo_master.html @@ -301,25 +301,6 @@
    • - {% if repo and repo.forks %} -
  • - - - Forks  - - {{ repo.forks |length }} - - -
    • - {% endif %} - {% if authenticated %} {% if g.repo_admin %}
  • diff --git a/pagure/templates/repo_stats.html b/pagure/templates/repo_stats.html index a9c01c5..e18f9b2 100644 --- a/pagure/templates/repo_stats.html +++ b/pagure/templates/repo_stats.html @@ -22,6 +22,12 @@
    • + + Forks + + {{ repo.forks |length }} +
    @@ -36,6 +42,43 @@ +
    +

    Forks list

    + {% if repo.forks %} +
    + {% for fork in repo.forks %} +
    +
    + + + {{ fork.user.user }}/{{ fork.name }} + +
    + + + {{ fork.user.default_email | avatar(20) | safe }} + {{ fork.user.fullname }} + forked this project + + + {{ fork.date_created|humanize }} + +
    +
    +
    + + {% endfor %} +
    + {% else %} +

    + This project has not been forked. +

    + {% endif %} +
    {% endblock %} @@ -112,23 +155,27 @@ commits_history_call = function() { commits_history(_stats_url, data); }; +toggle_forks = function() { + $("#forks_list").show(); +} $(document).ready(function() { $('.stats_btn').click(function(ev){ var _b = $("#data_stats"); _b.hide(); + $("#forks_list").hide(); if ($(this).attr('name') == 'issues') { issues_history_stats_plot_call(); } else if ($(this).attr('name') == 'authors') { commits_authors_call(); } else if ($(this).attr('name') == 'commits') { commits_history_call(); + } else if ($(this).attr('name') == 'forks') { + toggle_forks(); } }); - commits_history_call(); - }); {% endblock %} diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index 9b4b514..6a0168f 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -898,27 +898,6 @@ def view_tree(repo, identifier=None, username=None, namespace=None): ) -@APP.route('//forks/') -@APP.route('//forks') -@APP.route('///forks/') -@APP.route('///forks') -@APP.route('/fork///forks/') -@APP.route('/fork///forks') -@APP.route('/fork////forks/') -@APP.route('/fork////forks') -def view_forks(repo, username=None, namespace=None): - """ Presents all the forks of the project. - """ - repo = flask.g.repo - - return flask.render_template( - 'forks.html', - select='forks', - username=username, - repo=repo, - ) - - @APP.route('//releases/') @APP.route('//releases') @APP.route('///releases/') From 1a80864a0d7ae375d405a03c4501b011ee0b7241 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Nov 02 2017 09:41:39 +0000 Subject: [PATCH 12/12] Adjusts the tests now that the forks are in the stats tab Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/test_pagure_flask_ui_repo.py b/tests/test_pagure_flask_ui_repo.py index 0269d35..6494c03 100644 --- a/tests/test_pagure_flask_ui_repo.py +++ b/tests/test_pagure_flask_ui_repo.py @@ -1290,13 +1290,13 @@ class PagureFlaskRepotests(tests.Modeltests): def test_view_forks(self): """ Test the view_forks endpoint. """ - output = self.app.get('/foo/forks', follow_redirects=True) + output = self.app.get('/foo/stats', follow_redirects=True) self.assertEqual(output.status_code, 404) tests.create_projects(self.session) tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) - output = self.app.get('/test/forks', follow_redirects=True) + output = self.app.get('/test/stats', follow_redirects=True) self.assertEqual(output.status_code, 200) self.assertTrue('This project has not been forked.' in output.data)