From 77a243b5c89b260a323c768dddad24da5602a762 Mon Sep 17 00:00:00 2001 From: Josseline Perdomo Date: Apr 04 2021 22:08:46 +0000 Subject: [PATCH 1/2] Specify symbolic link to run all the python scripts --- diff --git a/create-all-time-userstats-csv.sh b/create-all-time-userstats-csv.sh index 97d9990..d616ede 100755 --- a/create-all-time-userstats-csv.sh +++ b/create-all-time-userstats-csv.sh @@ -10,8 +10,8 @@ STARTWEEK=158 ENDWEEK=$(( ( $(date +'%s') - $(date -ud '2012-01-01 0:0:0' +'%s') )/60/60/24/7 -1 )) -./new-and-old-users-report.py --csvh $(( $STARTWEEK - 1 )) | tee data/contributor-count.csv +python new-and-old-users-report.py --csvh $(( $STARTWEEK - 1 )) | tee data/contributor-count.csv for week in $(seq $STARTWEEK $ENDWEEK ); do - ./new-and-old-users-report.py --csv $week | tee -a data/contributor-count.csv + python new-and-old-users-report.py --csv $week | tee -a data/contributor-count.csv done diff --git a/get-weekly-user-stats.sh b/get-weekly-user-stats.sh index 67a379b..5efee40 100755 --- a/get-weekly-user-stats.sh +++ b/get-weekly-user-stats.sh @@ -1,11 +1,11 @@ #!/bin/sh -until ./weekly-user-activity.py org.fedoraproject.prod.irc.karma; do sleep 5; done -until ./weekly-user-activity.py org.fedoraproject.prod.git.receive; do sleep 5; done -until ./weekly-user-activity.py org.fedoraproject.prod.bodhi.update.comment; do sleep 5; done -until ./weekly-user-activity.py org.fedoraproject.prod.wiki.article.edit; do sleep 5; done -until ./weekly-user-activity.py org.fedoraproject.prod.infragit.receive; do sleep 5; done -until ./weekly-user-activity.py org.fedoraproject.prod.fedoratagger.rating.update; do sleep 5; done -until ./weekly-user-activity.py io.pagure.prod.pagure.git.receive; do sleep 5; done -until ./weekly-user-activity.py io.pagure.prod.pagure.issue.new; do sleep 5; done -until ./weekly-user-activity.py org.fedoraproject.prod.fedbadges.badge.award; do sleep 5; done +until python weekly-user-activity.py org.fedoraproject.prod.irc.karma; do sleep 5; done +until python weekly-user-activity.py org.fedoraproject.prod.git.receive; do sleep 5; done +until python weekly-user-activity.py org.fedoraproject.prod.bodhi.update.comment; do sleep 5; done +until python weekly-user-activity.py org.fedoraproject.prod.wiki.article.edit; do sleep 5; done +until python weekly-user-activity.py org.fedoraproject.prod.infragit.receive; do sleep 5; done +until python weekly-user-activity.py org.fedoraproject.prod.fedoratagger.rating.update; do sleep 5; done +until python weekly-user-activity.py io.pagure.prod.pagure.git.receive; do sleep 5; done +until python weekly-user-activity.py io.pagure.prod.pagure.issue.new; do sleep 5; done +until python weekly-user-activity.py org.fedoraproject.prod.fedbadges.badge.award; do sleep 5; done diff --git a/run.sh b/run.sh index 7bbfe8a..84ded08 100755 --- a/run.sh +++ b/run.sh @@ -1,6 +1,6 @@ #!/bin/bash ./get-weekly-user-stats.sh ./create-all-time-userstats-csv.sh -./generate-activity-charts.py -./generate-contributor-charts.py -./new-and-old-users-report.py +python generate-activity-charts.py +python generate-contributor-charts.py +python new-and-old-users-report.py From fd96d6e6786a3ea4e5a3cc5619a4d3066342413a Mon Sep 17 00:00:00 2001 From: Josseline Perdomo Date: Apr 04 2021 22:09:38 +0000 Subject: [PATCH 2/2] Merge branch 'main' of ssh://pagure.io/fedora-contributor-trends into update-python-run --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6760c2b --- /dev/null +++ b/.gitignore @@ -0,0 +1,58 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# JetBrains IDEs settings +.idea/ + +# VisualStudioCode settings +.vscode/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..398d69c --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2016-2021 Matthew Miller and others. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..90000ae --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +This is a hacky collection of scripts to create visualizations and hopefully +insight from contributor-triggered activity on Fedora's messaging bus, via +the ["datagrepper" service](https://apps.fedoraproject.org/datagrepper/). + +Results are currently run on a personal system belonging to Matthew Miller, +and published weekly there: https://mattdm.org/fedora/fedora-contributor-trends/ + +There's also a text report: https://mattdm.org/fedora/fedora-contributor-trends/report.txt + +You may also want to watch talk at DevConf 2016 where these metrics were +first presented: https://mattdm.org/fedora/2016devconf/ + +---- + +To get started, clone this repo and run `./run.sh`. You'll need the package +`python-fedmsg` installed. + +Note that this will try to gather data going back to the beginning of the +message bus, which is a long time and a lot of data, and it hits the service +pretty hard in doing so. The weekly data is therefore cached, so on later +runs only the last week is re-loaded. The first run may time out a lot and +even take several days to finish. This isn't ideal, of course, but it got me +up and running. + +You'll see that theme a lot here -- this is a sysadmin-hack kind of project, +not an elegant software engineer one. + +---- + +But that's not the end! I'd love for this to be better. In fact, there are +six main areas looking for improvement: + + + +1. The Fedora Message bus is in the midst of transitioning to new technology, which may make the way the current script gets data obsolete. See [this mailing list post](https://lists.fedoraproject.org/archives/list/infrastructure@lists.fedoraproject.org/message/6NRUH7EP6ERTBUEVTTXYLA25QUSHTKBE/) for possible plans. The upside is that some of the new options could be much better; the current code hits the server very hard. +2. The code really is hacky and ugly. It could use refactoring and beautification. +3. There are some data sources we're not using and could be: there are messages from pagure and will be ones from discourse that would be excellent candidates. +4. The visualizations I created for my presentation in 2016 are nice, but there are many other ways to look at the data. Come up with new ways to look at it which may show more insights. +5. Find an official place for this to run rather than on mattdm's personal server. +6. Simply put, the graphs and reports could be prettier. + +Any or all of these things would be an huge improvement. + +I originally designed this to answer these questions: + +* How many people are active in Fedora over time? +* How many of those contributors are highly or constantly active? +* Are we gaining new contributors? +* Are we retaining the new folks who show up? +* Are we retaining older contributors? + - I would actually like to see more on this, like the average length of contribution over time, and perhaps even patterns in the "shape" of contribution level (like, small activity at first and then growth; abrupt stops vs slowly ramping down, etc.) +* What percentage of work is done by a few people, vs. percentage where a lot of people contribute each a small amount +* What percentage of contributors work for Red Hat (or other companies that are paying people to work on Fedora)? + - this required some manual analysis, because there is no "I work for Red Hat" metadata + +Some things that aren't well-explored but could be: + +* What areas are new users most active in? +* Does activity in one area (like QA or Ask Fedora) flow to activity in other areas? +* Are the most active people the most vocal? ( + - We need https://pagure.io/fedora-infrastructure/issue/9576 for this; also, it's hard because Fedora Accounts don't map to mailing list IDs easily so I just avoided that. + +One thing that's an explicit non-goal: anything that is easily tied to a specific individual in the results, except maybe for very high level totals. + + +---- + +More? See https://pagure.io/fedora-contributor-trends/issues! \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8b6bc20 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fedmsg==1.1.2 +matplotlib==3.4.1 +numpy==1.20.2 +pandas==1.2.3 +requests==2.25.1 +pytz==2021.1 +pytest==6.2.2 \ No newline at end of file diff --git a/weekly-user-activity.py b/weekly-user-activity.py index 07bb8a3..1e1d1a0 100755 --- a/weekly-user-activity.py +++ b/weekly-user-activity.py @@ -134,7 +134,7 @@ with open("data/%s.bucketed-activity.csv" % (discriminant), "w") as bucketcsv: if os.path.exists(msgcachefile): - with open(msgcachefile, "r") as msgcache: + with open(msgcachefile, "rb") as msgcache: [firstseen, lastseen, weekinfo, weekbreakdown] = pickle.load(msgcache) print("(cached)")