From b47b84e6fa3b581278cbac758f999897f367f19c Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Oct 17 2017 22:15:40 +0000 Subject: Make fedmsg_vars_lib to share code Signed-off-by: Patrick Uiterwijk --- diff --git a/alt.fedoraproject.org/build/fedimg_vars.py b/alt.fedoraproject.org/build/fedimg_vars.py index e4e0d51..eeab54d 100755 --- a/alt.fedoraproject.org/build/fedimg_vars.py +++ b/alt.fedoraproject.org/build/fedimg_vars.py @@ -16,66 +16,12 @@ import functools import logging import os -import requests -import dogpile.cache +from fedimg_vars_lib import get_messages, sanity_check log = logging.getLogger('fedimg_vars') -base_url = 'https://apps.fedoraproject.org/datagrepper/raw' -topic = "org.fedoraproject.prod.fedimg.image.upload" - -session = requests.session() - -cache = dogpile.cache.make_region().configure( - "dogpile.cache.dbm", - # 'make clean' does not remove this cache, but we let the values expire - # once every this many seconds (once a day) - expiration_time=86400, - arguments={ - "filename": os.path.join(os.getcwd(), 'build/amis.cache') - }, -) - -def get_page(page, pages): - """ Retrieve the JSON for a particular page of datagrepper results """ - log.debug("Getting page %i of %s", page, pages) - response = session.get(base_url, params=dict( - topic=topic, - page=page, - # Get messages from eight weeks - delta=4838400, - rows_per_page=100, - )) - return response.json() - - -def retrieve_messages(): - """ Generator that yields messages from datagrepper """ - - # Get the first page - data = get_page(1, 'unknown') - for message in data['raw_messages']: - yield message - - more = functools.partial(get_page, pages=data['pages']) - - # Get all subsequent pages (if there are any...) - for page in range(1, data['pages']): - data = more(page + 1) - - for message in data['raw_messages']: - yield message - - -def get_messages(target): - """ Filter the messages on target. """ - for message in retrieve_messages(): - if target in str(message): - yield message - # We cache this guy on disk for 500s -@cache.cache_on_arguments() def collect(release): results = collections.defaultdict(dict) @@ -126,29 +72,3 @@ def collect(release): results[name][region] = ami return results - - -def sanity_check(globalvar, collected_fedimg_vars): - """ This is a sanity check just to make sure the datagrepper code is not - way off from what we had hand-typed before. - - Eventually, remove this. - """ - - names = [ - 'pre_HVM_base_AMI', - 'pre_GP2_HVM_base_AMI', - 'pre_HVM_atomic_AMI', - 'pre_GP2_HVM_atomic_AMI', - ] - for name in names: - handtyped = getattr(globalvar, name) - collected = collected_fedimg_vars[name] - - for key in handtyped: - if not key in collected: - log.warn("collected %r is missing %r" % (name, key)) - - for key in collected: - if not key in handtyped: - log.warn("handtyped %r is missing %r" % (name, key)) diff --git a/alt.fedoraproject.org/build/fedimg_vars_lib.py b/alt.fedoraproject.org/build/fedimg_vars_lib.py new file mode 120000 index 0000000..01e3ba3 --- /dev/null +++ b/alt.fedoraproject.org/build/fedimg_vars_lib.py @@ -0,0 +1 @@ +../../tools/fedimg_vars_lib.py \ No newline at end of file diff --git a/getfedora.org/build/fedimg_vars.py b/getfedora.org/build/fedimg_vars.py index 7a4c0cf..f6944c5 100755 --- a/getfedora.org/build/fedimg_vars.py +++ b/getfedora.org/build/fedimg_vars.py @@ -16,66 +16,12 @@ import functools import logging import os -import requests -import dogpile.cache +from fedimg_vars_lib import get_messages, sanity_check log = logging.getLogger('fedimg_vars') -base_url = 'https://apps.fedoraproject.org/datagrepper/raw' -topic = "org.fedoraproject.prod.fedimg.image.upload" - -session = requests.session() - -cache = dogpile.cache.make_region().configure( - "dogpile.cache.dbm", - # 'make clean' does not remove this cache, but we let the values expire - # once every this many seconds (once a day) - expiration_time=86400, - arguments={ - "filename": os.path.join(os.getcwd(), 'build/amis.cache') - }, -) - -def get_page(page, pages): - """ Retrieve the JSON for a particular page of datagrepper results """ - log.debug("Getting page %i of %s", page, pages) - response = session.get(base_url, params=dict( - topic=topic, - page=page, - # Get messages from 28 weeks (7 months) - delta=16934400, - rows_per_page=100, - )) - return response.json() - - -def retrieve_messages(): - """ Generator that yields messages from datagrepper """ - - # Get the first page - data = get_page(1, 'unknown') - for message in data['raw_messages']: - yield message - - more = functools.partial(get_page, pages=data['pages']) - - # Get all subsequent pages (if there are any...) - for page in range(1, data['pages']): - data = more(page + 1) - - for message in data['raw_messages']: - yield message - - -def get_messages(target): - """ Filter the messages on target. """ - for message in retrieve_messages(): - if target in str(message): - yield message - # We cache this guy on disk for 500s -@cache.cache_on_arguments() def collect(release): results = collections.defaultdict(dict) @@ -138,31 +84,3 @@ def collect(release): results[name][region] = ami return results - - -def sanity_check(globalvar, collected_fedimg_vars): - """ This is a sanity check just to make sure the datagrepper code is not - way off from what we had hand-typed before. - - Eventually, remove this. - """ - - names = [ - 'pre_HVM_base_AMI', - 'pre_GP2_HVM_base_AMI', - 'pre_PV_base_AMI', - 'pre_GP2_PV_base_AMI', - 'pre_HVM_atomic_AMI', - 'pre_GP2_HVM_atomic_AMI', - ] - for name in names: - handtyped = getattr(globalvar, name) - collected = collected_fedimg_vars[name] - - for key in handtyped: - if not key in collected: - log.warn("collected %r is missing %r" % (name, key)) - - for key in collected: - if not key in handtyped: - log.warn("handtyped %r is missing %r" % (name, key)) diff --git a/getfedora.org/build/fedimg_vars_lib.py b/getfedora.org/build/fedimg_vars_lib.py new file mode 120000 index 0000000..01e3ba3 --- /dev/null +++ b/getfedora.org/build/fedimg_vars_lib.py @@ -0,0 +1 @@ +../../tools/fedimg_vars_lib.py \ No newline at end of file diff --git a/tools/fedimg_vars_lib.py b/tools/fedimg_vars_lib.py new file mode 100755 index 0000000..4867bf5 --- /dev/null +++ b/tools/fedimg_vars_lib.py @@ -0,0 +1,113 @@ +#!/usr/bin/python +""" Shared functions between the two fedimg_vars.py scripts + +Search datagrepper to find the results. + +Deps: $ sudo dnf install python-requests + +Author: Ralph Bean +License: LGPLv2+ +""" + +from __future__ import print_function + +import collections +from datetime import datetime, timedelta +import functools +import logging +import json +import os + +import requests + +log = logging.getLogger('fedimg_vars') + +cache_file = '/tmp/fedora_websites_fedimg.cache' +dateformat = '%Y-%m-%dT%H:%MZ' +base_url = 'https://apps.fedoraproject.org/datagrepper/raw' +topic = "org.fedoraproject.prod.fedimg.image.upload" + +session = requests.session() + +def get_page(page, pages): + """ Retrieve the JSON for a particular page of datagrepper results """ + log.debug("Getting page %i of %s", page, pages) + response = session.get(base_url, params=dict( + topic=topic, + page=page, + # Get messages from 28 weeks (7 months) + delta=16934400, + rows_per_page=100, + )) + return response.json() + + +def retrieve_messages(): + """ Generator that yields messages from datagrepper """ + # Get the first page + data = get_page(1, 'unknown') + for message in data['raw_messages']: + yield message + + more = functools.partial(get_page, pages=data['pages']) + + # Get all subsequent pages (if there are any...) + for page in range(1, data['pages']): + data = more(page + 1) + + for message in data['raw_messages']: + yield message + + +def filter_messages(messages, target): + for message in messages: + if target in str(message): + yield message + + +def get_messages(target): + """ Filter the messages on target. """ + try: + with open(cache_file, 'r') as cf: + cache = json.load(cf) + cachetime = datetime.strptime(cache['timestamp'], dateformat) + if cachetime > (datetime.utcnow() - timedelta(days=1)): + return filter_messages(cache['messages'], target) + except: + log.info('No cache, loading from scratch') + + messages = list(retrieve_messages()) + with open(cache_file, 'w') as cf: + cache = {'timestamp': datetime.utcnow().strftime(dateformat), + 'messages': messages} + json.dump(cache, cf) + + return filter_messages(messages, target) + + +def sanity_check(globalvar, collected_fedimg_vars): + """ This is a sanity check just to make sure the datagrepper code is not + way off from what we had hand-typed before. + + Eventually, remove this. + """ + + names = [ + 'pre_HVM_base_AMI', + 'pre_GP2_HVM_base_AMI', + 'pre_PV_base_AMI', + 'pre_GP2_PV_base_AMI', + 'pre_HVM_atomic_AMI', + 'pre_GP2_HVM_atomic_AMI', + ] + for name in names: + handtyped = getattr(globalvar, name) + collected = collected_fedimg_vars[name] + + for key in handtyped: + if not key in collected: + log.warn("collected %r is missing %r" % (name, key)) + + for key in collected: + if not key in handtyped: + log.warn("handtyped %r is missing %r" % (name, key))