From 7dfa99e68c7d53e50e07221b1c7b2e035a00e3f0 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sep 18 2017 16:55:25 +0000 Subject: [PATCH 1/2] Introduce a configurable cache over other resources. This adds a cache on top of our query to resultsdb that currently does nothing at all. The backend for `dogpile.cache` is set to the `null` backend, which means nothing will be cached. This is a step in the road to get us to #77. --- diff --git a/greenwave/api_v1.py b/greenwave/api_v1.py index ca6fb82..71d2544 100644 --- a/greenwave/api_v1.py +++ b/greenwave/api_v1.py @@ -5,6 +5,7 @@ from flask import Blueprint, request, current_app, jsonify from werkzeug.exceptions import BadRequest, NotFound, UnsupportedMediaType from greenwave import __version__ from greenwave.policies import summarize_answers +from greenwave.resources import retrieve_results, retrieve_waivers from greenwave.utils import insert_headers api = (Blueprint('api_v1', __name__)) @@ -187,24 +188,10 @@ def make_decision(): if not subjects: raise BadRequest('Invalid subject, must be a list of dicts') answers = [] - timeout = current_app.config['REQUESTS_TIMEOUT'] for item in subjects: - # XXX make this more efficient than just fetching everything - params = item.copy() - params.update({'limit': '1000'}) - response = requests_session.get( - current_app.config['RESULTSDB_API_URL'] + '/results', - params=params, timeout=timeout) - response.raise_for_status() - results = response.json()['data'] + results = retrieve_results(item) if results: - response = requests_session.get( - current_app.config['WAIVERDB_API_URL'] + '/waivers/', - params={'product_version': product_version, - 'result_id': ','.join(str(result['id']) for result in results)}, - timeout=timeout) - response.raise_for_status() - waivers = response.json()['data'] + waivers = retrieve_waivers(product_version, results) else: waivers = [] for policy in applicable_policies: diff --git a/greenwave/app_factory.py b/greenwave/app_factory.py index b43c639..6f3f031 100644 --- a/greenwave/app_factory.py +++ b/greenwave/app_factory.py @@ -6,6 +6,7 @@ from greenwave.logger import init_logging from greenwave.api_v1 import api from greenwave.utils import json_error, load_config +from dogpile.cache import make_region from requests import ConnectionError, Timeout from werkzeug.exceptions import default_exceptions @@ -16,16 +17,23 @@ def create_app(config_obj=None): app.config.update(load_config(config_obj)) if app.config['PRODUCTION'] and app.secret_key == 'replace-me-with-something-random': raise Warning("You need to change the app.secret_key value for production") + # register error handlers for code in default_exceptions.iterkeys(): app.register_error_handler(code, json_error) app.register_error_handler(ConnectionError, json_error) app.register_error_handler(Timeout, json_error) + # initialize logging init_logging(app) + # register blueprints app.register_blueprint(api, url_prefix="/api/v1.0") app.add_url_rule('/healthcheck', view_func=healthcheck) + + # Initialize the cache. + app.cache = make_region().configure(**app.config['CACHE']) + return app diff --git a/greenwave/cache.py b/greenwave/cache.py new file mode 100644 index 0000000..aa2421d --- /dev/null +++ b/greenwave/cache.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: GPL-2.0+ + +import json + +from flask import current_app + + +def cache_key_generator(fn, arg): + """ Given a function and arguments, return a "cache key" for the value. + + The returned cache key should uniquely identify the function and arguments + passed to it. + """ + return "|".join([ + fn.__module__, + fn.__name__, + json.dumps(arg) + ]).encode('utf-8') + + +def cached(fn): + """ Cache the given function. + + This is a decorator. + + The return value of the given function is cached in the ``cache`` object + associated with the flask ``current_app``. + """ + + def wrapper(arg): + key = cache_key_generator(fn, arg) + creator = lambda: fn(arg) + return current_app.cache.get_or_create(key, creator) + wrapper.__module__ = fn.__module__ + wrapper.__name__ = fn.__name__ + wrapper.__doc__ = fn.__doc__ + return wrapper diff --git a/greenwave/config.py b/greenwave/config.py index ee38235..0d297dc 100644 --- a/greenwave/config.py +++ b/greenwave/config.py @@ -15,6 +15,8 @@ class Config(object): WAIVERDB_API_URL = 'https://waiverdb.fedoraproject.org/api/v1.0' REQUESTS_TIMEOUT = (6.1, 15) POLICIES_DIR = '/etc/greenwave/policies' + # By default, don't cache anything. + CACHE = {'backend': 'dogpile.cache.null'} class ProductionConfig(Config): diff --git a/greenwave/resources.py b/greenwave/resources.py new file mode 100644 index 0000000..11335f0 --- /dev/null +++ b/greenwave/resources.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-2.0+ +""" Greenwave resources. + +This module contains routines for interacting with other services (resultsdb, +waiverdb, etc..). + +""" + +import requests +from flask import current_app + +from greenwave.cache import cached + +requests_session = requests.Session() + + +@cached +def retrieve_results(item): + """ Retrieve cached results from resultsdb for a given item. """ + # XXX make this more efficient than just fetching everything + params = item.copy() + params.update({'limit': '1000'}) + timeout = current_app.config['REQUESTS_TIMEOUT'] + response = requests_session.get( + current_app.config['RESULTSDB_API_URL'] + '/results', + params=params, timeout=timeout) + response.raise_for_status() + return response.json()['data'] + + +# NOTE - not cached, for now. +def retrieve_waivers(product_version, results): + timeout = current_app.config['REQUESTS_TIMEOUT'] + response = requests_session.get( + current_app.config['WAIVERDB_API_URL'] + '/waivers/', + params={'product_version': product_version, + 'result_id': ','.join(str(result['id']) for result in results)}, + timeout=timeout) + response.raise_for_status() + return response.json()['data'] diff --git a/requirements.txt b/requirements.txt index 6f96835..bb138ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ flask requests PyYAML +dogpile.cache From 8fac2fed48fa998f062869ed8983869edd25ab7c Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sep 18 2017 17:18:08 +0000 Subject: [PATCH 2/2] Simplify and remove some code. --- diff --git a/greenwave/app_factory.py b/greenwave/app_factory.py index 6f3f031..cb86e47 100644 --- a/greenwave/app_factory.py +++ b/greenwave/app_factory.py @@ -4,9 +4,9 @@ from flask import Flask from greenwave.logger import init_logging from greenwave.api_v1 import api +from greenwave.cache import cache from greenwave.utils import json_error, load_config -from dogpile.cache import make_region from requests import ConnectionError, Timeout from werkzeug.exceptions import default_exceptions @@ -32,7 +32,8 @@ def create_app(config_obj=None): app.add_url_rule('/healthcheck', view_func=healthcheck) # Initialize the cache. - app.cache = make_region().configure(**app.config['CACHE']) + if not cache.is_configured: + cache.configure(**app.config['CACHE']) return app diff --git a/greenwave/cache.py b/greenwave/cache.py index aa2421d..27fb3b6 100644 --- a/greenwave/cache.py +++ b/greenwave/cache.py @@ -1,37 +1,9 @@ # SPDX-License-Identifier: GPL-2.0+ -import json +import dogpile.cache -from flask import current_app +# Our globally available cache region. Gets initialized in app_factory. +cache = dogpile.cache.make_region() - -def cache_key_generator(fn, arg): - """ Given a function and arguments, return a "cache key" for the value. - - The returned cache key should uniquely identify the function and arguments - passed to it. - """ - return "|".join([ - fn.__module__, - fn.__name__, - json.dumps(arg) - ]).encode('utf-8') - - -def cached(fn): - """ Cache the given function. - - This is a decorator. - - The return value of the given function is cached in the ``cache`` object - associated with the flask ``current_app``. - """ - - def wrapper(arg): - key = cache_key_generator(fn, arg) - creator = lambda: fn(arg) - return current_app.cache.get_or_create(key, creator) - wrapper.__module__ = fn.__module__ - wrapper.__name__ = fn.__name__ - wrapper.__doc__ = fn.__doc__ - return wrapper +# Provide a convenient alias for the key generator we want to use +key_generator = dogpile.cache.util.function_key_generator diff --git a/greenwave/resources.py b/greenwave/resources.py index 11335f0..2a95f2f 100644 --- a/greenwave/resources.py +++ b/greenwave/resources.py @@ -9,12 +9,12 @@ waiverdb, etc..). import requests from flask import current_app -from greenwave.cache import cached +from greenwave.cache import cache, key_generator requests_session = requests.Session() -@cached +@cache.cache_on_arguments(function_key_generator=key_generator) def retrieve_results(item): """ Retrieve cached results from resultsdb for a given item. """ # XXX make this more efficient than just fetching everything