From 67e695c6685c32ecbc11d23efff27003c3f99af4 Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Jan 14 2017 17:01:27 +0000 Subject: [PATCH 1/2] Handle the GET tags view by front ending the OSBS registry. fixes #4 Signed-off-by: Randy Barlow --- diff --git a/devel/ansible/roles/dev/tasks/main.yml b/devel/ansible/roles/dev/tasks/main.yml index 7bd4160..0e62c72 100644 --- a/devel/ansible/roles/dev/tasks/main.yml +++ b/devel/ansible/roles/dev/tasks/main.yml @@ -11,6 +11,7 @@ - python3-nose - python3-nose-cov - python3-PyYAML + - python3-requests - python3-sphinx - name: Install the .bashrc diff --git a/docs/configuration.rst b/docs/configuration.rst index 302f8a0..7efcb66 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -12,6 +12,9 @@ As fegistry is a `Flask `_ application, there is a set o that you can use in the config file. In addition to these settings, fegistry defines the following settings: +* ``BACKEND_REGISTRY``: The URL to the backend OSBS registry where the stable container images are + stored. fegistry will make requests against this registry for any tags or manifests that are not + found in fegistry's cache. Default: ``https://stable-registry.fedoraproject.org``. * ``LOG_LEVEL``: The logging level that fegistry should use when logging to syslog. This accepts any of the standard Python `logging levels `_, - and it will even upper case for you if you don't like shouting. + and it will even upper case for you if you don't like shouting. Default: ``WARNING``. diff --git a/fegistry.yaml.example b/fegistry.yaml.example index da9c41e..745162b 100644 --- a/fegistry.yaml.example +++ b/fegistry.yaml.example @@ -1,2 +1,3 @@ --- +# BACKEND_REGISTRY: https://stable-registry.fedoraproject.org # LOG_LEVEL: warning diff --git a/fegistry/config.py b/fegistry/config.py index 89c6207..df7e33c 100644 --- a/fegistry/config.py +++ b/fegistry/config.py @@ -22,7 +22,8 @@ import os import yaml -_DEFAULT_CONFIG = {'LOG_LEVEL': 'WARNING'} +_DEFAULT_CONFIG = {'BACKEND_REGISTRY': 'https://stable-registry.fedoraproject.org', + 'LOG_LEVEL': 'WARNING'} def load(app): diff --git a/fegistry/tests/test_views.py b/fegistry/tests/test_views.py index c1847ab..078008d 100644 --- a/fegistry/tests/test_views.py +++ b/fegistry/tests/test_views.py @@ -16,9 +16,11 @@ """This test suite contains tests on fegistry.views.""" import json +import mock import unittest import flask +import requests from fegistry import views @@ -51,3 +53,98 @@ class Testv2(ViewsTestCase): self.assertEqual(json.loads(response.get_data().decode('utf-8')), {}) self.assertEqual(response.headers['Docker-Distribution-API-Version'], 'registry/2.0') self.assertEqual(response.headers['Content-Type'], 'application/json') + + +class TestListTags(ViewsTestCase): + """This test class contains tests for the list_tags() function.""" + @mock.patch.dict('fegistry.views.app.config', {'BACKEND_REGISTRY': 'http://example.com'}) + @mock.patch('fegistry.views.requests.get') + def test_200(self, get): + """Ensure correct operation of the list_tags view when the backend gives a 200 status code. + """ + get.return_value = requests.Response() + get.return_value._content = \ + b'{"name":"fedora","tags":["24","latest","25","26","rawhide"]}\n' + get.return_value.headers = requests.structures.CaseInsensitiveDict({ + 'AppServer': 'proxy12.fedoraproject.org', 'AppTime': 'D=157698', + 'Connection': 'Keep-Alive', 'Content-Length': '61', + 'Content-Type': 'application/json; charset=utf-8', + 'Date': 'Thu, 12 Jan 2017 14:57:47 GMT', + 'Docker-Distribution-API-Version': 'registry/2.0', 'Keep-Alive': 'timeout=15, max=500', + 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux)', + 'Strict-Transport-Security': 'max-age=15768000; includeSubDomains; preload'}) + get.return_value.status_code = 200 + + response = self.app.get('/v2/fedora/tags/list') + + get.assert_called_once_with('http://example.com/v2/fedora/tags/list') + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.get_data().decode('utf-8')), + {'name': 'fedora', 'tags': ['24', 'latest', '25', '26', 'rawhide']}) + expected_headers = { + 'Content-Type': 'application/json; charset=utf-8', 'Content-Length': '61', + 'Docker-Distribution-API-Version': 'registry/2.0'} + self.assertEqual(dict(response.headers), expected_headers) + + @mock.patch.dict('fegistry.views.app.config', {'BACKEND_REGISTRY': 'http://example.com/'}) + @mock.patch('fegistry.views.requests.get') + def test_404(self, get): + """Ensure correct operation of the list_tags view when the backend gives a 404 status code. + """ + get.return_value = requests.Response() + get.return_value._content = ( + b'{"errors":[{"code":"NAME_UNKNOWN","message":"repository name not known to registry",' + b'"detail":{"name":"meaning/of/life"}}]}\n') + get.return_value.headers = requests.structures.CaseInsensitiveDict({ + 'AppServer': 'proxy12.fedoraproject.org', 'AppTime': 'D=157698', + 'Connection': 'Keep-Alive', 'Content-Length': '123', + 'Content-Type': 'application/json; charset=utf-8', + 'Date': 'Thu, 12 Jan 2017 14:57:47 GMT', + 'Docker-Distribution-API-Version': 'registry/2.0', 'Keep-Alive': 'timeout=15, max=500', + 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux)', + 'Strict-Transport-Security': 'max-age=15768000; includeSubDomains; preload'}) + get.return_value.status_code = 404 + + response = self.app.get('/v2/meaning/of/life/tags/list') + + get.assert_called_once_with('http://example.com/v2/meaning/of/life/tags/list') + self.assertEqual(response.status_code, 404) + self.assertEqual( + json.loads(response.get_data().decode('utf-8')), + {'errors': [{'code': 'NAME_UNKNOWN', 'message': 'repository name not known to registry', + 'detail': {'name': 'meaning/of/life'}}]}) + expected_headers = { + 'Content-Type': 'application/json; charset=utf-8', 'Content-Length': '123', + 'Docker-Distribution-API-Version': 'registry/2.0'} + self.assertEqual(dict(response.headers), expected_headers) + + @mock.patch.dict('fegistry.views.app.config', {'BACKEND_REGISTRY': 'http://example.com:1234'}) + @mock.patch('fegistry.views.requests.get') + def test_content_type_header_not_present(self, get): + """ + Ensure correct operation of the list_tags view when the backend doesn't give us a + Content-Type header. + """ + get.return_value = requests.Response() + get.return_value._content = \ + b'{"name":"fedora/cockpit","tags":["24","latest","25","26","rawhide"]}\n' + get.return_value.headers = requests.structures.CaseInsensitiveDict({ + 'AppServer': 'proxy12.fedoraproject.org', 'AppTime': 'D=157698', + 'Connection': 'Keep-Alive', 'Content-Length': '69', + 'Date': 'Thu, 12 Jan 2017 14:57:47 GMT', + 'Docker-Distribution-API-Version': 'registry/2.0', 'Keep-Alive': 'timeout=15, max=500', + 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux)', + 'Strict-Transport-Security': 'max-age=15768000; includeSubDomains; preload'}) + get.return_value.status_code = 200 + + response = self.app.get('/v2/fedora/cockpit/tags/list') + + get.assert_called_once_with('http://example.com:1234/v2/fedora/cockpit/tags/list') + self.assertEqual(response.status_code, 200) + self.assertEqual( + json.loads(response.get_data().decode('utf-8')), + {'name': 'fedora/cockpit', 'tags': ['24', 'latest', '25', '26', 'rawhide']}) + expected_headers = { + 'Content-Type': 'application/json; charset=utf-8', 'Content-Length': '69', + 'Docker-Distribution-API-Version': 'registry/2.0'} + self.assertEqual(dict(response.headers), expected_headers) diff --git a/fegistry/views.py b/fegistry/views.py index 3289bdb..8588ac0 100644 --- a/fegistry/views.py +++ b/fegistry/views.py @@ -13,7 +13,10 @@ # # You should have received a copy of the GNU General Public License # along with fegistry. If not, see . +from urllib import parse + import flask +import requests from fegistry import config @@ -46,3 +49,23 @@ def v2(): flask.Response: A JSON response """ return flask.json.jsonify({}) + + +@app.route('/v2//tags/list') +def list_tags(repo): + """ + Answer the GET /v2/repo/tags/list API call. + + Args: + repo (str): The repository to query for tags + + Returns: + flask.Response: A JSON response with the list of tags in the given repository. + """ + tags = requests.get( + parse.urljoin(app.config['BACKEND_REGISTRY'], '/v2/{}/tags/list'.format(repo))) + # We don't need to pass all the headers from the backend through, so let's just cherry pick the + # ones we want. + headers = {} + headers['Content-Type'] = tags.headers.get('Content-Type', 'application/json; charset=utf-8') + return (tags.text, tags.status_code, headers) diff --git a/setup.py b/setup.py index 8b12fd3..45186bb 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,6 @@ setup( long_description=README, classifiers=CLASSIFIERS, license=LICENSE, maintainer=MAINTAINER, maintainer_email=MAINTAINER_EMAIL, platforms=PLATFORMS, url=URL, keywords='fedora', packages=find_packages(exclude=('fegistry.tests', 'fegistry.tests.*')), - include_package_data=True, zip_safe=False, install_requires=['flask', 'PyYAML'], + include_package_data=True, zip_safe=False, install_requires=['flask', 'PyYAML', 'requests'], tests_require=['flake8', 'mock', 'nose', 'nose-cov'], test_suite="nose.collector") From 7ddf0bd912b1bb234683b148eb87a6cb46cc3d0a Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Jan 14 2017 17:01:30 +0000 Subject: [PATCH 2/2] Use dogpile cache to cache responses from the backend registry. This commit introduces a dependency on dogpile.cache, and uses it to cache responses from the backend registry. It adds three new settings that allow the admin to configure the cache. Signed-off-by: Randy Barlow --- diff --git a/devel/ansible/roles/dev/tasks/main.yml b/devel/ansible/roles/dev/tasks/main.yml index 0e62c72..dcbeffa 100644 --- a/devel/ansible/roles/dev/tasks/main.yml +++ b/devel/ansible/roles/dev/tasks/main.yml @@ -5,6 +5,7 @@ state: present with_items: - git + - python3-dogpile-cache - python3-flake8 - python3-flask - python3-mock diff --git a/docs/configuration.rst b/docs/configuration.rst index 7efcb66..076419a 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -15,6 +15,16 @@ settings: * ``BACKEND_REGISTRY``: The URL to the backend OSBS registry where the stable container images are stored. fegistry will make requests against this registry for any tags or manifests that are not found in fegistry's cache. Default: ``https://stable-registry.fedoraproject.org``. +* ``DOGPILE_CACHE_BACKEND``: The dogpile.cache backend you wish you use. You can peruse the + dogpile.cache documentation for the list of + `supported backends `_. + Default: ``dogpile.cache.memory_pickle``. +* ``DOGPILE_CACHE_EXPIRATION_TIME``: The time in seconds that values should remain in the cache. + Default: ``86400``. +* ``DOGPILE_CACHE_BACKEND_ARGUMENTS``: An associative array of settings that are relevant to the + dogpile.cache backend selected in ``DOGPILE_CACHE_BACKEND``. You can peruse the dogpile.cache + `documentation `_ for the + relevant options for your chosen backend. Default: ``{}``. * ``LOG_LEVEL``: The logging level that fegistry should use when logging to syslog. This accepts any of the standard Python `logging levels `_, and it will even upper case for you if you don't like shouting. Default: ``WARNING``. diff --git a/fegistry.yaml.example b/fegistry.yaml.example index 745162b..f156919 100644 --- a/fegistry.yaml.example +++ b/fegistry.yaml.example @@ -1,3 +1,6 @@ --- # BACKEND_REGISTRY: https://stable-registry.fedoraproject.org +# DOGPILE_CACHE_BACKEND: dogpile.cache.memory_pickle +# DOGPILE_CACHE_EXPIRATION_TIME: 86400 +# DOGPILE_CACHE_BACKEND_ARGUMENTS: {} # LOG_LEVEL: warning diff --git a/fegistry/config.py b/fegistry/config.py index df7e33c..f70cc40 100644 --- a/fegistry/config.py +++ b/fegistry/config.py @@ -22,8 +22,10 @@ import os import yaml -_DEFAULT_CONFIG = {'BACKEND_REGISTRY': 'https://stable-registry.fedoraproject.org', - 'LOG_LEVEL': 'WARNING'} +_DEFAULT_CONFIG = { + 'BACKEND_REGISTRY': 'https://stable-registry.fedoraproject.org', + 'DOGPILE_CACHE_BACKEND': 'dogpile.cache.memory_pickle', 'DOGPILE_CACHE_EXPIRATION_TIME': 86400, + 'DOGPILE_CACHE_BACKEND_ARGUMENTS': {}, 'LOG_LEVEL': 'WARNING'} def load(app): diff --git a/fegistry/tests/test_views.py b/fegistry/tests/test_views.py index 078008d..d6b76ca 100644 --- a/fegistry/tests/test_views.py +++ b/fegistry/tests/test_views.py @@ -32,6 +32,17 @@ class ViewsTestCase(unittest.TestCase): self.app = views.app.test_client() +class CachedViewTestCase(ViewsTestCase): + """A superclass for testing cached views. It invalidates the cache before and after the test.""" + def setUp(self): + super(CachedViewTestCase, self).setUp() + views.region.invalidate(hard=True) + + def tearDown(self): + super(CachedViewTestCase, self).tearDown() + views.region.invalidate(hard=True) + + class TestAddDockerHeaders(unittest.TestCase): """This test class contains tests on the add_docker_headers() function.""" def test_add_docker_headers(self): @@ -43,6 +54,135 @@ class TestAddDockerHeaders(unittest.TestCase): self.assertEqual(response.headers['Docker-Distribution-API-Version'], 'registry/2.0') +class TestGetFromBackendRegistry(CachedViewTestCase): + """This test class contains tests for the _get_from_backend_registry() function.""" + @mock.patch.dict('fegistry.views.app.config', {'BACKEND_REGISTRY': 'http://example.com:123/'}) + @mock.patch('fegistry.views.requests.get') + def test_200(self, get): + """ + Ensure correct operation of the _get_from_backend_registry() function when the backend gives + a 200 status code. + """ + get.return_value = requests.Response() + get.return_value._content = \ + b'{"name":"fedora","tags":["24","latest","25","26","rawhide"]}\n' + get.return_value.headers = requests.structures.CaseInsensitiveDict({ + 'AppServer': 'proxy12.fedoraproject.org', 'AppTime': 'D=157698', + 'Connection': 'Keep-Alive', 'Content-Length': '61', + 'Content-Type': 'application/json; charset=utf-8', + 'Date': 'Thu, 12 Jan 2017 14:57:47 GMT', + 'Docker-Distribution-API-Version': 'registry/2.0', 'Keep-Alive': 'timeout=15, max=500', + 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux)', + 'Strict-Transport-Security': 'max-age=15768000; includeSubDomains; preload'}) + get.return_value.status_code = 200 + + text, status_code, headers = views._get_from_backend_registry('/v2/fedora/tags/list') + + get.assert_called_once_with('http://example.com:123/v2/fedora/tags/list') + self.assertEqual(status_code, 200) + self.assertEqual(json.loads(text), + {'name': 'fedora', 'tags': ['24', 'latest', '25', '26', 'rawhide']}) + expected_headers = {'Content-Type': 'application/json; charset=utf-8'} + self.assertEqual(headers, expected_headers) + + @mock.patch.dict('fegistry.views.app.config', {'BACKEND_REGISTRY': 'http://example.com/'}) + @mock.patch('fegistry.views.requests.get') + def test_404(self, get): + """ + Ensure correct operation of the _get_from_backend_registry() function when the backend + gives a 404 status code. + """ + get.return_value = requests.Response() + get.return_value._content = ( + b'{"errors":[{"code":"NAME_UNKNOWN","message":"repository name not known to registry",' + b'"detail":{"name":"meaning/of/life"}}]}\n') + get.return_value.headers = requests.structures.CaseInsensitiveDict({ + 'AppServer': 'proxy12.fedoraproject.org', 'AppTime': 'D=157698', + 'Connection': 'Keep-Alive', 'Content-Length': '123', + 'Content-Type': 'application/json; charset=utf-8', + 'Date': 'Thu, 12 Jan 2017 14:57:47 GMT', + 'Docker-Distribution-API-Version': 'registry/2.0', 'Keep-Alive': 'timeout=15, max=500', + 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux)', + 'Strict-Transport-Security': 'max-age=15768000; includeSubDomains; preload'}) + get.return_value.status_code = 404 + + text, status_code, headers = views._get_from_backend_registry( + '/v2/meaning/of/life/manifests/latest') + + get.assert_called_once_with('http://example.com/v2/meaning/of/life/manifests/latest') + self.assertEqual(status_code, 404) + self.assertEqual( + json.loads(text), + {'errors': [{'code': 'NAME_UNKNOWN', 'message': 'repository name not known to registry', + 'detail': {'name': 'meaning/of/life'}}]}) + expected_headers = {'Content-Type': 'application/json; charset=utf-8'} + self.assertEqual(headers, expected_headers) + + @mock.patch.dict('fegistry.views.app.config', {'BACKEND_REGISTRY': 'http://example.com:1234'}) + @mock.patch('fegistry.views.requests.get') + def test_cache(self, get): + """Ensure that the function caches responses.""" + get.return_value = requests.Response() + get.return_value._content = \ + b'{"name":"fedora","tags":["24","latest","25","26","rawhide"]}\n' + get.return_value.headers = requests.structures.CaseInsensitiveDict({ + 'AppServer': 'proxy12.fedoraproject.org', 'AppTime': 'D=157698', + 'Connection': 'Keep-Alive', 'Content-Length': '61', + 'Content-Type': 'application/json; charset=utf-8', + 'Date': 'Thu, 12 Jan 2017 14:57:47 GMT', + 'Docker-Distribution-API-Version': 'registry/2.0', 'Keep-Alive': 'timeout=15, max=500', + 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux)', + 'Strict-Transport-Security': 'max-age=15768000; includeSubDomains; preload'}) + get.return_value.status_code = 200 + # Make one call to the function to get the response cached. + views._get_from_backend_registry('/v2/fedora/tags/list') + # Now let's alter what the backend would return so we can make sure the cached version is + # used. + get.return_value._content = b'This should not get used yet because the cache should be hit' + get.return_value.headers = requests.structures.CaseInsensitiveDict({'Not': 'Yet'}) + + text, status_code, headers = views._get_from_backend_registry('/v2/fedora/tags/list') + + get.assert_called_once_with('http://example.com:1234/v2/fedora/tags/list') + self.assertEqual(status_code, 200) + self.assertEqual(json.loads(text), + {'name': 'fedora', 'tags': ['24', 'latest', '25', '26', 'rawhide']}) + expected_headers = {'Content-Type': 'application/json; charset=utf-8'} + self.assertEqual(headers, expected_headers) + + @mock.patch.dict('fegistry.views.app.config', {'BACKEND_REGISTRY': 'http://example.com:1234'}) + @mock.patch('fegistry.views.requests.get') + def test_content_type_header_not_present(self, get): + """ + Ensure correct operation of the list_tags view when the backend doesn't give us a + Content-Type header. + """ + get.return_value = requests.Response() + get.return_value._content = \ + b'{"name":"fedora/cockpit","tags":["24","latest","25","26","rawhide"]}\n' + get.return_value.headers = requests.structures.CaseInsensitiveDict({ + 'AppServer': 'proxy12.fedoraproject.org', 'AppTime': 'D=157698', + 'Connection': 'Keep-Alive', 'Content-Length': '69', + 'Date': 'Thu, 12 Jan 2017 14:57:47 GMT', + 'Docker-Distribution-API-Version': 'registry/2.0', 'Keep-Alive': 'timeout=15, max=500', + 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux)', + 'Strict-Transport-Security': 'max-age=15768000; includeSubDomains; preload'}) + get.return_value.status_code = 200 + + text, status_code, headers = views._get_from_backend_registry( + '/v2/fedora/cockpit/manifests/latest') + + get.assert_called_once_with('http://example.com:1234/v2/fedora/cockpit/manifests/latest') + self.assertEqual(status_code, 200) + self.assertEqual( + json.loads(text), + {'name': 'fedora/cockpit', 'tags': ['24', 'latest', '25', '26', 'rawhide']}) + # Even though the backend didn't give us the Content-Type header, the function should have + # added it for us. + expected_headers = {'Content-Type': 'application/json; charset=utf-8'} + self.assertEqual(headers, expected_headers) + + class Testv2(ViewsTestCase): """This test class tests the v2() function.""" def test_v2(self): @@ -55,7 +195,7 @@ class Testv2(ViewsTestCase): self.assertEqual(response.headers['Content-Type'], 'application/json') -class TestListTags(ViewsTestCase): +class TestListTags(CachedViewTestCase): """This test class contains tests for the list_tags() function.""" @mock.patch.dict('fegistry.views.app.config', {'BACKEND_REGISTRY': 'http://example.com'}) @mock.patch('fegistry.views.requests.get') diff --git a/fegistry/views.py b/fegistry/views.py index 8588ac0..98ac986 100644 --- a/fegistry/views.py +++ b/fegistry/views.py @@ -15,6 +15,7 @@ # along with fegistry. If not, see . from urllib import parse +from dogpile.cache import make_region import flask import requests @@ -23,6 +24,10 @@ from fegistry import config app = flask.Flask(__name__) config.load(app) +region = make_region().configure( + app.config['DOGPILE_CACHE_BACKEND'], + expiration_time=app.config['DOGPILE_CACHE_EXPIRATION_TIME'], + arguments=app.config['DOGPILE_CACHE_BACKEND_ARGUMENTS']) @app.after_request @@ -62,10 +67,28 @@ def list_tags(repo): Returns: flask.Response: A JSON response with the list of tags in the given repository. """ - tags = requests.get( - parse.urljoin(app.config['BACKEND_REGISTRY'], '/v2/{}/tags/list'.format(repo))) + return _get_from_backend_registry('/v2/{}/tags/list'.format(repo)) + + +@region.cache_on_arguments() +def _get_from_backend_registry(path): + """ + Retrieve the given path with a GET request from the backend registry, returning a tuple of the + text, status code, and filtered headers (expressed as a dictionary). + + Args: + path (str): The path to retrieve from the backend registry + + Returns: + tuple: A 3-tuple of the text of the response (str), the status code (int), and a filtered + set of headers from the backend registry (dict). Currently, only the Content-Type + header is passed through, but the caller should be able to handle additional headers + that may be added in the future. + """ + response = requests.get(parse.urljoin(app.config['BACKEND_REGISTRY'], path)) # We don't need to pass all the headers from the backend through, so let's just cherry pick the # ones we want. headers = {} - headers['Content-Type'] = tags.headers.get('Content-Type', 'application/json; charset=utf-8') - return (tags.text, tags.status_code, headers) + headers['Content-Type'] = response.headers.get('Content-Type', + 'application/json; charset=utf-8') + return (response.text, response.status_code, headers)