From 0890ff7a1ce15005c2c3fdc8a6d7b1f2f8d9d597 Mon Sep 17 00:00:00 2001 From: Matt Jia Date: Feb 24 2017 08:40:15 +0000 Subject: implement HTTP API for creating new waivers --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d7cbf6c --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*.pyc +*.pyo +*.swp +*__pycache__* +conf/settings.py +*.sqlite +*.egg* +/env*/ +test_env +.cache diff --git a/README.md b/README.md index 1e2ecf6..e6c1501 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,35 @@ WaiverDB is a companion service to [ResultsDB](https://pagure.io/taskotron/resultsdb), for recording waivers against test results. + +## Quick development setup + +Set up a python virtualenv: + + $ sudo dnf install python-virtualenv + $ virtualenv env_waiverdb + $ source env_resultsdb/bin/activate + $ pip install -r requirements.txt + +Install the project: + + $ python setup.py develop + +Run the server: + + $ DEV=true python runapp.py + +The server is now running at and aPI calls can be sent to +. All data is stored inside `/var/tmp/waiverdb_db.sqlite`. + +## Adjusting configuration + +You can configure this app by copying `conf/settings.py.example` into +`conf/setting.py` and adjusting values as you see fit. It overrides default +values in `waiverdb/config.py`. + +## Running test suite + +You can run this test suite with the following command:: + + $ py.test tests/ diff --git a/conf/settings.py.example b/conf/settings.py.example new file mode 100644 index 0000000..4001f9a --- /dev/null +++ b/conf/settings.py.example @@ -0,0 +1,12 @@ +# Copy this file to `conf/settings.py` to put it into effect. It overrides the values defined +# in `waiverdb/config.py`. +SECRET_KEY = 'replace-me-with-something-random' +#SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://dbuser:dbpassword@dbhost:dbport/dbname' +SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/waiverdb_db.sqlite' +FILE_LOGGING = False +LOGFILE = '/var/log/waiverdb/waiverdb.log' +SYSLOG_LOGGING = False +STREAM_LOGGING = True +#SHOW_DB_URI = False +RUN_HOST= '0.0.0.0' +RUN_PORT = 5004 diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..9c43dcd --- /dev/null +++ b/pytest.ini @@ -0,0 +1,7 @@ +[pytest] + +minversion = 2.0 + +python_functions=test + +python_files=test_* diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0203797 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +# This is a list of pypi packages to be installed into virtualenv. Alternatively, +# you can install these as RPMs instead of pypi packages. + +Flask +Flask-RESTful +Flask-SQLAlchemy +SQLAlchemy + +pytest >= 2.4.2 diff --git a/runapp.py b/runapp.py new file mode 100644 index 0000000..77e766c --- /dev/null +++ b/runapp.py @@ -0,0 +1,25 @@ +#!/usr/bin/python +# +# runapp.py - script to facilitate running the waiverdb app from the CLI + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# + +from waiverdb.app import create_app, init_db + +if __name__ == '__main__': + app = create_app() + init_db(app) + app.run( + host=app.config['HOST'], + port=app.config['PORT'], + debug=app.config['DEBUG'], + ) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c306a87 --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup + +version = '1.0.0' + +setup(name='waiverdb', + version=version, + description='An engine for storing waivers against test results.', + author='Red Hat, Inc.', + author_email='qa-devel@lists.fedoraproject.org', + license='GPLv2+', + packages=['waiverdb'], + package_dir={'waiverdb': 'waiverdb'}, + #entry_points={ + # #TODO: register messaging plugins + #}, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/__init__.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..db596eb --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,72 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# + +import os +import pytest + +from waiverdb.app import create_app, init_db + +@pytest.fixture(scope='session') +def app(tmpdir_factory, request): + app = create_app('waiverdb.config.TestingConfig') + db_file = tmpdir_factory.mktemp('waiverdb').join('db.sqlite') + app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///%s' % db_file + # Establish an application context before running the tests. + ctx = app.app_context() + ctx.push() + + def teardown(): + ctx.pop() + + request.addfinalizer(teardown) + return app + +@pytest.fixture(scope='session') +def db(app, request): + """Session-wide test database.""" + db = init_db(app) + def teardown(): + db.drop_all() + request.addfinalizer(teardown) + return db + +@pytest.fixture(scope='function') +def session(db, request): + """Creates a new database session for a test.""" + connection = db.engine.connect() + transaction = connection.begin() + + # https://github.com/mitsuhiko/flask-sqlalchemy/issues/345 + class _dict(dict): + def __nonzero__(self): + return True + + options = dict(bind=connection, binds=_dict()) + session = db.create_scoped_session(options=options) + + db.session = session + + def teardown(): + transaction.rollback() + connection.close() + session.remove() + + request.addfinalizer(teardown) + return session + +@pytest.yield_fixture +def client(app): + """A Flask test client. An instance of :class:`flask.testing.TestClient` + by default. + """ + with app.test_client() as client: + yield client diff --git a/tests/test_api_v10.py b/tests/test_api_v10.py new file mode 100644 index 0000000..e9bb62b --- /dev/null +++ b/tests/test_api_v10.py @@ -0,0 +1,40 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +import pytest +import json + +def test_create_waiver(client, session): + data = { + 'result_id': 123, + 'product_version': 'fool-1', + 'waived': True, + 'comment': 'it broke', + } + r = client.post('/api/v1.0/waivers/', data=json.dumps(data), + content_type='application/json') + res_data = json.loads(r.data) + assert r.status_code == 201 + assert res_data['username'] == 'mjia' + assert res_data['result_id'] == 123 + assert res_data['product_version'] == 'fool-1' + assert res_data['waived'] == True + assert res_data['comment'] == 'it broke' + +def test_create_waiver_with_malformed_data(client): + data = { + 'result_id': 'wrong id', + } + r = client.post('/api/v1.0/waivers/', data=json.dumps(data), + content_type='application/json') + res_data = json.loads(r.data) + assert r.status_code == 400 + assert 'invalid literal for int()' in res_data['message']['result_id'] diff --git a/waiverdb/.gitignore b/waiverdb/.gitignore new file mode 100644 index 0000000..c55dfea --- /dev/null +++ b/waiverdb/.gitignore @@ -0,0 +1,7 @@ +*.pyc +*.pyo +*.swp +*__pycache__* +conf/settings.py +*.sqlite +*.egg* diff --git a/waiverdb/__init__.py b/waiverdb/__init__.py new file mode 100644 index 0000000..db41e89 --- /dev/null +++ b/waiverdb/__init__.py @@ -0,0 +1,10 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. diff --git a/waiverdb/api_v1.py b/waiverdb/api_v1.py new file mode 100644 index 0000000..95ab07f --- /dev/null +++ b/waiverdb/api_v1.py @@ -0,0 +1,48 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# + +from flask import Blueprint +from flask_restful import reqparse +from werkzeug.exceptions import HTTPException + +from waiverdb.models import db, Waiver +from waiverdb.utils import to_json + +api = Blueprint('api_v1', __name__) + +# RP contains request parsers (reqparse.RequestParser). +# Parsers are added in each 'resource section' for better readability +RP = {} + +RP['create_waiver'] = reqparse.RequestParser() +RP['create_waiver'].add_argument('result_id', type=int, required=True, location='json') +RP['create_waiver'].add_argument('waived', type=bool, required=True, location='json') +RP['create_waiver'].add_argument('product_version', type=str, required=True, location='json') +RP['create_waiver'].add_argument('comment', type=str, default=None, location='json') + +@api.route('/waivers/', methods=['POST']) +@to_json +def create_waiver(): + try: + args = RP['create_waiver'].parse_args() + except HTTPException as error: + return error.data, error.code + + # hardcode the username for now + username = 'mjia' + + waiver = Waiver(args['result_id'], username, args['product_version'], args['waived'], + args['comment']) + + db.session.add(waiver) + db.session.commit() + return waiver, 201 diff --git a/waiverdb/app.py b/waiverdb/app.py new file mode 100644 index 0000000..bb436d2 --- /dev/null +++ b/waiverdb/app.py @@ -0,0 +1,57 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +import os +from flask import Flask +from flask_sqlalchemy import SQLAlchemy +from waiverdb.logger import init_logging +from waiverdb.api_v1 import api as api_v1 +from waiverdb.models import db + +def load_default_config(app): + # Load default config, then override that with a config file + if os.getenv('DEV') == 'true': + default_config_obj = 'waiverdb.config.DevelopmentConfig' + default_config_file = os.getcwd() + '/conf/settings.py' + elif os.getenv('TEST') == 'true': + default_config_obj = 'waiverdb.config.TestingConfig' + default_config_file = os.getcwd() + '/conf/settings.py' + else: + default_config_obj = 'waiverdb.config.ProductionConfig' + default_config_file = '/etc/waiverdb/settings.py' + app.config.from_object(default_config_obj) + config_file = os.environ.get('WAIVERDB_CONFIG', default_config_file) + if os.path.exists(config_file): + app.config.from_pyfile(config_file) + +# applicaiton factory http://flask.pocoo.org/docs/0.12/patterns/appfactories/ +def create_app(config_obj=None): + app = Flask(__name__) + if config_obj: + app.config.from_object(config_obj) + else: + load_default_config(app) + if app.config['PRODUCTION'] and app.secret_key == 'replace-me-with-something-random': + raise Warning("You need to change the app.secret_key value for production") + if app.config['SHOW_DB_URI']: + app.logger.debug('using DBURI: %s' % app.config['SQLALCHEMY_DATABASE_URI']) + # initialize db + db.init_app(app) + # initialize logging + init_logging(app) + # register blueprints + app.register_blueprint(api_v1, url_prefix="/api/v1.0") + return app + +def init_db(app): + with app.app_context(): + db.create_all() + return db diff --git a/waiverdb/config.py b/waiverdb/config.py new file mode 100644 index 0000000..541784e --- /dev/null +++ b/waiverdb/config.py @@ -0,0 +1,39 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + + +class Config(object): + DEBUG = True + SQLALCHEMY_DATABASE_URI = 'sqlite://' + JOURNAL_LOGGING = False + HOST = '0.0.0.0' + PORT = 5004 + PRODUCTION = False + SHOW_DB_URI = False + SECRET_KEY = 'replace-me-with-something-random' + + +class ProductionConfig(Config): + DEBUG = False + PRODUCTION = True + + +class DevelopmentConfig(Config): + SQLALCHEMY_TRACK_MODIFICATIONS = True + TRAP_BAD_REQUEST_ERRORS = True + SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/waiverdb_db.sqlite' + SHOW_DB_URI = True + + +class TestingConfig(Config): + SQLALCHEMY_TRACK_MODIFICATIONS = True + TRAP_BAD_REQUEST_ERRORS = True + TESTING = True diff --git a/waiverdb/logger.py b/waiverdb/logger.py new file mode 100644 index 0000000..24b0d2a --- /dev/null +++ b/waiverdb/logger.py @@ -0,0 +1,38 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +import logging +import sys + +def log_to_stdout(app, level=logging.INFO): + fmt = '[%(filename)s:%(lineno)d] ' if app.debug else '%(module)-12s ' + fmt += '%(asctime)s %(levelname)-7s %(message)s' + datefmt = '%Y-%m-%d %H:%M:%S' + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setLevel(level) + stream_handler.setFormatter(logging.Formatter(fmt=fmt, datefmt=datefmt)) + app.logger.addHandler(stream_handler) + +def log_to_journal(app, level=logging.INFO): + try: + import systemd.journal + except: + raise ValueError("systemd.journal module is not installed") + journal_handler = systemd.journal.JournalHandler() + journal_handler.setLevel(level) + app.logger.addHandler(journal_handler) + +def init_logging(app): + log_level = logging.DEBUG if app.debug else logging.INFO + if app.config['JOURNAL_LOGGING']: + log_to_journal(app, level=log_level) + else: + log_to_stdout(app, level=log_level) diff --git a/waiverdb/models/__init__.py b/waiverdb/models/__init__.py new file mode 100644 index 0000000..5088532 --- /dev/null +++ b/waiverdb/models/__init__.py @@ -0,0 +1,13 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +from .base import db +from .waivers import Waiver diff --git a/waiverdb/models/base.py b/waiverdb/models/base.py new file mode 100644 index 0000000..de3d3fd --- /dev/null +++ b/waiverdb/models/base.py @@ -0,0 +1,15 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# + +from flask_sqlalchemy import SQLAlchemy + +db = SQLAlchemy() diff --git a/waiverdb/models/waivers.py b/waiverdb/models/waivers.py new file mode 100644 index 0000000..f9746ae --- /dev/null +++ b/waiverdb/models/waivers.py @@ -0,0 +1,45 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +import datetime +from .base import db + +class Waiver(db.Model): + id = db.Column(db.Integer, primary_key=True) + result_id = db.Column(db.Integer, nullable=False) + username = db.Column(db.String(255), nullable=False) + product_version = db.Column(db.String(200), nullable=False) + waived = db.Column(db.Boolean, nullable=False, default=False) + comment = db.Column(db.Text) + timestamp = db.Column(db.DateTime, default=datetime.datetime.utcnow) + + def __init__(self, result_id, username, product_version, waived=False, comment=None): + self.result_id = result_id + self.username = username + self.product_version = product_version + self.waived = waived + self.comment = comment + + def __repr__(self): + return '%s(result_id=%r, username=%r, product_version=%r, waived=%r)' % ( + self.__class__.__name__, self.result_id, self.username, + self.product_version, self.waived) + + def __json__(self): + return { + 'id': self.id, + 'result_id': self.result_id, + 'username': self.username, + 'product_version': self.product_version, + 'waived': self.waived, + 'comment': self.comment, + 'timestamp': self.timestamp.isoformat(), + } diff --git a/waiverdb/utils.py b/waiverdb/utils.py new file mode 100644 index 0000000..c4172d1 --- /dev/null +++ b/waiverdb/utils.py @@ -0,0 +1,38 @@ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +import functools +from flask import jsonify + +# https://github.com/miguelgrinberg/api-pycon2015/blob/master/api/decorators.py#L8-L30 +def to_json(f): + """This decorator generates a JSON response from a Python dictionary or + a SQLAlchemy model.""" + @functools.wraps(f) + def wrapped(*args, **kwargs): + rv = f(*args, **kwargs) + status_or_headers = None + headers = None + if isinstance(rv, tuple): + rv, status_or_headers, headers = rv + (None,) * (3 - len(rv)) + if isinstance(status_or_headers, (dict, list)): + headers, status_or_headers = status_or_headers, None + if not isinstance(rv, dict): + # assume it is a model, call its __json__() method + rv = rv.__json__() + + rv = jsonify(rv) + if status_or_headers is not None: + rv.status_code = status_or_headers + if headers is not None: + rv.headers.extend(headers) + return rv + return wrapped