From db6ee7567d07d8e76e3d10985271f0c1c1137a17 Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Jan 23 2019 20:01:31 +0000 Subject: [PATCH 1/2] initial factory pattern and trivial test --- diff --git a/execdb/__init__.py b/execdb/__init__.py index 405db7b..51995ad 100644 --- a/execdb/__init__.py +++ b/execdb/__init__.py @@ -28,95 +28,107 @@ import os # the version as used in setup.py __version__ = "0.0.11" - -# Flask App -app = Flask(__name__) -app.secret_key = 'replace-me-with-something-random' - - -# Load default config, then override that with a config file -if os.getenv('DEV') == 'true': - default_config_obj = 'execdb.config.DevelopmentConfig' - default_config_file = os.getcwd() + '/conf/settings.py' -elif os.getenv('TEST') == 'true': - default_config_obj = 'execdb.config.TestingConfig' - default_config_file = os.getcwd() + '/conf/settings.py' -else: - default_config_obj = 'execdb.config.ProductionConfig' - default_config_file = '/etc/execdb/settings.py' - -app.config.from_object(default_config_obj) - -config_file = os.environ.get('EXECDB_CONFIG', default_config_file) - -if os.path.exists(config_file): - app.config.from_pyfile(config_file) - -if app.config['PRODUCTION']: - if app.secret_key == 'replace-me-with-something-random': - raise Warning("You need to change the app.secret_key value for production") - -# setup logging -fmt = '[%(filename)s:%(lineno)d] ' if app.debug else '%(module)-12s ' -fmt += '%(asctime)s %(levelname)-7s %(message)s' -datefmt = '%Y-%m-%d %H:%M:%S' -loglevel = logging.DEBUG if app.debug else logging.INFO -formatter = logging.Formatter(fmt=fmt, datefmt=datefmt) - - -def setup_logging(): - root_logger = logging.getLogger('') - root_logger.setLevel(logging.DEBUG) - - if app.config['STREAM_LOGGING']: - print("doing stream logging") - stream_handler = logging.StreamHandler() - stream_handler.setLevel(loglevel) - stream_handler.setFormatter(formatter) - root_logger.addHandler(stream_handler) - app.logger.addHandler(stream_handler) - - if app.config['SYSLOG_LOGGING']: - print("doing syslog logging") - syslog_handler = logging.handlers.SysLogHandler( - address='/dev/log', - facility=logging.handlers.SysLogHandler.LOG_LOCAL4) - syslog_handler.setLevel(loglevel) - syslog_handler.setFormatter(formatter) - root_logger.addHandler(syslog_handler) - app.logger.addHandler(syslog_handler) - - if app.config['FILE_LOGGING'] and app.config['LOGFILE']: - print("doing file logging to %s" % app.config['LOGFILE']) - file_handler = logging.handlers.RotatingFileHandler( - app.config['LOGFILE'], - maxBytes=500000, - backupCount=5) - file_handler.setLevel(loglevel) - file_handler.setFormatter(formatter) - root_logger.addHandler(file_handler) - app.logger.addHandler(file_handler) - -setup_logging() - -if app.config['SHOW_DB_URI']: - app.logger.debug('using DBURI: %s' % app.config['SQLALCHEMY_DATABASE_URI']) - - -# database -db = SQLAlchemy(app) - -# setup login manager +#db = SQLAlchemy() login_manager = LoginManager() -login_manager.setup_app(app) -login_manager.login_view = 'login_page.login' - -# register blueprints -from execdb.controllers.main import main -app.register_blueprint(main) -from execdb.controllers.login_page import login_page -app.register_blueprint(login_page) -from execdb.controllers.admin import admin -app.register_blueprint(admin) +def create_app(test_config=None): + + # Flask App + app = Flask(__name__) + app.secret_key = 'replace-me-with-something-random' + + # Load default config, then override that with a config file + if os.getenv('DEV') == 'true': + default_config_obj = 'execdb.config.DevelopmentConfig' + default_config_file = os.getcwd() + '/conf/settings.py' + elif os.getenv('TEST') == 'true': + default_config_obj = 'execdb.config.TestingConfig' + default_config_file = os.getcwd() + '/conf/settings.py' + else: + default_config_obj = 'execdb.config.ProductionConfig' + default_config_file = '/etc/execdb/settings.py' + + config_file = os.environ.get('EXECDB_CONFIG', default_config_file) + + app.config.from_object(default_config_obj) + + # only load config from file if we don't have a test config + if test_config is None: + + if os.path.exists(config_file): + app.config.from_pyfile(config_file) + + if app.config['PRODUCTION']: + if app.secret_key == 'replace-me-with-something-random': + raise Warning("You need to change the app.secret_key value for production") + else: + app.config.update(test_config) + + + # setup logging + fmt = '[%(filename)s:%(lineno)d] ' if app.debug else '%(module)-12s ' + fmt += '%(asctime)s %(levelname)-7s %(message)s' + datefmt = '%Y-%m-%d %H:%M:%S' + loglevel = logging.DEBUG if app.debug else logging.INFO + formatter = logging.Formatter(fmt=fmt, datefmt=datefmt) + + def setup_logging(): + root_logger = logging.getLogger('') + root_logger.setLevel(logging.DEBUG) + + if app.config['STREAM_LOGGING']: + print("doing stream logging") + stream_handler = logging.StreamHandler() + stream_handler.setLevel(loglevel) + stream_handler.setFormatter(formatter) + root_logger.addHandler(stream_handler) + app.logger.addHandler(stream_handler) + + if app.config['SYSLOG_LOGGING']: + print("doing syslog logging") + syslog_handler = logging.handlers.SysLogHandler( + address='/dev/log', + facility=logging.handlers.SysLogHandler.LOG_LOCAL4) + syslog_handler.setLevel(loglevel) + syslog_handler.setFormatter(formatter) + root_logger.addHandler(syslog_handler) + app.logger.addHandler(syslog_handler) + + if app.config['FILE_LOGGING'] and app.config['LOGFILE']: + print("doing file logging to %s" % app.config['LOGFILE']) + file_handler = logging.handlers.RotatingFileHandler( + app.config['LOGFILE'], + maxBytes=500000, + backupCount=5) + file_handler.setLevel(loglevel) + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + app.logger.addHandler(file_handler) + + setup_logging() + + if app.config['SHOW_DB_URI']: + app.logger.debug('using DBURI: %s' % app.config['SQLALCHEMY_DATABASE_URI']) + + + # database + from execdb.models import db + db.init_app(app) + app.db = db + + # setup login manager + login_manager.setup_app(app) + login_manager.login_view = 'login_page.login' + + # register blueprints + from execdb.controllers.main import main + app.register_blueprint(main) + + from execdb.controllers.login_page import login_page + app.register_blueprint(login_page) + + from execdb.controllers.admin import admin + app.register_blueprint(admin) + + return app diff --git a/execdb/controllers/login_page.py b/execdb/controllers/login_page.py index 3c84bec..6e53af1 100644 --- a/execdb/controllers/login_page.py +++ b/execdb/controllers/login_page.py @@ -27,7 +27,8 @@ from wtforms.validators import Required from flask_login import login_user, logout_user, login_required, current_user, AnonymousUserMixin -from execdb import app, login_manager +#from execdb import app, login_manager +from execdb import login_manager from execdb.models.user import User login_page = Blueprint('login_page', __name__) diff --git a/execdb/controllers/main.py b/execdb/controllers/main.py index 623e888..be41c7d 100644 --- a/execdb/controllers/main.py +++ b/execdb/controllers/main.py @@ -17,7 +17,7 @@ # Authors: # Josef Skladanka -from flask import Blueprint, render_template, request, jsonify +from flask import Blueprint, render_template, request, jsonify, current_app import werkzeug.exceptions from sqlalchemy.orm import exc as orm_exc @@ -26,7 +26,9 @@ from werkzeug.exceptions import HTTPException from werkzeug.exceptions import BadRequest as JSONBadRequest -from execdb import app, db +#from execdb import app, db +#from execdb import db +from execdb.models import db from execdb.models.job import Job, BuildStep from sqlalchemy import desc @@ -36,8 +38,8 @@ import re from pprint import pformat main = Blueprint('main', __name__) -BB_URL = app.config['BUILDBOT_FRONTPAGE_URL'] -RESULTSDB_URL = app.config['RESULTSDB_FRONTPAGE_URL'] +#BB_URL = current_app.config['BUILDBOT_FRONTPAGE_URL'] +#RESULTSDB_URL = current_app.config['RESULTSDB_FRONTPAGE_URL'] RE_PAGE = re.compile(r"([?&])page=([0-9]+)") RP = {} @@ -110,7 +112,8 @@ def index(): return render_template('index.html', jobs=jobs, - buildbot_url=BB_URL, +# buildbot_url=BB_URL, + buildbot_url=current_app.config['BUILDBOT_FRONTPAGE_URL'], prev=prev, next=next) @@ -124,9 +127,11 @@ def show_job(uuid): job.t_triggered = str(job.t_triggered).split('.')[0] return render_template('show_job.html', job=job, - buildbot_url=BB_URL, - resultsdb_url=RESULTSDB_URL, - artifacts_base_url=app.config['ARTIFACTS_BASE_URL']) + #buildbot_url=BB_URL, + buildbot_url=current_app.config['BUILDBOT_FRONTPAGE_URL'], + #resultsdb_url=RESULTSDB_URL, + resultsdb_url=current_app.config['RESULTSDB_URL'], + artifacts_base_url=current_app.config['ARTIFACTS_BASE_URL']) @main.route('/jobs//steps', methods=['GET']) @@ -213,7 +218,7 @@ def process_event(data): if event not in known_events: # FIXME remove if 'uuid' in json.dumps(data): - app.logger.debug("UUID found in %s", event) + current_app.logger.debug("UUID found in %s", event) return 'Skipping event', 204 @@ -261,7 +266,7 @@ def process_event(data): # app.logger.debug("%s: %s" % (uuid, data['payload']['build']['steps'])) # app.logger.debug("%s - Build Started" % uuid) for step_info in data['payload']['build']['steps']: - # app.logger.debug("%s -- adding step %s"% (uuid, step_info['name'])) + # current_app.logger.debug("%s -- adding step %s"% (uuid, step_info['name'])) step = BuildStep(name=step_info['name']) step.job = job db.session.add(step) @@ -270,11 +275,11 @@ def process_event(data): elif event == 'stepStarted' and job.current_state == 'Running': step_info = data['payload']['step'] -# app.logger.debug("%s - Step Started - %s"% (uuid, step_info['name'])) +# current_app.logger.debug("%s - Step Started - %s"% (uuid, step_info['name'])) try: step = job.get_build_step(step_info['name']) except KeyError: - app.logger.debug("Job %s had missing step %s", job.uuid, step_info) + current_app.logger.debug("Job %s had missing step %s", job.uuid, step_info) step = BuildStep(name=step_info['name']) step.job = job @@ -283,11 +288,11 @@ def process_event(data): step.data = json.dumps(data['payload']) # FIXME - store sensible subset of data db.session.add(step) db.session.commit() -# app.logger.debug("%s - Step Started - %s - written to db"% (uuid, step_info['name'])) +# current_app.logger.debug("%s - Step Started - %s - written to db"% (uuid, step_info['name'])) elif event == 'stepFinished' and job.current_state == 'Running': step_info = data['payload']['step'] -# app.logger.debug("%s - Step Finished - %s"% (uuid, step_info['name'])) +# current_app.logger.debug("%s - Step Finished - %s"% (uuid, step_info['name'])) try: step = job.get_build_step(step_info['name']) except KeyError: @@ -303,13 +308,13 @@ def process_event(data): db.session.add(step) db.session.commit() -# app.logger.debug("%s - Step Finished - %s - written to db" % (uuid, step_info['name'])) +# current_app.logger.debug("%s - Step Finished - %s - written to db" % (uuid, step_info['name'])) elif event == 'buildFinished' and job.current_state == 'Running': job.finish() db.session.add(job) db.session.commit() -# app.logger.debug("%s - Build Finished " % uuid) +# current_app.logger.debug("%s - Build Finished " % uuid) @main.route('/buildbottest', methods=['POST']) @@ -319,20 +324,20 @@ def bb_push(): the steps for the job. """ # data are embedded in form field 'packets' - data = request.form - try: - data = request.form['packets'] - except werkzeug.exceptions.BadRequestKeyError: - return 'Field `packets` missing in request form.', 400 - data = json.loads(data) - - # app.logger.debug(pformat(data)) - - # multiple messages may be present in one 'packet' - for entry in data: - process_event(entry) -# app.logger.debug("%s %s, %s", entry['id'], entry['event'], process_event(entry)) - +# data = request.form +# try: +# data = request.form['packets'] +# except werkzeug.exceptions.BadRequestKeyError: +# return 'Field `packets` missing in request form.', 400 +# data = json.loads(data) +# +# # current_app.logger.debug(pformat(data)) +# +# # multiple messages may be present in one 'packet' +# for entry in data: +# process_event(entry) +## current_app.logger.debug("%s %s, %s", entry['id'], entry['event'], process_event(entry)) +# # plain 200 code needs to be returned - otherwise buildbot is # endlessly trying to re-send the message. # FIXME - add logging for non-200 responses diff --git a/execdb/models/__init__.py b/execdb/models/__init__.py index e69de29..cc45acb 100644 --- a/execdb/models/__init__.py +++ b/execdb/models/__init__.py @@ -0,0 +1,2 @@ +from flask_sqlalchemy import SQLAlchemy +db = SQLAlchemy() diff --git a/execdb/models/job.py b/execdb/models/job.py index 2e48f88..ca49bb1 100644 --- a/execdb/models/job.py +++ b/execdb/models/job.py @@ -17,7 +17,7 @@ # Authors: # Josef Skladanka -from execdb import db, app +from . import db import datetime import uuid diff --git a/execdb/models/user.py b/execdb/models/user.py index a5baf16..c78ad86 100644 --- a/execdb/models/user.py +++ b/execdb/models/user.py @@ -17,7 +17,8 @@ # Authors: # Josef Skladanka -from execdb import db +from . import db + from flask_login import UserMixin from werkzeug.security import generate_password_hash, check_password_hash diff --git a/tox.ini b/tox.ini index 256a3c1..2f5bc41 100644 --- a/tox.ini +++ b/tox.ini @@ -6,3 +6,9 @@ max-line-length=99 [pep8] max-line-length=99 + +[pytest] +minversion=2.0 +python_functions=test should +python_files=test_* functest_* +#addopts=--functional tests/ --cov-report=term-missing --cov execdb From b948b18872b37f2cfd205e07b09fc7038c11f50b Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Jan 24 2019 19:17:46 +0000 Subject: [PATCH 2/2] adapting execdb to new buildbot status messages - still needs cleanup --- diff --git a/execdb/__init__.py b/execdb/__init__.py index 51995ad..59a9ca9 100644 --- a/execdb/__init__.py +++ b/execdb/__init__.py @@ -49,7 +49,7 @@ def create_app(test_config=None): default_config_obj = 'execdb.config.ProductionConfig' default_config_file = '/etc/execdb/settings.py' - config_file = os.environ.get('EXECDB_CONFIG', default_config_file) + config_file = os.environ.get('EXECDB_CONFIG', default_config_file) app.config.from_object(default_config_obj) diff --git a/execdb/controllers/main.py b/execdb/controllers/main.py index be41c7d..5376970 100644 --- a/execdb/controllers/main.py +++ b/execdb/controllers/main.py @@ -34,6 +34,7 @@ from sqlalchemy import desc import json import re +from datetime import datetime from pprint import pformat @@ -130,7 +131,7 @@ def show_job(uuid): #buildbot_url=BB_URL, buildbot_url=current_app.config['BUILDBOT_FRONTPAGE_URL'], #resultsdb_url=RESULTSDB_URL, - resultsdb_url=current_app.config['RESULTSDB_URL'], + resultsdb_url=current_app.config['RESULTSDB_FRONTPAGE_URL'], artifacts_base_url=current_app.config['ARTIFACTS_BASE_URL']) @@ -142,7 +143,8 @@ def show_steps(uuid): return 'UUID not found', 404 steps = dict( - buildbot_url=BB_URL, + #buildbot_url=BB_URL, + buildbot_url=current_app.config['BUILDBOT_FRONTPAGE_URL'], steps=[], job_status=job.current_state, job_duration=str(job.build_took), @@ -316,13 +318,81 @@ def process_event(data): db.session.commit() # current_app.logger.debug("%s - Build Finished " % uuid) +def process_bb_status(status_data): + # grab uuid, build state, properties + build_properties = status_data['properties'] + uuid = build_properties['uuid'][0] + build_complete = status_data['complete'] -@main.route('/buildbottest', methods=['POST']) + current_app.logger.info("Processing data for job {} (complete: {})".format(uuid, build_complete)) + + # find job in db + try: + job = db.session.query(Job).filter(Job.uuid == uuid).one() + except orm_exc.NoResultFound: + current_app.logger.info("UUID {} not found".format(uuid)) + return 'UUID not found', 400 + + # if 'complete' is false, create job + if not build_complete: + current_app.logger.debug("%s -- adding job %s"% (uuid, status_data['number'])) + + job.t_build_started = datetime.fromtimestamp(status_data['started_at']) + job.t_triggered = datetime.fromtimestamp(status_data['started_at']) + + job.taskname = build_properties['taskname'][0] + job.item = build_properties['item'][0] + job.item_type = build_properties['item_type'][0] + job.arch = build_properties['arch'][0] + job.slavename = build_properties['slavename'][0] + job.link_build_log = '/builders/%s/builds/%s' % ( + status_data['buildrequest']['builderid'], + status_data['number']) + + db.session.add(job) + + db.session.commit() + + # if 'complete' is true, fill in buildsteps, finish job + else: + # add the completed time and state + + job.t_build_ended = datetime.fromtimestamp(status_data['complete_at']) + # add the build steps + for step_info in status_data['steps']: + + current_app.logger.debug("%s -- adding step %s"% (uuid, step_info['name'])) + + step = BuildStep(name=step_info['name']) + step.job = job + step.started_at = datetime.fromtimestamp(step_info['started_at']) + step.finished_at = datetime.fromtimestamp(step_info['complete_at']) + step.data = step_info['state_string'] + + # there doesn't seem to be a really reasonable way to tell if a step has failed but + # this should work well enough for now + if 'failed' in step_info['state_string']: + step.status = 'NOT OK' + else: + step.status = 'OK' + + db.session.add(step) + + db.session.commit() + + +@main.route('/buildbot', methods=['POST']) def bb_push(): """ Receives the post-push notifications from buildbot and fills in the steps for the job. """ + from pprint import pprint + + data = request.get_json() +# pprint(data) + + process_bb_status(data) # data are embedded in form field 'packets' # data = request.form # try: