From a3b40a4cd2f470219376cfa38b3c8d6bf99fec38 Mon Sep 17 00:00:00 2001 From: Giulia Naponiello Date: Sep 24 2018 02:57:22 +0000 Subject: Adding Prometheus /metrics endpoint and some data Adding the Prometheus endpoint /metrics so that when one day we will have a Prometheus instance running for our services the metrics will already be available. Added: * basic metrics automatically provided by Prometheus * counter for exceptions in the waiverdb consumer (exception receiving a new waiver) * counter for exceptions in the resultsdb consumer (exception receiving a new result) * counter for exceptions in the decision API * latency for the decision API --- diff --git a/greenwave.spec b/greenwave.spec index f923e29..d6d5d03 100644 --- a/greenwave.spec +++ b/greenwave.spec @@ -20,12 +20,14 @@ BuildRequires: python3-requests BuildRequires: python3-PyYAML BuildRequires: python3-dogpile-cache BuildRequires: python3-fedmsg +BuildRequires: python3-prometheus_client BuildArch: noarch Requires: python3-flask Requires: python3-requests Requires: python3-PyYAML Requires: python3-dogpile-cache Requires: python3-fedmsg +Requires: python3-prometheus_client %description Greenwave is a service for gating on automated tests by querying ResultsDB and diff --git a/greenwave/api_v1.py b/greenwave/api_v1.py index 4499a8e..1a869e6 100644 --- a/greenwave/api_v1.py +++ b/greenwave/api_v1.py @@ -1,12 +1,15 @@ # SPDX-License-Identifier: GPL-2.0+ -from flask import Blueprint, request, current_app, jsonify, url_for, redirect +from flask import Blueprint, request, current_app, jsonify, url_for, redirect, Response from werkzeug.exceptions import BadRequest, NotFound, UnsupportedMediaType, InternalServerError +from prometheus_client import generate_latest from greenwave import __version__ from greenwave.policies import summarize_answers, RemotePolicy, RemoteRule from greenwave.resources import ResultsRetriever, retrieve_waivers, retrieve_builds_in_update from greenwave.safe_yaml import SafeYAMLError from greenwave.utils import insert_headers, jsonp +from greenwave.monitoring import registry, decision_exception_counter, decision_latency + api = (Blueprint('api_v1', __name__)) @@ -157,6 +160,8 @@ def make_decision_options(): @api.route('/decision', methods=['POST']) +@decision_exception_counter.count_exceptions() +@decision_latency.time() @jsonp def make_decision(): """ @@ -431,3 +436,8 @@ def validate_gating_yaml_post(): raise BadRequest('No policies defined') return jsonify({'message': 'All OK'}) + + +@api.route('/metrics', methods=['GET']) +def metrics(): + return Response(generate_latest(registry)) diff --git a/greenwave/consumers/resultsdb.py b/greenwave/consumers/resultsdb.py index 05c87ad..b0ff8c9 100644 --- a/greenwave/consumers/resultsdb.py +++ b/greenwave/consumers/resultsdb.py @@ -20,6 +20,7 @@ import requests import greenwave.app_factory import greenwave.resources from greenwave.api_v1 import subject_type_identifier_to_list +from greenwave.monitoring import publish_decision_exceptions_result_counter import xmlrpc.client @@ -204,6 +205,7 @@ class ResultsDBHandler(fedmsg.consumers.FedmsgConsumer): self._publish_decision_changes(subject_type, subject_identifier, result_id, testcase) + @publish_decision_exceptions_result_counter.count_exceptions() def _publish_decision_changes(self, subject_type, subject_identifier, result_id, testcase): """ Process the given subject and publish a message if the decision is changed. diff --git a/greenwave/consumers/waiverdb.py b/greenwave/consumers/waiverdb.py index 309b013..9b2b2a3 100644 --- a/greenwave/consumers/waiverdb.py +++ b/greenwave/consumers/waiverdb.py @@ -18,6 +18,7 @@ import requests import greenwave.app_factory from greenwave.api_v1 import subject_type_identifier_to_list +from greenwave.monitoring import publish_decision_exceptions_waiver_counter requests_session = requests.Session() @@ -82,6 +83,7 @@ class WaiverDBHandler(fedmsg.consumers.FedmsgConsumer): self._publish_decision_changes('bodhi_update', updateid, msg['id'], product_version, testcase) + @publish_decision_exceptions_waiver_counter.count_exceptions() def _publish_decision_changes(self, subject_type, subject_identifier, waiver_id, product_version, testcase): for policy in self.flask_app.config['policies']: diff --git a/greenwave/monitoring.py b/greenwave/monitoring.py new file mode 100644 index 0000000..445f2ae --- /dev/null +++ b/greenwave/monitoring.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0+ + +import os +from prometheus_client import Counter, Histogram, multiprocess, CollectorRegistry + + +# tmp dir for Prometheus monitoring registry. +# Putting this here and not in the "create_app" function, because "create_app" imports the api +# ...so the check for this env variable would be made before the "create_app" can be able to set it +# This is executed only once at server starting, so it is not so bad for performace. +if not os.environ.get('prometheus_multiproc_dir'): + os.environ.setdefault('prometheus_multiproc_dir', '/tmp') +registry = CollectorRegistry() +multiprocess.MultiProcessCollector(registry) +# https://github.com/prometheus/client_python/issues/210 +# pylint: disable-msg=unexpected-keyword-arg,no-value-for-parameter +decision_exception_counter = Counter('total_decision_exceptions', ('All exceptions occurred in ' + 'Greenwave "decision" API'), + registry=registry) +decision_latency = Histogram('decision_latency', 'Decision latency', registry=registry) +publish_decision_exceptions_waiver_counter = Counter('publish_decision_exceptions_new_waiver', + ('All exceptions occurred in publishing a ' + 'message after a new waiver'), + registry=registry) +publish_decision_exceptions_result_counter = Counter('publish_decision_exceptions_new_result', + ('All exceptions occurred in publishing a ' + 'message after a new result'), + registry=registry) diff --git a/requirements.txt b/requirements.txt index 1121343..3387cae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ requests PyYAML dogpile.cache fedmsg[consumers] +prometheus_client