From 52e0f46f5bc1d8bd013c6ffa0abc19efb44c39b1 Mon Sep 17 00:00:00 2001
From: Giulia Naponiello <gnaponie@redhat.com>
Date: Aug 16 2018 10:00:19 +0000
Subject: Adding Prometheus /metrics endpoint and some data


Adding the Prometheus endpoint /metrics so that when one day we will
have a Prometheus instance running for our services the metrics will
already be available.
Added:
 * basic metrics automatically provided by Prometheus
 * counter for exceptions in the waiverdb consumer (exception receiving
   a new waiver)
 * counter for exceptions in the resultsdb consumer (exception receiving
   a new result)
 * counter for exceptions in the decision API
 * latency for the decision API

---

diff --git a/greenwave/api_v1.py b/greenwave/api_v1.py
index 9481185..4c1aa22 100644
--- a/greenwave/api_v1.py
+++ b/greenwave/api_v1.py
@@ -1,14 +1,26 @@
 # SPDX-License-Identifier: GPL-2.0+
 
-from flask import Blueprint, request, current_app, jsonify, url_for, redirect
+import time
+from flask import Blueprint, request, current_app, jsonify, url_for, redirect, Response
 from werkzeug.exceptions import BadRequest, NotFound, UnsupportedMediaType, InternalServerError
+from prometheus_client import generate_latest, Counter, Histogram
 from greenwave import __version__
 from greenwave.policies import summarize_answers, RemotePolicy, RemoteRule
 from greenwave.resources import retrieve_results, retrieve_waivers, retrieve_builds_in_update
 from greenwave.safe_yaml import SafeYAMLError
 from greenwave.utils import insert_headers, jsonp
 
+
 api = (Blueprint('api_v1', __name__))
+decision_exception_c = Counter('total_decision_exceptions', ('All exceptions occured in Greenwave'
+                                                             '"decision" API'))
+decision_latency = Histogram('decision_latency', 'Decision latency')
+publish_decision_exceptions_waiver_c = Counter('publish_decision_exceptions_new_waiver',
+                                               ('All exceptions occured in publishing a message '
+                                                'after a new waiver'))
+publish_decision_exceptions_result_c = Counter('publish_decision_exceptions_new_result',
+                                               ('All exceptions occured in publishing a message '
+                                                'after a new result'))
 
 
 def subject_list_to_type_identifier(subject):
@@ -157,6 +169,7 @@ def make_decision_options():
 
 
 @api.route('/decision', methods=['POST'])
+@decision_exception_c.count_exceptions()
 @jsonp
 def make_decision():
     """
@@ -270,6 +283,7 @@ def make_decision():
     :statuscode 400: Invalid data was given.
     """  # noqa: E501
 
+    t_start = time.time()
     if request.get_json():
         if ('product_version' not in request.get_json() or
                 not request.get_json()['product_version']):
@@ -372,6 +386,7 @@ def make_decision():
     resp = jsonify(res)
     resp = insert_headers(resp)
     resp.status_code = 200
+    decision_latency.observe(time.time() - t_start)
     return resp
 
 
@@ -424,3 +439,8 @@ def validate_gating_yaml_post():
         raise BadRequest('No policies defined')
 
     return jsonify({'message': 'All OK'})
+
+
+@api.route('/metrics', methods=['GET'])
+def metrics():
+    return Response(generate_latest())
diff --git a/greenwave/consumers/resultsdb.py b/greenwave/consumers/resultsdb.py
index 999a973..a0edae0 100644
--- a/greenwave/consumers/resultsdb.py
+++ b/greenwave/consumers/resultsdb.py
@@ -19,7 +19,7 @@ import requests
 import greenwave.app_factory
 import greenwave.cache
 import greenwave.resources
-from greenwave.api_v1 import subject_type_identifier_to_list
+from greenwave.api_v1 import subject_type_identifier_to_list, publish_decision_exceptions_result_c
 
 
 log = logging.getLogger(__name__)
@@ -127,6 +127,7 @@ class ResultsDBHandler(fedmsg.consumers.FedmsgConsumer):
                 self._publish_decision_changes(subject_type, subject_identifier,
                                                result_id, testcase)
 
+    @publish_decision_exceptions_result_c.count_exceptions()
     def _publish_decision_changes(self, subject_type, subject_identifier, result_id, testcase):
         """
         Process the given subject and publish a message if the decision is changed.
diff --git a/greenwave/consumers/waiverdb.py b/greenwave/consumers/waiverdb.py
index 309b013..f07ebee 100644
--- a/greenwave/consumers/waiverdb.py
+++ b/greenwave/consumers/waiverdb.py
@@ -17,7 +17,7 @@ import fedmsg.consumers
 import requests
 
 import greenwave.app_factory
-from greenwave.api_v1 import subject_type_identifier_to_list
+from greenwave.api_v1 import subject_type_identifier_to_list, publish_decision_exceptions_waiver_c
 
 requests_session = requests.Session()
 
@@ -82,6 +82,7 @@ class WaiverDBHandler(fedmsg.consumers.FedmsgConsumer):
                     self._publish_decision_changes('bodhi_update', updateid, msg['id'],
                                                    product_version, testcase)
 
+    @publish_decision_exceptions_waiver_c.count_exceptions()
     def _publish_decision_changes(self, subject_type, subject_identifier, waiver_id,
                                   product_version, testcase):
         for policy in self.flask_app.config['policies']:
diff --git a/requirements.txt b/requirements.txt
index 1121343..3387cae 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,4 @@ requests
 PyYAML
 dogpile.cache
 fedmsg[consumers]
+prometheus_client