From 18bc1665e18a5422bd00e0435c52b85e494b2ac1 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: May 02 2018 12:54:25 +0000
Subject: Add retry decorator to backend-frontend requests.


This is an addendum to 251c008372a3.

There, a retry decorator was added to try and work around some DNS issues when
we can't even establish a connection.  We covered connections from
greenwave/web to resultsdb and waiverdb, but I forgot about greenwave's
connection from the backend to greenwave/web.  This covers that third case.

It gets tricky like the comments says because the backend doesn't have access
to the frontend configuration - no application context.  We should think about
some kind of unified config for a future change.

---

diff --git a/greenwave/consumers/resultsdb.py b/greenwave/consumers/resultsdb.py
index 992bdd5..a297ccf 100644
--- a/greenwave/consumers/resultsdb.py
+++ b/greenwave/consumers/resultsdb.py
@@ -10,7 +10,6 @@ to the message bus about the newly satisfied/unsatisfied policy.
 """
 
 import collections
-import json
 import logging
 
 import dogpile.cache
@@ -147,27 +146,21 @@ class ResultsDBHandler(fedmsg.consumers.FedmsgConsumer):
         # result pushes any decisions over a threshold.
         for decision_context, product_versions in decision_contexts.items():
             for product_version in product_versions:
+                greenwave_url = self.fedmsg_config['greenwave_api_url'] + '/decision'
+
                 data = {
                     'decision_context': decision_context,
                     'product_version': product_version,
                     'subject': [subject],
                 }
-                response = requests_session.post(
-                    self.fedmsg_config['greenwave_api_url'] + '/decision',
-                    headers={'Content-Type': 'application/json'},
-                    data=json.dumps(data))
-                response.raise_for_status()
-                decision = response.json()
+                decision = greenwave.resources.retrieve_decision(greenwave_url, data)
+
                 # get old decision
                 data.update({
                     'ignore_result': [result_id],
                 })
-                response = requests_session.post(
-                    self.fedmsg_config['greenwave_api_url'] + '/decision',
-                    headers={'Content-Type': 'application/json'},
-                    data=json.dumps(data))
-                response.raise_for_status()
-                old_decision = response.json()
+                old_decision = greenwave.resources.retrieve_decision(greenwave_url, data)
+
                 if decision != old_decision:
                     decision.update({
                         'subject': [subject],
diff --git a/greenwave/resources.py b/greenwave/resources.py
index 2aac254..ef377e6 100644
--- a/greenwave/resources.py
+++ b/greenwave/resources.py
@@ -95,3 +95,16 @@ def retrieve_waivers(product_version, item):
         timeout=timeout)
     response.raise_for_status()
     return response.json()['data']
+
+
+# NOTE - not cached.
+@greenwave.utils.retry(timeout=300, interval=30, wait_on=urllib3.exceptions.NewConnectionError)
+def retrieve_decision(greenwave_url, data):
+    # TODO - get REQUESTS_TIMEOUT and REQUESTS_VERIFY here somehow.  This is usually
+    # called from the fedmsg-hub backend which doesn't have access to the flask
+    # application context.  We need to load the app context and config at backend
+    # startup to clean this up.
+    headers = {'Content-Type': 'application/json'}
+    response = requests_session.post(greenwave_url, headers=headers, data=json.dumps(data))
+    response.raise_for_status()
+    return response.json()