From 2641c4a117e702c31188653e357c9ee7504cbdfb Mon Sep 17 00:00:00 2001
From: Slavek Kabrda <bkabrda@redhat.com>
Date: Feb 23 2018 10:23:24 +0000
Subject: [PATCH 1/3] Check for worker readiness in subprocess not to leak redis connections


---

diff --git a/tests/__init__.py b/tests/__init__.py
index 9e4360b..b650ebd 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -335,29 +335,47 @@ class Modeltests(SimplePagureTest):
         # Using cocurrency 2 to test with some concurrency, but not be heavy
         # Using eventlet so that worker.terminate kills everything
         self.workerlog = open(os.path.join('.', 'worker.log'), 'w')
+        celery_exec = '/usr/bin/celery'
+        celery_env = {
+            'PAGURE_BROKER_URL': celery_broker_url,
+            'PAGURE_CONFIG': os.path.join(self.path, 'config'),
+            'PYTHONPATH': '.'
+        }
+        celery_cwd = os.path.normpath(
+            os.path.join(os.path.dirname(__file__),
+            '..')
+        )
         self.worker = subprocess.Popen(
-            ['/usr/bin/celery', '-A', 'pagure.lib.tasks', 'worker',
+            [celery_exec, '-A', 'pagure.lib.tasks', 'worker',
              '--loglevel=info', '--concurrency=2', '--pool=eventlet',
              '--without-gossip', '--without-mingle', '--quiet'],
-            env={'PAGURE_BROKER_URL': celery_broker_url,
-                 'PAGURE_CONFIG': os.path.join(self.path, 'config'),
-                 'PYTHONPATH': '.'},
-            cwd=os.path.normpath(os.path.join(os.path.dirname(__file__),
-                                              '..')),
+            env=celery_env,
+            cwd=celery_cwd,
             stdout=self.workerlog,
             stderr=self.workerlog)
         self.worker.poll()
         if self.worker.returncode is not None:
             raise Exception('Worker failed to start')
-        time.sleep(2)
-        # The below code seems to be leaking redis connection until
-        # Python starts raising OSError with too many open files
-        # This is probably related to https://github.com/celery/celery/issues/4465
-        # wait_start = time.time()
-        # while not pagure.lib.tasks.conn.control.ping(timeout=0.1):
-        #     time.sleep(0.1)
-        #     if time.time() - wait_start > 5:
-        #         raise Exception('Worker failed to initialize in 5 seconds')
+        # We could do the ping below in-process:
+        # pagure.lib.tasks.conn.control.ping(timeout=0.1)
+        # but if we try it, Python starts raising OSError
+        # with too many open files. This is probably related
+        # to https://github.com/celery/celery/issues/4465
+        wait_start = time.time()
+        while True:
+            time.sleep(0.1)
+            res = subprocess.call(
+                [celery_exec, '-A', 'pagure.lib.tasks',
+                 'inspect', '-t=0.1', 'ping'],
+                env=celery_env,
+                cwd=celery_cwd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE
+            )
+            if res == 0:
+                break
+            if time.time() - wait_start > 5:
+                raise Exception('Worker failed to initialize in 5 seconds')
 
         self.app.get = create_maybe_waiter(self.app.get, self.app.get)
         self.app.post = create_maybe_waiter(self.app.post, self.app.get)

From ced73cd8aeb9a30a2b4b95ed6afc8e3c815b3d63 Mon Sep 17 00:00:00 2001
From: Slavek Kabrda <bkabrda@redhat.com>
Date: Feb 23 2018 10:23:30 +0000
Subject: [PATCH 2/3] Fix lib tests by reloading tasks_services to get proper redis url


---

diff --git a/tests/__init__.py b/tests/__init__.py
index b650ebd..f1d5530 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -330,6 +330,7 @@ class Modeltests(SimplePagureTest):
         celery_broker_url = 'redis+socket://' + broker_url
         pagure_config['BROKER_URL'] = celery_broker_url
         reload(pagure.lib.tasks)
+        reload(pagure.lib.tasks_services)
 
         # Start a worker
         # Using cocurrency 2 to test with some concurrency, but not be heavy

From 4f464c0e378ee01df117396c954d0383f561aac3 Mon Sep 17 00:00:00 2001
From: Slavek Kabrda <bkabrda@redhat.com>
Date: Feb 26 2018 15:35:21 +0000
Subject: [PATCH 3/3] Close Celery connections in tearDown to not leak them


---

diff --git a/tests/__init__.py b/tests/__init__.py
index f1d5530..ca88171 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -393,6 +393,10 @@ class Modeltests(SimplePagureTest):
         self.worker = None
         self.workerlog.close()
         self.workerlog = None
+        # close the connections to redis before killing redis,
+        # otherwise we leak connections
+        pagure.lib.tasks.conn.close()
+        pagure.lib.tasks_services.conn.close()
         self.broker.kill()
         self.broker.wait()
         self.broker = None