From a581cb35528e8a7df3d55a47e850c30f47601a4e Mon Sep 17 00:00:00 2001
From: Alexander Scheel <ascheel@redhat.com>
Date: Jul 10 2017 18:36:13 +0000
Subject: Enable debugging for testsuites


Signed-off-by: Alexander Scheel <ascheel@redhat.com>

---

diff --git a/proxy/Makefile.am b/proxy/Makefile.am
index 5cd2255..2cad0ac 100644
--- a/proxy/Makefile.am
+++ b/proxy/Makefile.am
@@ -317,7 +317,7 @@ CLEANFILES = *.X */*.X */*/*.X \
     systemd/gssproxy.service
 
 check: all $(check_PROGRAMS)
-	$(srcdir)/tests/runtests.py
+	$(srcdir)/tests/runtests.py $(CHECKARGS)
 
 tests: check
 
diff --git a/proxy/tests/runtests.py b/proxy/tests/runtests.py
index 3f5d13d..d93d3e9 100755
--- a/proxy/tests/runtests.py
+++ b/proxy/tests/runtests.py
@@ -7,16 +7,37 @@ import signal
 import sys
 import traceback
 
+import testlib
 from testlib import *
 
 def parse_args():
     parser = argparse.ArgumentParser(description='GSS-Proxy Tests Environment')
     parser.add_argument('--path', default='%s/testdir' % os.getcwd(),
                         help="Directory in which tests are run")
-
-    return vars(parser.parse_args())
-
-if __name__ == '__main__':
+    parser.add_argument('--debug-all', default=False, action="store_true",
+                        help="Enable debugging for all test cases")
+    parser.add_argument('--debug-gssproxy', default=False, action="store_true",
+                        help="Enable debugging for gssproxy command")
+    parser.add_argument('--debug-cmd', default="gdb --args",
+                        help="Set the debugging command. Defaults to gdb " +
+                              "--args")
+    parser.add_argument('--debug-num', default=-1, type=int,
+                        help="Specify the testcase number to debug")
+    parser.add_argument('--timeout', default=15, type=int,
+                        help="Specify test case timeout limit")
+    parser.add_argument('--valgrind-cmd', default="valgrind " +
+                        "--track-origins=yes",
+                        help="Set the valgrind command. Defaults to " +
+                        "valgrind --track-origins=yes")
+    parser.add_argument('--force-valgrind', default=False, action="store_true",
+                        help="Force valgrind to be run on all test cases")
+
+    args = vars(parser.parse_args())
+    testlib_process_args(args)
+
+    return args
+
+def runtests_main(testfiles):
     args = parse_args()
 
     testdir = args['path']
@@ -41,6 +62,7 @@ if __name__ == '__main__':
         keysenv = setup_keys(testdir, kdcenv)
 
         gssapienv = setup_gssapi_env(testdir, kdcenv)
+        gssapienv['TERM'] = os.environ['TERM']
 
         gssproxylog = os.path.join(testdir, 'gssproxy.log')
 
@@ -59,23 +81,16 @@ if __name__ == '__main__':
         basicconf["gpid"] = gproc.pid
         basicconf["keysenv"] = keysenv
 
-        testnum = 0
-        testfiles = [f for f in os.listdir(os.path.dirname(sys.argv[0])) \
-                     if f.endswith(".py") and f.startswith("t_")]
-        testfiles.sort()
         print("Tests to be run: " + ", ".join(testfiles))
         for f in testfiles:
             fmod = f[:-len(".py")]
             t = importlib.__import__(fmod)
 
-            basicconf['prefix'] = '%02d' % testnum
-            logfile = os.path.join(testdir, "%02d_%s.log" % (testnum, fmod))
-            basicconf['logfile'] = open(logfile, 'a')
+            basicconf['prefix'] = str(testlib.cmd_index)
+            basicconf['logpath'] = testdir
             r = t.run(testdir, gssapienv, basicconf)
             if r != 0:
                 errored = True
-
-            testnum += 1
     except Exception:
         traceback.print_exc()
         errored = True
@@ -87,3 +102,15 @@ if __name__ == '__main__':
         if errored:
             sys.exit(1)
         sys.exit(0)
+
+if __name__ == "__main__":
+    print("\n")
+    print("To pass arguments to the test suite, use CHECKARGS:")
+    print("    make check CHECKARGS='--debug-num=<num>'")
+    print("A full set of available options can be seen with --help")
+    print("\n")
+
+    testfiles = [f for f in os.listdir(os.path.dirname(sys.argv[0])) \
+                 if f.endswith(".py") and f.startswith("t_")]
+    testfiles.sort()
+    runtests_main(testfiles)
diff --git a/proxy/tests/t_acquire.py b/proxy/tests/t_acquire.py
index 813ac6d..41de8ef 100755
--- a/proxy/tests/t_acquire.py
+++ b/proxy/tests/t_acquire.py
@@ -5,7 +5,7 @@ from testlib import *
 
 def run(testdir, env, conf, expected_failure=False):
     print("Testing basic acquire creds...", file=sys.stderr)
-    logfile = conf['logfile']
+    conf['prefix'] = str(cmd_index)
 
     svc_keytab = os.path.join(testdir, SVC_KTNAME)
     testenv = {'KRB5CCNAME': os.path.join(testdir, 't' + conf['prefix'] +
@@ -17,16 +17,10 @@ def run(testdir, env, conf, expected_failure=False):
                'GSSPROXY_BEHAVIOR': 'REMOTE_FIRST'}
     testenv.update(env)
 
-    cmd = ["./tests/t_acquire", conf['svc_name']]
-    print("[COMMAND]\n%s\n[ENVIRONMENT]\n%s\n" % (cmd, env), file=logfile)
-    logfile.flush()
+    cmd = "./tests/t_acquire " + conf['svc_name']
 
-    p1 = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=logfile,
-                          env=testenv, preexec_fn=os.setsid)
-    try:
-        p1.wait(10)
-    except subprocess.TimeoutExpired:
-        # p1.returncode is set to None here
-        pass
-    print_return(p1.returncode, "Acquire", expected_failure)
-    return p1.returncode if not expected_failure else int(not p1.returncode)
+    return run_testcase_cmd(testenv, conf, cmd, "Acquire", expected_failure)
+
+if __name__ == "__main__":
+    from runtests import runtests_main
+    runtests_main(["t_acquire.py"])
diff --git a/proxy/tests/t_basic.py b/proxy/tests/t_basic.py
index b4f6f9f..4e634e7 100755
--- a/proxy/tests/t_basic.py
+++ b/proxy/tests/t_basic.py
@@ -1,11 +1,24 @@
 #!/usr/bin/python3
 # Copyright (C) 2014,2015,2016 - GSS-Proxy contributors; see COPYING for the license
 
-from testlib import *
+import testlib
+import os
+import signal
+import subprocess
+import sys
+
 
 def run(testdir, env, conf, expected_failure=False):
     print("Testing basic init/accept context", file=sys.stderr)
-    logfile = conf['logfile']
+    conf['prefix'] = str(testlib.cmd_index)
+
+    init_logfile = os.path.join(conf['logpath'], "test_%d.log" %
+                                testlib.cmd_index)
+    init_logfile = open(init_logfile, 'a')
+
+    accept_logfile = os.path.join(conf['logpath'], "test_%d.log" %
+                                  (testlib.cmd_index + 1))
+    accept_logfile = open(accept_logfile, 'a')
 
     svcenv = {'KRB5_KTNAME': conf['keytab'],
               'KRB5CCNAME': os.path.join(testdir, 't' + conf['prefix'] +
@@ -20,6 +33,9 @@ def run(testdir, env, conf, expected_failure=False):
     else:
         init_cmd = ["./tests/t_init", conf['svc_name']]
 
+    init_cmd = " ".join(init_cmd)
+    accept_cmd = " ".join(["./tests/t_accept"])
+
     clienv = {'KRB5CCNAME': os.path.join(testdir, 't' + conf['prefix'] +
                                                   '_init.ccache'),
               'KRB5_TRACE': os.path.join(testdir, 't' + conf['prefix'] +
@@ -28,27 +44,94 @@ def run(testdir, env, conf, expected_failure=False):
               'GSSPROXY_BEHAVIOR': 'REMOTE_FIRST'}
     clienv.update(env)
 
-    print("[SVCENV]\n%s\n[CLIENV]\n%s\nCLI NAME: %s\n" % (
-          svcenv, clienv, client_name), file=logfile)
+    print("[CLIENV]\n%s\nCLI NAME: %s\n" % (
+          clienv, client_name), file=init_logfile)
+
+    print("[SRVENV]\n%s\n" % (svcenv), file=accept_logfile)
+
+    init_logfile.flush()
+    accept_logfile.flush()
 
     pipe0 = os.pipe()
     pipe1 = os.pipe()
 
+    if testlib.debug_cmd_index == testlib.cmd_index:
+        p1 = subprocess.Popen(init_cmd,
+                              stdin=pipe0[0], stdout=pipe1[1],
+                              stderr=init_logfile, env=clienv,
+                              preexec_fn=os.setsid, shell=True,
+                              executable="/bin/bash")
+        print("PID: %d\n" % p1.pid)
+        print("Attach and start debugging, then press enter to start t_init.")
+        input()
+
+        p2 = subprocess.Popen(["./tests/t_accept"],
+                              stdin=pipe1[0], stdout=pipe0[1],
+                              stderr=accept_logfile, env=svcenv,
+                              preexec_fn=os.setsid, shell=True,
+                              executable="/bin/bash")
+        print("To resume tests if hung, kill pid %d\n" % p2.pid)
+        p2.wait()
+
+        init_logfile.close()
+        accept_logfile.close()
+
+        testlib.cmd_index += 2
+        return int(expected_failure)
+    elif testlib.debug_cmd_index == testlib.cmd_index+1:
+        p2 = subprocess.Popen(["./tests/t_accept"],
+                              stdin=pipe1[0], stdout=pipe0[1],
+                              stderr=accept_logfile, env=svcenv,
+                              preexec_fn=os.setsid, shell=True,
+                              executable="/bin/bash")
+        print("PID: %d\n" % p2.pid)
+        print("Attach and start debugging, then press enter to start t_init.")
+        input()
+
+        p1 = subprocess.Popen(init_cmd,
+                              stdin=pipe0[0], stdout=pipe1[1],
+                              stderr=init_logfile, env=clienv,
+                              preexec_fn=os.setsid, shell=True,
+                              executable="/bin/bash")
+        print("To resume tests if hung, kill pid %d\n" % p1.pid)
+        p1.wait()
+
+        init_logfile.close()
+        accept_logfile.close()
+
+        testlib.cmd_index += 2
+        return int(expected_failure)
+
+    if testlib.valgrind_everywhere:
+        accept_cmd = testlib.valgrind_cmd + accept_cmd
+        init_cmd = testlib.valgrind_cmd + init_cmd
+    p2 = subprocess.Popen(accept_cmd,
+                          stdin=pipe1[0], stdout=pipe0[1],
+                          stderr=accept_logfile, env=svcenv,
+                          preexec_fn=os.setsid, shell=True,
+                          executable="/bin/bash")
     p1 = subprocess.Popen(init_cmd,
                           stdin=pipe0[0], stdout=pipe1[1],
-                          stderr=logfile, env=clienv, preexec_fn=os.setsid)
-    p2 = subprocess.Popen(["./tests/t_accept"],
-                          stdin=pipe1[0], stdout=pipe0[1],
-                          stderr=logfile, env=svcenv, preexec_fn=os.setsid)
+                          stderr=init_logfile, env=clienv,
+                          preexec_fn=os.setsid, shell=True,
+                          executable="/bin/bash")
 
     try:
-        p1.wait(10)
-        p2.wait(10)
+        p1.wait(testlib.testcase_wait)
+        p2.wait(testlib.testcase_wait)
     except subprocess.TimeoutExpired:
         # {p1,p2}.returncode are set to None here
-        pass
-    print_return(p1.returncode, "Init", expected_failure)
-    print_return(p2.returncode, "Accept", expected_failure)
+        if not expected_failure:
+            testlib.print_warning("warning", "timeout")
+    init_logfile.close()
+    accept_logfile.close()
+    testlib.print_return(p1.returncode, testlib.cmd_index,
+                         "(%d) Init" % testlib.cmd_index,
+                         expected_failure)
+    testlib.print_return(p2.returncode, testlib.cmd_index + 1,
+                         "(%d) Accept" % (testlib.cmd_index + 1),
+                         expected_failure)
+    testlib.cmd_index += 2
     try:
         os.killpg(p1.pid, signal.SIGTERM)
         os.killpg(p2.pid, signal.SIGTERM)
@@ -59,3 +142,7 @@ def run(testdir, env, conf, expected_failure=False):
     elif p2.returncode != 0:
         return p2.returncode if not expected_failure else int(not p2.returncode)
     return int(expected_failure)
+
+if __name__ == "__main__":
+    from runtests import runtests_main
+    runtests_main(["t_basic.py"])
diff --git a/proxy/tests/t_cred_store.py b/proxy/tests/t_cred_store.py
index 76a5ad0..253233e 100755
--- a/proxy/tests/t_cred_store.py
+++ b/proxy/tests/t_cred_store.py
@@ -5,7 +5,10 @@ from testlib import *
 
 def run(testdir, env, conf):
     print("Testing cred store extensions...", file=sys.stderr)
-    logfile = conf["logfile"]
+    conf['prefix'] = str(cmd_index)
+
+    logfile = os.path.join(conf["logpath"], "test_%d.log" % cmd_index)
+    logfile = open(logfile, 'a')
 
     ccache = "FILE:" + os.path.join(testdir, "t" + conf["prefix"] +
                                     "_cred_store.ccache")
@@ -26,16 +29,10 @@ def run(testdir, env, conf):
     testenv.update(env)
     temp_ccache = "FILE:" + os.path.join(testdir, "t" + conf["prefix"] +
                                          "_temp.ccache")
-    cmd = ["./tests/t_cred_store", ccache, temp_ccache]
-    print("[COMMAND]\n%s\n[ENVIRONMENT]\n%s\n" % (cmd, testenv), file=logfile)
-    logfile.flush()
+    cmd = " ".join(["./tests/t_cred_store", ccache, temp_ccache])
+
+    return run_testcase_cmd(testenv, conf, cmd, "Cred store")
 
-    p1 = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=logfile,
-                          env=testenv, preexec_fn=os.setsid)
-    try:
-        p1.wait()
-    except subprocess.TimeoutExpired:
-        # p1.returncode is set to None here
-        pass
-    print_return(p1.returncode, "Cred store", False)
-    return p1.returncode
+if __name__ == "__main__":
+    from runtests import runtests_main
+    runtests_main(["t_cred_store.py"])
diff --git a/proxy/tests/t_impersonate.py b/proxy/tests/t_impersonate.py
index 29f9a41..86775e0 100755
--- a/proxy/tests/t_impersonate.py
+++ b/proxy/tests/t_impersonate.py
@@ -35,8 +35,7 @@ IMPERSONATE_CONF_TEMPLATE = '''
 '''
 
 def run_cmd(testdir, env, conf, name, socket, cmd, keytab, expected_failure):
-
-    logfile = conf['logfile']
+    conf['prefix'] = str(cmd_index)
     testenv = env.copy()
     testenv.update({'KRB5CCNAME': os.path.join(testdir, 't' + conf['prefix'] +
                                                '_impersonate.ccache'),
@@ -47,19 +46,7 @@ def run_cmd(testdir, env, conf, name, socket, cmd, keytab, expected_failure):
                     'GSSPROXY_SOCKET': socket,
                     'GSSPROXY_BEHAVIOR': 'REMOTE_FIRST'})
 
-    print("\nTesting: [%s]" % (name,), file=logfile)
-    print("[COMMAND]\n%s\n[ENVIRONMENT]\n%s\n" % (cmd, testenv), file=logfile)
-    logfile.flush()
-
-    p1 = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=logfile,
-                          env=testenv, preexec_fn=os.setsid)
-    try:
-        p1.wait(10)
-    except subprocess.TimeoutExpired:
-        # p1.returncode is set to None here
-        pass
-    print_return(p1.returncode, name, expected_failure)
-    return p1.returncode if not expected_failure else int(not p1.returncode)
+    return run_testcase_cmd(testenv, conf, cmd, name, expected_failure)
 
 def run(testdir, env, conf):
     print("Testing impersonate creds...", file=sys.stderr)
@@ -77,56 +64,56 @@ def run(testdir, env, conf):
     # Test all permitted
     msg = "Impersonate"
     socket = os.path.join(testdir, 'impersonate.socket')
-    cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
-           path_prefix + 'impersonate.cache']
+    cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
+           path_prefix + 'impersonate.cache'])
     r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, False)
     rets.append(r)
 
     #Test self fail
     msg = "Impersonate fail self"
     socket = os.path.join(testdir, 'impersonate-proxyonly.socket')
-    cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
-           path_prefix + 'impersonate.cache']
+    cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
+           path_prefix + 'impersonate.cache'])
     r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, True)
     rets.append(r)
 
     #Test proxy fail
     msg = "Impersonate fail proxy"
     socket = os.path.join(testdir, 'impersonate-selfonly.socket')
-    cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
-           path_prefix + 'impersonate.cache']
+    cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
+           path_prefix + 'impersonate.cache'])
     r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, True)
     rets.append(r)
 
     #Test s4u2self half succeed
     msg = "s4u2self delegation"
     socket = os.path.join(testdir, 'impersonate-selfonly.socket')
-    cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
-           path_prefix + 'impersonate.cache', 's4u2self']
+    cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
+           path_prefix + 'impersonate.cache', 's4u2self'])
     r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, False)
     rets.append(r)
 
     #Test proxy to self succeed
     msg = "Impersonate to self"
     socket = os.path.join(testdir, 'impersonate-selfonly.socket')
-    cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, HOST_GSS,
-           path_prefix + 'impersonate.cache', 's4u2proxy']
+    cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, HOST_GSS,
+           path_prefix + 'impersonate.cache', 's4u2proxy'])
     r = run_cmd(testdir, env, conf, msg, socket, cmd, SVC_KTNAME, False)
     rets.append(r)
 
     #Test s4u2proxy half fail
     msg = "s4u2proxy fail"
     socket = os.path.join(testdir, 'impersonate-selfonly.socket')
-    cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
-           path_prefix + 'impersonate.cache', 's4u2proxy']
+    cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
+           path_prefix + 'impersonate.cache', 's4u2proxy'])
     r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, True)
     rets.append(r)
 
     #Test s4u2proxy half succeed
     msg = "s4u2proxy"
     socket = os.path.join(testdir, 'impersonate-proxyonly.socket')
-    cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
-           path_prefix + 'impersonate.cache', 's4u2proxy']
+    cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,
+           path_prefix + 'impersonate.cache', 's4u2proxy'])
     r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, False)
     rets.append(r)
 
@@ -139,3 +126,7 @@ def run(testdir, env, conf):
     if len(e) > 0:
         return e[0]
     return 0
+
+if __name__ == "__main__":
+    from runtests import runtests_main
+    runtests_main(["t_impersonate.py"])
diff --git a/proxy/tests/t_interpose.py b/proxy/tests/t_interpose.py
index b8a38fa..c9b9a22 100755
--- a/proxy/tests/t_interpose.py
+++ b/proxy/tests/t_interpose.py
@@ -5,7 +5,9 @@ from testlib import *
 
 def run(testdir, env, conf):
     print("Testing interposer...", file=sys.stderr)
-    logfile = conf['logfile']
+    conf['prefix'] = str(cmd_index)
+    logfile = os.path.join(conf['logpath'], "test_%d.log" % cmd_index)
+    logfile = open(logfile, 'a')
 
     ienv = {"KRB5CCNAME": os.path.join(testdir, 'interpose_ccache'),
             "KRB5_KTNAME": os.path.join(testdir, SVC_KTNAME)}
@@ -19,10 +21,9 @@ def run(testdir, env, conf):
     if ksetup.returncode != 0:
         raise ValueError('Kinit %s failed' % USR_NAME)
 
-    itest = subprocess.Popen(["./interposetest", "-t",
-                              "host@%s" % WRAP_HOSTNAME],
-                             stdout=logfile, stderr=logfile,
-                             env=ienv)
-    itest.wait()
-    print_return(itest.returncode, "Interpose", False)
-    return itest.returncode
+    cmd = " ".join(["./interposetest", "-t", "host@%s" % WRAP_HOSTNAME])
+    return run_testcase_cmd(ienv, conf, cmd, "Interpose")
+
+if __name__ == "__main__":
+    from runtests import runtests_main
+    runtests_main(["t_interpose.py"])
diff --git a/proxy/tests/t_multi_key.py b/proxy/tests/t_multi_key.py
index 3e78d1b..c08930d 100755
--- a/proxy/tests/t_multi_key.py
+++ b/proxy/tests/t_multi_key.py
@@ -19,6 +19,7 @@ from testlib import *
 
 def run(testdir, env, conf):
     setup_multi_keys(testdir, env)
+    conf['prefix'] = str(cmd_index)
     prefix = conf["prefix"]
 
     print("Testing multiple keys Keytab with first principal",
@@ -56,3 +57,7 @@ def run(testdir, env, conf):
     elif r2 != 0:
         return r2
     return 0
+
+if __name__ == "__main__":
+    from runtests import runtests_main
+    runtests_main(["t_multi_key.py"])
diff --git a/proxy/tests/t_program.py b/proxy/tests/t_program.py
index 1f71f07..37b0c60 100644
--- a/proxy/tests/t_program.py
+++ b/proxy/tests/t_program.py
@@ -22,6 +22,7 @@ GSSPROXY_PROGRAM = '''
 '''
 
 def run(testdir, env, conf):
+    conf['prefix'] = str(cmd_index)
     prefix = conf["prefix"]
     retval = 0
 
@@ -47,5 +48,9 @@ def run(testdir, env, conf):
     os.kill(conf["gpid"], signal.SIGHUP)
     time.sleep(1)
 
-    print_return(retval, "Program", False)
+    print_return(retval, -1, "Program", False)
     return retval
+
+if __name__ == "__main__":
+    from runtests import runtests_main
+    runtests_main(["t_program.py"])
diff --git a/proxy/tests/t_reloading.py b/proxy/tests/t_reloading.py
index 5933ce3..e49f683 100755
--- a/proxy/tests/t_reloading.py
+++ b/proxy/tests/t_reloading.py
@@ -5,6 +5,7 @@ from testlib import *
 from t_basic import run as run_basic_test
 
 def run(testdir, env, basicconf):
+    basicconf['prefix'] = str(cmd_index)
     prefix = basicconf['prefix']
     keysenv = basicconf["keysenv"]
 
@@ -56,3 +57,7 @@ def run(testdir, env, basicconf):
     if len(e) > 0:
         return e[0]
     return 0
+
+if __name__ == "__main__":
+    from runtests import runtests_main
+    runtests_main(["t_reloading.py"])
diff --git a/proxy/tests/t_setcredopt.py b/proxy/tests/t_setcredopt.py
index af10d74..1655852 100755
--- a/proxy/tests/t_setcredopt.py
+++ b/proxy/tests/t_setcredopt.py
@@ -5,10 +5,13 @@ from testlib import *
 
 def run(testdir, env, conf):
     print("Testing setting credential options...", file=sys.stderr)
+    conf['prefix'] = str(cmd_index)
     path_prefix = os.path.join(testdir, 't' + conf['prefix'] + '_')
     init_ccache = path_prefix + 'sco_init.ccache'
 
-    logfile = conf['logfile']
+    logfile = os.path.join(conf['logpath'], "test_%d.log" % cmd_index)
+    logfile = open(logfile, 'a')
+
     testenv = env.copy()
     testenv.update({'KRB5CCNAME': init_ccache})
 
@@ -21,22 +24,15 @@ def run(testdir, env, conf):
         raise ValueError("Kinit %s failed" % USR_NAME)
 
 
-    cmd = ["./tests/t_setcredopt", USR_NAME, HOST_GSS, init_ccache]
+    cmd = " ".join(["./tests/t_setcredopt", USR_NAME, HOST_GSS, init_ccache])
 
     testenv.update({'KRB5CCNAME': path_prefix + 'sco.ccache',
                     'KRB5_KTNAME': os.path.join(testdir, PROXY_KTNAME),
                     'KRB5_TRACE': path_prefix + 'sco.trace',
                     'GSSPROXY_BEHAVIOR': 'REMOTE_FIRST'})
 
-    print("[COMMAND]\n%s\n[ENVIRONMENT]\n%s\n" % (cmd, testenv), file=logfile)
-    logfile.flush()
-
-    p1 = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=logfile,
-                          env=testenv, preexec_fn=os.setsid)
-    try:
-        p1.wait(10)
-    except subprocess.TimeoutExpired:
-        # p1.returncode is set to None here
-        pass
-    print_return(p1.returncode, "Set cred options", False)
-    return p1.returncode
+    return run_testcase_cmd(testenv, conf, cmd, "Set cred options")
+
+if __name__ == "__main__":
+    from runtests import runtests_main
+    runtests_main(["t_setcredopt.py"])
diff --git a/proxy/tests/testlib.py b/proxy/tests/testlib.py
index 781275a..05c734d 100755
--- a/proxy/tests/testlib.py
+++ b/proxy/tests/testlib.py
@@ -11,6 +11,17 @@ import subprocess
 import sys
 import time
 
+testcase_wait = 15
+cmd_index = 0
+
+debug_all = False
+debug_gssproxy = False
+debug_cmd = "gdb --args"
+debug_cmd_index = -1
+
+valgrind_cmd = "valgrind", "--track-origins=yes"
+valgrind_everywhere = False
+
 try:
     from colorama import Fore, Style
 
@@ -28,6 +39,20 @@ except ImportError:
     def format_key(status, key):
         return "[" + key + "]"
 
+def testlib_process_args(args):
+    global debug_all, debug_cmd, debug_cmd_index, debug_gssproxy
+    global testcase_wait, valgrind_cmd, valgrind_everywhere
+
+    testcase_wait = args['timeout']
+    debug_cmd_index = args['debug_num']
+
+    debug_all = args['debug_all']
+    debug_cmd = args['debug_cmd'] + " "
+    debug_gssproxy = args['debug_gssproxy']
+
+    valgrind_cmd = args['valgrind_cmd'] + " "
+    valgrind_everywhere = args['force_valgrind']
+
 def print_keyed(status, key, text, io):
     print("%s %s" % (format_key(status, key), text), file=io)
 
@@ -42,7 +67,7 @@ def print_failure(key, text, io=sys.stderr):
 def print_warning(key, text, io=sys.stderr):
     print_keyed("other", key, text, io)
 
-def print_return(ret, name, expected_failure):
+def print_return(ret, num, name, expected_failure):
     key = "PASS"
     expected = "zero" if not expected_failure else "nonzero"
     if (ret == 0 and expected_failure) or \
@@ -54,9 +79,68 @@ def print_return(ret, name, expected_failure):
     else:
         print_failure(key, "%s test returned %s (expected %s)" %
                       (name, str(ret), expected))
+        if num != -1:
+            print_warning("INFO", "To debug this test case, run:\n" +
+                          ("    make check CHECKARGS='--debug-num=%d'" % num))
 
 WRAP_HOSTNAME = "kdc.gssproxy.dev"
 
+def run_testcase_cmd(env, conf, cmd, name, expected_failure=False, wait=True):
+    global testcase_wait, debug_cmd_index, cmd_index
+    global valgrind_everywhere, valgrind_cmd, debug_all
+
+    logfile = os.path.join(conf['logpath'], "test_%d.log" % cmd_index)
+    logfile = open(logfile, 'w')
+
+    print("[NAME]\n%s\n[COMMAND %d]\n%s\n[ENVIRONMENT]\n%s\n\n" % (name,
+          cmd_index, cmd, env), file=logfile)
+    logfile.flush()
+
+    testenv = env.copy()
+
+    if debug_all or debug_cmd_index == cmd_index:
+        return rundebug_cmd(testenv, conf, cmd, name, expected_failure)
+
+    run_cmd = cmd
+    if valgrind_everywhere:
+        run_cmd = valgrind_cmd + cmd
+
+    p1 = subprocess.Popen(run_cmd, stderr=subprocess.STDOUT, stdout=logfile,
+                          env=testenv, preexec_fn=os.setsid, shell=True,
+                          executable="/bin/bash")
+
+    if not wait:
+        cmd_index += 1
+        conf['prefix'] = str(cmd_index)
+        return p1
+
+    try:
+        p1.wait(testcase_wait)
+    except subprocess.TimeoutExpired:
+        # p1.returncode is set to None here
+        if not expected_failure:
+            print_warning("warning", "timeout")
+
+    logfile.close()
+    print_return(p1.returncode, cmd_index, "(%d) %s" % (cmd_index, name),
+                 expected_failure)
+    cmd_index += 1
+    conf['prefix'] = str(cmd_index)
+    return p1.returncode if not expected_failure else int(not p1.returncode)
+
+def rundebug_cmd(env, conf, cmd, name, expected_failure=False):
+    global debug_cmd, cmd_index
+
+    run_cmd = debug_cmd + cmd
+
+    returncode = subprocess.call(run_cmd, env=env, shell=True,
+                                 executable="/bin/bash")
+
+    print_return(returncode, cmd_index, "(%d) %s" % (cmd_index, name),
+                 expected_failure)
+    cmd_index += 1
+
+    return returncode if not expected_failure else int(not returncode)
 
 def setup_wrappers(base):
 
@@ -633,6 +717,7 @@ def update_gssproxy_conf(testdir, env, template):
         f.write(text)
 
 def setup_gssproxy(testdir, logfile, env):
+    global debug_gssproxy, valgrind_cmd
 
     gssproxy = os.path.join(testdir, 'gssproxy')
     if os.path.exists(gssproxy):
@@ -646,9 +731,22 @@ def setup_gssproxy(testdir, logfile, env):
 
     socket = os.path.join(gssproxy, 'gp.sock')
     conf = os.path.join(gssproxy, 'gp.conf')
-    gproc = subprocess.Popen(["valgrind", "--track-origins=yes",
-                              "./gssproxy", "-i", "-s", socket, "-c", conf],
+
+    cmd = "./gssproxy -i -s " + socket + " -c " + conf
+
+    full_command = valgrind_cmd + cmd
+
+    if debug_gssproxy:
+        full_command = cmd
+
+    gproc = subprocess.Popen(full_command,
                              stdout=logfile, stderr=logfile,
-                             env=gpenv, preexec_fn=os.setsid)
+                             env=gpenv, preexec_fn=os.setsid, shell=True,
+                             executable="/bin/bash")
+
+    if debug_gssproxy:
+        print("PID: %d" % (gproc.pid))
+        print("Attach and start debugging, then press enter to continue.")
+        input()
 
     return gproc, socket

