PR#35: Add a new runtests and combine it with upload script - kernel-tests

kernel-tests

#35 Add a new runtests and combine it with upload script

Opened 4 years ago by jcline. Modified 4 years ago

jcline/kernel-tests update-test-submission into master

Add a new runtests and combine it with upload script

Jeremy Cline • 4 years ago

6037b63

fedora_submit.py

file removed

-55

		`@@ -1,55 +0,0 @@`
		`- #!/usr/bin/env python3`
		`- #`
		`- # Licensed under the terms of the GNU GPL License version 2`
		`-`
		`- from __future__ import print_function`
		`- from fedora.client import OpenIdBaseClient`
		`- import getpass`
		`- import sys, getopt`
		`-`
		`- username = ''`
		`- password = ''`
		`- log = ''`
		`-`
		`- argv = sys.argv[1:]`
		`- try:`
		`- opts, args = getopt.getopt(argv,"hu:p:l:",["user=","password=", "logfile="])`
		`- except getopt.GetoptError:`
		`- print('fedora_submit.py -u <fasuser> [-p <password>] -l <logfile>')`
		`- sys.exit(2)`
		`- for opt, arg in opts:`
		`- if opt == '-h':`
		`- print('fedora_submit.py -u <fasuser> [-p <password>] -l <logfile>')`
		`- sys.exit()`
		`- elif opt in ("-u", "--user"):`
		`- username = arg`
		`- elif opt in ("-p", "--password"):`
		`- password = arg`
		`- elif opt in ("-l", "--logfile"):`
		`- log = arg`
		`-`
		`- if username == '' or log == '':`
		`- print('fedora_submit.py -u <fasuser> [-p <password>] -l <logfile>')`
		`- sys.exit(2)`
		`- if password == '':`
		`- password = getpass.getpass('FAS password: ')`
		`-`
		`- submitclient = OpenIdBaseClient(`
		`- base_url='https://apps.fedoraproject.org/kerneltest/',`
		`- login_url='https://apps.fedoraproject.org/kerneltest/login',`
		`- username=username,`
		`- )`
		`-`
		`- submitclient.login(`
		`- submitclient.username,`
		`- password=password`
		`- )`
		`-`
		`- req = submitclient.send_request(`
		`- 'https://apps.fedoraproject.org/kerneltest/upload/anonymous',`
		`- verb='POST',`
		`- auth=True,`
		`- files= { 'test_result': ('logfile', open(log, 'rb'), 'text/x-log'),}`
		`- )`
		`-`
		`- print(req.message)`

runtests

file added

+183

		`@@ -0,0 +1,183 @@`
		`+ #!/usr/bin/env python3`
		`+`
		`+ import argparse`
		`+ import subprocess`
		`+ import json`
		`+ import os`
		`+ import sys`
		`+ import time`
		`+`
		`+ from openidc_client import OpenIDCClient, requestsauth`
		`+ import requests`
		`+`
		`+ BASE_URL = "https://apps.fedoraproject.org/kerneltest"`
		`+ ID_PROVIDER = "https://id.fedoraproject.org/openidc/"`
		`+ APP_IDENTIFIER = "kerneltest"`
		`+ SCOPES = [`
		`+ "openid",`
		`+ "https://github.com/jmflinuxtx/kerneltest-harness/oidc/upload_test_run",`
		`+ ]`
		`+ ID_PROVIDER_MAPPING = {`
		`+ "Authorization": "Authorization",`
		`+ "Token": "Token",`
		`+ "UserInfo": "UserInfo",`
		`+ }`
		`+ CLIENT_ID = "kerneltest"`
		`+ CLIENT_SECRET = "notsecret"`
		`+ CACHEDIR = "~/.cache/kerneltest"`
		`+`
		`+`
		`+ def upload_results(results, anonymous=False):`
		`+ client = OpenIDCClient(`
		`+ APP_IDENTIFIER,`
		`+ ID_PROVIDER,`
		`+ ID_PROVIDER_MAPPING,`
		`+ CLIENT_ID,`
		`+ client_secret=CLIENT_SECRET,`
		`+ cachedir=CACHEDIR,`
		`+ )`
		`+ auth = None if anonymous else requestsauth.OpenIDCClientAuther(client, SCOPES)`
		`+ response = requests.post(BASE_URL + "/api/v1/results/", auth=auth, json=results)`
		`+ if response.status_code != 201:`
		`+ print(response.status_code)`
		`+ print(response.text)`
		`+`
		`+`
		`+ parser = argparse.ArgumentParser(description="Run and submit kernel tests")`
		`+ parser.add_argument("--logfile", help="The log file from a previous test run")`
		`+ parser.add_argument(`
		`+ "--upload", action="store_true", help="Upload the results to Fedora's servers"`
		`+ )`
		`+ parser.add_argument(`
		`+ "--anonymous", action="store_true", help="Upload the results anonymously"`
		`+ )`
		`+ parser.add_argument(`
		`+ "--minimal", action="store_true", help="Run only the minimal test suite"`
		`+ )`
		`+ parser.add_argument("--stress", action="store_true", help="Run the stress test suite")`
		`+ parser.add_argument(`
		`+ "--performance", action="store_true", help="Run the performance test suite"`
		`+ )`
		`+ parser.add_argument(`
		`+ "--destructive", action="store_true", help="Run the destructive test suite"`
		`+ )`
		`+ parser.add_argument(`
		`+ "--thirdparty", action="store_true", help="Run the third-party test suite"`
		`+ )`
		`+ parser.add_argument(`
		`+ "--secureboot", action="store_true", help="Check the Secure Boot signature"`
		`+ )`
		`+ parser.add_argument("--disable-retest", help="Disable re-testing.")`
		`+ args = parser.parse_args()`
		`+`
		`+`
		`+ if args.logfile:`
		`+ with open(args.logfile) as fd:`
		`+ results = json.load(fd)`
		`+ try:`
		`+ upload_results(results, args.anonymous)`
		`+ sys.exit(0)`
		`+ except Exception:`
		`+ sys.exit(1)`
		`+`
		`+ encoding = sys.getdefaultencoding()`
		`+`
		`+ result = subprocess.run(["uname", "-r"], encoding=encoding, capture_output=True)`
		`+ rpm_version = result.stdout.strip()`
		`+ kernel_version, release_and_arch = rpm_version.split("-")`
		`+ build_release, arch = release_and_arch.rsplit(".", 1)`
		`+ with open("/etc/os-release") as fd:`
		`+ for line in fd.readlines():`
		`+ if line.startswith("VERSION_ID="):`
		`+ fedora_version = int(line.split("=")[1])`
		`+ break`
		`+ else:`
		`+ raise Exception("No VERSION_ID found in /etc/os-release")`
		`+`
		`+ test_results = {`
		`+ "kernel_version": kernel_version,`
		`+ "build_release": build_release,`
		`+ "arch": arch,`
		`+ "fedora_version": fedora_version,`
		`+ "tests": [],`
		`+ }`
		`+`
		`+`
		`+ directories = ["minimal", "default"]`
		`+ if args.minimal:`
		`+ directories = ["minimal"]`
		`+ if args.destructive:`
		`+ directories.append("destructive")`
		`+ if args.performance:`
		`+ directories.append("performance")`
		`+ if args.secureboot:`
		`+ directories.append("secureboot")`
		`+ if args.stress:`
		`+ directories.append("stress")`
		`+ if args.thirdparty:`
		`+ directories.append("thirdparty")`
		`+ if args.stress:`
		`+ directories.append("stress")`
		`+`
		`+ # run test, check return code`
		`+ # 0 = pass`
		`+ # 3 = skip`
		`+ # 4 - warn`
		`+ # anything else is fail`
		`+ root_dir = os.getcwd()`
		`+ for directory in directories:`
		`+ for dirpath, dirnames, filenames in os.walk(directory):`
		`+ if "runtest.sh" in filenames:`
		`+ os.chdir(dirpath)`
		`+ print("Running " + dirpath)`
		`+ result = subprocess.run(`
		`+ ["./runtest.sh"], encoding=encoding, shell=True, capture_output=True`
		`+ )`
		`+ os.chdir(root_dir)`
		`+ test_results["tests"].append(`
		`+ {`
		`+ "name": dirpath,`
		`+ "passed": result.returncode == 0,`
		`+ "waived": result.returncode == 3,`
		`+ "details": result.stdout,`
		`+ }`
		`+ )`
		`+`
		`+ logdir = os.path.join(root_dir, "logs", str(fedora_version), rpm_version)`
		`+ logfile = os.path.join(`
		`+ logdir, "{suite}-{date}.log".format(suite="-".join(directories), date=time.time())`
		`+ )`
		`+`
		`+ os.makedirs(logdir, exist_ok=True)`
		`+ with open(logfile, "w") as fd:`
		`+ json.dump(test_results, fd, sort_keys=True, indent=4)`
		`+`
		`+ if args.upload:`
		`+ try:`
		`+ upload_results(test_results, args.anonymous)`
		`+ except Exception as e:`
		`+ print("Failed to upload results: " + str(e))`
		`+`
		`+ print("Your log file is located at: " + logfile)`
		`+ print("Submit your results to: https://apps.fedoraproject.org/kerneltest/")`
		`+`
		`+ print("The following information is not submitted with your log;")`
		`+ print("it is for informational purposes only.")`
		`+`
		`+ if os.path.exists("/usr/bin/pesign"):`
		`+ sign_result = subprocess.run(`
		`+ ["pesign", "-i", "/boot/vmlinuz-{}".format(rpm_version), "-S"]`
		`+ )`
		`+ print(sign_result)`
		`+`
		`+ print("Vulnerability status:")`
		`+ vulns = "/sys/devices/system/cpu/vulnerabilities/"`
		`+ for vuln in os.listdir(vulns):`
		`+ with open(os.path.join(vulns, vuln)) as fd:`
		`+ print("{}: {}".format(vuln, fd.read().strip()))`
		`+`
		`+`
		`+ if all([test["passed"] for test in test_results["tests"]]):`
		`+ sys.exit(0)`
		`+ else:`
		`+ sys.exit(1)`

runtests.sh

file removed

-211

		`@@ -1,211 +0,0 @@`
		`- #!/bin/bash`
		`- #`
		`- # Licensed under the terms of the GNU GPL License version 2.`
		`-`
		`- date=$(date +%s)`
		`- logdir=$(pwd)/logs`
		`- logfile=$logdir/kernel-test-$date.log.txt`
		`- verbose=n`
		`- testset=default`
		`- cleanrun=PASS`
		`- failedtests=None`
		`- warntests=None`
		`- commit=n`
		`- commithook=/usr/bin/true`
		`- disable_retest=n`
		`- thirdparty=n`
		`-`
		`- if [ -f ./.config ]; then`
		`- source ./.config`
		`- else`
		`- echo "No .config file found. You can cp config.example .config and edit as needed for easier log submission."`
		`- fi`
		`-`
		`- kver=$(uname -r)`
		`- release=$(cat /etc/redhat-release)`
		`-`
		`- # Check for pre-requisites.`
		`- if [ ! -f /usr/bin/gcc ]; then`
		`- echo " Could not locate gcc. The Fedora kernel test suite requires gcc."`
		`- exit`
		`- fi`
		`-`
		`- # Unset MALLOC_CHECK_ and MALLOC_PERTURB_. Some tests might not work`
		`- # well with those active (e.g. libhugetlbfs).`
		`- unset MALLOC_CHECK_`
		`- unset MALLOC_PERTURB_`
		`-`
		`- if [ ! -d "$logdir" ] ; then`
		`- mkdir $logdir`
		`- fi`
		`-`
		`- args=y`
		`-`
		`- while [ $args = y ]`
		`- do`
		`- case "$1" in`
		`- -v)`
		`- #TO DO: Implement verbose behavior.`
		`- verbose=y`
		`- shift 1`
		`- ;;`
		`- -t)`
		`- testset=$2`
		`- shift 2`
		`- ;;`
		`- *)`
		`- args=n`
		`- esac`
		`- done`
		`-`
		`-`
		`- if [ "$disable_retest" == "y" ]; then`
		`- # Check if wanted test has been executed with current kernel`
		`- # version.`
		`- for file in $(find $logdir -name \*.log.txt); do`
		`- version_tested=$(cat $file \| sed -n 3p \| cut -d ' ' -f 2)`
		`- test_executed=$(cat $file \| sed -n 2p \| cut -d ' ' -f 3)`
		`-`
		`- if [ "$version_tested" == "$kver" -a "$test_executed" == "$testset" ]; then`
		`- echo "The current kernel was already tested with this test, abort."`
		`- exit 0`
		`- fi`
		`- done`
		`- fi`
		`-`
		`- case $testset in`
		`- minimal)`
		`- dirlist="minimal"`
		`- ;;`
		`- default)`
		`- dirlist="minimal default"`
		`- ;;`
		`- stress)`
		`- dirlist="minimal default stress"`
		`- ;;`
		`- destructive)`
		`- echo "You have specified the destructive test set."`
		`- echo "This test may cause damage to your system."`
		`- echo "Are you sure that you wish to continue?"`
		`- read continue`
		`- if [ $continue == 'y' ] ; then`
		`- dirlist="minimal default destructive"`
		`- else`
		`- dirlist="minimal default"`
		`- testset=default`
		`- fi`
		`- ;;`
		`- performance)`
		`- dirlist="performance"`
		`- ;;`
		`- *)`
		`- echo "Supported test sets are minimal, default, stress, destructive or performance."`
		`- exit 1`
		`- esac`
		`-`
		`- # Test Secure Boot?`
		`- if [ "$checksig" == "y" ]; then`
		`- dirlist="secureboot $dirlist"`
		`- fi`
		`-`
		`- # Test Third Party Modules?`
		`- if [ "$thirdparty" == "y" ]; then`
		`- dirlist="$dirlist thirdparty"`
		`- fi`
		`-`
		`- # Basic logfile headers.`
		`- echo "Date: $(date)" > $logfile`
		`- echo "Test set: $testset" >> $logfile`
		`- echo "Kernel: $kver" >> $logfile`
		`- echo "Release: $release" >> $logfile`
		`- echo "Result: RESULTHOLDER" >> $logfile`
		`- echo "============================================================" >>$logfile`
		`-`
		`-`
		`-`
		`- # Start running tests.`
		`- echo "Test suite called with $testset"`
		`-`
		`- for dir in $dirlist`
		`- do`
		`- for test in $(find ./$dir/ -name runtest.sh)`
		`- do`
		`- testdir=$(dirname $test)`
		`- pushd $testdir &>/dev/null`
		`- #TO DO: Purpose file test name format.`
		`- testname=$testdir`
		`- echo "Starting test $testname" >> $logfile`
		`-`
		`- if [ "$testset" == "performance" ]; then`
		`- ./runtest.sh >>$logfile`
		`- elif [ "$dir" == "secureboot" ]; then`
		`- ./runtest.sh "$validsig" &>>$logfile`
		`- else`
		`- ./runtest.sh &>>$logfile`
		`- fi`
		`- complete=$?`
		`- case $complete in`
		`- 0)`
		`- result=PASS`
		`- ;;`
		`- 3)`
		`- result=SKIP`
		`- ;;`
		`- 4)`
		`- result=WARN`
		`- ;;`
		`- *)`
		`- result=FAIL`
		`- esac`
		`- printf "%-65s%-8s\n" "$testname" "$result"`
		`- if [ "$result" == "FAIL" ]; then`
		`- cleanrun=FAIL`
		`- if [ "$failedtests" == "None" ]; then`
		`- failedtests="$testname"`
		`- else`
		`- failedtests="$failedtests $testname"`
		`- fi`
		`- fi`
		`- if [ "$result" == "WARN" ]; then`
		`- if [ "$cleanrun" == "PASS" ]; then`
		`- cleanrun=WARN`
		`- fi`
		`- if [ "$warntests" == "None" ]; then`
		`- warntests="$testname"`
		`- else`
		`- warntests="$warntests $testname"`
		`- fi`
		`- fi`
		`- popd &>/dev/null`
		`- done`
		`- done`
		`-`
		`- # Fix up logfile headers.`
		`- sed -i "s,RESULTHOLDER,$cleanrun\nFailed Tests: $failedtests\nWarned Tests: $warntests,g" $logfile`
		`- printf "\n%-65s%-8s\n" "Test suite complete" "$cleanrun"`
		`-`
		`- if [ "$commit" == "y" ]; then`
		`- printf "\nYour log file is being submitted...\n"`
		`- $commithook`
		`- else`
		`- printf "\nYour log file is located at: $logfile\n"`
		`- printf "Submit your results to: https://apps.fedoraproject.org/kerneltest/\n"`
		`- fi`
		`-`
		`- echo "The following information is not submitted with your log;"`
		`- echo "it is for informational purposes only."`
		`-`
		`- if [ -f /usr/bin/pesign ]; then`
		`- echo "Checking for kernel signature:"`
		`- /usr/bin/pesign -i /boot/vmlinuz-$kver -S \| grep "common name"`
		`- fi`
		`-`
		`- echo "Vulnerability status:"`
		`- grep . /sys/devices/system/cpu/vulnerabilities/*`
		`-`
		`- if [ "$cleanrun" == "FAIL" ]; then`
		`- exit 1`
		`- else`
		`- exit 0`
		`- fi`
		`-`

jcline commented 4 years ago

I have no memory of writing this so I don't recall if it works or not, but in theory this should submit results to the version of kerneltests-harness that supports OpenID Connect.

gtwilliams commented 4 years ago

Not sure what your intent is, but I merged your pull request and ran sudo ./runtests. My test results were not uploaded. When I run runtests.sh before this change, my results are automatically uploaded. (I also see an indication of pass/fail on the console when I run runtests.sh before this change.

Maybe I do not understand the purpose of the pull request.

garry@tfr$ sudo ./runtests
[sudo] password for garry:
Running default/memfd
Running default/insert_leap_second
Running default/selinux-dac-controls
Running default/modsign
Running default/mq-memory-corruption
Running default/paxtest
Running default/libhugetlbfs
Running default/sysfs-perms
Running default/cachedrop
Running default/posix_timers
Running default/stack-randomness
Running default/timer-overhead
Your log file is located at: /home/garry/src/kernel-tests/logs/32/5.6.10-300.fc32.x86_64/minimal-default-1588903174.0848687.log
Submit your results to: https://apps.fedoraproject.org/kerneltest/
The following information is not submitted with your log;
it is for informational purposes only.
Vulnerability status:
spectre_v2: Mitigation: Full generic retpoline, IBPB: conditional, IBRS_FW, STIBP: conditional, RSB filling
itlb_multihit: KVM: Mitigation: Split huge pages
mds: Mitigation: Clear CPU buffers; SMT vulnerable
l1tf: Mitigation: PTE Inversion; VMX: conditional cache flushes, SMT vulnerable
spec_store_bypass: Mitigation: Speculative Store Bypass disabled via prctl and seccomp
tsx_async_abort: Not affected
spectre_v1: Mitigation: usercopy/swapgs barriers and __user pointer sanitization
meltdown: Mitigation: PTI
garry@tfr$

jforbes commented 4 years ago

That is because this pull request has nothing to do with the code that is currently running on https://apps.fedoraproject.org/kerneltest/
It is required for a rewrite of the app which is currently in staging. Provided testing works out, this code would be merged at the time that the front end code is promoted from staging.