From b74b7e87bb1a0529adcf9088cfd08e3ac0b04c44 Mon Sep 17 00:00:00 2001 From: Jouni Malinen Date: Sun, 27 Oct 2013 01:05:45 +0300 Subject: [PATCH] tests: Add support for sqlite results database This is more convenient to use directly than going through the text-based results file. Signed-hostap: Jouni Malinen --- tests/hwsim/README | 15 +++++++++++++++ tests/hwsim/run-all.sh | 19 ++++++++++++++----- tests/hwsim/run-tests.py | 46 +++++++++++++++++++++++++++++++++++++++++++--- tests/hwsim/start.sh | 1 + 4 files changed, 73 insertions(+), 8 deletions(-) diff --git a/tests/hwsim/README b/tests/hwsim/README index 90782d9..8110a7a 100644 --- a/tests/hwsim/README +++ b/tests/hwsim/README @@ -175,3 +175,18 @@ line is a convenient way of verifying functionality. run-tests.py will automatically import all test cases from the test_*.py files in this directory. All functions starting with the "test_" prefix in these files are assumed to be test cases. + + +Results database +---------------- + +run-tests.py can be requested to write results from the execution of +each test case into an sqlite database. The "-S " and +"-b " command line arguments can be used to do that. The +database must have been prepared before this, e.g., with following: + +cat | sqlite3 /tmp/example.db < logs/last-debug @@ -11,7 +20,7 @@ if [ "x$1" = "xconcurrent-valgrind" ]; then DATE=`ls -1tr logs | tail -1 | cut -f1 -d-` rm logs/last-debug for i in autogo discovery grpform; do - ./run-tests.py -l logs/$DATE-run-$i -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1 + ./run-tests.py -l logs/$DATE-run-$i $DB -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1 cat logs/$DATE-run-$i >> logs/last-debug done ./stop-wifi.sh @@ -32,7 +41,7 @@ elif [ "x$1" = "xconcurrent" ]; then DATE=`ls -1tr logs | tail -1 | cut -f1 -d-` rm logs/last-debug for i in autogo discovery grpform; do - ./run-tests.py -l logs/$DATE-run-$i -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1 + ./run-tests.py -l logs/$DATE-run-$i $DB -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1 cat logs/$DATE-run-$i >> logs/last-debug done ./stop-wifi.sh @@ -46,7 +55,7 @@ elif [ "x$1" = "xvalgrind" ]; then exit 1 fi DATE=`ls -1tr logs | tail -1 | cut -f1 -d-` - ./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt || errors=1 + ./run-tests.py -l logs/$DATE-run $DB -e logs/$DATE-failed -r logs/results.txt || errors=1 cat logs/$DATE-run > logs/last-debug ./stop-wifi.sh failures=`grep "ERROR SUMMARY" logs/$DATE-valgrind-* | grep -v " 0 errors" | wc -l` @@ -64,7 +73,7 @@ elif [ "x$1" = "xtrace" ]; then exit 1 fi DATE=`ls -1tr logs | tail -1 | cut -f1 -d-` - sudo trace-cmd record -o logs/$DATE-trace.dat -e mac80211 -e cfg80211 su $USER -c "./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt" || errors=1 + sudo trace-cmd record -o logs/$DATE-trace.dat -e mac80211 -e cfg80211 su $USER -c "./run-tests.py -l logs/$DATE-run $DB -e logs/$DATE-failed -r logs/results.txt" || errors=1 if [ -e logs/$DATE-failed ]; then error=1 fi @@ -81,7 +90,7 @@ else exit 1 fi DATE=`ls -1tr logs | tail -1 | cut -f1 -d-` - ./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt || errors=1 + ./run-tests.py -l logs/$DATE-run $DB -e logs/$DATE-failed -r logs/results.txt || errors=1 cat logs/$DATE-run > logs/last-debug ./stop-wifi.sh if [ $errors -gt 0 ]; then diff --git a/tests/hwsim/run-tests.py b/tests/hwsim/run-tests.py index 8a4c9f0..9aaec02 100755 --- a/tests/hwsim/run-tests.py +++ b/tests/hwsim/run-tests.py @@ -10,6 +10,7 @@ import os import re import sys import time +import sqlite3 from datetime import datetime import logging @@ -29,11 +30,29 @@ def reset_devs(dev, apdev): for ap in apdev: hapd.remove(ap['ifname']) +def report(conn, build, commit, run, test, result, diff): + if conn: + if not build: + build = '' + if not commit: + commit = '' + sql = "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES('" + test.replace('test_', '', 1) + "', '" + result + "', " + str(run) + ", " + str(time.time()) + ", " + str(diff.total_seconds()) + ", '" + build + "', '" + commit + "')" + try: + conn.execute(sql) + conn.commit() + except Exception, e: + print "sqlite: " + str(e) + print "sql: " + sql + def main(): test_file = None error_file = None log_file = None results_file = None + conn = None + run = None + build = None + commit = None idx = 1 print_res = False if len(sys.argv) > 1 and sys.argv[1] == '-d': @@ -60,9 +79,22 @@ def main(): elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-f': test_file = sys.argv[idx + 1] idx = idx + 2 + elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-S': + conn = sqlite3.connect(sys.argv[idx + 1]) + idx = idx + 2 + elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-b': + build = sys.argv[idx + 1] + idx = idx + 2 else: break + if conn: + run = str(int(time.time())) + with open("commit") as f: + val = f.readlines() + if len(val) > 0: + commit = val[0].rstrip() + tests = [] for t in os.listdir("."): m = re.match(r'(test_.*)\.py$', t) @@ -125,6 +157,9 @@ def main(): logger.info("Failed to issue TEST-START before " + t.__name__ + " for " + d.ifname) logger.info(e) print "FAIL " + t.__name__ + " - could not start test" + if conn: + conn.close() + conn = None sys.exit(1) try: if t.func_code.co_argcount > 1: @@ -135,11 +170,12 @@ def main(): diff = end - start if res == "skip": skipped.append(t.__name__) - result = "SKIP " + result = "SKIP" else: passed.append(t.__name__) - result = "PASS " - result = result + t.__name__ + " " + result = "PASS" + report(conn, build, commit, run, t.__name__, result, diff) + result = result + " " + t.__name__ + " " result = result + str(diff.total_seconds()) + " " + str(end) logger.info(result) if log_file or print_res: @@ -154,6 +190,7 @@ def main(): diff = end - start logger.info(e) failed.append(t.__name__) + report(conn, build, commit, run, t.__name__, "FAIL", diff) result = "FAIL " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end) logger.info(result) if log_file: @@ -173,6 +210,9 @@ def main(): if not test_filter: reset_devs(dev, apdev) + if conn: + conn.close() + if len(failed): logger.info("passed " + str(len(passed)) + " test case(s)") logger.info("skipped " + str(len(skipped)) + " test case(s)") diff --git a/tests/hwsim/start.sh b/tests/hwsim/start.sh index cc9dfcb..9931ee5 100755 --- a/tests/hwsim/start.sh +++ b/tests/hwsim/start.sh @@ -35,6 +35,7 @@ else fi $DIR/stop-wifi.sh +git show -s --format=%H > commit sudo modprobe mac80211_hwsim radios=5 if [ "$CONCURRENT" = "y" ]; then sudo iw wlan0 interface add sta0 type station -- 2.1.4