tests/hwsim/run-tests.py

   1 #!/usr/bin/python
   2 #
   3 # AP tests
   4 # Copyright (c) 2013, Jouni Malinen <j@w1.fi>
   5 #
   6 # This software may be distributed under the terms of the BSD license.
   7 # See README for more details.
   8
   9 import os
  10 import re
  11 import sys
  12 import time
  13 import sqlite3
  14 from datetime import datetime
  15
  16 import logging
  17 logger = logging.getLogger(__name__)
  18
  19 sys.path.append('../../wpaspy')
  20
  21 from wpasupplicant import WpaSupplicant
  22 from hostapd import HostapdGlobal
  23
  24 def reset_devs(dev, apdev):
  25     hapd = HostapdGlobal()
  26     for d in dev:
  27         try:
  28             d.reset()
  29         except Exception, e:
  30             logger.info("Failed to reset device " + d.ifname)
  31             print str(e)
  32     for ap in apdev:
  33         hapd.remove(ap['ifname'])
  34
  35 def report(conn, build, commit, run, test, result, diff):
  36     if conn:
  37         if not build:
  38             build = ''
  39         if not commit:
  40             commit = ''
  41         sql = "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES('" + test.replace('test_', '', 1) + "', '" + result + "', " + str(run) + ", " + str(time.time()) + ", " + str(diff.total_seconds()) + ", '" + build + "', '" + commit + "')"
  42         try:
  43             conn.execute(sql)
  44             conn.commit()
  45         except Exception, e:
  46             print "sqlite: " + str(e)
  47             print "sql: " + sql
  48
  49 def main():
  50     test_file = None
  51     error_file = None
  52     log_file = None
  53     results_file = None
  54     conn = None
  55     run = None
  56     build = None
  57     commit = None
  58     idx = 1
  59     print_res = False
  60     if len(sys.argv) > 1 and sys.argv[1] == '-d':
  61         logging.basicConfig(level=logging.DEBUG)
  62         idx = idx + 1
  63     elif len(sys.argv) > 1 and sys.argv[1] == '-q':
  64         logging.basicConfig(level=logging.WARNING)
  65         print_res = True
  66         idx = idx + 1
  67     elif len(sys.argv) > 2 and sys.argv[1] == '-l':
  68         log_file = sys.argv[2]
  69         logging.basicConfig(filename=log_file,level=logging.DEBUG)
  70         idx = idx + 2
  71     else:
  72         logging.basicConfig(level=logging.INFO)
  73
  74     while len(sys.argv) > idx:
  75         if len(sys.argv) > idx + 1 and sys.argv[idx] == '-e':
  76             error_file = sys.argv[idx + 1]
  77             idx = idx + 2
  78         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-r':
  79             results_file = sys.argv[idx + 1]
  80             idx = idx + 2
  81         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-f':
  82             test_file = sys.argv[idx + 1]
  83             idx = idx + 2
  84         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-S':
  85             conn = sqlite3.connect(sys.argv[idx + 1])
  86             idx = idx + 2
  87         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-b':
  88             build = sys.argv[idx + 1]
  89             idx = idx + 2
  90         else:
  91             break
  92
  93     if conn:
  94         run = str(int(time.time()))
  95         try:
  96             with open("commit") as f:
  97                 val = f.readlines()
  98                 if len(val) > 0:
  99                     commit = val[0].rstrip()
 100         except IOError:
 101             pass
 102
 103     tests = []
 104     for t in os.listdir("."):
 105         m = re.match(r'(test_.*)\.py$', t)
 106         if m:
 107             if test_file and test_file not in t:
 108                 continue
 109             logger.debug("Import test cases from " + t)
 110             mod = __import__(m.group(1))
 111             for s in dir(mod):
 112                 if s.startswith("test_"):
 113                     func = mod.__dict__.get(s)
 114                     tests.append(func)
 115
 116     if len(sys.argv) > idx and sys.argv[idx] == '-L':
 117         for t in tests:
 118             print t.__name__ + " - " + t.__doc__
 119             if conn:
 120                 sql = 'INSERT OR REPLACE INTO tests(test,description) VALUES ("' + t.__name__.replace('test_', '', 1) + '", "' + t.__doc__ + '")';
 121                 try:
 122                     conn.execute(sql)
 123                 except Exception, e:
 124                     print "sqlite: " + str(e)
 125                     print "sql: " + sql
 126         if conn:
 127             conn.commit()
 128             conn.close()
 129         sys.exit(0)
 130
 131     if len(sys.argv) > idx:
 132         test_filter = sys.argv[idx]
 133     else:
 134         test_filter = None
 135
 136     dev0 = WpaSupplicant('wlan0', '/tmp/wpas-wlan0')
 137     dev1 = WpaSupplicant('wlan1', '/tmp/wpas-wlan1')
 138     dev2 = WpaSupplicant('wlan2', '/tmp/wpas-wlan2')
 139     dev = [ dev0, dev1, dev2 ]
 140     apdev = [ ]
 141     apdev.append({"ifname": 'wlan3', "bssid": "02:00:00:00:03:00"})
 142     apdev.append({"ifname": 'wlan4', "bssid": "02:00:00:00:04:00"})
 143
 144     for d in dev:
 145         if not d.ping():
 146             logger.info(d.ifname + ": No response from wpa_supplicant")
 147             return
 148         logger.info("DEV: " + d.ifname + ": " + d.p2p_dev_addr())
 149     for ap in apdev:
 150         logger.info("APDEV: " + ap['ifname'])
 151
 152     passed = []
 153     skipped = []
 154     failed = []
 155
 156     for t in tests:
 157         if test_filter:
 158             if test_filter != t.__name__:
 159                 continue
 160         reset_devs(dev, apdev)
 161         logger.info("START " + t.__name__)
 162         if log_file:
 163             print "START " + t.__name__
 164             sys.stdout.flush()
 165         if t.__doc__:
 166             logger.info("Test: " + t.__doc__)
 167         start = datetime.now()
 168         for d in dev:
 169             try:
 170                 d.request("NOTE TEST-START " + t.__name__)
 171             except Exception, e:
 172                 logger.info("Failed to issue TEST-START before " + t.__name__ + " for " + d.ifname)
 173                 logger.info(e)
 174                 print "FAIL " + t.__name__ + " - could not start test"
 175                 if conn:
 176                     conn.close()
 177                     conn = None
 178                 sys.exit(1)
 179         try:
 180             if t.func_code.co_argcount > 1:
 181                 res = t(dev, apdev)
 182             else:
 183                 res = t(dev)
 184             end = datetime.now()
 185             diff = end - start
 186             if res == "skip":
 187                 skipped.append(t.__name__)
 188                 result = "SKIP"
 189             else:
 190                 passed.append(t.__name__)
 191                 result = "PASS"
 192             report(conn, build, commit, run, t.__name__, result, diff)
 193             result = result + " " + t.__name__ + " "
 194             result = result + str(diff.total_seconds()) + " " + str(end)
 195             logger.info(result)
 196             if log_file or print_res:
 197                 print result
 198                 sys.stdout.flush()
 199             if results_file:
 200                 f = open(results_file, 'a')
 201                 f.write(result + "\n")
 202                 f.close()
 203         except Exception, e:
 204             end = datetime.now()
 205             diff = end - start
 206             logger.info(e)
 207             failed.append(t.__name__)
 208             report(conn, build, commit, run, t.__name__, "FAIL", diff)
 209             result = "FAIL " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end)
 210             logger.info(result)
 211             if log_file:
 212                 print result
 213                 sys.stdout.flush()
 214             if results_file:
 215                 f = open(results_file, 'a')
 216                 f.write(result + "\n")
 217                 f.close()
 218         for d in dev:
 219             try:
 220                 d.request("NOTE TEST-STOP " + t.__name__)
 221             except Exception, e:
 222                 logger.info("Failed to issue TEST-STOP after " + t.__name__ + " for " + d.ifname)
 223                 logger.info(e)
 224
 225     if not test_filter:
 226         reset_devs(dev, apdev)
 227
 228     if conn:
 229         conn.close()
 230
 231     if len(failed):
 232         logger.info("passed " + str(len(passed)) + " test case(s)")
 233         logger.info("skipped " + str(len(skipped)) + " test case(s)")
 234         logger.info("failed tests: " + str(failed))
 235         if error_file:
 236             f = open(error_file, 'w')
 237             f.write(str(failed) + '\n')
 238             f.close()
 239         sys.exit(1)
 240     logger.info("passed all " + str(len(passed)) + " test case(s)")
 241     if len(skipped):
 242         logger.info("skipped " + str(len(skipped)) + " test case(s)")
 243     if log_file:
 244         print "passed all " + str(len(passed)) + " test case(s)"
 245         if len(skipped):
 246             print "skipped " + str(len(skipped)) + " test case(s)"
 247
 248 if __name__ == "__main__":
 249     main()