tests/hwsim/run-tests.py

   1 #!/usr/bin/python
   2 #
   3 # AP tests
   4 # Copyright (c) 2013, Jouni Malinen <j@w1.fi>
   5 #
   6 # This software may be distributed under the terms of the BSD license.
   7 # See README for more details.
   8
   9 import os
  10 import re
  11 import sys
  12 import time
  13 import sqlite3
  14 from datetime import datetime
  15
  16 import logging
  17 logger = logging.getLogger(__name__)
  18
  19 sys.path.append('../../wpaspy')
  20
  21 from wpasupplicant import WpaSupplicant
  22 from hostapd import HostapdGlobal
  23
  24 def reset_devs(dev, apdev):
  25     hapd = HostapdGlobal()
  26     for d in dev:
  27         try:
  28             d.reset()
  29         except Exception, e:
  30             logger.info("Failed to reset device " + d.ifname)
  31             print str(e)
  32     for ap in apdev:
  33         hapd.remove(ap['ifname'])
  34
  35 def report(conn, build, commit, run, test, result, diff):
  36     if conn:
  37         if not build:
  38             build = ''
  39         if not commit:
  40             commit = ''
  41         sql = "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES(?, ?, ?, ?, ?, ?, ?)"
  42         params = (test.replace('test_', '', 1), result, run, time.time(), diff.total_seconds(), build, commit)
  43         try:
  44             conn.execute(sql, params)
  45             conn.commit()
  46         except Exception, e:
  47             print "sqlite: " + str(e)
  48             print "sql: %r" % (params, )
  49
  50 def main():
  51     test_file = None
  52     error_file = None
  53     log_file = None
  54     results_file = None
  55     conn = None
  56     run = None
  57     build = None
  58     commit = None
  59     idx = 1
  60     print_res = False
  61     if len(sys.argv) > 1 and sys.argv[1] == '-d':
  62         logging.basicConfig(level=logging.DEBUG)
  63         idx = idx + 1
  64     elif len(sys.argv) > 1 and sys.argv[1] == '-q':
  65         logging.basicConfig(level=logging.WARNING)
  66         print_res = True
  67         idx = idx + 1
  68     elif len(sys.argv) > 2 and sys.argv[1] == '-l':
  69         log_file = sys.argv[2]
  70         logging.basicConfig(filename=log_file,level=logging.DEBUG)
  71         idx = idx + 2
  72     else:
  73         logging.basicConfig(level=logging.INFO)
  74
  75     while len(sys.argv) > idx:
  76         if len(sys.argv) > idx + 1 and sys.argv[idx] == '-e':
  77             error_file = sys.argv[idx + 1]
  78             idx = idx + 2
  79         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-r':
  80             results_file = sys.argv[idx + 1]
  81             idx = idx + 2
  82         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-f':
  83             test_file = sys.argv[idx + 1]
  84             idx = idx + 2
  85         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-S':
  86             conn = sqlite3.connect(sys.argv[idx + 1])
  87             idx = idx + 2
  88         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-b':
  89             build = sys.argv[idx + 1]
  90             idx = idx + 2
  91         else:
  92             break
  93
  94     if conn:
  95         run = str(int(time.time()))
  96         try:
  97             with open("commit") as f:
  98                 val = f.readlines()
  99                 if len(val) > 0:
 100                     commit = val[0].rstrip()
 101         except IOError:
 102             pass
 103
 104     tests = []
 105     for t in os.listdir("."):
 106         m = re.match(r'(test_.*)\.py$', t)
 107         if m:
 108             if test_file and test_file not in t:
 109                 continue
 110             logger.debug("Import test cases from " + t)
 111             mod = __import__(m.group(1))
 112             for s in dir(mod):
 113                 if s.startswith("test_"):
 114                     func = mod.__dict__.get(s)
 115                     tests.append(func)
 116
 117     if len(sys.argv) > idx and sys.argv[idx] == '-L':
 118         for t in tests:
 119             print t.__name__ + " - " + t.__doc__
 120             if conn:
 121                 sql = 'INSERT OR REPLACE INTO tests(test,description) VALUES (?, ?)'
 122                 params = (t.__name__.replace('test_', '', 1), t.__doc__)
 123                 try:
 124                     conn.execute(sql, params)
 125                 except Exception, e:
 126                     print "sqlite: " + str(e)
 127                     print "sql: %r" % (params,)
 128         if conn:
 129             conn.commit()
 130             conn.close()
 131         sys.exit(0)
 132
 133     if len(sys.argv) > idx:
 134         test_filter = sys.argv[idx]
 135     else:
 136         test_filter = None
 137
 138     dev0 = WpaSupplicant('wlan0', '/tmp/wpas-wlan0')
 139     dev1 = WpaSupplicant('wlan1', '/tmp/wpas-wlan1')
 140     dev2 = WpaSupplicant('wlan2', '/tmp/wpas-wlan2')
 141     dev = [ dev0, dev1, dev2 ]
 142     apdev = [ ]
 143     apdev.append({"ifname": 'wlan3', "bssid": "02:00:00:00:03:00"})
 144     apdev.append({"ifname": 'wlan4', "bssid": "02:00:00:00:04:00"})
 145
 146     for d in dev:
 147         if not d.ping():
 148             logger.info(d.ifname + ": No response from wpa_supplicant")
 149             return
 150         logger.info("DEV: " + d.ifname + ": " + d.p2p_dev_addr())
 151     for ap in apdev:
 152         logger.info("APDEV: " + ap['ifname'])
 153
 154     passed = []
 155     skipped = []
 156     failed = []
 157
 158     for t in tests:
 159         if test_filter:
 160             if test_filter != t.__name__:
 161                 continue
 162         reset_devs(dev, apdev)
 163         logger.info("START " + t.__name__)
 164         if log_file:
 165             print "START " + t.__name__
 166             sys.stdout.flush()
 167         if t.__doc__:
 168             logger.info("Test: " + t.__doc__)
 169         start = datetime.now()
 170         for d in dev:
 171             try:
 172                 d.request("NOTE TEST-START " + t.__name__)
 173             except Exception, e:
 174                 logger.info("Failed to issue TEST-START before " + t.__name__ + " for " + d.ifname)
 175                 logger.info(e)
 176                 print "FAIL " + t.__name__ + " - could not start test"
 177                 if conn:
 178                     conn.close()
 179                     conn = None
 180                 sys.exit(1)
 181         try:
 182             if t.func_code.co_argcount > 1:
 183                 res = t(dev, apdev)
 184             else:
 185                 res = t(dev)
 186             end = datetime.now()
 187             diff = end - start
 188             if res == "skip":
 189                 skipped.append(t.__name__)
 190                 result = "SKIP"
 191             else:
 192                 passed.append(t.__name__)
 193                 result = "PASS"
 194             report(conn, build, commit, run, t.__name__, result, diff)
 195             result = result + " " + t.__name__ + " "
 196             result = result + str(diff.total_seconds()) + " " + str(end)
 197             logger.info(result)
 198             if log_file or print_res:
 199                 print result
 200                 sys.stdout.flush()
 201             if results_file:
 202                 f = open(results_file, 'a')
 203                 f.write(result + "\n")
 204                 f.close()
 205         except Exception, e:
 206             end = datetime.now()
 207             diff = end - start
 208             logger.info(e)
 209             failed.append(t.__name__)
 210             report(conn, build, commit, run, t.__name__, "FAIL", diff)
 211             result = "FAIL " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end)
 212             logger.info(result)
 213             if log_file:
 214                 print result
 215                 sys.stdout.flush()
 216             if results_file:
 217                 f = open(results_file, 'a')
 218                 f.write(result + "\n")
 219                 f.close()
 220         for d in dev:
 221             try:
 222                 d.request("NOTE TEST-STOP " + t.__name__)
 223             except Exception, e:
 224                 logger.info("Failed to issue TEST-STOP after " + t.__name__ + " for " + d.ifname)
 225                 logger.info(e)
 226
 227     if not test_filter:
 228         reset_devs(dev, apdev)
 229
 230     if conn:
 231         conn.close()
 232
 233     if len(failed):
 234         logger.info("passed " + str(len(passed)) + " test case(s)")
 235         logger.info("skipped " + str(len(skipped)) + " test case(s)")
 236         logger.info("failed tests: " + str(failed))
 237         if error_file:
 238             f = open(error_file, 'w')
 239             f.write(str(failed) + '\n')
 240             f.close()
 241         sys.exit(1)
 242     logger.info("passed all " + str(len(passed)) + " test case(s)")
 243     if len(skipped):
 244         logger.info("skipped " + str(len(skipped)) + " test case(s)")
 245     if log_file:
 246         print "passed all " + str(len(passed)) + " test case(s)"
 247         if len(skipped):
 248             print "skipped " + str(len(skipped)) + " test case(s)"
 249
 250 if __name__ == "__main__":
 251     main()