tests/hwsim/run-tests.py

   1 #!/usr/bin/python
   2 #
   3 # AP tests
   4 # Copyright (c) 2013, Jouni Malinen <j@w1.fi>
   5 #
   6 # This software may be distributed under the terms of the BSD license.
   7 # See README for more details.
   8
   9 import os
  10 import re
  11 import sys
  12 import time
  13 import sqlite3
  14 from datetime import datetime
  15
  16 import logging
  17 logger = logging.getLogger(__name__)
  18
  19 from wpasupplicant import WpaSupplicant
  20 from hostapd import HostapdGlobal
  21
  22 def reset_devs(dev, apdev):
  23     hapd = HostapdGlobal()
  24     for d in dev:
  25         try:
  26             d.reset()
  27         except Exception, e:
  28             logger.info("Failed to reset device " + d.ifname)
  29             print str(e)
  30     for ap in apdev:
  31         hapd.remove(ap['ifname'])
  32
  33 def report(conn, build, commit, run, test, result, diff):
  34     if conn:
  35         if not build:
  36             build = ''
  37         if not commit:
  38             commit = ''
  39         sql = "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES('" + test.replace('test_', '', 1) + "', '" + result + "', " + str(run) + ", " + str(time.time()) + ", " + str(diff.total_seconds()) + ", '" + build + "', '" + commit + "')"
  40         try:
  41             conn.execute(sql)
  42             conn.commit()
  43         except Exception, e:
  44             print "sqlite: " + str(e)
  45             print "sql: " + sql
  46
  47 def main():
  48     test_file = None
  49     error_file = None
  50     log_file = None
  51     results_file = None
  52     conn = None
  53     run = None
  54     build = None
  55     commit = None
  56     idx = 1
  57     print_res = False
  58     if len(sys.argv) > 1 and sys.argv[1] == '-d':
  59         logging.basicConfig(level=logging.DEBUG)
  60         idx = idx + 1
  61     elif len(sys.argv) > 1 and sys.argv[1] == '-q':
  62         logging.basicConfig(level=logging.WARNING)
  63         print_res = True
  64         idx = idx + 1
  65     elif len(sys.argv) > 2 and sys.argv[1] == '-l':
  66         log_file = sys.argv[2]
  67         logging.basicConfig(filename=log_file,level=logging.DEBUG)
  68         idx = idx + 2
  69     else:
  70         logging.basicConfig(level=logging.INFO)
  71
  72     while len(sys.argv) > idx:
  73         if len(sys.argv) > idx + 1 and sys.argv[idx] == '-e':
  74             error_file = sys.argv[idx + 1]
  75             idx = idx + 2
  76         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-r':
  77             results_file = sys.argv[idx + 1]
  78             idx = idx + 2
  79         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-f':
  80             test_file = sys.argv[idx + 1]
  81             idx = idx + 2
  82         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-S':
  83             conn = sqlite3.connect(sys.argv[idx + 1])
  84             idx = idx + 2
  85         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-b':
  86             build = sys.argv[idx + 1]
  87             idx = idx + 2
  88         else:
  89             break
  90
  91     if conn:
  92         run = str(int(time.time()))
  93         with open("commit") as f:
  94             val = f.readlines()
  95             if len(val) > 0:
  96                 commit = val[0].rstrip()
  97
  98     tests = []
  99     for t in os.listdir("."):
 100         m = re.match(r'(test_.*)\.py$', t)
 101         if m:
 102             if test_file and test_file not in t:
 103                 continue
 104             logger.debug("Import test cases from " + t)
 105             mod = __import__(m.group(1))
 106             for s in dir(mod):
 107                 if s.startswith("test_"):
 108                     func = mod.__dict__.get(s)
 109                     tests.append(func)
 110
 111     if len(sys.argv) > 1 and sys.argv[1] == '-L':
 112         for t in tests:
 113             print t.__name__ + " - " + t.__doc__
 114         sys.exit(0)
 115
 116     if len(sys.argv) > idx:
 117         test_filter = sys.argv[idx]
 118     else:
 119         test_filter = None
 120
 121     dev0 = WpaSupplicant('wlan0', '/tmp/wpas-wlan0')
 122     dev1 = WpaSupplicant('wlan1', '/tmp/wpas-wlan1')
 123     dev2 = WpaSupplicant('wlan2', '/tmp/wpas-wlan2')
 124     dev = [ dev0, dev1, dev2 ]
 125     apdev = [ ]
 126     apdev.append({"ifname": 'wlan3', "bssid": "02:00:00:00:03:00"})
 127     apdev.append({"ifname": 'wlan4', "bssid": "02:00:00:00:04:00"})
 128
 129     for d in dev:
 130         if not d.ping():
 131             logger.info(d.ifname + ": No response from wpa_supplicant")
 132             return
 133         logger.info("DEV: " + d.ifname + ": " + d.p2p_dev_addr())
 134     for ap in apdev:
 135         logger.info("APDEV: " + ap['ifname'])
 136
 137     passed = []
 138     skipped = []
 139     failed = []
 140
 141     for t in tests:
 142         if test_filter:
 143             if test_filter != t.__name__:
 144                 continue
 145         reset_devs(dev, apdev)
 146         logger.info("START " + t.__name__)
 147         if log_file:
 148             print "START " + t.__name__
 149             sys.stdout.flush()
 150         if t.__doc__:
 151             logger.info("Test: " + t.__doc__)
 152         start = datetime.now()
 153         for d in dev:
 154             try:
 155                 d.request("NOTE TEST-START " + t.__name__)
 156             except Exception, e:
 157                 logger.info("Failed to issue TEST-START before " + t.__name__ + " for " + d.ifname)
 158                 logger.info(e)
 159                 print "FAIL " + t.__name__ + " - could not start test"
 160                 if conn:
 161                     conn.close()
 162                     conn = None
 163                 sys.exit(1)
 164         try:
 165             if t.func_code.co_argcount > 1:
 166                 res = t(dev, apdev)
 167             else:
 168                 res = t(dev)
 169             end = datetime.now()
 170             diff = end - start
 171             if res == "skip":
 172                 skipped.append(t.__name__)
 173                 result = "SKIP"
 174             else:
 175                 passed.append(t.__name__)
 176                 result = "PASS"
 177             report(conn, build, commit, run, t.__name__, result, diff)
 178             result = result + " " + t.__name__ + " "
 179             result = result + str(diff.total_seconds()) + " " + str(end)
 180             logger.info(result)
 181             if log_file or print_res:
 182                 print result
 183                 sys.stdout.flush()
 184             if results_file:
 185                 f = open(results_file, 'a')
 186                 f.write(result + "\n")
 187                 f.close()
 188         except Exception, e:
 189             end = datetime.now()
 190             diff = end - start
 191             logger.info(e)
 192             failed.append(t.__name__)
 193             report(conn, build, commit, run, t.__name__, "FAIL", diff)
 194             result = "FAIL " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end)
 195             logger.info(result)
 196             if log_file:
 197                 print result
 198                 sys.stdout.flush()
 199             if results_file:
 200                 f = open(results_file, 'a')
 201                 f.write(result + "\n")
 202                 f.close()
 203         for d in dev:
 204             try:
 205                 d.request("NOTE TEST-STOP " + t.__name__)
 206             except Exception, e:
 207                 logger.info("Failed to issue TEST-STOP after " + t.__name__ + " for " + d.ifname)
 208                 logger.info(e)
 209
 210     if not test_filter:
 211         reset_devs(dev, apdev)
 212
 213     if conn:
 214         conn.close()
 215
 216     if len(failed):
 217         logger.info("passed " + str(len(passed)) + " test case(s)")
 218         logger.info("skipped " + str(len(skipped)) + " test case(s)")
 219         logger.info("failed tests: " + str(failed))
 220         if error_file:
 221             f = open(error_file, 'w')
 222             f.write(str(failed) + '\n')
 223             f.close()
 224         sys.exit(1)
 225     logger.info("passed all " + str(len(passed)) + " test case(s)")
 226     if len(skipped):
 227         logger.info("skipped " + str(len(skipped)) + " test case(s)")
 228     if log_file:
 229         print "passed all " + str(len(passed)) + " test case(s)"
 230         if len(skipped):
 231             print "skipped " + str(len(skipped)) + " test case(s)"
 232
 233 if __name__ == "__main__":
 234     main()