tests/hwsim/run-tests.py

   1 #!/usr/bin/python
   2 #
   3 # AP tests
   4 # Copyright (c) 2013, Jouni Malinen <j@w1.fi>
   5 #
   6 # This software may be distributed under the terms of the BSD license.
   7 # See README for more details.
   8
   9 import os
  10 import re
  11 import sys
  12 import time
  13 from datetime import datetime
  14 import argparse
  15
  16 import logging
  17 logger = logging.getLogger(__name__)
  18
  19 sys.path.append('../../wpaspy')
  20
  21 from wpasupplicant import WpaSupplicant
  22 from hostapd import HostapdGlobal
  23
  24 def reset_devs(dev, apdev):
  25     hapd = HostapdGlobal()
  26     for d in dev:
  27         try:
  28             d.reset()
  29         except Exception, e:
  30             logger.info("Failed to reset device " + d.ifname)
  31             print str(e)
  32     for ap in apdev:
  33         hapd.remove(ap['ifname'])
  34
  35 def report(conn, build, commit, run, test, result, diff):
  36     if conn:
  37         if not build:
  38             build = ''
  39         if not commit:
  40             commit = ''
  41         sql = "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES(?, ?, ?, ?, ?, ?, ?)"
  42         params = (test.replace('test_', '', 1), result, run, time.time(), diff.total_seconds(), build, commit)
  43         try:
  44             conn.execute(sql, params)
  45             conn.commit()
  46         except Exception, e:
  47             print "sqlite: " + str(e)
  48             print "sql: %r" % (params, )
  49
  50 def main():
  51     tests = []
  52     test_modules = []
  53     for t in os.listdir("."):
  54         m = re.match(r'(test_.*)\.py$', t)
  55         if m:
  56             logger.debug("Import test cases from " + t)
  57             mod = __import__(m.group(1))
  58             test_modules.append(mod.__name__)
  59             for s in dir(mod):
  60                 if s.startswith("test_"):
  61                     func = mod.__dict__.get(s)
  62                     tests.append(func)
  63     test_names = list(set([t.__name__ for t in tests]))
  64
  65     run = None
  66     commit = None
  67     print_res = False
  68
  69     parser = argparse.ArgumentParser(description='hwsim test runner')
  70     group = parser.add_mutually_exclusive_group()
  71     group.add_argument('-d', const=logging.DEBUG, action='store_const',
  72                        dest='loglevel', default=logging.INFO,
  73                        help="verbose debug output")
  74     group.add_argument('-q', const=logging.WARNING, action='store_const',
  75                        dest='loglevel', help="be quiet")
  76     group.add_argument('-l', metavar='<filename>', dest='logfile',
  77                        help='debug log filename')
  78
  79     parser.add_argument('-e', metavar="<filename>", dest='errorfile',
  80                         help='error filename')
  81     parser.add_argument('-r', metavar="<filename>", dest='resultsfile',
  82                         help='results filename')
  83     parser.add_argument('-S', metavar='<sqlite3 db>', dest='database',
  84                         help='database to write results to')
  85     parser.add_argument('-b', metavar='<build>', dest='build', help='build ID')
  86     parser.add_argument('-L', action='store_true', dest='update_tests_db',
  87                         help='List tests (and update descriptions in DB)')
  88     parser.add_argument('-f', dest='testmodule', metavar='<test module>',
  89                         help='execute only tests from this test module',
  90                         type=str, choices=[[]] + test_modules)
  91     parser.add_argument('tests', metavar='<test>', nargs='*', type=str,
  92                         help='tests to run (only valid without -f)',
  93                         choices=[[]] + test_names)
  94
  95     args = parser.parse_args()
  96
  97     if args.tests and args.testmodule:
  98         print 'Invalid arguments - both test module and tests given'
  99         sys.exit(2)
 100
 101     if args.logfile:
 102         logging.basicConfig(filename=args.logfile,
 103                             level=logging.DEBUG)
 104         log_to_file = True
 105     else:
 106         logging.basicConfig(level=args.loglevel)
 107         log_to_file = False
 108         if args.loglevel == logging.WARNING:
 109             print_res = True
 110
 111     error_file = args.errorfile
 112     results_file = args.resultsfile
 113
 114     if args.database:
 115         import sqlite3
 116         conn = sqlite3.connect(args.database)
 117     else:
 118         conn = None
 119
 120     if conn:
 121         run = str(int(time.time()))
 122         try:
 123             with open("commit") as f:
 124                 val = f.readlines()
 125                 if len(val) > 0:
 126                     commit = val[0].rstrip()
 127         except IOError:
 128             pass
 129
 130     if args.update_tests_db:
 131         for t in tests:
 132             print t.__name__ + " - " + t.__doc__
 133             if conn:
 134                 sql = 'INSERT OR REPLACE INTO tests(test,description) VALUES (?, ?)'
 135                 params = (t.__name__.replace('test_', '', 1), t.__doc__)
 136                 try:
 137                     conn.execute(sql, params)
 138                 except Exception, e:
 139                     print "sqlite: " + str(e)
 140                     print "sql: %r" % (params,)
 141         if conn:
 142             conn.commit()
 143             conn.close()
 144         sys.exit(0)
 145
 146
 147     dev0 = WpaSupplicant('wlan0', '/tmp/wpas-wlan0')
 148     dev1 = WpaSupplicant('wlan1', '/tmp/wpas-wlan1')
 149     dev2 = WpaSupplicant('wlan2', '/tmp/wpas-wlan2')
 150     dev = [ dev0, dev1, dev2 ]
 151     apdev = [ ]
 152     apdev.append({"ifname": 'wlan3', "bssid": "02:00:00:00:03:00"})
 153     apdev.append({"ifname": 'wlan4', "bssid": "02:00:00:00:04:00"})
 154
 155     for d in dev:
 156         if not d.ping():
 157             logger.info(d.ifname + ": No response from wpa_supplicant")
 158             return
 159         logger.info("DEV: " + d.ifname + ": " + d.p2p_dev_addr())
 160     for ap in apdev:
 161         logger.info("APDEV: " + ap['ifname'])
 162
 163     passed = []
 164     skipped = []
 165     failed = []
 166
 167     for t in tests:
 168         if args.tests:
 169             if not t.__name__ in args.tests:
 170                 continue
 171         if args.testmodule:
 172             if not t.__module__ == args.testmodule:
 173                 continue
 174         reset_devs(dev, apdev)
 175         logger.info("START " + t.__name__)
 176         if log_to_file:
 177             print "START " + t.__name__
 178             sys.stdout.flush()
 179         if t.__doc__:
 180             logger.info("Test: " + t.__doc__)
 181         start = datetime.now()
 182         for d in dev:
 183             try:
 184                 d.request("NOTE TEST-START " + t.__name__)
 185             except Exception, e:
 186                 logger.info("Failed to issue TEST-START before " + t.__name__ + " for " + d.ifname)
 187                 logger.info(e)
 188                 print "FAIL " + t.__name__ + " - could not start test"
 189                 if conn:
 190                     conn.close()
 191                     conn = None
 192                 sys.exit(1)
 193         try:
 194             if t.func_code.co_argcount > 1:
 195                 res = t(dev, apdev)
 196             else:
 197                 res = t(dev)
 198             end = datetime.now()
 199             diff = end - start
 200             if res == "skip":
 201                 skipped.append(t.__name__)
 202                 result = "SKIP"
 203             else:
 204                 passed.append(t.__name__)
 205                 result = "PASS"
 206             report(conn, args.build, commit, run, t.__name__, result, diff)
 207             result = result + " " + t.__name__ + " "
 208             result = result + str(diff.total_seconds()) + " " + str(end)
 209             logger.info(result)
 210             if log_to_file or print_res:
 211                 print result
 212                 sys.stdout.flush()
 213             if results_file:
 214                 f = open(results_file, 'a')
 215                 f.write(result + "\n")
 216                 f.close()
 217         except Exception, e:
 218             end = datetime.now()
 219             diff = end - start
 220             logger.info(e)
 221             failed.append(t.__name__)
 222             report(conn, args.build, commit, run, t.__name__, "FAIL", diff)
 223             result = "FAIL " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end)
 224             logger.info(result)
 225             if log_to_file:
 226                 print result
 227                 sys.stdout.flush()
 228             if results_file:
 229                 f = open(results_file, 'a')
 230                 f.write(result + "\n")
 231                 f.close()
 232         for d in dev:
 233             try:
 234                 d.request("NOTE TEST-STOP " + t.__name__)
 235             except Exception, e:
 236                 logger.info("Failed to issue TEST-STOP after " + t.__name__ + " for " + d.ifname)
 237                 logger.info(e)
 238
 239     if not args.tests:
 240         reset_devs(dev, apdev)
 241
 242     if conn:
 243         conn.close()
 244
 245     if len(failed):
 246         logger.info("passed " + str(len(passed)) + " test case(s)")
 247         logger.info("skipped " + str(len(skipped)) + " test case(s)")
 248         logger.info("failed tests: " + str(failed))
 249         if error_file:
 250             f = open(error_file, 'w')
 251             f.write(str(failed) + '\n')
 252             f.close()
 253         sys.exit(1)
 254     logger.info("passed all " + str(len(passed)) + " test case(s)")
 255     if len(skipped):
 256         logger.info("skipped " + str(len(skipped)) + " test case(s)")
 257     if log_to_file:
 258         print "passed all " + str(len(passed)) + " test case(s)"
 259         if len(skipped):
 260             print "skipped " + str(len(skipped)) + " test case(s)"
 261
 262 if __name__ == "__main__":
 263     main()