tests/hwsim/run-tests.py

   1 #!/usr/bin/python
   2 #
   3 # AP tests
   4 # Copyright (c) 2013, Jouni Malinen <j@w1.fi>
   5 #
   6 # This software may be distributed under the terms of the BSD license.
   7 # See README for more details.
   8
   9 import os
  10 import re
  11 import sys
  12 import time
  13 from datetime import datetime
  14 import argparse
  15
  16 import logging
  17 logger = logging.getLogger(__name__)
  18
  19 sys.path.append('../../wpaspy')
  20
  21 from wpasupplicant import WpaSupplicant
  22 from hostapd import HostapdGlobal
  23
  24 def reset_devs(dev, apdev):
  25     hapd = HostapdGlobal()
  26     for d in dev:
  27         try:
  28             d.reset()
  29         except Exception, e:
  30             logger.info("Failed to reset device " + d.ifname)
  31             print str(e)
  32     for ap in apdev:
  33         hapd.remove(ap['ifname'])
  34
  35 def report(conn, build, commit, run, test, result, diff):
  36     if conn:
  37         if not build:
  38             build = ''
  39         if not commit:
  40             commit = ''
  41         sql = "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES(?, ?, ?, ?, ?, ?, ?)"
  42         params = (test.replace('test_', '', 1), result, run, time.time(), diff.total_seconds(), build, commit)
  43         try:
  44             conn.execute(sql, params)
  45             conn.commit()
  46         except Exception, e:
  47             print "sqlite: " + str(e)
  48             print "sql: %r" % (params, )
  49
  50 def main():
  51     tests = []
  52     test_modules = []
  53     for t in os.listdir("."):
  54         m = re.match(r'(test_.*)\.py$', t)
  55         if m:
  56             logger.debug("Import test cases from " + t)
  57             mod = __import__(m.group(1))
  58             test_modules.append(mod.__name__)
  59             for s in dir(mod):
  60                 if s.startswith("test_"):
  61                     func = mod.__dict__.get(s)
  62                     tests.append(func)
  63     test_names = list(set([t.__name__ for t in tests]))
  64
  65     run = None
  66     print_res = False
  67
  68     parser = argparse.ArgumentParser(description='hwsim test runner')
  69     group = parser.add_mutually_exclusive_group()
  70     group.add_argument('-d', const=logging.DEBUG, action='store_const',
  71                        dest='loglevel', default=logging.INFO,
  72                        help="verbose debug output")
  73     group.add_argument('-q', const=logging.WARNING, action='store_const',
  74                        dest='loglevel', help="be quiet")
  75     group.add_argument('-l', metavar='<filename>', dest='logfile',
  76                        help='debug log filename')
  77
  78     parser.add_argument('-e', metavar="<filename>", dest='errorfile',
  79                         help='error filename')
  80     parser.add_argument('-r', metavar="<filename>", dest='resultsfile',
  81                         help='results filename')
  82     parser.add_argument('-S', metavar='<sqlite3 db>', dest='database',
  83                         help='database to write results to')
  84     parser.add_argument('--commit', metavar='<commit id>',
  85                         help='commit ID, only for database')
  86     parser.add_argument('-b', metavar='<build>', dest='build', help='build ID')
  87     parser.add_argument('-L', action='store_true', dest='update_tests_db',
  88                         help='List tests (and update descriptions in DB)')
  89     parser.add_argument('-f', dest='testmodules', metavar='<test module>',
  90                         help='execute only tests from these test modules',
  91                         type=str, choices=[[]] + test_modules, nargs='+')
  92     parser.add_argument('tests', metavar='<test>', nargs='*', type=str,
  93                         help='tests to run (only valid without -f)',
  94                         choices=[[]] + test_names)
  95
  96     args = parser.parse_args()
  97
  98     if args.tests and args.testmodules:
  99         print 'Invalid arguments - both test module and tests given'
 100         sys.exit(2)
 101
 102     if args.logfile:
 103         logging.basicConfig(filename=args.logfile,
 104                             level=logging.DEBUG)
 105         log_to_file = True
 106     else:
 107         logging.basicConfig(level=args.loglevel)
 108         log_to_file = False
 109         if args.loglevel == logging.WARNING:
 110             print_res = True
 111
 112     error_file = args.errorfile
 113     results_file = args.resultsfile
 114
 115     if args.database:
 116         import sqlite3
 117         conn = sqlite3.connect(args.database)
 118     else:
 119         conn = None
 120
 121     if conn:
 122         run = str(int(time.time()))
 123
 124     if args.update_tests_db:
 125         for t in tests:
 126             print t.__name__ + " - " + t.__doc__
 127             if conn:
 128                 sql = 'INSERT OR REPLACE INTO tests(test,description) VALUES (?, ?)'
 129                 params = (t.__name__.replace('test_', '', 1), t.__doc__)
 130                 try:
 131                     conn.execute(sql, params)
 132                 except Exception, e:
 133                     print "sqlite: " + str(e)
 134                     print "sql: %r" % (params,)
 135         if conn:
 136             conn.commit()
 137             conn.close()
 138         sys.exit(0)
 139
 140
 141     dev0 = WpaSupplicant('wlan0', '/tmp/wpas-wlan0')
 142     dev1 = WpaSupplicant('wlan1', '/tmp/wpas-wlan1')
 143     dev2 = WpaSupplicant('wlan2', '/tmp/wpas-wlan2')
 144     dev = [ dev0, dev1, dev2 ]
 145     apdev = [ ]
 146     apdev.append({"ifname": 'wlan3', "bssid": "02:00:00:00:03:00"})
 147     apdev.append({"ifname": 'wlan4', "bssid": "02:00:00:00:04:00"})
 148
 149     for d in dev:
 150         if not d.ping():
 151             logger.info(d.ifname + ": No response from wpa_supplicant")
 152             return
 153         logger.info("DEV: " + d.ifname + ": " + d.p2p_dev_addr())
 154     for ap in apdev:
 155         logger.info("APDEV: " + ap['ifname'])
 156
 157     passed = []
 158     skipped = []
 159     failed = []
 160
 161     for t in tests:
 162         if args.tests:
 163             if not t.__name__ in args.tests:
 164                 continue
 165         if args.testmodules:
 166             if not t.__module__ in args.testmodules:
 167                 continue
 168         reset_devs(dev, apdev)
 169         logger.info("START " + t.__name__)
 170         if log_to_file:
 171             print "START " + t.__name__
 172             sys.stdout.flush()
 173         if t.__doc__:
 174             logger.info("Test: " + t.__doc__)
 175         start = datetime.now()
 176         for d in dev:
 177             try:
 178                 d.request("NOTE TEST-START " + t.__name__)
 179             except Exception, e:
 180                 logger.info("Failed to issue TEST-START before " + t.__name__ + " for " + d.ifname)
 181                 logger.info(e)
 182                 print "FAIL " + t.__name__ + " - could not start test"
 183                 if conn:
 184                     conn.close()
 185                     conn = None
 186                 sys.exit(1)
 187         try:
 188             if t.func_code.co_argcount > 1:
 189                 res = t(dev, apdev)
 190             else:
 191                 res = t(dev)
 192             end = datetime.now()
 193             diff = end - start
 194             if res == "skip":
 195                 skipped.append(t.__name__)
 196                 result = "SKIP"
 197             else:
 198                 passed.append(t.__name__)
 199                 result = "PASS"
 200             report(conn, args.build, args.commit, run, t.__name__, result, diff)
 201             result = result + " " + t.__name__ + " "
 202             result = result + str(diff.total_seconds()) + " " + str(end)
 203             logger.info(result)
 204             if log_to_file or print_res:
 205                 print result
 206                 sys.stdout.flush()
 207             if results_file:
 208                 f = open(results_file, 'a')
 209                 f.write(result + "\n")
 210                 f.close()
 211         except Exception, e:
 212             end = datetime.now()
 213             diff = end - start
 214             logger.info(e)
 215             failed.append(t.__name__)
 216             report(conn, args.build, args.commit, run, t.__name__, "FAIL", diff)
 217             result = "FAIL " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end)
 218             logger.info(result)
 219             if log_to_file:
 220                 print result
 221                 sys.stdout.flush()
 222             if results_file:
 223                 f = open(results_file, 'a')
 224                 f.write(result + "\n")
 225                 f.close()
 226         for d in dev:
 227             try:
 228                 d.request("NOTE TEST-STOP " + t.__name__)
 229             except Exception, e:
 230                 logger.info("Failed to issue TEST-STOP after " + t.__name__ + " for " + d.ifname)
 231                 logger.info(e)
 232
 233     if not args.tests:
 234         reset_devs(dev, apdev)
 235
 236     if conn:
 237         conn.close()
 238
 239     if len(failed):
 240         logger.info("passed " + str(len(passed)) + " test case(s)")
 241         logger.info("skipped " + str(len(skipped)) + " test case(s)")
 242         logger.info("failed tests: " + str(failed))
 243         if error_file:
 244             f = open(error_file, 'w')
 245             f.write(str(failed) + '\n')
 246             f.close()
 247         sys.exit(1)
 248     logger.info("passed all " + str(len(passed)) + " test case(s)")
 249     if len(skipped):
 250         logger.info("skipped " + str(len(skipped)) + " test case(s)")
 251     if log_to_file:
 252         print "passed all " + str(len(passed)) + " test case(s)"
 253         if len(skipped):
 254             print "skipped " + str(len(skipped)) + " test case(s)"
 255
 256 if __name__ == "__main__":
 257     main()