tests/remote/run-tests.py

   1 #!/usr/bin/env python2
   2 #
   3 # Remote test case executor
   4 # Copyright (c) 2016, Tieto Corporation
   5 #
   6 # This software may be distributed under the terms of the BSD license.
   7 # See README for more details.
   8
   9 import os
  10 import re
  11 import sys
  12 import time
  13 import traceback
  14 import getopt
  15 from datetime import datetime
  16
  17 import logging
  18 logger = logging.getLogger()
  19
  20 scriptsdir = os.path.dirname(os.path.realpath(sys.modules[__name__].__file__))
  21 sys.path.append(os.path.join(scriptsdir, '..', '..', 'wpaspy'))
  22 sys.path.append(os.path.join(scriptsdir, '..', 'hwsim'))
  23
  24 import wpaspy
  25 import config
  26 from test_devices import show_devices
  27 from test_devices import check_devices
  28 from rutils import TestSkip
  29 from utils import HwsimSkip
  30 from hwsim_wrapper import run_hwsim_test
  31
  32 def usage():
  33     print "USAGE: " + sys.argv[0] + " -t devices"
  34     print "USAGE: " + sys.argv[0] + " -t check_devices"
  35     print "USAGE: " + sys.argv[0] + " -d <dut_name> -t <all|sanity|tests_to_run> [-r <ref_name>] [-c <cfg_file.py>] [-m <all|monitor_name>] [-h hwsim_tests][-R][-T][-P][-v]"
  36     print "USAGE: " + sys.argv[0]
  37
  38 def get_devices(devices, duts, refs, monitors):
  39     for dut in duts:
  40         config.get_device(devices, dut, lock=True)
  41     for ref in refs:
  42         config.get_device(devices, ref, lock=True)
  43     for monitor in monitors:
  44         if monitor == "all":
  45             continue
  46         if monitor in duts:
  47             continue
  48         if monitor in refs:
  49             continue
  50         config.get_device(devices, monitor, lock=True)
  51
  52 def put_devices(devices, duts, refs, monitors):
  53     for dut in duts:
  54         config.put_device(devices, dut)
  55     for ref in refs:
  56         config.put_device(devices, ref)
  57     for monitor in monitors:
  58         if monitor == "all":
  59             continue
  60         if monitor in duts:
  61             continue
  62         if monitor in refs:
  63             continue
  64         config.put_device(devices, monitor)
  65
  66 def main():
  67     duts = []
  68     refs = []
  69     monitors = []
  70     filter_keys = []
  71     requested_tests = ["help"]
  72     requested_hwsim_tests = []
  73     hwsim_tests = []
  74     cfg_file = "cfg.py"
  75     log_dir = "./logs/"
  76     verbose = False
  77     trace = False
  78     restart = False
  79     perf = False
  80
  81     # parse input parameters
  82     try:
  83         opts, args = getopt.getopt(sys.argv[1:], "d:r:t:l:k:c:m:h:vRPT",
  84                                    ["dut=", "ref=", "tests=", "log-dir=",
  85                                     "cfg=", "key=", "monitor=", "hwsim="])
  86     except getopt.GetoptError as err:
  87         print(err)
  88         usage()
  89         sys.exit(2)
  90
  91     for option, argument in opts:
  92         if option == "-v":
  93             verbose = True
  94         elif option == "-R":
  95             restart = True
  96         elif option == "-T":
  97             trace = True
  98         elif option == "-P":
  99             perf = True
 100         elif option in ("-d", "--dut"):
 101             duts.append(argument)
 102         elif option in ("-r", "--ref"):
 103             refs.append(argument)
 104         elif option in ("-t", "--tests"):
 105             requested_tests = re.split('; | |, ', argument)
 106         elif option in ("-l", "--log-dir"):
 107             log_dir = argument
 108         elif option in ("-k", "--key"):
 109             filter_keys.append(argument)
 110         elif option in ("-m", "--monitor"):
 111             monitors.append(argument)
 112         elif option in ("-c", "--cfg"):
 113             cfg_file = argument
 114         elif option in ("-h", "--hwsim"):
 115             requested_hwsim_tests = re.split('; | |, ', argument)
 116         else:
 117             assert False, "unhandled option"
 118
 119     # get env configuration
 120     setup_params = config.get_setup_params(cfg_file)
 121     devices = config.get_devices(cfg_file)
 122
 123     # put logs in log_dir
 124     symlink = os.path.join(log_dir, "current");
 125     if os.path.exists(symlink):
 126         os.unlink(symlink)
 127     log_dir = os.path.join(log_dir, time.strftime("%Y_%m_%d_%H_%M_%S"))
 128     if not os.path.exists(log_dir):
 129         os.makedirs(log_dir)
 130     os.symlink(os.path.join("../", log_dir), symlink)
 131
 132     # setup restart/trace/perf request
 133     setup_params['local_log_dir'] = log_dir
 134     setup_params['restart_device'] = restart
 135     setup_params['trace'] = trace
 136     setup_params['perf'] = perf
 137
 138     # configure logger
 139     logger.setLevel(logging.DEBUG)
 140
 141     stdout_handler = logging.StreamHandler()
 142     stdout_handler.setLevel(logging.WARNING)
 143     if verbose:
 144         stdout_handler.setLevel(logging.DEBUG)
 145     logger.addHandler(stdout_handler)
 146
 147     formatter = logging.Formatter('%(asctime)s - %(message)s')
 148     file_name = os.path.join(log_dir, 'run-tests.log')
 149     log_handler = logging.FileHandler(file_name)
 150     log_handler.setLevel(logging.DEBUG)
 151     log_handler.setFormatter(formatter)
 152     logger.addHandler(log_handler)
 153
 154     # import available tests
 155     tests = []
 156     failed = []
 157     test_modules = []
 158     files = os.listdir(scriptsdir)
 159     for t in files:
 160         m = re.match(r'(test_.*)\.py$', t)
 161         if m:
 162             mod = __import__(m.group(1))
 163             test_modules.append(mod.__name__.replace('test_', '', 1))
 164             for key,val in mod.__dict__.iteritems():
 165                 if key.startswith("test_"):
 166                     tests.append(val)
 167     test_names = list(set([t.__name__.replace('test_', '', 1) for t in tests]))
 168
 169     # import test_*
 170     files = os.listdir("../hwsim/")
 171     for t in files:
 172         m = re.match(r'(test_.*)\.py$', t)
 173         if m:
 174             mod = __import__(m.group(1))
 175             test_modules.append(mod.__name__.replace('test_', '', 1))
 176             for key,val in mod.__dict__.iteritems():
 177                 if key.startswith("test_"):
 178                     hwsim_tests.append(val)
 179
 180     # setup hwsim tests
 181     hwsim_tests_to_run = []
 182     if len(requested_hwsim_tests) > 0:
 183         # apply filters
 184         for filter_key in filter_keys:
 185             filtered_tests = []
 186             for hwsim_test in hwsim_tests:
 187                 if re.search(filter_key, hwsim_test.__name__):
 188                     filtered_tests.append(hwsim_test)
 189             hwsim_tests = filtered_tests
 190
 191         # setup hwsim_test we should run
 192         if requested_hwsim_tests[0] == "all":
 193             hwsim_tests_to_run = hwsim_tests
 194         elif requested_hwsim_tests[0] == "remote":
 195             hwsim_tests_to_run = [t for t in hwsim_tests
 196                                   if hasattr(t, "remote_compatible") and
 197                                      t.remote_compatible]
 198         else:
 199             for test in requested_hwsim_tests:
 200                 t = None
 201                 for tt in hwsim_tests:
 202                     name = tt.__name__.replace('test_', '', 1)
 203                     if name == test and tt.func_code.co_argcount <= 2:
 204                         t = tt
 205                         break
 206                 if not t:
 207                     logger.warning("hwsim test case: " + test + " NOT-FOUND")
 208                     continue
 209                 hwsim_tests_to_run.append(t)
 210
 211     # sort the list
 212     test_names.sort()
 213     tests.sort()
 214
 215     # print help
 216     if requested_tests[0] == "help" and len(requested_hwsim_tests) == 0:
 217         usage()
 218         print "\nAvailable Devices:"
 219         for device in devices:
 220             print "\t", device['name']
 221         print "\nAvailable tests:"
 222         for test in test_names:
 223             print "\t", test
 224         print "\nAvailable hwsim tests:"
 225         for hwsim_test in hwsim_tests:
 226             print "\t", hwsim_test.__name__.replace('test_', '', 1)
 227         return
 228
 229     # show/check devices
 230     if requested_tests[0] == "devices":
 231         show_devices(devices, setup_params)
 232         return
 233
 234     # apply filters
 235     for filter_key in filter_keys:
 236         filtered_tests = []
 237         for test in tests:
 238             if re.search(filter_key, test.__name__):
 239                 filtered_tests.append(test)
 240         tests = filtered_tests
 241
 242     # setup test we should run
 243     tests_to_run = []
 244     if requested_tests[0] == "all":
 245         tests_to_run = tests
 246     if requested_tests[0] == "help":
 247         pass
 248     elif requested_tests[0] == "sanity":
 249         for test in tests:
 250             if test.__name__.startswith("test_sanity_"):
 251                 tests_to_run.append(test)
 252     else:
 253         for test in requested_tests:
 254             t = None
 255             for tt in tests:
 256                 name = tt.__name__.replace('test_', '', 1)
 257                 if name == test:
 258                     t = tt
 259                     break
 260             if not t:
 261                 logger.warning("test case: " + test + " NOT-FOUND")
 262                 continue
 263             tests_to_run.append(t)
 264
 265     # lock devices
 266     try:
 267         get_devices(devices, duts, refs, monitors)
 268     except Exception, e:
 269         logger.warning("get devices failed: " + str(e))
 270         logger.info(traceback.format_exc())
 271         put_devices(devices, duts, refs, monitors)
 272         return
 273     except:
 274         logger.warning("get devices failed")
 275         logger.info(traceback.format_exc())
 276         put_devices(devices, duts, refs, monitors)
 277         return
 278
 279     # now run test cases
 280     for dut in duts:
 281         logger.warning("DUT: " + str(dut))
 282     for ref in refs:
 283         logger.warning("REF: " + str(ref))
 284     for monitor in monitors:
 285         logger.warning("MON: " + str(monitor))
 286
 287     # run check_devices at begining
 288     logger.warning("RUN check_devices")
 289     try:
 290         check_devices(devices, setup_params, refs, duts, monitors)
 291     except Exception, e:
 292         logger.warning("FAILED: " + str(e))
 293         logger.info(traceback.format_exc())
 294         put_devices(devices, duts, refs, monitors)
 295         return
 296     except:
 297         logger.warning("FAILED")
 298         logger.info(traceback.format_exc())
 299         put_devices(devices, duts, refs, monitors)
 300         return
 301     logger.warning("PASS")
 302
 303     test_no = 1
 304     for test in tests_to_run:
 305         try:
 306             start = datetime.now()
 307             setup_params['tc_name'] = test.__name__.replace('test_', '', 1)
 308             logger.warning("START - " + setup_params['tc_name'] + " (" + str(test_no) + "/" + str(len(tests_to_run)) + ")")
 309             if test.__doc__:
 310                 logger.info("Test: " + test.__doc__)
 311
 312             # run tc
 313             res = test(devices, setup_params, refs, duts, monitors)
 314
 315             end = datetime.now()
 316             logger.warning("PASS (" + res + ") - " + str((end - start).total_seconds()) + "s")
 317         except KeyboardInterrupt:
 318             put_devices(devices, duts, refs, monitors)
 319             raise
 320         except TestSkip, e:
 321             end = datetime.now()
 322             logger.warning("SKIP (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
 323         except Exception, e:
 324             end = datetime.now()
 325             logger.warning("FAILED (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
 326             logger.info(traceback.format_exc())
 327             failed.append(test.__name__.replace('test_', '', 1))
 328         except:
 329             end = datetime.now()
 330             logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
 331             logger.info(traceback.format_exc())
 332             failed.append(test.__name__.replace('test_', '', 1))
 333         test_no += 1
 334
 335     test_no = 1
 336     for hwsim_test in hwsim_tests_to_run:
 337         try:
 338             start = datetime.now()
 339             setup_params['tc_name'] = hwsim_test.__name__.replace('test_', '', 1)
 340             logger.warning("START - " + setup_params['tc_name'] + " (" + str(test_no) + "/" + str(len(hwsim_tests_to_run)) + ")")
 341             res = run_hwsim_test(devices, setup_params, refs, duts, monitors, hwsim_test)
 342             end = datetime.now()
 343             logger.warning("PASS (" + res + ") - " + str((end - start).total_seconds()) + "s")
 344         except KeyboardInterrupt:
 345             put_devices(devices, duts, refs, monitors)
 346             raise
 347         except HwsimSkip,e:
 348             end = datetime.now()
 349             logger.warning("SKIP (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
 350             failed.append(hwsim_test.__name__.replace('test_', '', 1))
 351         except Exception, e:
 352             end = datetime.now()
 353             logger.warning("FAILED (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
 354             logger.info(traceback.format_exc())
 355             failed.append(hwsim_test.__name__.replace('test_', '', 1))
 356         except:
 357             end = datetime.now()
 358             logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
 359             logger.info(traceback.format_exc())
 360             failed.append(hwsim_test.__name__.replace('test_', '', 1))
 361         test_no += 1
 362
 363     # unlock devices
 364     put_devices(devices, duts, refs, monitors)
 365
 366     if len(failed) > 0:
 367         logger.warning("Failed test cases:")
 368         for test in failed:
 369             logger.warning("\t" + test)
 370
 371
 372 if __name__ == "__main__":
 373         main()