tests: Add run-tests.py --shuffle-test

This optional argument can be used to randomize the order in which the
test cases are run. This can provide more coverage on testing
interactions of common use cases in various different sequences. Such
issues have already been found even with the fixed order of test cases,
but being able to reorder the tests makes this more efficient.

Signed-hostap: Jouni Malinen <j@w1.fi>

diff --git a/tests/hwsim/run-tests.py b/tests/hwsim/run-tests.py
index bd2d093..594c495 100755 (executable)
--- a/tests/hwsim/run-tests.py
+++ b/tests/hwsim/run-tests.py
@@ -121,6 +121,9 @@ def main():
                         help='collect tracing per test case (in log directory)')
     parser.add_argument('-D', action='store_true', dest='dmesg',
                         help='collect dmesg per test case (in log directory)')
+    parser.add_argument('--shuffle-tests', action='store_true',
+                        dest='shuffle_tests',
+                        help='Shuffle test cases to randomize order')
     parser.add_argument('-f', dest='testmodules', metavar='<test module>',
                         help='execute only tests from these test modules',
                         type=str, choices=[[]] + test_modules, nargs='+')
@@ -228,6 +231,10 @@ def main():
             name = t.__name__.replace('test_', '', 1)
             report(conn, False, args.build, args.commit, run, name, 'NOTRUN', 0)
 
+    if args.shuffle_tests:
+        from random import shuffle
+        shuffle(tests_to_run)
+
     for t in tests_to_run:
         name = t.__name__.replace('test_', '', 1)
         if log_handler: