Changeset 391 for python/trunk/Lib/test/regrtest.py

Timestamp:

Mar 19, 2014, 11:31:01 PM (11 years ago)

Author:

dmik

Message:

python: Merge vendor 2.7.6 to trunk.

Location:

python/trunk

Files:

• . (modified) (1 prop)
• Lib/test/regrtest.py (modified) (44 diffs)

python/trunk/Lib/test/regrtest.py

@@ -1 +1 @@
 #! /usr/bin/env python
 
-"""Regression test.
-
-This will find all modules whose name is "test_*" in the test
-directory, and run them.  Various command line options provide
-additional facilities.
-
-Command line options:
-
--v: verbose    -- run tests in verbose mode with output to stdout
--w: verbose2   -- re-run failed tests in verbose mode
--q: quiet      -- don't print anything except if a test fails
--x: exclude    -- arguments are tests to *exclude*
--s: single     -- run only a single test (see below)
--S: slow       -- print the slowest 10 tests
--r: random     -- randomize test execution order
--f: fromfile   -- read names of tests to run from a file (see below)
--l: findleaks  -- if GC is available detect tests that leak memory
--u: use        -- specify which special resource intensive tests to run
--h: help       -- print this text and exit
--t: threshold  -- call gc.set_threshold(N)
--T: coverage   -- turn on code coverage using the trace module
--D: coverdir   -- Directory where coverage files are put
--N: nocoverdir -- Put coverage files alongside modules
--L: runleaks   -- run the leaks(1) command just before exit
--R: huntrleaks -- search for reference leaks (needs debug build, v. slow)
--M: memlimit   -- run very large memory-consuming tests
-
-If non-option arguments are present, they are names for tests to run,
-unless -x is given, in which case they are names for tests not to run.
-If no test names are given, all tests are run.
-
--T turns on code coverage tracing with the trace module.
-
--D specifies the directory where coverage files are put.
-
--N Put coverage files alongside modules.
-
--s means to run only a single test and exit.  This is useful when
-doing memory analysis on the Python interpreter (which tend to consume
-too many resources to run the full regression test non-stop).  The
-file /tmp/pynexttest is read to find the next test to run.  If this
-file is missing, the first test_*.py file in testdir or on the command
-line is used.  (actually tempfile.gettempdir() is used instead of
-/tmp).
+"""
+Usage:
+
+python -m test.regrtest [options] [test_name1 [test_name2 ...]]
+python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]]
+
+
+If no arguments or options are provided, finds all files matching
+the pattern "test_*" in the Lib/test subdirectory and runs
+them in alphabetical order (but see -M and -u, below, for exceptions).
+
+For more rigorous testing, it is useful to use the following
+command line:
+
+python -E -tt -Wd -3 -m test.regrtest [options] [test_name1 ...]
+
+
+Options:
+
+-h/--help       -- print this text and exit
+
+Verbosity
+
+-v/--verbose    -- run tests in verbose mode with output to stdout
+-w/--verbose2   -- re-run failed tests in verbose mode
+-W/--verbose3   -- re-run failed tests in verbose mode immediately
+-q/--quiet      -- no output unless one or more tests fail
+-S/--slow       -- print the slowest 10 tests
+   --header     -- print header with interpreter info
+
+Selecting tests
+
+-r/--randomize  -- randomize test execution order (see below)
+   --randseed   -- pass a random seed to reproduce a previous random run
+-f/--fromfile   -- read names of tests to run from a file (see below)
+-x/--exclude    -- arguments are tests to *exclude*
+-s/--single     -- single step through a set of tests (see below)
+-u/--use RES1,RES2,...
+                -- specify which special resource intensive tests to run
+-M/--memlimit LIMIT
+                -- run very large memory-consuming tests
+
+Special runs
+
+-l/--findleaks  -- if GC is available detect tests that leak memory
+-L/--runleaks   -- run the leaks(1) command just before exit
+-R/--huntrleaks RUNCOUNTS
+                -- search for reference leaks (needs debug build, v. slow)
+-j/--multiprocess PROCESSES
+                -- run PROCESSES processes at once
+-T/--coverage   -- turn on code coverage tracing using the trace module
+-D/--coverdir DIRECTORY
+                -- Directory where coverage files are put
+-N/--nocoverdir -- Put coverage files alongside modules
+-t/--threshold THRESHOLD
+                -- call gc.set_threshold(THRESHOLD)
+-F/--forever    -- run the specified tests in a loop, until an error happens
+
+
+Additional Option Details:
+
+-r randomizes test execution order. You can use --randseed=int to provide a
+int seed value for the randomizer; this is useful for reproducing troublesome
+test orders.
+
+-s On the first invocation of regrtest using -s, the first test file found
+or the first test file given on the command line is run, and the name of
+the next test is recorded in a file named pynexttest.  If run from the
+Python build directory, pynexttest is located in the 'build' subdirectory,
+otherwise it is located in tempfile.gettempdir().  On subsequent runs,
+the test in pynexttest is run, and the next test is written to pynexttest.
+When the last test has been run, pynexttest is deleted.  In this way it
+is possible to single step through the test files.  This is useful when
+doing memory analysis on the Python interpreter, which process tends to
+consume too many resources to run the full regression test non-stop.
 
 -f reads the names of tests from the file given as f's argument, one
@@ -61 +92 @@
 of times further it is run and 'fname' is the name of the file the
 reports are written to.  These parameters all have defaults (5, 4 and
-"reflog.txt" respectively), so the minimal invocation is '-R ::'.
+"reflog.txt" respectively), and the minimal invocation is '-R :'.
 
 -M runs tests that require an exorbitant amount of memory. These tests
@@ -92 +123 @@
                 state and output modes.
 
-    lib2to3 -   Run the tests for 2to3 (They take a while.)
-
     largefile - It is okay to run some test that may create huge
                 files.  These tests can take a long time and may
@@ -107 +136 @@
                 verifies compliance with standards.
 
-    compiler -  Test the compiler package by compiling all the source
-                in the standard library and test suite.  This takes
-                a long time.  Enabling this resource also allows
-                test_tokenize to verify round-trip lexing on every
-                file in the test library.
+    cpu -       Used for certain CPU-heavy tests.
 
     subprocess  Run all tests for the subprocess module.
 
     urlfetch -  It is okay to download files required on testing.
+
+    gui -       Run tests that require a running GUI.
+
+    xpickle -   Test pickle and cPickle against Python 2.4, 2.5 and 2.6 to
+                test backwards compatibility. These tests take a long time
+                to run.
 
 To enable all resources except one, use '-uall,-<resource>'.  For
@@ -122 +153 @@
 """
 
-import cStringIO
+import StringIO
 import getopt
+import json
 import os
 import random
 import re
+import shutil
 import sys
 import time
 import traceback
 import warnings
-
-# I see no other way to suppress these warnings;
-# putting them in test_grammar.py has no effect:
-warnings.filterwarnings("ignore", "hex/oct constants", FutureWarning,
-                        ".*test.test_grammar$")
-if sys.maxint > 0x7fffffff:
-    # Also suppress them in <string>, because for 64-bit platforms,
-    # that's where test_grammar.py hides them.
-    warnings.filterwarnings("ignore", "hex/oct constants", FutureWarning,
-                            "<string>")
-
-# Ignore ImportWarnings that only occur in the source tree,
-# (because of modules with the same name as source-directories in Modules/)
-for mod in ("ctypes", "gzip", "zipfile", "tarfile", "encodings.zlib_codec",
-            "test.test_zipimport", "test.test_zlib", "test.test_zipfile",
-            "test.test_codecs", "test.string_tests"):
-    warnings.filterwarnings(module=".*%s$" % (mod,),
-                            action="ignore", category=ImportWarning)
+import unittest
+import tempfile
+import imp
+import platform
+import sysconfig
+
+
+# Some times __path__ and __file__ are not absolute (e.g. while running from
+# Lib/) and, if we change the CWD to run the tests in a temporary dir, some
+# imports might fail.  This affects only the modules imported before os.chdir().
+# These modules are searched first in sys.path[0] (so '' -- the CWD) and if
+# they are found in the CWD their __file__ and __path__ will be relative (this
+# happens before the chdir).  All the modules imported after the chdir, are
+# not found in the CWD, and since the other paths in sys.path[1:] are absolute
+# (site.py absolutize them), the __file__ and __path__ will be absolute too.
+# Therefore it is necessary to absolutize manually the __file__ and __path__ of
+# the packages to prevent later imports to fail when the CWD is different.
+for module in sys.modules.itervalues():
+    if hasattr(module, '__path__'):
+        module.__path__ = [os.path.abspath(path) for path in module.__path__]
+    if hasattr(module, '__file__'):
+        module.__file__ = os.path.abspath(module.__file__)
+
 
 # MacOSX (a.k.a. Darwin) has a default stack size that is too small
@@ -166 +204 @@
         resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard))
 
+# Test result constants.
+PASSED = 1
+FAILED = 0
+ENV_CHANGED = -1
+SKIPPED = -2
+RESOURCE_DENIED = -3
+INTERRUPTED = -4
+
 from test import test_support
 
 RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', 'bsddb',
-                  'decimal', 'compiler', 'subprocess', 'urlfetch')
+                  'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui',
+                  'xpickle')
+
+TEMPDIR = os.path.abspath(tempfile.gettempdir())
 
 
@@ -181 +230 @@
          exclude=False, single=False, randomize=False, fromfile=None,
          findleaks=False, use_resources=None, trace=False, coverdir='coverage',
-         runleaks=False, huntrleaks=False, verbose2=False, print_slow=False):
+         runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
+         random_seed=None, use_mp=None, verbose3=False, forever=False,
+         header=False):
     """Execute a test suite.
 
@@ -199 +250 @@
 
     The other default arguments (verbose, quiet, exclude,
-    single, randomize, findleaks, use_resources, trace, coverdir, and
-    print_slow) allow programmers calling main() directly to set the
-    values that would normally be set by flags on the command line.
+    single, randomize, findleaks, use_resources, trace, coverdir,
+    print_slow, and random_seed) allow programmers calling main()
+    directly to set the values that would normally be set by flags
+    on the command line.
     """
 
     test_support.record_original_stdout(sys.stdout)
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'hvqxsSrf:lu:t:TD:NLR:wM:',
-                                   ['help', 'verbose', 'quiet', 'exclude',
-                                    'single', 'slow', 'random', 'fromfile',
-                                    'findleaks', 'use=', 'threshold=', 'trace',
-                                    'coverdir=', 'nocoverdir', 'runleaks',
-                                    'huntrleaks=', 'verbose2', 'memlimit=',
-                                    ])
+        opts, args = getopt.getopt(sys.argv[1:], 'hvqxsSrf:lu:t:TD:NLR:FwWM:j:',
+            ['help', 'verbose', 'verbose2', 'verbose3', 'quiet',
+             'exclude', 'single', 'slow', 'randomize', 'fromfile=', 'findleaks',
+             'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir',
+             'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=',
+             'multiprocess=', 'slaveargs=', 'forever', 'header'])
     except getopt.error, msg:
         usage(2, msg)
 
     # Defaults
+    if random_seed is None:
+        random_seed = random.randrange(10000000)
     if use_resources is None:
         use_resources = []
@@ -226 +279 @@
         elif o in ('-w', '--verbose2'):
             verbose2 = True
+        elif o in ('-W', '--verbose3'):
+            verbose3 = True
         elif o in ('-q', '--quiet'):
             quiet = True;
@@ -237 +292 @@
         elif o in ('-r', '--randomize'):
             randomize = True
+        elif o == '--randseed':
+            random_seed = int(a)
         elif o in ('-f', '--fromfile'):
             fromfile = a
@@ -254 +311 @@
         elif o in ('-R', '--huntrleaks'):
             huntrleaks = a.split(':')
-            if len(huntrleaks) != 3:
+            if len(huntrleaks) not in (2, 3):
                 print a, huntrleaks
-                usage(2, '-R takes three colon-separated arguments')
-            if len(huntrleaks[0]) == 0:
+                usage(2, '-R takes 2 or 3 colon-separated arguments')
+            if not huntrleaks[0]:
                 huntrleaks[0] = 5
             else:
                 huntrleaks[0] = int(huntrleaks[0])
-            if len(huntrleaks[1]) == 0:
+            if not huntrleaks[1]:
                 huntrleaks[1] = 4
             else:
                 huntrleaks[1] = int(huntrleaks[1])
-            if len(huntrleaks[2]) == 0:
-                huntrleaks[2] = "reflog.txt"
+            if len(huntrleaks) == 2 or not huntrleaks[2]:
+                huntrleaks[2:] = ["reflog.txt"]
         elif o in ('-M', '--memlimit'):
             test_support.set_memlimit(a)
@@ -286 +343 @@
                 elif r not in use_resources:
                     use_resources.append(r)
+        elif o in ('-F', '--forever'):
+            forever = True
+        elif o in ('-j', '--multiprocess'):
+            use_mp = int(a)
+        elif o == '--header':
+            header = True
+        elif o == '--slaveargs':
+            args, kwargs = json.loads(a)
+            try:
+                result = runtest(*args, **kwargs)
+            except BaseException, e:
+                result = INTERRUPTED, e.__class__.__name__
+            print   # Force a newline (just in case)
+            print json.dumps(result)
+            sys.exit(0)
         else:
-            print >>sys.stderr, ("No handler for option {0}.  Please "
+            print >>sys.stderr, ("No handler for option {}.  Please "
                 "report this as a bug at http://bugs.python.org.").format(o)
             sys.exit(1)
     if single and fromfile:
         usage(2, "-s and -f don't go together!")
+    if use_mp and trace:
+        usage(2, "-T and -j don't go together!")
+    if use_mp and findleaks:
+        usage(2, "-l and -j don't go together!")
 
     good = []
@@ -297 +373 @@
     skipped = []
     resource_denieds = []
+    environment_changed = []
+    interrupted = False
 
     if findleaks:
@@ -312 +390 @@
 
     if single:
-        from tempfile import gettempdir
-        filename = os.path.join(gettempdir(), 'pynexttest')
+        filename = os.path.join(TEMPDIR, 'pynexttest')
         try:
             fp = open(filename, 'r')
-            next = fp.read().strip()
-            tests = [next]
+            next_test = fp.read().strip()
+            tests = [next_test]
             fp.close()
         except IOError:
@@ -324 +401 @@
     if fromfile:
         tests = []
-        fp = open(fromfile)
+        fp = open(os.path.join(test_support.SAVEDCWD, fromfile))
         for line in fp:
             guts = line.split() # assuming no test has whitespace in its name
@@ -332 +409 @@
 
     # Strip .py extensions.
-    if args:
-        args = map(removepy, args)
-    if tests:
-        tests = map(removepy, tests)
+    removepy(args)
+    removepy(tests)
 
     stdtests = STDTESTS[:]
-    nottests = NOTTESTS[:]
+    nottests = NOTTESTS.copy()
     if exclude:
         for arg in args:
             if arg in stdtests:
                 stdtests.remove(arg)
-        nottests[:0] = args
+            nottests.add(arg)
         args = []
-    tests = tests or args or findtests(testdir, stdtests, nottests)
+
+    # For a partial run, we do not need to clutter the output.
+    if verbose or header or not (quiet or single or tests or args):
+        # Print basic platform information
+        print "==", platform.python_implementation(), \
+                    " ".join(sys.version.split())
+        print "==  ", platform.platform(aliased=True), \
+                      "%s-endian" % sys.byteorder
+        print "==  ", os.getcwd()
+        print "Testing with flags:", sys.flags
+
+    alltests = findtests(testdir, stdtests, nottests)
+    selected = tests or args or alltests
     if single:
-        tests = tests[:1]
+        selected = selected[:1]
+        try:
+            next_single_test = alltests[alltests.index(selected[0])+1]
+        except IndexError:
+            next_single_test = None
     if randomize:
-        random.shuffle(tests)
+        random.seed(random_seed)
+        print "Using random seed", random_seed
+        random.shuffle(selected)
     if trace:
         import trace
         tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix],
                              trace=False, count=True)
+
     test_times = []
-    test_support.verbose = verbose      # Tell tests to be moderately quiet
     test_support.use_resources = use_resources
     save_modules = sys.modules.keys()
-    for test in tests:
-        if not quiet:
-            print test
-            sys.stdout.flush()
-        if trace:
-            # If we're tracing code coverage, then we don't exit with status
-            # if on a false return value from main.
-            tracer.runctx('runtest(test, verbose, quiet,'
-                          '        test_times, testdir)',
-                          globals=globals(), locals=vars())
-        else:
+
+    def accumulate_result(test, result):
+        ok, test_time = result
+        test_times.append((test_time, test))
+        if ok == PASSED:
+            good.append(test)
+        elif ok == FAILED:
+            bad.append(test)
+        elif ok == ENV_CHANGED:
+            bad.append(test)
+            environment_changed.append(test)
+        elif ok == SKIPPED:
+            skipped.append(test)
+        elif ok == RESOURCE_DENIED:
+            skipped.append(test)
+            resource_denieds.append(test)
+
+    if forever:
+        def test_forever(tests=list(selected)):
+            while True:
+                for test in tests:
+                    yield test
+                    if bad:
+                        return
+        tests = test_forever()
+    else:
+        tests = iter(selected)
+
+    if use_mp:
+        try:
+            from threading import Thread
+        except ImportError:
+            print "Multiprocess option requires thread support"
+            sys.exit(2)
+        from Queue import Queue
+        from subprocess import Popen, PIPE
+        debug_output_pat = re.compile(r"\[\d+ refs\]$")
+        output = Queue()
+        def tests_and_args():
+            for test in tests:
+                args_tuple = (
+                    (test, verbose, quiet),
+                    dict(huntrleaks=huntrleaks, use_resources=use_resources)
+                )
+                yield (test, args_tuple)
+        pending = tests_and_args()
+        opt_args = test_support.args_from_interpreter_flags()
+        base_cmd = [sys.executable] + opt_args + ['-m', 'test.regrtest']
+        def work():
+            # A worker thread.
             try:
-                ok = runtest(test, verbose, quiet, test_times,
-                             testdir, huntrleaks)
-            except KeyboardInterrupt:
-                # print a newline separate from the ^C
-                print
-                break
-            except:
+                while True:
+                    try:
+                        test, args_tuple = next(pending)
+                    except StopIteration:
+                        output.put((None, None, None, None))
+                        return
+                    # -E is needed by some tests, e.g. test_import
+                    popen = Popen(base_cmd + ['--slaveargs', json.dumps(args_tuple)],
+                                   stdout=PIPE, stderr=PIPE,
+                                   universal_newlines=True,
+                                   close_fds=(os.name != 'nt'))
+                    stdout, stderr = popen.communicate()
+                    # Strip last refcount output line if it exists, since it
+                    # comes from the shutdown of the interpreter in the subcommand.
+                    stderr = debug_output_pat.sub("", stderr)
+                    stdout, _, result = stdout.strip().rpartition("\n")
+                    if not result:
+                        output.put((None, None, None, None))
+                        return
+                    result = json.loads(result)
+                    if not quiet:
+                        stdout = test+'\n'+stdout
+                    output.put((test, stdout.rstrip(), stderr.rstrip(), result))
+            except BaseException:
+                output.put((None, None, None, None))
                 raise
-            if ok > 0:
-                good.append(test)
-            elif ok == 0:
-                bad.append(test)
+        workers = [Thread(target=work) for i in range(use_mp)]
+        for worker in workers:
+            worker.start()
+        finished = 0
+        try:
+            while finished < use_mp:
+                test, stdout, stderr, result = output.get()
+                if test is None:
+                    finished += 1
+                    continue
+                if stdout:
+                    print stdout
+                if stderr:
+                    print >>sys.stderr, stderr
+                sys.stdout.flush()
+                sys.stderr.flush()
+                if result[0] == INTERRUPTED:
+                    assert result[1] == 'KeyboardInterrupt'
+                    raise KeyboardInterrupt   # What else?
+                accumulate_result(test, result)
+        except KeyboardInterrupt:
+            interrupted = True
+            pending.close()
+        for worker in workers:
+            worker.join()
+    else:
+        for test in tests:
+            if not quiet:
+                print test
+                sys.stdout.flush()
+            if trace:
+                # If we're tracing code coverage, then we don't exit with status
+                # if on a false return value from main.
+                tracer.runctx('runtest(test, verbose, quiet)',
+                              globals=globals(), locals=vars())
             else:
-                skipped.append(test)
-                if ok == -2:
-                    resource_denieds.append(test)
-        if findleaks:
-            gc.collect()
-            if gc.garbage:
-                print "Warning: test created", len(gc.garbage),
-                print "uncollectable object(s)."
-                # move the uncollectable objects somewhere so we don't see
-                # them again
-                found_garbage.extend(gc.garbage)
-                del gc.garbage[:]
-        # Unload the newly imported modules (best effort finalization)
-        for module in sys.modules.keys():
-            if module not in save_modules and module.startswith("test."):
-                test_support.unload(module)
-
-    # The lists won't be sorted if running with -r
-    good.sort()
-    bad.sort()
-    skipped.sort()
-
+                try:
+                    result = runtest(test, verbose, quiet, huntrleaks)
+                    accumulate_result(test, result)
+                    if verbose3 and result[0] == FAILED:
+                        print "Re-running test %r in verbose mode" % test
+                        runtest(test, True, quiet, huntrleaks)
+                except KeyboardInterrupt:
+                    interrupted = True
+                    break
+                except:
+                    raise
+            if findleaks:
+                gc.collect()
+                if gc.garbage:
+                    print "Warning: test created", len(gc.garbage),
+                    print "uncollectable object(s)."
+                    # move the uncollectable objects somewhere so we don't see
+                    # them again
+                    found_garbage.extend(gc.garbage)
+                    del gc.garbage[:]
+            # Unload the newly imported modules (best effort finalization)
+            for module in sys.modules.keys():
+                if module not in save_modules and module.startswith("test."):
+                    test_support.unload(module)
+
+    if interrupted:
+        # print a newline after ^C
+        print
+        print "Test suite interrupted by signal SIGINT."
+        omitted = set(selected) - set(good) - set(bad) - set(skipped)
+        print count(len(omitted), "test"), "omitted:"
+        printlist(omitted)
     if good and not quiet:
-        if not bad and not skipped and len(good) > 1:
+        if not bad and not skipped and not interrupted and len(good) > 1:
             print "All",
         print count(len(good), "test"), "OK."
@@ -415 +606 @@
             print "%s: %.1fs" % (test, time)
     if bad:
-        print count(len(bad), "test"), "failed:"
-        printlist(bad)
+        bad = set(bad) - set(environment_changed)
+        if bad:
+            print count(len(bad), "test"), "failed:"
+            printlist(bad)
+        if environment_changed:
+            print "{} altered the execution environment:".format(
+                count(len(environment_changed), "test"))
+            printlist(environment_changed)
     if skipped and not quiet:
         print count(len(skipped), "test"), "skipped:"
@@ -442 +639 @@
             try:
                 test_support.verbose = True
-                ok = runtest(test, True, quiet, test_times, testdir,
-                             huntrleaks)
+                ok = runtest(test, True, quiet, huntrleaks)
             except KeyboardInterrupt:
                 # print a newline separate from the ^C
@@ -452 +648 @@
 
     if single:
-        alltests = findtests(testdir, stdtests, nottests)
-        for i in range(len(alltests)):
-            if tests[0] == alltests[i]:
-                if i == len(alltests) - 1:
-                    os.unlink(filename)
-                else:
-                    fp = open(filename, 'w')
-                    fp.write(alltests[i+1] + '\n')
-                    fp.close()
-                break
+        if next_single_test:
+            with open(filename, 'w') as fp:
+                fp.write(next_single_test + '\n')
         else:
             os.unlink(filename)
@@ -472 +661 @@
         os.system("leaks %d" % os.getpid())
 
-    sys.exit(len(bad) > 0)
+    sys.exit(len(bad) > 0 or interrupted)
 
 
@@ -485 +674 @@
     'test_doctest',
     'test_doctest2',
-   ]
-
-NOTTESTS = [
+]
+
+NOTTESTS = {
     'test_support',
     'test_future1',
     'test_future2',
-    ]
+}
 
 def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
     """Return a list of all applicable test modules."""
-    if not testdir: testdir = findtestdir()
+    testdir = findtestdir(testdir)
     names = os.listdir(testdir)
     tests = []
+    others = set(stdtests) | nottests
     for name in names:
-        if name[:5] == "test_" and name[-3:] == os.extsep+"py":
-            modname = name[:-3]
-            if modname not in stdtests and modname not in nottests:
-                tests.append(modname)
-    tests.sort()
-    return stdtests + tests
-
-def runtest(test, verbose, quiet, test_times,
-            testdir=None, huntrleaks=False):
+        modname, ext = os.path.splitext(name)
+        if modname[:5] == "test_" and ext == ".py" and modname not in others:
+            tests.append(modname)
+    return stdtests + sorted(tests)
+
+def runtest(test, verbose, quiet,
+            huntrleaks=False, use_resources=None):
     """Run a single test.
 
@@ -514 +702 @@
     quiet -- if true, don't print 'skipped' messages (probably redundant)
     test_times -- a list of (time, test_name) pairs
-    testdir -- test directory
     huntrleaks -- run multiple times to test for leaks; requires a debug
                   build; a triple corresponding to -R's three arguments
-    Return:
-        -2  test skipped because resource denied
-        -1  test skipped for some other reason
-         0  test failed
-         1  test passed
+    Returns one of the test result constants:
+        INTERRUPTED      KeyboardInterrupt when run under -j
+        RESOURCE_DENIED  test skipped because resource denied
+        SKIPPED          test skipped for some other reason
+        ENV_CHANGED      test failed because it changed the execution environment
+        FAILED           test failed
+        PASSED           test passed
     """
 
+    test_support.verbose = verbose  # Tell tests to be moderately quiet
+    if use_resources is not None:
+        test_support.use_resources = use_resources
     try:
-        return runtest_inner(test, verbose, quiet, test_times,
-                             testdir, huntrleaks)
+        return runtest_inner(test, verbose, quiet, huntrleaks)
     finally:
         cleanup_test_droppings(test, verbose)
 
-def runtest_inner(test, verbose, quiet, test_times,
-                  testdir=None, huntrleaks=False):
+
+# Unit tests are supposed to leave the execution environment unchanged
+# once they complete.  But sometimes tests have bugs, especially when
+# tests fail, and the changes to environment go on to mess up other
+# tests.  This can cause issues with buildbot stability, since tests
+# are run in random order and so problems may appear to come and go.
+# There are a few things we can save and restore to mitigate this, and
+# the following context manager handles this task.
+
+class saved_test_environment:
+    """Save bits of the test environment and restore them at block exit.
+
+        with saved_test_environment(testname, verbose, quiet):
+            #stuff
+
+    Unless quiet is True, a warning is printed to stderr if any of
+    the saved items was changed by the test.  The attribute 'changed'
+    is initially False, but is set to True if a change is detected.
+
+    If verbose is more than 1, the before and after state of changed
+    items is also printed.
+    """
+
+    changed = False
+
+    def __init__(self, testname, verbose=0, quiet=False):
+        self.testname = testname
+        self.verbose = verbose
+        self.quiet = quiet
+
+    # To add things to save and restore, add a name XXX to the resources list
+    # and add corresponding get_XXX/restore_XXX functions.  get_XXX should
+    # return the value to be saved and compared against a second call to the
+    # get function when test execution completes.  restore_XXX should accept
+    # the saved value and restore the resource using it.  It will be called if
+    # and only if a change in the value is detected.
+    #
+    # Note: XXX will have any '.' replaced with '_' characters when determining
+    # the corresponding method names.
+
+    resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr',
+                 'os.environ', 'sys.path', 'asyncore.socket_map',
+                 'test_support.TESTFN',
+                )
+
+    def get_sys_argv(self):
+        return id(sys.argv), sys.argv, sys.argv[:]
+    def restore_sys_argv(self, saved_argv):
+        sys.argv = saved_argv[1]
+        sys.argv[:] = saved_argv[2]
+
+    def get_cwd(self):
+        return os.getcwd()
+    def restore_cwd(self, saved_cwd):
+        os.chdir(saved_cwd)
+
+    def get_sys_stdout(self):
+        return sys.stdout
+    def restore_sys_stdout(self, saved_stdout):
+        sys.stdout = saved_stdout
+
+    def get_sys_stderr(self):
+        return sys.stderr
+    def restore_sys_stderr(self, saved_stderr):
+        sys.stderr = saved_stderr
+
+    def get_sys_stdin(self):
+        return sys.stdin
+    def restore_sys_stdin(self, saved_stdin):
+        sys.stdin = saved_stdin
+
+    def get_os_environ(self):
+        return id(os.environ), os.environ, dict(os.environ)
+    def restore_os_environ(self, saved_environ):
+        os.environ = saved_environ[1]
+        os.environ.clear()
+        os.environ.update(saved_environ[2])
+
+    def get_sys_path(self):
+        return id(sys.path), sys.path, sys.path[:]
+    def restore_sys_path(self, saved_path):
+        sys.path = saved_path[1]
+        sys.path[:] = saved_path[2]
+
+    def get_asyncore_socket_map(self):
+        asyncore = sys.modules.get('asyncore')
+        # XXX Making a copy keeps objects alive until __exit__ gets called.
+        return asyncore and asyncore.socket_map.copy() or {}
+    def restore_asyncore_socket_map(self, saved_map):
+        asyncore = sys.modules.get('asyncore')
+        if asyncore is not None:
+            asyncore.close_all(ignore_all=True)
+            asyncore.socket_map.update(saved_map)
+
+    def get_test_support_TESTFN(self):
+        if os.path.isfile(test_support.TESTFN):
+            result = 'f'
+        elif os.path.isdir(test_support.TESTFN):
+            result = 'd'
+        else:
+            result = None
+        return result
+    def restore_test_support_TESTFN(self, saved_value):
+        if saved_value is None:
+            if os.path.isfile(test_support.TESTFN):
+                os.unlink(test_support.TESTFN)
+            elif os.path.isdir(test_support.TESTFN):
+                shutil.rmtree(test_support.TESTFN)
+
+    def resource_info(self):
+        for name in self.resources:
+            method_suffix = name.replace('.', '_')
+            get_name = 'get_' + method_suffix
+            restore_name = 'restore_' + method_suffix
+            yield name, getattr(self, get_name), getattr(self, restore_name)
+
+    def __enter__(self):
+        self.saved_values = dict((name, get()) for name, get, restore
+                                                   in self.resource_info())
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        saved_values = self.saved_values
+        del self.saved_values
+        for name, get, restore in self.resource_info():
+            current = get()
+            original = saved_values.pop(name)
+            # Check for changes to the resource's value
+            if current != original:
+                self.changed = True
+                restore(original)
+                if not self.quiet:
+                    print >>sys.stderr, (
+                          "Warning -- {} was modified by {}".format(
+                                                 name, self.testname))
+                    if self.verbose > 1:
+                        print >>sys.stderr, (
+                              "  Before: {}\n  After:  {} ".format(
+                                                  original, current))
+            # XXX (ncoghlan): for most resources (e.g. sys.path) identity
+            # matters at least as much as value. For others (e.g. cwd),
+            # identity is irrelevant. Should we add a mechanism to check
+            # for substitution in the cases where it matters?
+        return False
+
+
+def runtest_inner(test, verbose, quiet, huntrleaks=False):
     test_support.unload(test)
-    if not testdir:
-        testdir = findtestdir()
     if verbose:
         capture_stdout = None
     else:
-        capture_stdout = cStringIO.StringIO()
-
+        capture_stdout = StringIO.StringIO()
+
+    test_time = 0.0
+    refleak = False  # True if the test leaked references.
     try:
         save_stdout = sys.stdout
@@ -550 +886 @@
                 # Always import it from the test package
                 abstest = 'test.' + test
-            start_time = time.time()
-            the_package = __import__(abstest, globals(), locals(), [])
-            the_module = getattr(the_package, test)
-            # Old tests run to completion simply as a side-effect of
-            # being imported.  For tests based on unittest or doctest,
-            # explicitly invoke their test_main() function (if it exists).
-            indirect_test = getattr(the_module, "test_main", None)
-            if indirect_test is not None:
-                indirect_test()
-            if huntrleaks:
-                dash_R(the_module, test, indirect_test, huntrleaks)
-            test_time = time.time() - start_time
-            test_times.append((test_time, test))
+            with saved_test_environment(test, verbose, quiet) as environment:
+                start_time = time.time()
+                the_package = __import__(abstest, globals(), locals(), [])
+                the_module = getattr(the_package, test)
+                # Old tests run to completion simply as a side-effect of
+                # being imported.  For tests based on unittest or doctest,
+                # explicitly invoke their test_main() function (if it exists).
+                indirect_test = getattr(the_module, "test_main", None)
+                if indirect_test is not None:
+                    indirect_test()
+                if huntrleaks:
+                    refleak = dash_R(the_module, test, indirect_test,
+                        huntrleaks)
+                test_time = time.time() - start_time
         finally:
             sys.stdout = save_stdout
@@ -569 +906 @@
             print test, "skipped --", msg
             sys.stdout.flush()
-        return -2
-    except (ImportError, test_support.TestSkipped), msg:
+        return RESOURCE_DENIED, test_time
+    except unittest.SkipTest, msg:
         if not quiet:
             print test, "skipped --", msg
             sys.stdout.flush()
-        return -1
+        return SKIPPED, test_time
     except KeyboardInterrupt:
         raise
     except test_support.TestFailed, msg:
-        print "test", test, "failed --", msg
-        sys.stdout.flush()
-        return 0
+        print >>sys.stderr, "test", test, "failed --", msg
+        sys.stderr.flush()
+        return FAILED, test_time
     except:
         type, value = sys.exc_info()[:2]
-        print "test", test, "crashed --", str(type) + ":", value
-        sys.stdout.flush()
+        print >>sys.stderr, "test", test, "crashed --", str(type) + ":", value
+        sys.stderr.flush()
         if verbose:
-            traceback.print_exc(file=sys.stdout)
-            sys.stdout.flush()
-        return 0
+            traceback.print_exc(file=sys.stderr)
+            sys.stderr.flush()
+        return FAILED, test_time
     else:
+        if refleak:
+            return FAILED, test_time
+        if environment.changed:
+            return ENV_CHANGED, test_time
         # Except in verbose mode, tests should not print anything
         if verbose or huntrleaks:
-            return 1
+            return PASSED, test_time
         output = capture_stdout.getvalue()
         if not output:
-            return 1
+            return PASSED, test_time
         print "test", test, "produced unexpected output:"
         print "*" * 70
@@ -601 +942 @@
         print "*" * 70
         sys.stdout.flush()
-        return 0
+        return FAILED, test_time
 
 def cleanup_test_droppings(testname, verbose):
-    import shutil
+    import stat
+    import gc
+
+    # First kill any dangling references to open files etc.
+    gc.collect()
 
     # Try to clean up junk commonly left behind.  While tests shouldn't leave
@@ -629 +974 @@
             print "%r left behind %s %r" % (testname, kind, name)
         try:
+            # if we have chmod, fix possible permissions problems
+            # that might prevent cleanup
+            if (hasattr(os, 'chmod')):
+                os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
             nuker(name)
         except Exception, msg:
@@ -635 +984 @@
 
 def dash_R(the_module, test, indirect_test, huntrleaks):
+    """Run a test multiple times, looking for reference leaks.
+
+    Returns:
+        False if the test didn't leak references; True if we detected refleaks.
+    """
     # This code is hackish and inelegant, but it seems to do the job.
-    import copy_reg, _abcoll, io
+    import copy_reg, _abcoll, _pyio
 
     if not hasattr(sys, 'gettotalrefcount'):
@@ -646 +1000 @@
     ps = copy_reg.dispatch_table.copy()
     pic = sys.path_importer_cache.copy()
+    try:
+        import zipimport
+    except ImportError:
+        zdc = None # Run unmodified on platforms without zipimport support
+    else:
+        zdc = zipimport._zip_directory_cache.copy()
     abcs = {}
-    modules = _abcoll, io
+    modules = _abcoll, _pyio
     for abc in [getattr(mod, a) for mod in modules for a in mod.__all__]:
         # XXX isinstance(abc, ABCMeta) leads to infinite recursion
@@ -660 +1020 @@
     else:
         def run_the_test():
-            reload(the_module)
+            imp.reload(the_module)
 
     deltas = []
     nwarmup, ntracked, fname = huntrleaks
+    fname = os.path.join(test_support.SAVEDCWD, fname)
     repcount = nwarmup + ntracked
     print >> sys.stderr, "beginning", repcount, "repetitions"
     print >> sys.stderr, ("1234567890"*(repcount//10 + 1))[:repcount]
-    dash_R_cleanup(fs, ps, pic, abcs)
+    dash_R_cleanup(fs, ps, pic, zdc, abcs)
     for i in range(repcount):
-        rc = sys.gettotalrefcount()
+        rc_before = sys.gettotalrefcount()
         run_the_test()
         sys.stderr.write('.')
-        dash_R_cleanup(fs, ps, pic, abcs)
+        dash_R_cleanup(fs, ps, pic, zdc, abcs)
+        rc_after = sys.gettotalrefcount()
         if i >= nwarmup:
-            deltas.append(sys.gettotalrefcount() - rc - 2)
+            deltas.append(rc_after - rc_before)
     print >> sys.stderr
     if any(deltas):
         msg = '%s leaked %s references, sum=%s' % (test, deltas, sum(deltas))
         print >> sys.stderr, msg
-        refrep = open(fname, "a")
-        print >> refrep, msg
-        refrep.close()
-
-def dash_R_cleanup(fs, ps, pic, abcs):
+        with open(fname, "a") as refrep:
+            print >> refrep, msg
+            refrep.flush()
+        return True
+    return False
+
+def dash_R_cleanup(fs, ps, pic, zdc, abcs):
     import gc, copy_reg
     import _strptime, linecache
@@ -702 +1066 @@
     sys.path_importer_cache.clear()
     sys.path_importer_cache.update(pic)
+    try:
+        import zipimport
+    except ImportError:
+        pass # Run unmodified on platforms without zipimport support
+    else:
+        zipimport._zip_directory_cache.clear()
+        zipimport._zip_directory_cache.update(zdc)
 
     # clear type cache
@@ -725 +1096 @@
     struct._clearcache()
     doctest.master = None
+    try:
+        import ctypes
+    except ImportError:
+        # Don't worry about resetting the cache if ctypes is not supported
+        pass
+    else:
+        ctypes._reset_cache()
 
     # Collect cyclic trash.
     gc.collect()
 
-def findtestdir():
-    if __name__ == '__main__':
-        file = sys.argv[0]
-    else:
-        file = __file__
-    testdir = os.path.dirname(file) or os.curdir
-    return testdir
-
-def removepy(name):
-    if name.endswith(os.extsep + "py"):
-        name = name[:-3]
-    return name
+def findtestdir(path=None):
+    return path or os.path.dirname(__file__) or os.curdir
+
+def removepy(names):
+    if not names:
+        return
+    for idx, name in enumerate(names):
+        basename, ext = os.path.splitext(name)
+        if ext == '.py':
+            names[idx] = basename
 
 def count(n, word):
@@ -758 +1134 @@
     from textwrap import fill
     blanks = ' ' * indent
-    print fill(' '.join(map(str, x)), width,
+    # Print the sorted list: 'x' may be a '--random' list or a set()
+    print fill(' '.join(str(elt) for elt in sorted(x)), width,
                initial_indent=blanks, subsequent_indent=blanks)
 
@@ -768 +1145 @@
 #         The _ExpectedSkips constructor adds this to the set of expected
 #         skips if not os.path.supports_unicode_filenames.
-#     test_socket_ssl
-#         Controlled by test_socket_ssl.skip_expected.  Requires the network
-#         resource, and a socket module with ssl support.
 #     test_timeout
 #         Controlled by test_timeout.skip_expected.  Requires the network
@@ -821 +1195 @@
         test_ossaudiodev
         """,
-   'mac':
-        """
-        test_atexit
-        test_bsddb
-        test_bsddb185
-        test_bsddb3
-        test_bz2
-        test_commands
-        test_crypt
-        test_curses
-        test_dbm
-        test_dl
-        test_fcntl
-        test_fork1
-        test_epoll
-        test_grp
-        test_ioctl
-        test_largefile
-        test_locale
-        test_kqueue
-        test_mmap
-        test_openpty
-        test_ossaudiodev
-        test_poll
-        test_popen
-        test_popen2
-        test_posix
-        test_pty
-        test_pwd
-        test_resource
-        test_signal
-        test_sundry
-        test_tarfile
-        test_timing
-        """,
     'unixware7':
         """
@@ -947 +1286 @@
         test_curses
         test_epoll
+        test_gdb
         test_gdbm
         test_largefile
@@ -1072 +1412 @@
         test_pep277
         test_pty
-        test_socket_ssl
         test_socketserver
         test_tcl
+        test_tk
+        test_ttk_guionly
+        test_ttk_textonly
         test_timeout
         test_urllibnet
@@ -1092 +1434 @@
         test_ossaudiodev
         test_tcl
+        test_tk
+        test_ttk_guionly
+        test_ttk_textonly
         test_zipimport
         test_zlib
@@ -1097 +1442 @@
     'openbsd3':
         """
+        test_ascii_formatd
         test_bsddb
         test_bsddb3
@@ -1108 +1454 @@
         test_pep277
         test_tcl
+        test_tk
+        test_ttk_guionly
+        test_ttk_textonly
         test_multiprocessing
         """,
     'netbsd3':
         """
+        test_ascii_formatd
         test_bsddb
         test_bsddb185
@@ -1124 +1474 @@
         test_pep277
         test_tcl
+        test_tk
+        test_ttk_guionly
+        test_ttk_textonly
         test_multiprocessing
         """,
@@ -1148 +1501 @@
                 self.expected.add('test_pep277')
 
-            try:
-                from test import test_socket_ssl
-            except ImportError:
-                pass
-            else:
-                if test_socket_ssl.skip_expected:
-                    self.expected.add('test_socket_ssl')
-
             if test_timeout.skip_expected:
                 self.expected.add('test_timeout')
@@ -1162 +1507 @@
                 self.expected.add('test_imageop')
 
-            if not sys.platform in ("mac", "darwin"):
+            if sys.platform != "darwin":
                 MAC_ONLY = ["test_macos", "test_macostools", "test_aepack",
                             "test_plistlib", "test_scriptpackages",
@@ -1177 +1522 @@
                 WIN_ONLY = ["test_unicode_file", "test_winreg",
                             "test_winsound", "test_startfile",
-                            "test_sqlite"]
+                            "test_sqlite", "test_msilib"]
                 for skip in WIN_ONLY:
                     self.expected.add(skip)
@@ -1210 +1555 @@
 
 if __name__ == '__main__':
-    # Remove regrtest.py's own directory from the module search path.  This
-    # prevents relative imports from working, and relative imports will screw
-    # up the testing framework.  E.g. if both test.test_support and
-    # test_support are imported, they will not contain the same globals, and
-    # much of the testing framework relies on the globals in the
-    # test.test_support module.
-    mydir = os.path.abspath(os.path.normpath(os.path.dirname(sys.argv[0])))
-    i = len(sys.path)
-    while i >= 0:
-        i -= 1
-        if os.path.abspath(os.path.normpath(sys.path[i])) == mydir:
-            del sys.path[i]
-    main()
+    # findtestdir() gets the dirname out of __file__, so we have to make it
+    # absolute before changing the working directory.
+    # For example __file__ may be relative when running trace or profile.
+    # See issue #9323.
+    __file__ = os.path.abspath(__file__)
+
+    # sanity check
+    assert __file__ == os.path.abspath(sys.argv[0])
+
+    # When tests are run from the Python build directory, it is best practice
+    # to keep the test files in a subfolder.  It eases the cleanup of leftover
+    # files using command "make distclean".
+    if sysconfig.is_python_build():
+        TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build')
+        TEMPDIR = os.path.abspath(TEMPDIR)
+        if not os.path.exists(TEMPDIR):
+            os.mkdir(TEMPDIR)
+
+    # Define a writable temp dir that will be used as cwd while running
+    # the tests. The name of the dir includes the pid to allow parallel
+    # testing (see the -j option).
+    TESTCWD = 'test_python_{}'.format(os.getpid())
+
+    TESTCWD = os.path.join(TEMPDIR, TESTCWD)
+
+    # Run the tests in a context manager that temporary changes the CWD to a
+    # temporary and writable directory. If it's not possible to create or
+    # change the CWD, the original CWD will be used. The original CWD is
+    # available from test_support.SAVEDCWD.
+    with test_support.temp_cwd(TESTCWD, quiet=True):
+        main()