[libcxx] Allow use of ShTest in libc++ tests along with other changes.

Summary:
This patch allows the use of LIT's ShTest format in the libc++ test suite. ShTests have the suffix '.sh.cpp'. It also introduces a series of other changes. These changes are:

- More functionality including parsing test metadata has been moved into LIT.
- LibcxxTestFormat now supports multi-part suffixes.
- the `CXXCompiler` functionality has been used to shrink the size of LibcxxTestFormat. 
- The recursive loading of the site config has been turned into `libcxx.test.config.loadSiteConfig` so it can be used with libc++abi.
- Temporary files are now created in the build directory of libc++. This follows how it is down in ShTest.
- `not.py` was added as a utility executable that mirrors the functionality of LLVM's `not` executable. 
- The first ShTest test was added under test/libcxx/double_include.sh.cpp


Reviewers: jroelofs, danalbert

Reviewed By: danalbert

Subscribers: cfe-commits

Differential Revision: http://reviews.llvm.org/D7073

git-svn-id: https://llvm.org/svn/llvm-project/libcxx/trunk@226844 91177308-0d34-0410-b5e6-96231b3b80d8

9 files changed, 484 insertions, 198 deletions

diff --git a/test/libcxx/compiler.py b/test/libcxx/compiler.py
index b47ecbd21..ca546f97f 100644
--- a/test/libcxx/compiler.py
+++ b/test/libcxx/compiler.py
@@ -1,13 +1,15 @@
-
+import os
 import lit.util
+import libcxx.util
 
 
 class CXXCompiler(object):
-    def __init__(self, path, flags=[], compile_flags=[], link_flags=[], use_ccache=False):
+    def __init__(self, path, flags=None, compile_flags=None, link_flags=None,
+                 use_ccache=False):
         self.path = path
-        self.flags = list(flags)
-        self.compile_flags = list(compile_flags)
-        self.link_flags = list(link_flags)
+        self.flags = list(flags or [])
+        self.compile_flags = list(compile_flags or [])
+        self.link_flags = list(link_flags or [])
         self.use_ccache = use_ccache
         self.type = None
         self.version = None
@@ -36,66 +38,85 @@ class CXXCompiler(object):
         self.type = compiler_type
         self.version = (major_ver, minor_ver, patchlevel)
 
-    def _basicCmd(self, infiles, out, is_link=False):
+    def _basicCmd(self, source_files, out, is_link=False):
         cmd = []
         if self.use_ccache and not is_link:
             cmd += ['ccache']
         cmd += [self.path]
         if out is not None:
             cmd += ['-o', out]
-        if isinstance(infiles, list):
-            cmd += infiles
-        elif isinstance(infiles, str):
-            cmd += [infiles]
+        if isinstance(source_files, list):
+            cmd += source_files
+        elif isinstance(source_files, str):
+            cmd += [source_files]
         else:
-            raise TypeError('infiles must be a string or list')
+            raise TypeError('source_files must be a string or list')
         return cmd
 
-    def preprocessCmd(self, infiles, out=None, flags=[]):
-        cmd = self._basicCmd(infiles, out) + ['-x', 'c++', '-E']
+    def preprocessCmd(self, source_files, out=None, flags=[]):
+        cmd = self._basicCmd(source_files, out) + ['-x', 'c++', '-E']
         cmd += self.flags + self.compile_flags + flags
         return cmd
 
-    def compileCmd(self, infiles, out=None, flags=[]):
-        cmd = self._basicCmd(infiles, out) + ['-x', 'c++', '-c']
+    def compileCmd(self, source_files, out=None, flags=[]):
+        cmd = self._basicCmd(source_files, out) + ['-x', 'c++', '-c']
         cmd += self.flags + self.compile_flags + flags
         return cmd
 
-    def linkCmd(self, infiles, out=None, flags=[]):
-        cmd = self._basicCmd(infiles, out, is_link=True)
+    def linkCmd(self, source_files, out=None, flags=[]):
+        cmd = self._basicCmd(source_files, out, is_link=True)
         cmd += self.flags + self.link_flags + flags
         return cmd
 
-    def compileLinkCmd(self, infiles, out=None, flags=[]):
-        cmd = self._basicCmd(infiles, out, is_link=True) + ['-x', 'c++']
+    def compileLinkCmd(self, source_files, out=None, flags=[]):
+        cmd = self._basicCmd(source_files, out, is_link=True) + ['-x', 'c++']
         cmd += self.flags + self.compile_flags + self.link_flags + flags
         return cmd
 
-    def preprocess(self, infiles, out=None, flags=[], env=None, cwd=None):
-        cmd = self.preprocessCmd(infiles, out, flags)
+    def preprocess(self, source_files, out=None, flags=[], env=None, cwd=None):
+        cmd = self.preprocessCmd(source_files, out, flags)
         out, err, rc = lit.util.executeCommand(cmd, env=env, cwd=cwd)
         return cmd, out, err, rc
 
-    def compile(self, infiles, out=None, flags=[], env=None, cwd=None):
-        cmd = self.compileCmd(infiles, out, flags)
+    def compile(self, source_files, out=None, flags=[], env=None, cwd=None):
+        cmd = self.compileCmd(source_files, out, flags)
         out, err, rc = lit.util.executeCommand(cmd, env=env, cwd=cwd)
         return cmd, out, err, rc
 
-    def link(self, infiles, out=None, flags=[], env=None, cwd=None):
-        cmd = self.linkCmd(infiles, out, flags)
+    def link(self, source_files, out=None, flags=[], env=None, cwd=None):
+        cmd = self.linkCmd(source_files, out, flags)
         out, err, rc = lit.util.executeCommand(cmd, env=env, cwd=cwd)
         return cmd, out, err, rc
 
-    def compileLink(self, infiles, out=None, flags=[], env=None, cwd=None):
-        cmd = self.compileLinkCmd(infiles, out, flags)
+    def compileLink(self, source_files, out=None, flags=[], env=None,
+                    cwd=None):
+        cmd = self.compileLinkCmd(source_files, out, flags)
         out, err, rc = lit.util.executeCommand(cmd, env=env, cwd=cwd)
         return cmd, out, err, rc
 
-    def dumpMacros(self, infiles=None, flags=[], env=None, cwd=None):
-        if infiles is None:
-            infiles = '/dev/null'
+    def compileLinkTwoSteps(self, source_file, out=None, object_file=None,
+                            flags=[], env=None, cwd=None):
+        if not isinstance(source_file, str):
+            raise TypeError('This function only accepts a single input file')
+        if object_file is None:
+            # Create, use and delete a temporary object file if none is given.
+            with_fn = lambda: libcxx.util.guardedTempFilename(suffix='.o')
+        else:
+            # Otherwise wrap the filename in a context manager function.
+            with_fn = lambda: libcxx.util.nullContext(object_file)
+        with with_fn() as object_file:
+            cmd, output, err, rc = self.compile(source_file, object_file,
+                                                flags=flags, env=env, cwd=cwd)
+            if rc != 0:
+                return cmd, output, err, rc
+            return self.link(object_file, out=out, flags=flags, env=env,
+                             cwd=cwd)
+
+    def dumpMacros(self, source_files=None, flags=[], env=None, cwd=None):
+        if source_files is None:
+            source_files = os.devnull
         flags = ['-dM'] + flags
-        cmd, out, err, rc = self.preprocess(infiles, flags=flags, env=env,
+        cmd, out, err, rc = self.preprocess(source_files, flags=flags, env=env,
                                             cwd=cwd)
         if rc != 0:
             return None
diff --git a/test/libcxx/double_include.sh.cpp b/test/libcxx/double_include.sh.cpp
new file mode 100644
index 000000000..3f787572f
--- /dev/null
+++ b/test/libcxx/double_include.sh.cpp
@@ -0,0 +1,111 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// Test that we can include each header in two TU's and link them together.
+
+// RUN: %cxx -c %s -o %t.first.o %flags %compile_flags
+// RUN: %cxx -c %s -o %t.second.o -DWITH_MAIN %flags %compile_flags
+// RUN: %cxx -o %t.exe %t.first.o %t.second.o %flags %link_flags
+// RUN: %run
+
+#include <algorithm>
+#include <array>
+#include <bitset>
+#include <cassert>
+#include <ccomplex>
+#include <cctype>
+#include <cerrno>
+#include <cfenv>
+#include <cfloat>
+#include <chrono>
+#include <cinttypes>
+#include <ciso646>
+#include <climits>
+#include <clocale>
+#include <cmath>
+#include <codecvt>
+#include <complex>
+#include <complex.h>
+#include <condition_variable>
+#include <csetjmp>
+#include <csignal>
+#include <cstdarg>
+#include <cstdbool>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctgmath>
+#include <ctime>
+#include <cwchar>
+#include <cwctype>
+#include <deque>
+#include <exception>
+#include <experimental/dynarray>
+#include <experimental/optional>
+#include <experimental/string_view>
+#include <experimental/type_traits>
+#include <experimental/utility>
+#include <ext/hash_map>
+#include <ext/hash_set>
+#include <forward_list>
+#include <fstream>
+#include <functional>
+#include <initializer_list>
+#include <iomanip>
+#include <ios>
+#include <iosfwd>
+#include <iostream>
+#include <istream>
+#include <iterator>
+#include <limits>
+#include <list>
+#include <locale>
+#include <map>
+#include <memory>
+#include <new>
+#include <numeric>
+#include <ostream>
+#include <queue>
+#include <random>
+#include <ratio>
+#include <regex>
+#include <scoped_allocator>
+#include <set>
+#include <sstream>
+#include <stack>
+#include <stdexcept>
+#include <streambuf>
+#include <string>
+#include <strstream>
+#include <system_error>
+#include <tgmath.h>
+#include <tuple>
+#include <typeindex>
+#include <typeinfo>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <valarray>
+#include <vector>
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+#include <atomic>
+#include <future>
+#include <mutex>
+#include <shared_mutex>
+#include <thread>
+#endif
+
+#if defined(WITH_MAIN)
+int main() {}
+#endif
diff --git a/test/libcxx/selftest/not_test.sh.cpp b/test/libcxx/selftest/not_test.sh.cpp
new file mode 100644
index 000000000..5b8348f0e
--- /dev/null
+++ b/test/libcxx/selftest/not_test.sh.cpp
@@ -0,0 +1,17 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// RUN: %build
+// RUN: not %run
+
+int main()
+{
+  return 1;
+}
diff --git a/test/libcxx/selftest/test.fail.cpp b/test/libcxx/selftest/test.fail.cpp
new file mode 100644
index 000000000..2ad608bab
--- /dev/null
+++ b/test/libcxx/selftest/test.fail.cpp
@@ -0,0 +1,11 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#error This test should not compile.
diff --git a/test/libcxx/selftest/test.pass.cpp b/test/libcxx/selftest/test.pass.cpp
new file mode 100644
index 000000000..9a59227ab
--- /dev/null
+++ b/test/libcxx/selftest/test.pass.cpp
@@ -0,0 +1,13 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+int main()
+{
+}
diff --git a/test/libcxx/selftest/test.sh.cpp b/test/libcxx/selftest/test.sh.cpp
new file mode 100644
index 000000000..14dc2db80
--- /dev/null
+++ b/test/libcxx/selftest/test.sh.cpp
@@ -0,0 +1,16 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// RUN: %build
+// RUN: %run
+
+int main()
+{
+}
diff --git a/test/libcxx/test/config.py b/test/libcxx/test/config.py
index e2cba77a2..55ba93493 100644
--- a/test/libcxx/test/config.py
+++ b/test/libcxx/test/config.py
@@ -1,6 +1,7 @@
 import locale
 import os
 import platform
+import pkgutil
 import re
 import shlex
 import sys
@@ -12,6 +13,34 @@ from libcxx.test.format import LibcxxTestFormat
 from libcxx.compiler import CXXCompiler
 
 
+def loadSiteConfig(lit_config, config, param_name, env_name):
+    # We haven't loaded the site specific configuration (the user is
+    # probably trying to run on a test file directly, and either the site
+    # configuration hasn't been created by the build system, or we are in an
+    # out-of-tree build situation).
+    site_cfg = lit_config.params.get(param_name,
+                                     os.environ.get(env_name))
+    if not site_cfg:
+        lit_config.warning('No site specific configuration file found!'
+                           ' Running the tests in the default configuration.')
+    elif not os.path.isfile(site_cfg):
+        lit_config.fatal(
+            "Specified site configuration file does not exist: '%s'" %
+            site_cfg)
+    else:
+        lit_config.note('using site specific configuration at %s' % site_cfg)
+        ld_fn = lit_config.load_config
+
+        # Null out the load_config function so that lit.site.cfg doesn't
+        # recursively load a config even if it tries.
+        # TODO: This is one hell of a hack. Fix it.
+        def prevent_reload_fn(*args, **kwargs):
+            pass
+        lit_config.load_config = prevent_reload_fn
+        ld_fn(config, site_cfg)
+        lit_config.load_config = ld_fn
+
+
 class Configuration(object):
     # pylint: disable=redefined-outer-name
     def __init__(self, lit_config, config):
@@ -26,6 +55,7 @@ class Configuration(object):
         self.use_system_cxx_lib = False
         self.use_clang_verify = False
         self.long_tests = None
+        self.execute_external = False
 
         if platform.system() not in ('Darwin', 'FreeBSD', 'Linux'):
             self.lit_config.fatal("unrecognized system")
@@ -57,18 +87,21 @@ class Configuration(object):
         self.configure_cxx_library_root()
         self.configure_use_system_cxx_lib()
         self.configure_use_clang_verify()
+        self.configure_execute_external()
         self.configure_ccache()
         self.configure_env()
         self.configure_compile_flags()
         self.configure_link_flags()
         self.configure_sanitizer()
+        self.configure_substitutions()
         self.configure_features()
 
     def print_config_info(self):
         # Print the final compile and link flags.
         self.lit_config.note('Using compiler: %s' % self.cxx.path)
         self.lit_config.note('Using flags: %s' % self.cxx.flags)
-        self.lit_config.note('Using compile flags: %s' % self.cxx.compile_flags)
+        self.lit_config.note('Using compile flags: %s'
+                             % self.cxx.compile_flags)
         self.lit_config.note('Using link flags: %s' % self.cxx.link_flags)
         # Print as list to prevent "set([...])" from being printed.
         self.lit_config.note('Using available_features: %s' %
@@ -79,6 +112,7 @@ class Configuration(object):
         return LibcxxTestFormat(
             self.cxx,
             self.use_clang_verify,
+            self.execute_external,
             exec_env=self.env)
 
     def configure_cxx(self):
@@ -106,7 +140,6 @@ class Configuration(object):
             self.config.available_features.add('%s-%s.%s' % (
                 cxx_type, maj_v, min_v))
 
-
     def configure_src_root(self):
         self.libcxx_src_root = self.get_lit_conf(
             'libcxx_src_root', os.path.dirname(self.config.test_source_root))
@@ -140,6 +173,22 @@ class Configuration(object):
             self.lit_config.note(
                 "inferred use_clang_verify as: %r" % self.use_clang_verify)
 
+    def configure_execute_external(self):
+        # Choose between lit's internal shell pipeline runner and a real shell.
+        # If LIT_USE_INTERNAL_SHELL is in the environment, we use that as the
+        # default value. Otherwise we default to internal on Windows and
+        # external elsewhere, as bash on Windows is usually very slow.
+        use_lit_shell_default = os.environ.get('LIT_USE_INTERNAL_SHELL')
+        if use_lit_shell_default is not None:
+            use_lit_shell_default = use_lit_shell_default != '0'
+        else:
+            use_lit_shell_default = sys.platform == 'win32'
+        # Check for the command line parameter using the default value if it is
+        # not present.
+        use_lit_shell = self.get_lit_bool('use_lit_shell',
+                                          use_lit_shell_default)
+        self.execute_external = not use_lit_shell
+
     def configure_ccache(self):
         use_ccache = self.get_lit_bool('use_ccache', False)
         if use_ccache:
@@ -233,12 +282,12 @@ class Configuration(object):
             self.config.available_features.add('long_tests')
 
     def configure_compile_flags(self):
-      no_default_flags = self.get_lit_bool('no_default_flags', False)
-      if not no_default_flags:
-        self.configure_default_compile_flags()
-      # Configure extra flags
-      compile_flags_str = self.get_lit_conf('compile_flags', '')
-      self.cxx.compile_flags += shlex.split(compile_flags_str)
+        no_default_flags = self.get_lit_bool('no_default_flags', False)
+        if not no_default_flags:
+            self.configure_default_compile_flags()
+        # Configure extra flags
+        compile_flags_str = self.get_lit_conf('compile_flags', '')
+        self.cxx.compile_flags += shlex.split(compile_flags_str)
 
     def configure_default_compile_flags(self):
         # Try and get the std version from the command line. Fall back to
@@ -283,9 +332,10 @@ class Configuration(object):
             self.cxx.flags += ['-target', self.config.target_triple]
 
     def configure_compile_flags_header_includes(self):
-        self.cxx.compile_flags += ['-I' + self.libcxx_src_root + '/test/support']
-        libcxx_headers = self.get_lit_conf('libcxx_headers',
-                                           self.libcxx_src_root + '/include')
+        self.cxx.compile_flags += [
+            '-I' + os.path.join(self.libcxx_src_root, 'test/support')]
+        libcxx_headers = self.get_lit_conf(
+            'libcxx_headers', os.path.join(self.libcxx_src_root, 'include'))
         if not os.path.isdir(libcxx_headers):
             self.lit_config.fatal("libcxx_headers='%s' is not a directory."
                                   % libcxx_headers)
@@ -316,16 +366,16 @@ class Configuration(object):
     def configure_link_flags(self):
         no_default_flags = self.get_lit_bool('no_default_flags', False)
         if not no_default_flags:
-          self.cxx.link_flags += ['-nodefaultlibs']
+            self.cxx.link_flags += ['-nodefaultlibs']
 
-          # Configure library path
-          self.configure_link_flags_cxx_library_path()
-          self.configure_link_flags_abi_library_path()
+            # Configure library path
+            self.configure_link_flags_cxx_library_path()
+            self.configure_link_flags_abi_library_path()
 
-          # Configure libraries
-          self.configure_link_flags_cxx_library()
-          self.configure_link_flags_abi_library()
-          self.configure_extra_library_flags()
+            # Configure libraries
+            self.configure_link_flags_cxx_library()
+            self.configure_link_flags_abi_library()
+            self.configure_extra_library_flags()
 
         link_flags_str = self.get_lit_conf('link_flags', '')
         self.cxx.link_flags += shlex.split(link_flags_str)
@@ -425,7 +475,8 @@ class Configuration(object):
             elif san == 'Memory' or san == 'MemoryWithOrigins':
                 self.cxx.flags += ['-fsanitize=memory']
                 if san == 'MemoryWithOrigins':
-                    self.cxx.compile_flags += ['-fsanitize-memory-track-origins']
+                    self.cxx.compile_flags += [
+                        '-fsanitize-memory-track-origins']
                 if llvm_symbolizer is not None:
                     self.env['MSAN_SYMBOLIZER_PATH'] = llvm_symbolizer
                 self.config.available_features.add('msan')
@@ -442,6 +493,45 @@ class Configuration(object):
                 self.lit_config.fatal('unsupported value for '
                                       'use_sanitizer: {0}'.format(san))
 
+    def configure_substitutions(self):
+        sub = self.config.substitutions
+        # Configure compiler substitions
+        sub.append(('%cxx', self.cxx.path))
+        # Configure flags substitutions
+        flags_str = ' '.join(self.cxx.flags)
+        compile_flags_str = ' '.join(self.cxx.compile_flags)
+        link_flags_str = ' '.join(self.cxx.link_flags)
+        all_flags = '%s %s %s' % (flags_str, compile_flags_str, link_flags_str)
+        sub.append(('%flags', flags_str))
+        sub.append(('%compile_flags', compile_flags_str))
+        sub.append(('%link_flags', link_flags_str))
+        sub.append(('%all_flags', all_flags))
+        # Add compile and link shortcuts
+        compile_str = (self.cxx.path + ' -o %t.o %s -c ' + flags_str
+                       + compile_flags_str)
+        link_str = (self.cxx.path + ' -o %t.exe %t.o ' + flags_str
+                    + link_flags_str)
+        assert type(link_str) is str
+        build_str = self.cxx.path + ' -o %t.exe %s ' + all_flags
+        sub.append(('%compile', compile_str))
+        sub.append(('%link', link_str))
+        sub.append(('%build', build_str))
+        # Configure exec prefix substitutions.
+        exec_env_str = 'env ' if len(self.env) != 0 else ''
+        for k, v in self.env.items():
+            exec_env_str += ' %s=%s' % (k, v)
+        # Configure run env substitution.
+        exec_str = ''
+        if self.lit_config.useValgrind:
+            exec_str = ' '.join(self.lit_config.valgrindArgs) + exec_env_str
+        sub.append(('%exec', exec_str))
+        # Configure run shortcut
+        sub.append(('%run', exec_str + ' %t.exe'))
+        # Configure not program substitions
+        not_py = os.path.join(self.libcxx_src_root, 'utils', 'not', 'not.py')
+        not_str = '%s %s' % (sys.executable, not_py)
+        sub.append(('not', not_str))
+
     def configure_triple(self):
         # Get or infer the target triple.
         self.config.target_triple = self.get_lit_conf('target_triple')
diff --git a/test/libcxx/test/format.py b/test/libcxx/test/format.py
index e162e1f72..cb3623015 100644
--- a/test/libcxx/test/format.py
+++ b/test/libcxx/test/format.py
@@ -1,12 +1,15 @@
 import errno
 import os
-import tempfile
 import time
 
-import lit.formats  # pylint: disable=import-error
+import lit.Test        # pylint: disable=import-error
+import lit.TestRunner  # pylint: disable=import-error
+import lit.util        # pylint: disable=import-error
 
+import libcxx.util
 
-class LibcxxTestFormat(lit.formats.FileBasedTest):
+
+class LibcxxTestFormat(object):
     """
     Custom test format handler for use with the test format use by libc++.
 
@@ -14,13 +17,31 @@ class LibcxxTestFormat(lit.formats.FileBasedTest):
       FOO.pass.cpp - Executable test which should compile, run, and exit with
                      code 0.
       FOO.fail.cpp - Negative test case which is expected to fail compilation.
+      FOO.sh.cpp   - A test that uses LIT's ShTest format.
     """
 
-    def __init__(self, cxx, use_verify_for_fail, exec_env):
+    def __init__(self, cxx, use_verify_for_fail, execute_external, exec_env):
         self.cxx = cxx
         self.use_verify_for_fail = use_verify_for_fail
+        self.execute_external = execute_external
         self.exec_env = dict(exec_env)
 
+    # TODO: Move this into lit's FileBasedTest
+    def getTestsInDirectory(self, testSuite, path_in_suite,
+                            litConfig, localConfig):
+        source_path = testSuite.getSourcePath(path_in_suite)
+        for filename in os.listdir(source_path):
+            # Ignore dot files and excluded tests.
+            if filename.startswith('.') or filename in localConfig.excludes:
+                continue
+
+            filepath = os.path.join(source_path, filename)
+            if not os.path.isdir(filepath):
+                if any([filename.endswith(ext)
+                        for ext in localConfig.suffixes]):
+                    yield lit.Test.Test(testSuite, path_in_suite + (filename,),
+                                        localConfig)
+
     def execute(self, test, lit_config):
         while True:
             try:
@@ -31,154 +52,94 @@ class LibcxxTestFormat(lit.formats.FileBasedTest):
                 time.sleep(0.1)
 
     def _execute(self, test, lit_config):
-        # Extract test metadata from the test file.
-        requires = []
-        unsupported = []
-        use_verify = False
-        with open(test.getSourcePath()) as f:
-            for ln in f:
-                if 'XFAIL:' in ln:
-                    items = ln[ln.index('XFAIL:') + 6:].split(',')
-                    test.xfails.extend([s.strip() for s in items])
-                elif 'REQUIRES:' in ln:
-                    items = ln[ln.index('REQUIRES:') + 9:].split(',')
-                    requires.extend([s.strip() for s in items])
-                elif 'UNSUPPORTED:' in ln:
-                    items = ln[ln.index('UNSUPPORTED:') + 12:].split(',')
-                    unsupported.extend([s.strip() for s in items])
-                elif 'USE_VERIFY' in ln and self.use_verify_for_fail:
-                    use_verify = True
-                elif not ln.strip().startswith("//") and ln.strip():
-                    # Stop at the first non-empty line that is not a C++
-                    # comment.
-                    break
-
-        # Check that we have the required features.
-        #
-        # FIXME: For now, this is cribbed from lit.TestRunner, to avoid
-        # introducing a dependency there. What we more ideally would like to do
-        # is lift the "requires" handling to be a core lit framework feature.
-        missing_required_features = [
-            f for f in requires
-            if f not in test.config.available_features
-        ]
-        if missing_required_features:
-            return (lit.Test.UNSUPPORTED,
-                    "Test requires the following features: %s" % (
-                        ', '.join(missing_required_features),))
-
-        unsupported_features = [f for f in unsupported
-                                if f in test.config.available_features]
-        if unsupported_features:
-            return (lit.Test.UNSUPPORTED,
-                    "Test is unsupported with the following features: %s" % (
-                        ', '.join(unsupported_features),))
-
-        # Evaluate the test.
-        return self._evaluate_test(test, use_verify, lit_config)
-
-    def _make_report(self, cmd, out, err, rc):  # pylint: disable=no-self-use
-        report = "Command: %s\n" % cmd
-        report += "Exit Code: %d\n" % rc
-        if out:
-            report += "Standard Output:\n--\n%s--\n" % out
-        if err:
-            report += "Standard Error:\n--\n%s--\n" % err
-        report += '\n'
-        return cmd, report, rc
-
-    def _compile(self, output_path, source_path, use_verify=False):
-        extra_flags = []
-        if use_verify:
-            extra_flags += ['-Xclang', '-verify']
-        return self.cxx.compile(source_path, out=output_path, flags=extra_flags)
+        name = test.path_in_suite[-1]
+        is_sh_test = name.endswith('.sh.cpp')
+        is_pass_test = name.endswith('.pass.cpp')
+        is_fail_test = name.endswith('.fail.cpp')
+
+        res = lit.TestRunner.parseIntegratedTestScript(
+            test, require_script=is_sh_test)
+        # Check if a result for the test was returned. If so return that
+        # result.
+        if isinstance(res, lit.Test.Result):
+            return res
+        if lit_config.noExecute:
+            return lit.Test.Result(lit.Test.PASS)
+        # res is not an instance of lit.test.Result. Expand res into its parts.
+        script, tmpBase, execDir = res
+        # Check that we don't have run lines on tests that don't support them.
+        if not is_sh_test and len(script) != 0:
+            lit_config.fatal('Unsupported RUN line found in test %s' % name)
+
+        # Dispatch the test based on its suffix.
+        if is_sh_test:
+            return lit.TestRunner._runShTest(test, lit_config,
+                                             self.execute_external, script,
+                                             tmpBase, execDir)
+        elif is_fail_test:
+            return self._evaluate_fail_test(test)
+        elif is_pass_test:
+            return self._evaluate_pass_test(test, tmpBase, execDir, lit_config)
+        else:
+            # No other test type is supported
+            assert False
 
-    def _link(self, exec_path, object_path):
-        return self.cxx.link(object_path, out=exec_path)
+    def _clean(self, exec_path):  # pylint: disable=no-self-use
+        libcxx.util.cleanFile(exec_path)
 
-    def _compile_and_link(self, exec_path, source_path):
-        object_file = tempfile.NamedTemporaryFile(suffix=".o", delete=False)
-        object_path = object_file.name
-        object_file.close()
+    def _evaluate_pass_test(self, test, tmpBase, execDir, lit_config):
+        source_path = test.getSourcePath()
+        exec_path = tmpBase + '.exe'
+        object_path = tmpBase + '.o'
+        # Create the output directory if it does not already exist.
+        lit.util.mkdir_p(os.path.dirname(tmpBase))
         try:
-            cmd, out, err, rc = self.cxx.compile(source_path, out=object_path)
+            # Compile the test
+            cmd, out, err, rc = self.cxx.compileLinkTwoSteps(
+                source_path, out=exec_path, object_file=object_path,
+                cwd=execDir)
+            compile_cmd = cmd
+            if rc != 0:
+                report = libcxx.util.makeReport(cmd, out, err, rc)
+                report += "Compilation failed unexpectedly!"
+                return lit.Test.FAIL, report
+            # Run the test
+            cmd = []
+            if self.exec_env:
+                cmd += ['env']
+                cmd += ['%s=%s' % (k, v) for k, v in self.exec_env.items()]
+            if lit_config.useValgrind:
+                cmd = lit_config.valgrindArgs + cmd
+            cmd += [exec_path]
+            out, err, rc = lit.util.executeCommand(
+                cmd, cwd=os.path.dirname(source_path))
             if rc != 0:
-                return cmd, out, err, rc
-            return self.cxx.link(object_path, out=exec_path)
+                report = libcxx.util.makeReport(cmd, out, err, rc)
+                report = "Compiled With: %s\n%s" % (compile_cmd, report)
+                report += "Compiled test failed unexpectedly!"
+                return lit.Test.FAIL, report
+            return lit.Test.PASS, ''
         finally:
-            try:
-                os.remove(object_path)
-            except OSError:
-                pass
+            # Note that cleanup of exec_file happens in `_clean()`. If you
+            # override this, cleanup is your reponsibility.
+            libcxx.util.cleanFile(object_path)
+            self._clean(exec_path)
 
-    def _build(self, exec_path, source_path, compile_only=False,
-               use_verify=False):
-        if compile_only:
-            cmd, out, err, rc = self._compile(exec_path, source_path,
-                                              use_verify)
-        else:
-            assert not use_verify
-            cmd, out, err, rc = self._compile_and_link(exec_path, source_path)
-        return self._make_report(cmd, out, err, rc)
-
-    def _clean(self, exec_path):  # pylint: disable=no-self-use
-        try:
-            os.remove(exec_path)
-        except OSError:
-            pass
-
-    def _run(self, exec_path, lit_config, in_dir=None):
-        cmd = []
-        if self.exec_env:
-            cmd.append('env')
-            cmd.extend('%s=%s' % (name, value)
-                       for name, value in self.exec_env.items())
-        cmd.append(exec_path)
-        if lit_config.useValgrind:
-            cmd = lit_config.valgrindArgs + cmd
-        out, err, rc = lit.util.executeCommand(cmd, cwd=in_dir)
-        return self._make_report(cmd, out, err, rc)
-
-    def _evaluate_test(self, test, use_verify, lit_config):
-        name = test.path_in_suite[-1]
+    def _evaluate_fail_test(self, test):
         source_path = test.getSourcePath()
-        source_dir = os.path.dirname(source_path)
-
-        # Check what kind of test this is.
-        assert name.endswith('.pass.cpp') or name.endswith('.fail.cpp')
-        expected_compile_fail = name.endswith('.fail.cpp')
-
-        # If this is a compile (failure) test, build it and check for failure.
-        if expected_compile_fail:
-            cmd, report, rc = self._build('/dev/null', source_path,
-                                          compile_only=True,
-                                          use_verify=use_verify)
-            expected_rc = 0 if use_verify else 1
-            if rc == expected_rc:
-                return lit.Test.PASS, ""
-            else:
-                return (lit.Test.FAIL,
-                        report + 'Expected compilation to fail!\n')
+        # TODO: Move the checking of USE_VERIFY into
+        # lit.TestRunner.parseIntegratedTestScript by adding support for custom
+        # tags.
+        with open(source_path, 'r') as f:
+            contents = f.read()
+        use_verify = 'USE_VERIFY' in contents and self.use_verify_for_fail
+        extra_flags = ['-Xclang', '-verify'] if use_verify else []
+        cmd, out, err, rc = self.cxx.compile(source_path, out=os.devnull,
+                                             flags=extra_flags)
+        expected_rc = 0 if use_verify else 1
+        if rc == expected_rc:
+            return lit.Test.PASS, ''
         else:
-            exec_file = tempfile.NamedTemporaryFile(suffix="exe", delete=False)
-            exec_path = exec_file.name
-            exec_file.close()
-
-            try:
-                cmd, report, rc = self._build(exec_path, source_path)
-                compile_cmd = cmd
-                if rc != 0:
-                    report += "Compilation failed unexpectedly!"
-                    return lit.Test.FAIL, report
-
-                cmd, report, rc = self._run(exec_path, lit_config,
-                                            source_dir)
-                if rc != 0:
-                    report = "Compiled With: %s\n%s" % (compile_cmd, report)
-                    report += "Compiled test failed unexpectedly!"
-                    return lit.Test.FAIL, report
-            finally:
-                # Note that cleanup of exec_file happens in `_clean()`. If you
-                # override this, cleanup is your reponsibility.
-                self._clean(exec_path)
-        return lit.Test.PASS, ""
+            report = libcxx.util.makeReport(cmd, out, err, rc)
+            return (lit.Test.FAIL,
+                    report + 'Expected compilation to fail!\n')
diff --git a/test/libcxx/util.py b/test/libcxx/util.py
new file mode 100644
index 000000000..28f2619d6
--- /dev/null
+++ b/test/libcxx/util.py
@@ -0,0 +1,46 @@
+from contextlib import contextmanager
+import os
+import tempfile
+
+
+def cleanFile(filename):
+    try:
+        os.remove(filename)
+    except OSError:
+        pass
+
+
+@contextmanager
+def guardedTempFilename(suffix='', prefix='', dir=None):
+    # Creates and yeilds a temporary filename within a with statement. The file
+    # is removed upon scope exit.
+    handle, name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir)
+    os.close(handle)
+    yield name
+    cleanFile(name)
+
+
+@contextmanager
+def guardedFilename(name):
+    # yeilds a filename within a with statement. The file is removed upon scope
+    # exit.
+    yield name
+    cleanFile(name)
+
+
+@contextmanager
+def nullContext(value):
+    # yeilds a variable within a with statement. No action is taken upon scope
+    # exit.
+    yield value
+
+
+def makeReport(cmd, out, err, rc):
+    report = "Command: %s\n" % cmd
+    report += "Exit Code: %d\n" % rc
+    if out:
+        report += "Standard Output:\n--\n%s--\n" % out
+    if err:
+        report += "Standard Error:\n--\n%s--\n" % err
+    report += '\n'
+    return report