Simplify par Launcher

Instead of copying some of Py_Main and changing it, use Py_Main
directly, paired with a __main__.py script that uses the runpy module to
trampoline into the user code.

Test: build/soong/python/tests/runtest.sh
Change-Id: I7f5e1e38b63b4e2ce83a85c08953a5e67c7b8788

5 files changed, 29 insertions, 355 deletions

diff --git a/Android.bp b/Android.bp
index 5497151366..2a56885aca 100644
--- a/Android.bp
+++ b/Android.bp
@@ -225,7 +225,6 @@ cc_binary {
     ],
     srcs: [
         "Launcher/launcher_main.cpp",
-        "Launcher/launcher_internal.cpp",
         "Modules/gcmodule.c",
         "Modules/getpath.c",
         "Modules/config.c",
@@ -241,9 +240,7 @@ cc_binary {
         "Modules/_weakref.c",
         "Modules/zipimport.c",
         "Modules/symtablemodule.c",
-    ],
-    local_include_dirs: [
-        "Launcher",
+        "Modules/main.c",
     ],
     // NOTE: Please update Modules/config.c if new lib get added in the static_libs.
     static_libs: [
@@ -315,21 +312,14 @@ cc_binary {
     target: {
         linux_glibc_x86_64: {
             host_ldlibs: ["-lutil"],
-            static_libs: [
-                "libsqlite",
-            ],
         },
-        darwin_x86_64: {
-            static_libs: [
-                "libsqlite",
-            ],
+        host: {
+            static_libs: ["libsqlite"],
         },
         // Use shared libsqlite for device side, otherwise
         // the executable size will be really huge.
         android: {
-            shared_libs: [
-                "libsqlite",
-            ],
+            shared_libs: ["libsqlite"],
         },
     },
 }
diff --git a/Launcher/launcher_internal.cpp b/Launcher/launcher_internal.cpp
deleted file mode 100644
index 7d5b25adb6..0000000000
--- a/Launcher/launcher_internal.cpp
+++ /dev/null
@@ -1,187 +0,0 @@
-// Copyright 2017 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "launcher_internal.h"
-
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-extern "C" {
-// Cpython built-in C functions.
-/*
-   read_directory(archive) -> files dict (new reference)
-
-   Given a path to a Zip archive, build a dict, mapping file names
-   (local to the archive, using SEP as a separator) to toc entries.
-*/
-PyObject *read_directory(const char *archive);
-
-/* Given a path to a Zip file and a toc_entry, return the (uncompressed)
-   data as a new reference. */
-PyObject *get_data(const char *archive, PyObject *toc_entry);
-}
-
-namespace android {
-namespace cpython2 {
-namespace python_launcher {
-namespace internal {
-
-int RunModule(const char *module, int set_argv0) {
-  PyObject *runpy, *runmodule, *runargs, *result;
-  runpy = PyImport_ImportModule("runpy");
-  if (runpy == NULL) {
-    fprintf(stderr, "Could not import runpy module\n");
-    return -1;
-  }
-  runmodule = PyObject_GetAttrString(runpy, "_run_module_as_main");
-  if (runmodule == NULL) {
-    fprintf(stderr, "Could not access runpy._run_module_as_main\n");
-    Py_DECREF(runpy);
-    return -1;
-  }
-  runargs = Py_BuildValue("(si)", module, set_argv0);
-  if (runargs == NULL) {
-    fprintf(stderr,
-            "Could not create arguments for runpy._run_module_as_main\n");
-    Py_DECREF(runpy);
-    Py_DECREF(runmodule);
-    return -1;
-  }
-  result = PyObject_Call(runmodule, runargs, NULL);
-  if (result == NULL) {
-    PyErr_Print();
-  }
-  Py_DECREF(runpy);
-  Py_DECREF(runmodule);
-  Py_DECREF(runargs);
-  if (result == NULL) {
-    return -1;
-  }
-  Py_DECREF(result);
-  return 0;
-}
-
-std::string GetEntryPointFilePath(const char *launcher_path) {
-  PyObject *files;
-  files = read_directory(launcher_path);
-  if (files == NULL) {
-    return std::string();
-  }
-  PyObject *toc_entry;
-  // Return value: Borrowed reference.
-  toc_entry = PyDict_GetItemString(files, ENTRYPOINT_FILE);
-  if (toc_entry == NULL) {
-    Py_DECREF(files);
-    return std::string();
-  }
-  PyObject *py_data;
-  py_data = get_data(launcher_path, toc_entry);
-  if (py_data == NULL) {
-    Py_DECREF(files);
-    return std::string();
-  }
-  // PyString_AsString returns a NUL-terminated representation of the "py_data",
-  // "data" must not be modified in any way. And it must not be deallocated.
-  char *data = PyString_AsString(py_data);
-  if (data == NULL) {
-    Py_DECREF(py_data);
-    Py_DECREF(files);
-    return std::string();
-  }
-
-  char *res = strdup(data); /* deep copy of data */
-  Py_DECREF(py_data);
-  Py_DECREF(files);
-
-  return std::string(res);
-}
-
-int RunModuleNameFromEntryPoint(const char *launcher_path, std::string entrypoint) {
-  if (entrypoint.empty()) {
-    return -1;
-  }
-  // Has to pass to free to avoid a memory leak after use.
-  char *arr = strdup(entrypoint.c_str());
-
-  if (AddPathToPythonSysPath(launcher_path) < 0) {
-    free(arr);
-    return -1;
-  }
-  int ret =  RunModule(arr, 0);
-  free(arr);
-  return ret;
-}
-
-int AddPathToPythonSysPath(const char *path) {
-  if (path == NULL) {
-    return -1;
-  }
-  PyObject *py_path;
-  py_path = PyString_FromString(path);
-  if (py_path == NULL) {
-    return -1;
-  }
-  PyObject *sys_path;
-  // Return value: Borrowed reference.
-  sys_path = PySys_GetObject(const_cast<char*>("path"));
-  if (sys_path == NULL) {
-    Py_DECREF(py_path);
-    return -1;
-  }
-  PyList_Insert(sys_path, 0, py_path);
-  Py_DECREF(py_path);
-  return 0;
-}
-
-int RunMainFromImporter(const char *launcher_path) {
-  PyObject *py_launcher_path, *importer;
-  py_launcher_path = PyString_FromString(launcher_path);
-  if (py_launcher_path == NULL) {
-    return -1;
-  }
-  importer = PyImport_GetImporter(py_launcher_path);
-  if (importer == NULL) {
-    Py_DECREF(py_launcher_path);
-    return -1;
-  }
-  if (importer != Py_None && importer->ob_type != &PyNullImporter_Type) {
-    /* Launcher path is usable as an import source, so
-       put it in sys.path[0] and import __main__ */
-    if (AddPathToPythonSysPath(launcher_path) < 0) {
-      Py_DECREF(importer);
-      Py_DECREF(py_launcher_path);
-      return -1;
-    }
-  }
-  Py_DECREF(importer);
-  Py_DECREF(py_launcher_path);
-  return RunModule("__main__", 0);
-}
-}  // namespace internal
-
-int RunEntryPointOrMainModule(const char *launcher_path) {
-  std::string entrypoint = internal::GetEntryPointFilePath(launcher_path);
-  if (entrypoint.empty()) {
-    // If entry point can not be found or can not be executed, we try to
-    // run __main__.py within the .par file.
-    fprintf(stderr, "Cannot find valid entry point to execute par file!\n");
-    fprintf(stdout, "Start trying to run __main__ module within par file.\n");
-    return internal::RunMainFromImporter(launcher_path);
-  }
-  return internal::RunModuleNameFromEntryPoint(launcher_path, entrypoint);
-}
-}  // namespace python_launcher
-}  // namespace cpython2
-}  // namespace android
diff --git a/Launcher/launcher_internal.h b/Launcher/launcher_internal.h
deleted file mode 100644
index aa5f2dd9a5..0000000000
--- a/Launcher/launcher_internal.h
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2017 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef ANDROID_CPYTHON2_PYTHON_LAUNCHER_INTERNAL_H
-#define ANDROID_CPYTHON2_PYTHON_LAUNCHER_INTERNAL_H
-
-#include <Python.h>
-
-#include <string>
-
-namespace android {
-namespace cpython2 {
-namespace python_launcher {
-
-namespace internal{
-#define ENTRYPOINT_FILE "entry_point.txt"
-
-// Use "runpy" module to locate and run Python script using Python module
-// namespace rather than the filesystem.
-// The caller owns "module" pointer, which cannot be NULL.
-int RunModule(const char *module, int set_argv0);
-
-// Get valid entrypoint file path.
-// The caller owns "launcher_path" pointer, which cannot be NULL.
-// Return non-empty string as success. Otherwise, return empty string.
-std::string GetEntryPointFilePath(const char *launcher_path);
-
-// Run the Python script embedded in ENTRYPOINT_FILE.
-// The caller owns "launcher_path" pointer, which cannot be NULL.
-int RunModuleNameFromEntryPoint(const char *launcher_path, std::string entrypoint);
-
-// Add path to Python sys.path list.
-// The caller owns "path" pointer, which cannot be NULL.
-int AddPathToPythonSysPath(const char *path);
-
-// Run __main__ module within the hermetic .par file.
-// The caller owns "launcher_path" pointer, which cannot be NULL.
-// Return 0 as success. Otherwise, return -1.
-int RunMainFromImporter(const char *launcher_path);
-}  // namespace internal
-
-// Try to run the Python script embedded in ENTRYPOINT_FILE. Otherwise,
-// run __main__ module as fallback.
-// The caller owns "launcher_path" pointer, which cannot be NULL.
-int RunEntryPointOrMainModule(const char *launcher_path);
-}  // namespace python_launcher
-}  // namespace cpython2
-}  // namespace android
-
-#endif  // ANDROID_CPYTHON2_PYTHON_LAUNCHER_INTERNAL_H
diff --git a/Launcher/launcher_main.cpp b/Launcher/launcher_main.cpp
index a49998cd5a..09a9d3969f 100644
--- a/Launcher/launcher_main.cpp
+++ b/Launcher/launcher_main.cpp
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "launcher_internal.h"
-
 #include <Python.h>
 #include <android-base/file.h>
 #include <osdefs.h>
@@ -22,103 +20,39 @@
 #include <string>
 
 int main(int argc, char *argv[]) {
-  int result = 0 /* Used to mark if current program runs with success/failure. */;
-
-  // Clear PYTHONPATH and PYTHONHOME so Python doesn't attempt to check the local
-  // disk for Python modules to load. The value of PYTHONHOME will replace "prefix"
-  // and "exe_prefix" based on the description in getpath.c.
-  // Please don't use PYTHONPATH and PYTHONHOME within user program.
-  // TODO(nanzhang): figure out if unsetenv("PYTHONPATH") is better.
-  unsetenv(const_cast<char *>("PYTHONPATH"));
-  // TODO(nanzhang): figure out if Py_SetPythonHome() is better.
-  unsetenv(const_cast<char *>("PYTHONHOME"));
   // PYTHONEXECUTABLE is only used on MacOs X, when the Python interpreter
   // embedded in an application bundle. It is not sure that we have this use case
-  // for Android hermetic Python. So override this environment variable to empty
-  // for now to make our self-contained environment more strict.
+  // for Android hermetic Python. So remove this environment variable to make
+  // our self-contained environment more strict.
   // For user (.py) program, it can access hermetic .par file path through
   // sys.argv[0].
   unsetenv(const_cast<char *>("PYTHONEXECUTABLE"));
 
-  // Resolving absolute path based on argv[0] is not reliable since it may
-  // include something unusable, too bad.
-  // android::base::GetExecutablePath() also handles for Darwin/Windows.
-  std::string executable_path = android::base::GetExecutablePath();
-
-  argv[0] = strdup(executable_path.c_str());
-  // argv[0] is used for setting internal path, and Python sys.argv[0]. It
-  // should not exceed MAXPATHLEN defined for CPython.
-  if (!argv[0] || strlen(argv[0]) > MAXPATHLEN) {
-    fprintf(stderr, "The executable path %s is NULL or of invalid length.\n", argv[0]);
-    return 1;
-  }
-
-  // For debugging/logging purpose, set stdin/stdout/stderr unbuffered through
-  // environment variable.
-  // TODO(nanzhang): Set Py_VerboseFlag if more debugging requests needed.
-  const char *unbuffered_env = getenv("PYTHONUNBUFFERED");
-  if (unbuffered_env && unbuffered_env[0]) {
-    #if defined(MS_WINDOWS) || defined(__CYGWIN__)
-      _setmode(fileno(stdin), O_BINARY);
-      _setmode(fileno(stdout), O_BINARY);
-    #endif
-    #ifdef HAVE_SETVBUF
-      setvbuf(stdin,  (char *)NULL, _IONBF, BUFSIZ);
-      setvbuf(stdout, (char *)NULL, _IONBF, BUFSIZ);
-      setvbuf(stderr, (char *)NULL, _IONBF, BUFSIZ);
-    #else /* !HAVE_SETVBUF */
-      setbuf(stdin,  (char *)NULL);
-      setbuf(stdout, (char *)NULL);
-      setbuf(stderr, (char *)NULL);
-    #endif /* !HAVE_SETVBUF */
-  }
-  //For debugging/logging purpose, Warning control.
-  //Python’s warning machinery by default prints warning messages to sys.stderr.
-  //The full form of argument is:action:message:category:module:line
-  char *warnings_env = getenv("PYTHONWARNINGS");
-  if (warnings_env && warnings_env[0]) {
-      char *warnings_buf, *warning;
-
-      // Note: "new" operation; we need free this chuck of data after use.
-      warnings_buf = new char[strlen(warnings_env) + 1];
-      if (warnings_buf == NULL)
-          Py_FatalError(
-             "not enough memory to copy PYTHONWARNINGS");
-      strcpy(warnings_buf, warnings_env);
-      for (warning = strtok(warnings_buf, ",");
-           warning != NULL;
-           warning = strtok(NULL, ","))
-          PySys_AddWarnOption(warning);
-      delete[] warnings_buf;
-  }
-
   // Always enable Python "-s" option. We don't need user-site directories,
   // everything's supposed to be hermetic.
   Py_NoUserSiteDirectory = 1;
 
-  Py_SetProgramName(argv[0]);
-  Py_Initialize();
-  PySys_SetArgvEx(argc, argv, 0);
+  // Ignore PYTHONPATH and PYTHONHOME from the environment.
+  Py_IgnoreEnvironmentFlag = 1;
 
-  // Set sys.executable to None. The real executable is available as
-  // sys.argv[0], and too many things assume sys.executable is a regular Python
-  // binary, which isn't available. By setting it to None we get clear errors
-  // when people try to use it.
-  if (PySys_SetObject(const_cast<char *>("executable"), Py_None) < 0) {
-    PyErr_Print();
-    result = 1;
-    goto error;
-  }
+  Py_DontWriteBytecodeFlag = 1;
 
-  result = android::cpython2::python_launcher::RunEntryPointOrMainModule(argv[0]);
-  if (result < 0) {
-    PyErr_Print();
-    goto error;
-  }
+  // Resolving absolute path based on argv[0] is not reliable since it may
+  // include something unusable, too bad.
+  // android::base::GetExecutablePath() also handles for Darwin/Windows.
+  std::string executable_path = android::base::GetExecutablePath();
 
-error:
-  Py_Finalize();
+  int new_argc = argc + 1;
+  char **new_argv = reinterpret_cast<char**>(calloc(new_argc, sizeof(*argv)));
+
+  // Inject the path to our binary into argv[1] so the Py_Main won't parse any
+  // other options, and will execute the __main__.py script inside the zip file
+  // attached to our executable.
+  new_argv[0] = argv[0];
+  new_argv[1] = strdup(executable_path.c_str());
+  for (int i = 1; i < argc; i++) {
+    new_argv[i+1] = argv[i];
+  }
 
-  free(argv[0]);
-  exit(abs(result));
+  return Py_Main(new_argc, new_argv);
 }
diff --git a/Modules/zipimport.c b/Modules/zipimport.c
index 0a7cd0ffc0..1691773ec1 100644
--- a/Modules/zipimport.c
+++ b/Modules/zipimport.c
@@ -43,11 +43,9 @@ struct _zipimporter {
 static PyObject *ZipImportError;
 static PyObject *zip_directory_cache = NULL;
 
-// GOOGLE(nanzhang): Changed two functions below to be visible to launcher so
-// that launcher can access the zip metadata section.
 /* forward decls */
-PyObject *read_directory(const char *archive);
-PyObject *get_data(const char *archive, PyObject *toc_entry);
+static PyObject *read_directory(const char *archive);
+static PyObject *get_data(const char *archive, PyObject *toc_entry);
 static PyObject *get_module_code(ZipImporter *self, char *fullname,
                                  int *p_ispackage, char **p_modpath);
 
@@ -704,7 +702,7 @@ set_file_error(const char *archive, int eof)
    Directories can be recognized by the trailing SEP in the name,
    data_size and file_offset are 0.
 */
-PyObject *
+static PyObject *
 read_directory(const char *archive)
 {
     PyObject *files = NULL;
@@ -914,7 +912,7 @@ get_decompress_func(void)
 
 /* Given a path to a Zip file and a toc_entry, return the (uncompressed)
    data as a new reference. */
-PyObject *
+static PyObject *
 get_data(const char *archive, PyObject *toc_entry)
 {
     PyObject *raw_data = NULL, *data, *decompress;