[C++] Run EmitNode in parallel

1 files changed, 49 insertions, 28 deletions

diff --git a/ninja.cc b/ninja.cc
index 6925cbb..edf320e 100644
--- a/ninja.cc
+++ b/ninja.cc
@@ -40,6 +40,7 @@
 #include "string_piece.h"
 #include "stringprintf.h"
 #include "strutil.h"
+#include "thread_pool.h"
 #include "timeutil.h"
 #include "var.h"
 #include "version.h"
@@ -169,6 +170,8 @@ bool GetDepfileFromCommand(string* cmd, string* out) {
 struct NinjaNode {
   const DepNode* node;
   vector<Command*> commands;
+  int rule_id;
+  string str;
 };
 
 class NinjaGenerator {
@@ -240,6 +243,7 @@ class NinjaGenerator {
     NinjaNode* nn = new NinjaNode;
     nn->node = node;
     ce_.Eval(node, &nn->commands);
+    nn->rule_id = nn->commands.empty() ? -1 : rule_id_++;
     nodes_.push_back(nn);
 
     for (DepNode* d : node->deps) {
@@ -250,10 +254,6 @@ class NinjaGenerator {
     }
   }
 
-  string GenRuleName() {
-    return StringPrintf("rule%d", rule_id_++);
-  }
-
   StringPiece TranslateCommand(const char* in, string* cmd_buf) {
     const size_t orig_size = cmd_buf->size();
     bool prev_backslash = false;
@@ -481,12 +481,13 @@ class NinjaGenerator {
     return result;
   }
 
-  void EmitDepfile(const DepNode* node, string* cmd_buf) {
+  void EmitDepfile(NinjaNode* nn, string* cmd_buf) {
+    const DepNode* node = nn->node;
     string depfile;
     if (!GetDepfile(node, cmd_buf, &depfile))
       return;
-    fprintf(fp_, " depfile = %s\n", depfile.c_str());
-    fprintf(fp_, " deps = gcc\n");
+    nn->str += StringPrintf(" depfile = %s\n", depfile.c_str());
+    nn->str += StringPrintf(" deps = gcc\n");
   }
 
   void EmitNode(NinjaNode* nn) {
@@ -496,32 +497,33 @@ class NinjaGenerator {
     string rule_name = "phony";
     bool use_local_pool = false;
     if (!commands.empty()) {
-      rule_name = GenRuleName();
-      fprintf(fp_, "rule %s\n", rule_name.c_str());
+      rule_name = StringPrintf("rule%d", nn->rule_id);
+      nn->str += StringPrintf("rule %s\n", rule_name.c_str());
 
       string description = "build $out";
       string cmd_buf;
-      use_local_pool |= GenShellScript(node->output.c_str(), commands, &cmd_buf, &description);
-      fprintf(fp_, " description = %s\n", description.c_str());
-      EmitDepfile(node, &cmd_buf);
+      use_local_pool |= GenShellScript(node->output.c_str(), commands,
+                                       &cmd_buf, &description);
+      nn->str += StringPrintf(" description = %s\n", description.c_str());
+      EmitDepfile(nn, &cmd_buf);
 
       // It seems Linux is OK with ~130kB and Mac's limit is ~250kB.
       // TODO: Find this number automatically.
       if (cmd_buf.size() > 100 * 1000) {
-        fprintf(fp_, " rspfile = $out.rsp\n");
-        fprintf(fp_, " rspfile_content = %s\n", cmd_buf.c_str());
-        fprintf(fp_, " command = %s $out.rsp\n", shell_.c_str());
+        nn->str += StringPrintf(" rspfile = $out.rsp\n");
+        nn->str += StringPrintf(" rspfile_content = %s\n", cmd_buf.c_str());
+        nn->str += StringPrintf(" command = %s $out.rsp\n", shell_.c_str());
       } else {
         EscapeShell(&cmd_buf);
-        fprintf(fp_, " command = %s -c \"%s\"\n",
+        nn->str += StringPrintf(" command = %s -c \"%s\"\n",
                 shell_.c_str(), cmd_buf.c_str());
       }
       if (node->is_restat) {
-        fprintf(fp_, " restat = 1\n");
+        nn->str += StringPrintf(" restat = 1\n");
       }
     }
 
-    EmitBuild(node, rule_name, use_local_pool);
+    EmitBuild(nn, rule_name, use_local_pool);
   }
 
   string EscapeBuildTarget(Symbol s) const {
@@ -572,29 +574,31 @@ class NinjaGenerator {
     s->swap(r);
   }
 
-  void EmitBuild(const DepNode* node, const string& rule_name,
+  void EmitBuild(NinjaNode* nn, const string& rule_name,
                  bool use_local_pool) {
+    const DepNode* node = nn->node;
     string target = EscapeBuildTarget(node->output);
-    fprintf(fp_, "build %s: %s",
+    nn->str += StringPrintf("build %s: %s",
             target.c_str(),
             rule_name.c_str());
     vector<Symbol> order_onlys;
     if (node->is_phony) {
-      fprintf(fp_, " _kati_always_build_");
+      nn->str += StringPrintf(" _kati_always_build_");
     }
     for (DepNode* d : node->deps) {
-      fprintf(fp_, " %s", EscapeBuildTarget(d->output).c_str());
+      nn->str += StringPrintf(" %s", EscapeBuildTarget(d->output).c_str());
     }
     if (!node->order_onlys.empty()) {
-      fprintf(fp_, " ||");
+      nn->str += StringPrintf(" ||");
       for (DepNode* d : node->order_onlys) {
-        fprintf(fp_, " %s", EscapeBuildTarget(d->output).c_str());
+        nn->str += StringPrintf(" %s", EscapeBuildTarget(d->output).c_str());
       }
     }
-    fprintf(fp_, "\n");
+    nn->str += StringPrintf("\n");
     if (use_local_pool)
-      fprintf(fp_, " pool = local_pool\n");
+      nn->str += StringPrintf(" pool = local_pool\n");
     if (node->is_default_target) {
+      unique_lock<mutex> lock(mu_);
       default_target_ = node;
     }
   }
@@ -649,8 +653,23 @@ class NinjaGenerator {
 
     EmitRegenRules(orig_args);
 
+    unique_ptr<ThreadPool> tp(NewThreadPool(g_flags.num_jobs));
+    CHECK(g_flags.num_jobs);
+    int num_nodes_per_task = nodes_.size() / (g_flags.num_jobs * 10) + 1;
+    int num_tasks = nodes_.size() / num_nodes_per_task + 1;
+    for (int i = 0; i < num_tasks; i++) {
+      tp->Submit([this, i, num_nodes_per_task]() {
+          int l = min(num_nodes_per_task * (i + 1),
+                      static_cast<int>(nodes_.size()));
+          for (int j = num_nodes_per_task * i; j < l; j++) {
+            EmitNode(nodes_[j]);
+          }
+        });
+    }
+    tp->Wait();
+
     for (NinjaNode* node : nodes_) {
-      EmitNode(node);
+      fprintf(fp_, "%s", node->str.c_str());
     }
 
     unordered_set<Symbol> used_env_vars(Vars::used_env_vars());
@@ -813,8 +832,10 @@ class NinjaGenerator {
   string shell_;
   map<string, string> used_envs_;
   string kati_binary_;
-  double start_time_;
+  const double start_time_;
   vector<NinjaNode*> nodes_;
+
+  mutex mu_;
   const DepNode* default_target_;
 };