[C++] Set CPU affinity

Sticking to a single processor improves the performance while
we are running only a single thread.

9 files changed, 105 insertions, 26 deletions

diff --git a/Android.bp b/Android.bp
index 432505e..fea1241 100644
--- a/Android.bp
+++ b/Android.bp
@@ -15,6 +15,7 @@
 cc_library_host_static {
     name: "libckati",
     srcs: [
+        "affinity.cc",
         "command.cc",
         "condvar.cc",
         "dep.cc",
diff --git a/Makefile.ckati b/Makefile.ckati
index cf5bab8..38e1feb 100644
--- a/Makefile.ckati
+++ b/Makefile.ckati
@@ -22,6 +22,7 @@ KATI_INTERMEDIATES_PATH ?= .
 KATI_BIN_PATH ?= .
 
 KATI_CXX_SRCS := \
+	affinity.cc \
 	command.cc \
 	condvar.cc \
 	dep.cc \
diff --git a/affinity.cc b/affinity.cc
new file mode 100644
index 0000000..0743f94
--- /dev/null
+++ b/affinity.cc
@@ -0,0 +1,50 @@
+// Copyright 2016 Google Inc. All rights reserved
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "affinity.h"
+
+#include "flags.h"
+#include "log.h"
+
+#ifdef __linux__
+
+#include <sched.h>
+
+void SetAffinityForSingleThread() {
+  cpu_set_t cs;
+  CPU_ZERO(&cs);
+  int n = g_flags.num_cpus / 2;
+  CPU_SET(n, &cs);
+  if (n > 1)
+    CPU_SET(n + 1, &cs);
+  if (sched_setaffinity(0, sizeof(cs), &cs) < 0)
+    WARN("sched_setaffinity: %s", strerror(errno));
+}
+
+void SetAffinityForMultiThread() {
+  cpu_set_t cs;
+  CPU_ZERO(&cs);
+  for (int i = 0; i < g_flags.num_cpus; i++) {
+    CPU_SET(i, &cs);
+  }
+  if (sched_setaffinity(0, sizeof(cs), &cs) < 0)
+    WARN("sched_setaffinity: %s", strerror(errno));
+}
+
+#else
+
+void SetAffinityForSingleThread() {}
+void SetAffinityForMultiThread() {}
+
+#endif
diff --git a/affinity.h b/affinity.h
new file mode 100644
index 0000000..e6f6adc
--- /dev/null
+++ b/affinity.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Google Inc. All rights reserved
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef AFFINITY_H_
+#define AFFINITY_H_
+
+void SetAffinityForSingleThread();
+void SetAffinityForMultiThread();
+
+#endif
diff --git a/flags.cc b/flags.cc
index 06772f0..73c7105 100644
--- a/flags.cc
+++ b/flags.cc
@@ -49,7 +49,7 @@ static bool ParseCommandLineOptionWithArg(StringPiece option,
 
 void Flags::Parse(int argc, char** argv) {
   subkati_args.push_back(argv[0]);
-  num_jobs = sysconf(_SC_NPROCESSORS_ONLN);
+  num_jobs = num_cpus = sysconf(_SC_NPROCESSORS_ONLN);
   const char* num_jobs_str;
 
   for (int i = 1; i < argc; i++) {
diff --git a/flags.h b/flags.h
index d2fe923..275a3ee 100644
--- a/flags.h
+++ b/flags.h
@@ -44,6 +44,7 @@ struct Flags {
   const char* makefile;
   const char* ninja_dir;
   const char* ninja_suffix;
+  int num_cpus;
   int num_jobs;
   int remote_num_jobs;
   vector<const char*> subkati_args;
diff --git a/main.cc b/main.cc
index 0c0dfd0..c732dda 100644
--- a/main.cc
+++ b/main.cc
@@ -21,6 +21,7 @@
 #include <time.h>
 #include <unistd.h>
 
+#include "affinity.h"
 #include "dep.h"
 #include "eval.h"
 #include "exec.h"
@@ -133,6 +134,8 @@ static int Run(const vector<Symbol>& targets,
     ClearGlobCache();
   }
 
+  SetAffinityForSingleThread();
+
   MakefileCacheManager* cache_mgr = NewMakefileCacheManager();
 
   Vars* vars = new Vars();
diff --git a/ninja.cc b/ninja.cc
index 8b6a076..ccc51f2 100644
--- a/ninja.cc
+++ b/ninja.cc
@@ -22,6 +22,7 @@
 #include <unistd.h>
 
 #include <map>
+#include <sstream>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -480,16 +481,16 @@ class NinjaGenerator {
     return result;
   }
 
-  void EmitDepfile(NinjaNode* nn, string* cmd_buf, string* o) {
+  void EmitDepfile(NinjaNode* nn, string* cmd_buf, ostringstream* o) {
     const DepNode* node = nn->node;
     string depfile;
     if (!GetDepfile(node, cmd_buf, &depfile))
       return;
-    *o += StringPrintf(" depfile = %s\n", depfile.c_str());
-    *o += StringPrintf(" deps = gcc\n");
+    *o << " depfile = " << depfile << "\n";
+    *o << " deps = gcc\n";
   }
 
-  void EmitNode(NinjaNode* nn, string* o) {
+  void EmitNode(NinjaNode* nn, ostringstream* o) {
     const DepNode* node = nn->node;
     const vector<Command*>& commands = nn->commands;
 
@@ -497,28 +498,27 @@ class NinjaGenerator {
     bool use_local_pool = false;
     if (!commands.empty()) {
       rule_name = StringPrintf("rule%d", nn->rule_id);
-      *o += StringPrintf("rule %s\n", rule_name.c_str());
+      *o << "rule " << rule_name << "\n";
 
       string description = "build $out";
       string cmd_buf;
       use_local_pool |= GenShellScript(node->output.c_str(), commands,
                                        &cmd_buf, &description);
-      *o += StringPrintf(" description = %s\n", description.c_str());
+      *o << " description = " << description << "\n";
       EmitDepfile(nn, &cmd_buf, o);
 
       // It seems Linux is OK with ~130kB and Mac's limit is ~250kB.
       // TODO: Find this number automatically.
       if (cmd_buf.size() > 100 * 1000) {
-        *o += StringPrintf(" rspfile = $out.rsp\n");
-        *o += StringPrintf(" rspfile_content = %s\n", cmd_buf.c_str());
-        *o += StringPrintf(" command = %s $out.rsp\n", shell_.c_str());
+        *o << " rspfile = $out.rsp\n";
+        *o << " rspfile_content = " << cmd_buf << "\n";
+        *o << " command = " << shell_ << " $out.rsp\n";
       } else {
         EscapeShell(&cmd_buf);
-        *o += StringPrintf(" command = %s -c \"%s\"\n",
-                shell_.c_str(), cmd_buf.c_str());
+        *o << " command = " << shell_ << " -c \"" << cmd_buf << "\"\n";
       }
       if (node->is_restat) {
-        *o += StringPrintf(" restat = 1\n");
+        *o << " restat = 1\n";
       }
     }
 
@@ -574,28 +574,26 @@ class NinjaGenerator {
   }
 
   void EmitBuild(NinjaNode* nn, const string& rule_name,
-                 bool use_local_pool, string* o) {
+                 bool use_local_pool, ostringstream* o) {
     const DepNode* node = nn->node;
     string target = EscapeBuildTarget(node->output);
-    *o += StringPrintf("build %s: %s",
-            target.c_str(),
-            rule_name.c_str());
+    *o << "build " << target << ": " << rule_name;
     vector<Symbol> order_onlys;
     if (node->is_phony) {
-      *o += StringPrintf(" _kati_always_build_");
+      *o << " _kati_always_build_";
     }
     for (DepNode* d : node->deps) {
-      *o += StringPrintf(" %s", EscapeBuildTarget(d->output).c_str());
+      *o << " " << EscapeBuildTarget(d->output).c_str();
     }
     if (!node->order_onlys.empty()) {
-      *o += StringPrintf(" ||");
+      *o << " ||";
       for (DepNode* d : node->order_onlys) {
-        *o += StringPrintf(" %s", EscapeBuildTarget(d->output).c_str());
+        *o << " " << EscapeBuildTarget(d->output).c_str();
       }
     }
-    *o += StringPrintf("\n");
+    *o << "\n";
     if (use_local_pool)
-      *o += StringPrintf(" pool = local_pool\n");
+      *o << " pool = local_pool\n";
     if (node->is_default_target) {
       unique_lock<mutex> lock(mu_);
       default_target_ = node;
@@ -656,7 +654,7 @@ class NinjaGenerator {
     CHECK(g_flags.num_jobs);
     int num_nodes_per_task = nodes_.size() / (g_flags.num_jobs * 10) + 1;
     int num_tasks = nodes_.size() / num_nodes_per_task + 1;
-    vector<string> bufs(num_tasks);
+    vector<ostringstream> bufs(num_tasks);
     for (int i = 0; i < num_tasks; i++) {
       tp->Submit([this, i, num_nodes_per_task, &bufs]() {
           int l = min(num_nodes_per_task * (i + 1),
@@ -668,8 +666,8 @@ class NinjaGenerator {
     }
     tp->Wait();
 
-    for (const string& buf : bufs) {
-      fprintf(fp_, "%s", buf.c_str());
+    for (const ostringstream& buf : bufs) {
+      fprintf(fp_, "%s", buf.str().c_str());
     }
 
     unordered_set<Symbol> used_env_vars(Vars::used_env_vars());
diff --git a/thread_pool.cc b/thread_pool.cc
index ba7ee00..791b718 100644
--- a/thread_pool.cc
+++ b/thread_pool.cc
@@ -17,6 +17,7 @@
 #include <stack>
 #include <vector>
 
+#include "affinity.h"
 #include "condvar.h"
 #include "mutex.h"
 #include "thread.h"
@@ -25,6 +26,7 @@ class ThreadPoolImpl : public ThreadPool {
  public:
   explicit ThreadPoolImpl(int num_threads)
       : is_waiting_(false) {
+    SetAffinityForMultiThread();
     threads_.reserve(num_threads);
     for (int i = 0; i < num_threads; i++) {
       threads_.push_back(thread([this]() { Loop(); }));
@@ -50,6 +52,8 @@ class ThreadPoolImpl : public ThreadPool {
     for (thread& th : threads_) {
       th.join();
     }
+
+    SetAffinityForSingleThread();
   }
 
  private: