Implement the `file` function to read and write files

This allows us to do file reading and writing without $(shell). Besides
being simpler, this also allows faster regen times, since we can just
stat the files to be read, or directly write to the files that need to
be written.

7 files changed, 379 insertions, 34 deletions

diff --git a/func.cc b/func.cc
index 623e56e..1c666ec 100644
--- a/func.cc
+++ b/func.cc
@@ -17,9 +17,11 @@
 #include "func.h"
 
 #include <errno.h>
+#include <fcntl.h>
 #include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <sys/stat.h>
 #include <unistd.h>
 
 #include <algorithm>
@@ -571,6 +573,7 @@ void ShellFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
   ShellFuncImpl(shell, cmd, &out, &fc);
   if (ShouldStoreCommandResult(cmd)) {
     CommandResult* cr = new CommandResult();
+    cr->op = (fc == NULL) ? CommandOp::SHELL : CommandOp::FIND,
     cr->shell = shell;
     cr->cmd = cmd;
     cr->find.reset(fc);
@@ -686,6 +689,124 @@ void ErrorFunc(const vector<Value*>& args, Evaluator* ev, string*) {
   ev->Error(StringPrintf("*** %s.", a.c_str()));
 }
 
+static void FileReadFunc(Evaluator* ev, const string& filename, string* s) {
+  int fd = open(filename.c_str(), O_RDONLY);
+  if (fd < 0) {
+    if (errno == ENOENT) {
+      if (ShouldStoreCommandResult(filename)) {
+        CommandResult* cr = new CommandResult();
+        cr->op = CommandOp::READ_MISSING;
+        cr->cmd = filename;
+        g_command_results.push_back(cr);
+      }
+      return;
+    } else {
+      ev->Error("*** open failed.");
+    }
+  }
+
+  struct stat st;
+  if (fstat(fd, &st) < 0) {
+    ev->Error("*** fstat failed.");
+  }
+
+  size_t len = st.st_size;
+  string out;
+  out.resize(len);
+  ssize_t r = HANDLE_EINTR(read(fd, &out[0], len));
+  if (r != static_cast<ssize_t>(len)) {
+    ev->Error("*** read failed.");
+  }
+
+  if (close(fd) < 0) {
+    ev->Error("*** close failed.");
+  }
+
+  if (out.back() == '\n') {
+    out.pop_back();
+  }
+
+  if (ShouldStoreCommandResult(filename)) {
+    CommandResult* cr = new CommandResult();
+    cr->op = CommandOp::READ;
+    cr->cmd = filename;
+    g_command_results.push_back(cr);
+  }
+  *s += out;
+}
+
+static void FileWriteFunc(Evaluator* ev, const string& filename, bool append, string text) {
+  FILE* f = fopen(filename.c_str(), append ? "ab" : "wb");
+  if (f == NULL) {
+    ev->Error("*** fopen failed.");
+  }
+
+  if (fwrite(&text[0], text.size(), 1, f) != 1) {
+    ev->Error("*** fwrite failed.");
+  }
+
+  if (fclose(f) != 0) {
+    ev->Error("*** fclose failed.");
+  }
+
+  if (ShouldStoreCommandResult(filename)) {
+    CommandResult* cr = new CommandResult();
+    cr->op = CommandOp::WRITE;
+    cr->cmd = filename;
+    cr->result = text;
+    g_command_results.push_back(cr);
+  }
+}
+
+void FileFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
+  if (ev->avoid_io()) {
+    ev->Error("*** $(file ...) is not supported in rules.");
+  }
+
+  string arg = args[0]->Eval(ev);
+  StringPiece filename = TrimSpace(arg);
+
+  if (filename.size() <= 1) {
+    ev->Error("*** Missing filename");
+  }
+
+  if (filename[0] == '<') {
+    filename = TrimLeftSpace(filename.substr(1));
+    if (!filename.size()) {
+      ev->Error("*** Missing filename");
+    }
+    if (args.size() > 1) {
+      ev->Error("*** invalid argument");
+    }
+
+    FileReadFunc(ev, filename.as_string(), s);
+  } else if (filename[0] == '>') {
+    bool append = false;
+    if (filename[1] == '>') {
+      append = true;
+      filename = filename.substr(2);
+    } else {
+      filename = filename.substr(1);
+    }
+    filename = TrimLeftSpace(filename);
+    if (!filename.size()) {
+      ev->Error("*** Missing filename");
+    }
+
+    string text;
+    if (args.size() > 1) {
+      text = args[1]->Eval(ev);
+      if (text.size() == 0 || text.back() != '\n') {
+        text.push_back('\n');
+      }
+    }
+
+    FileWriteFunc(ev, filename.as_string(), append, text);
+  } else {
+    ev->Error(StringPrintf("*** Invalid file operation: %s.  Stop.", filename.as_string().c_str()));
+  }
+}
+
 FuncInfo g_func_infos[] = {
   { "patsubst", &PatsubstFunc, 3, 3, false, false },
   { "strip", &StripFunc, 1, 1, false, false },
@@ -727,6 +848,8 @@ FuncInfo g_func_infos[] = {
   { "info", &InfoFunc, 1, 1, false, false },
   { "warning", &WarningFunc, 1, 1, false, false },
   { "error", &ErrorFunc, 1, 1, false, false },
+
+  { "file", &FileFunc, 2, 1, false, false },
 };
 
 unordered_map<StringPiece, FuncInfo*>* g_func_info_map;
diff --git a/func.h b/func.h
index e78deb7..ab6affb 100644
--- a/func.h
+++ b/func.h
@@ -41,7 +41,17 @@ FuncInfo* GetFuncInfo(StringPiece name);
 
 struct FindCommand;
 
+enum struct CommandOp {
+  SHELL,
+  FIND,
+  READ,
+  READ_MISSING,
+  WRITE,
+  APPEND,
+};
+
 struct CommandResult {
+  CommandOp op;
   string shell;
   string cmd;
   unique_ptr<FindCommand> find;
diff --git a/ninja.cc b/ninja.cc
index 90fe404..59e8712 100644
--- a/ninja.cc
+++ b/ninja.cc
@@ -736,31 +736,27 @@ class NinjaGenerator {
     const vector<CommandResult*>& crs = GetShellCommandResults();
     DumpInt(fp, crs.size());
     for (CommandResult* cr : crs) {
+      DumpInt(fp, static_cast<int>(cr->op));
       DumpString(fp, cr->shell);
       DumpString(fp, cr->cmd);
       DumpString(fp, cr->result);
-      if (!cr->find.get()) {
-        // Always re-run this command.
-        DumpInt(fp, 0);
-        continue;
-      }
 
-      DumpInt(fp, 1);
-
-      vector<string> missing_dirs;
-      for (StringPiece fd : cr->find->finddirs) {
-        const string& d = ConcatDir(cr->find->chdir, fd);
-        if (!Exists(d))
-          missing_dirs.push_back(d);
-      }
-      DumpInt(fp, missing_dirs.size());
-      for (const string& d : missing_dirs) {
-        DumpString(fp, d);
-      }
+      if (cr->op == CommandOp::FIND) {
+        vector<string> missing_dirs;
+        for (StringPiece fd : cr->find->finddirs) {
+          const string& d = ConcatDir(cr->find->chdir, fd);
+          if (!Exists(d))
+            missing_dirs.push_back(d);
+        }
+        DumpInt(fp, missing_dirs.size());
+        for (const string& d : missing_dirs) {
+          DumpString(fp, d);
+        }
 
-      DumpInt(fp, cr->find->read_dirs->size());
-      for (StringPiece s : *cr->find->read_dirs) {
-        DumpString(fp, ConcatDir(cr->find->chdir, s));
+        DumpInt(fp, cr->find->read_dirs->size());
+        for (StringPiece s : *cr->find->read_dirs) {
+          DumpString(fp, ConcatDir(cr->find->chdir, s));
+        }
       }
     }
 
diff --git a/regen.cc b/regen.cc
index f5ed50a..bbae7c8 100644
--- a/regen.cc
+++ b/regen.cc
@@ -23,6 +23,7 @@
 
 #include "fileutil.h"
 #include "find.h"
+#include "func.h"
 #include "io.h"
 #include "log.h"
 #include "ninja.h"
@@ -52,12 +53,12 @@ class StampChecker {
   };
 
   struct ShellResult {
+    CommandOp op;
     string shell;
     string cmd;
     string result;
     vector<string> missing_dirs;
     vector<string> read_dirs;
-    bool has_condition;
   };
 
  public:
@@ -231,22 +232,22 @@ class StampChecker {
     for (int i = 0; i < num_crs; i++) {
       ShellResult* sr = new ShellResult;
       commands_.push_back(sr);
+      sr->op = static_cast<CommandOp>(LOAD_INT(fp));
       LOAD_STRING(fp, &sr->shell);
       LOAD_STRING(fp, &sr->cmd);
       LOAD_STRING(fp, &sr->result);
-      sr->has_condition = LOAD_INT(fp);
-      if (!sr->has_condition)
-        continue;
 
-      int num_missing_dirs = LOAD_INT(fp);
-      for (int j = 0; j < num_missing_dirs; j++) {
-        LOAD_STRING(fp, &s);
-        sr->missing_dirs.push_back(s);
-      }
-      int num_read_dirs = LOAD_INT(fp);
-      for (int j = 0; j < num_read_dirs; j++) {
-        LOAD_STRING(fp, &s);
-        sr->read_dirs.push_back(s);
+      if (sr->op == CommandOp::FIND) {
+        int num_missing_dirs = LOAD_INT(fp);
+        for (int j = 0; j < num_missing_dirs; j++) {
+          LOAD_STRING(fp, &s);
+          sr->missing_dirs.push_back(s);
+        }
+        int num_read_dirs = LOAD_INT(fp);
+        for (int j = 0; j < num_read_dirs; j++) {
+          LOAD_STRING(fp, &s);
+          sr->read_dirs.push_back(s);
+        }
       }
     }
 
@@ -292,7 +293,7 @@ class StampChecker {
   }
 
   bool ShouldRunCommand(const ShellResult* sr) {
-    if (!sr->has_condition)
+    if (sr->op != CommandOp::FIND)
       return true;
 
     COLLECT_STATS("stat time (regen)");
@@ -324,6 +325,54 @@ class StampChecker {
   }
 
   bool CheckShellResult(const ShellResult* sr, string* err) {
+    if (sr->op == CommandOp::READ_MISSING) {
+      if (Exists(sr->cmd)) {
+        if (g_flags.dump_kati_stamp)
+          printf("file %s: dirty\n", sr->cmd.c_str());
+        else
+          *err = StringPrintf("$(file <%s) was changed, regenerating...\n",
+                              sr->cmd.c_str());
+        return true;
+      }
+      if (g_flags.dump_kati_stamp)
+        printf("file %s: clean\n", sr->cmd.c_str());
+      return false;
+    }
+
+    if (sr->op == CommandOp::READ) {
+      double ts = GetTimestamp(sr->cmd);
+      if (gen_time_ < ts) {
+        if (g_flags.dump_kati_stamp)
+          printf("file %s: dirty\n", sr->cmd.c_str());
+        else
+          *err = StringPrintf("$(file <%s) was changed, regenerating...\n",
+                              sr->cmd.c_str());
+        return true;
+      }
+      if (g_flags.dump_kati_stamp)
+        printf("file %s: clean\n", sr->cmd.c_str());
+      return false;
+    }
+
+    if (sr->op == CommandOp::WRITE || sr->op == CommandOp::APPEND) {
+      FILE* f = fopen(sr->cmd.c_str(), (sr->op == CommandOp::WRITE) ? "wb" : "ab");
+      if (f == NULL) {
+        PERROR("fopen");
+      }
+
+      if (fwrite(&sr->result[0], sr->result.size(), 1, f) != 1) {
+        PERROR("fwrite");
+      }
+
+      if (fclose(f) != 0) {
+        PERROR("fclose");
+      }
+
+      if (g_flags.dump_kati_stamp)
+        printf("file %s: clean (write)\n", sr->cmd.c_str());
+      return false;
+    }
+
     if (!ShouldRunCommand(sr)) {
       if (g_flags.regen_debug)
         printf("shell %s: clean (no rerun)\n", sr->cmd.c_str());
diff --git a/testcase/file_func.sh b/testcase/file_func.sh
new file mode 100755
index 0000000..72935dc
--- /dev/null
+++ b/testcase/file_func.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Copyright 2016 Google Inc. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+mk="$@"
+
+echo "PASS" >testfile
+
+cat <<EOF > Makefile
+ifdef KATI
+SUPPORTS_FILE := 1
+endif
+ifneq (,\$(filter 4.2%,\$(MAKE_VERSION)))
+SUPPORTS_FILE := 1
+endif
+
+ifdef SUPPORTS_FILE
+  \$(file >testwrite,PASS)
+  \$(info Read not found: \$(if \$(file <notfound),FAIL,PASS))
+  \$(info Read: \$(file < testfile))
+  \$(info Read back: \$(file <testwrite))
+else
+  # Make <4 does not support \$(file ...)
+  \$(info Read not found: PASS)
+  \$(info Read: PASS)
+  \$(info Read back: PASS)
+endif
+
+.PHONY: all
+all:
+EOF
+
+${mk} 2>&1
diff --git a/testcase/ninja_regen_filefunc_read.sh b/testcase/ninja_regen_filefunc_read.sh
new file mode 100755
index 0000000..7fc3a9f
--- /dev/null
+++ b/testcase/ninja_regen_filefunc_read.sh
@@ -0,0 +1,74 @@
+#!/bin/sh
+#
+# Copyright 2016 Google Inc. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+log=/tmp/log
+mk="$@"
+
+sleep_if_necessary() {
+  if [ x$(uname) != x"Linux" -o x"${TRAVIS}" != x"" ]; then
+    sleep "$@"
+  fi
+}
+
+cat <<EOF > Makefile
+A := \$(file <file_a)
+all:
+	echo foo
+EOF
+
+${mk} 2> ${log}
+if [ -e ninja.sh ]; then
+  ./ninja.sh
+fi
+
+${mk} 2> ${log}
+if [ -e ninja.sh ]; then
+  if grep regenerating ${log}; then
+    echo 'Should not be regenerated'
+  fi
+  ./ninja.sh
+fi
+
+echo regen >file_a
+
+${mk} 2> ${log}
+if [ -e ninja.sh ]; then
+  if ! grep regenerating ${log} >/dev/null; then
+    echo 'Should be regenerated'
+  fi
+  ./ninja.sh
+fi
+
+${mk} 2> ${log}
+if [ -e ninja.sh ]; then
+  if grep regenerating ${log}; then
+    echo 'Should not be regenerated'
+  fi
+  ./ninja.sh
+fi
+
+sleep_if_necessary 1
+echo regen >>file_a
+
+${mk} 2> ${log}
+if [ -e ninja.sh ]; then
+  if ! grep regenerating ${log} >/dev/null; then
+    echo 'Should be regenerated'
+  fi
+  ./ninja.sh
+fi
diff --git a/testcase/ninja_regen_filefunc_write.sh b/testcase/ninja_regen_filefunc_write.sh
new file mode 100755
index 0000000..cb438cc
--- /dev/null
+++ b/testcase/ninja_regen_filefunc_write.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+# Copyright 2016 Google Inc. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+log=/tmp/log
+mk="$@"
+
+cat <<EOF > Makefile
+\$(file >file_a,test)
+all:
+	echo foo
+EOF
+
+${mk} 2> ${log}
+if [ -e ninja.sh ]; then
+  if [ ! -f file_a ]; then
+    echo 'file_a does not exist'
+  fi
+  ./ninja.sh
+  rm file_a
+fi
+
+${mk} 2> ${log}
+if [ -e ninja.sh ]; then
+  if grep regenerating ${log}; then
+    echo 'Should not be regenerated'
+  fi
+  if [ ! -f file_a ]; then
+    echo 'file_a does not exist'
+  fi
+  ./ninja.sh
+fi