From f06d8019e99ae6aee1d2881f30315aee7b544cfb Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Mon, 3 Oct 2016 00:16:07 -0700 Subject: Implement the `file` function to read and write files This allows us to do file reading and writing without $(shell). Besides being simpler, this also allows faster regen times, since we can just stat the files to be read, or directly write to the files that need to be written. --- func.cc | 123 +++++++++++++++++++++++++++++++++ func.h | 10 +++ ninja.cc | 36 +++++----- regen.cc | 77 +++++++++++++++++---- testcase/file_func.sh | 47 +++++++++++++ testcase/ninja_regen_filefunc_read.sh | 74 ++++++++++++++++++++ testcase/ninja_regen_filefunc_write.sh | 46 ++++++++++++ 7 files changed, 379 insertions(+), 34 deletions(-) create mode 100755 testcase/file_func.sh create mode 100755 testcase/ninja_regen_filefunc_read.sh create mode 100755 testcase/ninja_regen_filefunc_write.sh diff --git a/func.cc b/func.cc index 623e56e..1c666ec 100644 --- a/func.cc +++ b/func.cc @@ -17,9 +17,11 @@ #include "func.h" #include +#include #include #include #include +#include #include #include @@ -571,6 +573,7 @@ void ShellFunc(const vector& args, Evaluator* ev, string* s) { ShellFuncImpl(shell, cmd, &out, &fc); if (ShouldStoreCommandResult(cmd)) { CommandResult* cr = new CommandResult(); + cr->op = (fc == NULL) ? CommandOp::SHELL : CommandOp::FIND, cr->shell = shell; cr->cmd = cmd; cr->find.reset(fc); @@ -686,6 +689,124 @@ void ErrorFunc(const vector& args, Evaluator* ev, string*) { ev->Error(StringPrintf("*** %s.", a.c_str())); } +static void FileReadFunc(Evaluator* ev, const string& filename, string* s) { + int fd = open(filename.c_str(), O_RDONLY); + if (fd < 0) { + if (errno == ENOENT) { + if (ShouldStoreCommandResult(filename)) { + CommandResult* cr = new CommandResult(); + cr->op = CommandOp::READ_MISSING; + cr->cmd = filename; + g_command_results.push_back(cr); + } + return; + } else { + ev->Error("*** open failed."); + } + } + + struct stat st; + if (fstat(fd, &st) < 0) { + ev->Error("*** fstat failed."); + } + + size_t len = st.st_size; + string out; + out.resize(len); + ssize_t r = HANDLE_EINTR(read(fd, &out[0], len)); + if (r != static_cast(len)) { + ev->Error("*** read failed."); + } + + if (close(fd) < 0) { + ev->Error("*** close failed."); + } + + if (out.back() == '\n') { + out.pop_back(); + } + + if (ShouldStoreCommandResult(filename)) { + CommandResult* cr = new CommandResult(); + cr->op = CommandOp::READ; + cr->cmd = filename; + g_command_results.push_back(cr); + } + *s += out; +} + +static void FileWriteFunc(Evaluator* ev, const string& filename, bool append, string text) { + FILE* f = fopen(filename.c_str(), append ? "ab" : "wb"); + if (f == NULL) { + ev->Error("*** fopen failed."); + } + + if (fwrite(&text[0], text.size(), 1, f) != 1) { + ev->Error("*** fwrite failed."); + } + + if (fclose(f) != 0) { + ev->Error("*** fclose failed."); + } + + if (ShouldStoreCommandResult(filename)) { + CommandResult* cr = new CommandResult(); + cr->op = CommandOp::WRITE; + cr->cmd = filename; + cr->result = text; + g_command_results.push_back(cr); + } +} + +void FileFunc(const vector& args, Evaluator* ev, string* s) { + if (ev->avoid_io()) { + ev->Error("*** $(file ...) is not supported in rules."); + } + + string arg = args[0]->Eval(ev); + StringPiece filename = TrimSpace(arg); + + if (filename.size() <= 1) { + ev->Error("*** Missing filename"); + } + + if (filename[0] == '<') { + filename = TrimLeftSpace(filename.substr(1)); + if (!filename.size()) { + ev->Error("*** Missing filename"); + } + if (args.size() > 1) { + ev->Error("*** invalid argument"); + } + + FileReadFunc(ev, filename.as_string(), s); + } else if (filename[0] == '>') { + bool append = false; + if (filename[1] == '>') { + append = true; + filename = filename.substr(2); + } else { + filename = filename.substr(1); + } + filename = TrimLeftSpace(filename); + if (!filename.size()) { + ev->Error("*** Missing filename"); + } + + string text; + if (args.size() > 1) { + text = args[1]->Eval(ev); + if (text.size() == 0 || text.back() != '\n') { + text.push_back('\n'); + } + } + + FileWriteFunc(ev, filename.as_string(), append, text); + } else { + ev->Error(StringPrintf("*** Invalid file operation: %s. Stop.", filename.as_string().c_str())); + } +} + FuncInfo g_func_infos[] = { { "patsubst", &PatsubstFunc, 3, 3, false, false }, { "strip", &StripFunc, 1, 1, false, false }, @@ -727,6 +848,8 @@ FuncInfo g_func_infos[] = { { "info", &InfoFunc, 1, 1, false, false }, { "warning", &WarningFunc, 1, 1, false, false }, { "error", &ErrorFunc, 1, 1, false, false }, + + { "file", &FileFunc, 2, 1, false, false }, }; unordered_map* g_func_info_map; diff --git a/func.h b/func.h index e78deb7..ab6affb 100644 --- a/func.h +++ b/func.h @@ -41,7 +41,17 @@ FuncInfo* GetFuncInfo(StringPiece name); struct FindCommand; +enum struct CommandOp { + SHELL, + FIND, + READ, + READ_MISSING, + WRITE, + APPEND, +}; + struct CommandResult { + CommandOp op; string shell; string cmd; unique_ptr find; diff --git a/ninja.cc b/ninja.cc index 90fe404..59e8712 100644 --- a/ninja.cc +++ b/ninja.cc @@ -736,31 +736,27 @@ class NinjaGenerator { const vector& crs = GetShellCommandResults(); DumpInt(fp, crs.size()); for (CommandResult* cr : crs) { + DumpInt(fp, static_cast(cr->op)); DumpString(fp, cr->shell); DumpString(fp, cr->cmd); DumpString(fp, cr->result); - if (!cr->find.get()) { - // Always re-run this command. - DumpInt(fp, 0); - continue; - } - DumpInt(fp, 1); - - vector missing_dirs; - for (StringPiece fd : cr->find->finddirs) { - const string& d = ConcatDir(cr->find->chdir, fd); - if (!Exists(d)) - missing_dirs.push_back(d); - } - DumpInt(fp, missing_dirs.size()); - for (const string& d : missing_dirs) { - DumpString(fp, d); - } + if (cr->op == CommandOp::FIND) { + vector missing_dirs; + for (StringPiece fd : cr->find->finddirs) { + const string& d = ConcatDir(cr->find->chdir, fd); + if (!Exists(d)) + missing_dirs.push_back(d); + } + DumpInt(fp, missing_dirs.size()); + for (const string& d : missing_dirs) { + DumpString(fp, d); + } - DumpInt(fp, cr->find->read_dirs->size()); - for (StringPiece s : *cr->find->read_dirs) { - DumpString(fp, ConcatDir(cr->find->chdir, s)); + DumpInt(fp, cr->find->read_dirs->size()); + for (StringPiece s : *cr->find->read_dirs) { + DumpString(fp, ConcatDir(cr->find->chdir, s)); + } } } diff --git a/regen.cc b/regen.cc index f5ed50a..bbae7c8 100644 --- a/regen.cc +++ b/regen.cc @@ -23,6 +23,7 @@ #include "fileutil.h" #include "find.h" +#include "func.h" #include "io.h" #include "log.h" #include "ninja.h" @@ -52,12 +53,12 @@ class StampChecker { }; struct ShellResult { + CommandOp op; string shell; string cmd; string result; vector missing_dirs; vector read_dirs; - bool has_condition; }; public: @@ -231,22 +232,22 @@ class StampChecker { for (int i = 0; i < num_crs; i++) { ShellResult* sr = new ShellResult; commands_.push_back(sr); + sr->op = static_cast(LOAD_INT(fp)); LOAD_STRING(fp, &sr->shell); LOAD_STRING(fp, &sr->cmd); LOAD_STRING(fp, &sr->result); - sr->has_condition = LOAD_INT(fp); - if (!sr->has_condition) - continue; - int num_missing_dirs = LOAD_INT(fp); - for (int j = 0; j < num_missing_dirs; j++) { - LOAD_STRING(fp, &s); - sr->missing_dirs.push_back(s); - } - int num_read_dirs = LOAD_INT(fp); - for (int j = 0; j < num_read_dirs; j++) { - LOAD_STRING(fp, &s); - sr->read_dirs.push_back(s); + if (sr->op == CommandOp::FIND) { + int num_missing_dirs = LOAD_INT(fp); + for (int j = 0; j < num_missing_dirs; j++) { + LOAD_STRING(fp, &s); + sr->missing_dirs.push_back(s); + } + int num_read_dirs = LOAD_INT(fp); + for (int j = 0; j < num_read_dirs; j++) { + LOAD_STRING(fp, &s); + sr->read_dirs.push_back(s); + } } } @@ -292,7 +293,7 @@ class StampChecker { } bool ShouldRunCommand(const ShellResult* sr) { - if (!sr->has_condition) + if (sr->op != CommandOp::FIND) return true; COLLECT_STATS("stat time (regen)"); @@ -324,6 +325,54 @@ class StampChecker { } bool CheckShellResult(const ShellResult* sr, string* err) { + if (sr->op == CommandOp::READ_MISSING) { + if (Exists(sr->cmd)) { + if (g_flags.dump_kati_stamp) + printf("file %s: dirty\n", sr->cmd.c_str()); + else + *err = StringPrintf("$(file <%s) was changed, regenerating...\n", + sr->cmd.c_str()); + return true; + } + if (g_flags.dump_kati_stamp) + printf("file %s: clean\n", sr->cmd.c_str()); + return false; + } + + if (sr->op == CommandOp::READ) { + double ts = GetTimestamp(sr->cmd); + if (gen_time_ < ts) { + if (g_flags.dump_kati_stamp) + printf("file %s: dirty\n", sr->cmd.c_str()); + else + *err = StringPrintf("$(file <%s) was changed, regenerating...\n", + sr->cmd.c_str()); + return true; + } + if (g_flags.dump_kati_stamp) + printf("file %s: clean\n", sr->cmd.c_str()); + return false; + } + + if (sr->op == CommandOp::WRITE || sr->op == CommandOp::APPEND) { + FILE* f = fopen(sr->cmd.c_str(), (sr->op == CommandOp::WRITE) ? "wb" : "ab"); + if (f == NULL) { + PERROR("fopen"); + } + + if (fwrite(&sr->result[0], sr->result.size(), 1, f) != 1) { + PERROR("fwrite"); + } + + if (fclose(f) != 0) { + PERROR("fclose"); + } + + if (g_flags.dump_kati_stamp) + printf("file %s: clean (write)\n", sr->cmd.c_str()); + return false; + } + if (!ShouldRunCommand(sr)) { if (g_flags.regen_debug) printf("shell %s: clean (no rerun)\n", sr->cmd.c_str()); diff --git a/testcase/file_func.sh b/testcase/file_func.sh new file mode 100755 index 0000000..72935dc --- /dev/null +++ b/testcase/file_func.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# +# Copyright 2016 Google Inc. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +mk="$@" + +echo "PASS" >testfile + +cat < Makefile +ifdef KATI +SUPPORTS_FILE := 1 +endif +ifneq (,\$(filter 4.2%,\$(MAKE_VERSION))) +SUPPORTS_FILE := 1 +endif + +ifdef SUPPORTS_FILE + \$(file >testwrite,PASS) + \$(info Read not found: \$(if \$(file &1 diff --git a/testcase/ninja_regen_filefunc_read.sh b/testcase/ninja_regen_filefunc_read.sh new file mode 100755 index 0000000..7fc3a9f --- /dev/null +++ b/testcase/ninja_regen_filefunc_read.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# +# Copyright 2016 Google Inc. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +log=/tmp/log +mk="$@" + +sleep_if_necessary() { + if [ x$(uname) != x"Linux" -o x"${TRAVIS}" != x"" ]; then + sleep "$@" + fi +} + +cat < Makefile +A := \$(file ${log} +if [ -e ninja.sh ]; then + ./ninja.sh +fi + +${mk} 2> ${log} +if [ -e ninja.sh ]; then + if grep regenerating ${log}; then + echo 'Should not be regenerated' + fi + ./ninja.sh +fi + +echo regen >file_a + +${mk} 2> ${log} +if [ -e ninja.sh ]; then + if ! grep regenerating ${log} >/dev/null; then + echo 'Should be regenerated' + fi + ./ninja.sh +fi + +${mk} 2> ${log} +if [ -e ninja.sh ]; then + if grep regenerating ${log}; then + echo 'Should not be regenerated' + fi + ./ninja.sh +fi + +sleep_if_necessary 1 +echo regen >>file_a + +${mk} 2> ${log} +if [ -e ninja.sh ]; then + if ! grep regenerating ${log} >/dev/null; then + echo 'Should be regenerated' + fi + ./ninja.sh +fi diff --git a/testcase/ninja_regen_filefunc_write.sh b/testcase/ninja_regen_filefunc_write.sh new file mode 100755 index 0000000..cb438cc --- /dev/null +++ b/testcase/ninja_regen_filefunc_write.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# +# Copyright 2016 Google Inc. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +log=/tmp/log +mk="$@" + +cat < Makefile +\$(file >file_a,test) +all: + echo foo +EOF + +${mk} 2> ${log} +if [ -e ninja.sh ]; then + if [ ! -f file_a ]; then + echo 'file_a does not exist' + fi + ./ninja.sh + rm file_a +fi + +${mk} 2> ${log} +if [ -e ninja.sh ]; then + if grep regenerating ${log}; then + echo 'Should not be regenerated' + fi + if [ ! -f file_a ]; then + echo 'file_a does not exist' + fi + ./ninja.sh +fi -- cgit v1.2.3