Ninja
build_log.cc
Go to the documentation of this file.
00001 // Copyright 2011 Google Inc. All Rights Reserved.
00002 //
00003 // Licensed under the Apache License, Version 2.0 (the "License");
00004 // you may not use this file except in compliance with the License.
00005 // You may obtain a copy of the License at
00006 //
00007 //     http://www.apache.org/licenses/LICENSE-2.0
00008 //
00009 // Unless required by applicable law or agreed to in writing, software
00010 // distributed under the License is distributed on an "AS IS" BASIS,
00011 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00012 // See the License for the specific language governing permissions and
00013 // limitations under the License.
00014 
00015 #include "build_log.h"
00016 
00017 #include <errno.h>
00018 #include <stdlib.h>
00019 #include <string.h>
00020 
00021 #ifndef _WIN32
00022 #define __STDC_FORMAT_MACROS
00023 #include <inttypes.h>
00024 #include <unistd.h>
00025 #endif
00026 
00027 #include "build.h"
00028 #include "graph.h"
00029 #include "metrics.h"
00030 #include "util.h"
00031 
00032 // Implementation details:
00033 // Each run's log appends to the log file.
00034 // To load, we run through all log entries in series, throwing away
00035 // older runs.
00036 // Once the number of redundant entries exceeds a threshold, we write
00037 // out a new file and replace the existing one with it.
00038 
00039 namespace {
00040 
00041 const char kFileSignature[] = "# ninja log v%d\n";
00042 const int kOldestSupportedVersion = 4;
00043 const int kCurrentVersion = 5;
00044 
00045 // 64bit MurmurHash2, by Austin Appleby
00046 #if defined(_MSC_VER)
00047 #define BIG_CONSTANT(x) (x)
00048 #else   // defined(_MSC_VER)
00049 #define BIG_CONSTANT(x) (x##LLU)
00050 #endif // !defined(_MSC_VER)
00051 inline
00052 uint64_t MurmurHash64A(const void* key, size_t len) {
00053   static const uint64_t seed = 0xDECAFBADDECAFBADull;
00054   const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
00055   const int r = 47;
00056   uint64_t h = seed ^ (len * m);
00057   const unsigned char* data = (const unsigned char*)key;
00058   while (len >= 8) {
00059     uint64_t k;
00060     memcpy(&k, data, sizeof k);
00061     k *= m;
00062     k ^= k >> r;
00063     k *= m;
00064     h ^= k;
00065     h *= m;
00066     data += 8;
00067     len -= 8;
00068   }
00069   switch (len & 7)
00070   {
00071   case 7: h ^= uint64_t(data[6]) << 48;
00072   case 6: h ^= uint64_t(data[5]) << 40;
00073   case 5: h ^= uint64_t(data[4]) << 32;
00074   case 4: h ^= uint64_t(data[3]) << 24;
00075   case 3: h ^= uint64_t(data[2]) << 16;
00076   case 2: h ^= uint64_t(data[1]) << 8;
00077   case 1: h ^= uint64_t(data[0]);
00078           h *= m;
00079   };
00080   h ^= h >> r;
00081   h *= m;
00082   h ^= h >> r;
00083   return h;
00084 }
00085 #undef BIG_CONSTANT
00086 
00087 
00088 }  // namespace
00089 
00090 // static
00091 uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) {
00092   return MurmurHash64A(command.str_, command.len_);
00093 }
00094 
00095 BuildLog::LogEntry::LogEntry(const string& output)
00096   : output(output) {}
00097 
00098 BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash,
00099   int start_time, int end_time, TimeStamp restat_mtime)
00100   : output(output), command_hash(command_hash),
00101     start_time(start_time), end_time(end_time), restat_mtime(restat_mtime)
00102 {}
00103 
00104 BuildLog::BuildLog()
00105   : log_file_(NULL), needs_recompaction_(false) {}
00106 
00107 BuildLog::~BuildLog() {
00108   Close();
00109 }
00110 
00111 bool BuildLog::OpenForWrite(const string& path, string* err) {
00112   if (needs_recompaction_) {
00113     if (!Recompact(path, err))
00114       return false;
00115   }
00116 
00117   log_file_ = fopen(path.c_str(), "ab");
00118   if (!log_file_) {
00119     *err = strerror(errno);
00120     return false;
00121   }
00122   setvbuf(log_file_, NULL, _IOLBF, BUFSIZ);
00123   SetCloseOnExec(fileno(log_file_));
00124 
00125   // Opening a file in append mode doesn't set the file pointer to the file's
00126   // end on Windows. Do that explicitly.
00127   fseek(log_file_, 0, SEEK_END);
00128 
00129   if (ftell(log_file_) == 0) {
00130     if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) {
00131       *err = strerror(errno);
00132       return false;
00133     }
00134   }
00135 
00136   return true;
00137 }
00138 
00139 bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
00140                              TimeStamp restat_mtime) {
00141   string command = edge->EvaluateCommand(true);
00142   uint64_t command_hash = LogEntry::HashCommand(command);
00143   for (vector<Node*>::iterator out = edge->outputs_.begin();
00144        out != edge->outputs_.end(); ++out) {
00145     const string& path = (*out)->path();
00146     Entries::iterator i = entries_.find(path);
00147     LogEntry* log_entry;
00148     if (i != entries_.end()) {
00149       log_entry = i->second;
00150     } else {
00151       log_entry = new LogEntry(path);
00152       entries_.insert(Entries::value_type(log_entry->output, log_entry));
00153     }
00154     log_entry->command_hash = command_hash;
00155     log_entry->start_time = start_time;
00156     log_entry->end_time = end_time;
00157     log_entry->restat_mtime = restat_mtime;
00158 
00159     if (log_file_) {
00160       if (!WriteEntry(log_file_, *log_entry))
00161         return false;
00162     }
00163   }
00164   return true;
00165 }
00166 
00167 void BuildLog::Close() {
00168   if (log_file_)
00169     fclose(log_file_);
00170   log_file_ = NULL;
00171 }
00172 
00173 struct LineReader {
00174   explicit LineReader(FILE* file)
00175     : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {
00176       memset(buf_, 0, sizeof(buf_));
00177   }
00178 
00179   // Reads a \n-terminated line from the file passed to the constructor.
00180   // On return, *line_start points to the beginning of the next line, and
00181   // *line_end points to the \n at the end of the line. If no newline is seen
00182   // in a fixed buffer size, *line_end is set to NULL. Returns false on EOF.
00183   bool ReadLine(char** line_start, char** line_end) {
00184     if (line_start_ >= buf_end_ || !line_end_) {
00185       // Buffer empty, refill.
00186       size_t size_read = fread(buf_, 1, sizeof(buf_), file_);
00187       if (!size_read)
00188         return false;
00189       line_start_ = buf_;
00190       buf_end_ = buf_ + size_read;
00191     } else {
00192       // Advance to next line in buffer.
00193       line_start_ = line_end_ + 1;
00194     }
00195 
00196     line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
00197     if (!line_end_) {
00198       // No newline. Move rest of data to start of buffer, fill rest.
00199       size_t already_consumed = line_start_ - buf_;
00200       size_t size_rest = (buf_end_ - buf_) - already_consumed;
00201       memmove(buf_, line_start_, size_rest);
00202 
00203       size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_);
00204       buf_end_ = buf_ + size_rest + read;
00205       line_start_ = buf_;
00206       line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
00207     }
00208 
00209     *line_start = line_start_;
00210     *line_end = line_end_;
00211     return true;
00212   }
00213 
00214  private:
00215   FILE* file_;
00216   char buf_[256 << 10];
00217   char* buf_end_;  // Points one past the last valid byte in |buf_|.
00218 
00219   char* line_start_;
00220   // Points at the next \n in buf_ after line_start, or NULL.
00221   char* line_end_;
00222 };
00223 
00224 bool BuildLog::Load(const string& path, string* err) {
00225   METRIC_RECORD(".ninja_log load");
00226   FILE* file = fopen(path.c_str(), "r");
00227   if (!file) {
00228     if (errno == ENOENT)
00229       return true;
00230     *err = strerror(errno);
00231     return false;
00232   }
00233 
00234   int log_version = 0;
00235   int unique_entry_count = 0;
00236   int total_entry_count = 0;
00237 
00238   LineReader reader(file);
00239   char* line_start = 0;
00240   char* line_end = 0;
00241   while (reader.ReadLine(&line_start, &line_end)) {
00242     if (!log_version) {
00243       sscanf(line_start, kFileSignature, &log_version);
00244 
00245       if (log_version < kOldestSupportedVersion) {
00246         *err = ("build log version invalid, perhaps due to being too old; "
00247                 "starting over");
00248         fclose(file);
00249         unlink(path.c_str());
00250         // Don't report this as a failure.  An empty build log will cause
00251         // us to rebuild the outputs anyway.
00252         return true;
00253       }
00254     }
00255 
00256     // If no newline was found in this chunk, read the next.
00257     if (!line_end)
00258       continue;
00259 
00260     const char kFieldSeparator = '\t';
00261 
00262     char* start = line_start;
00263     char* end = (char*)memchr(start, kFieldSeparator, line_end - start);
00264     if (!end)
00265       continue;
00266     *end = 0;
00267 
00268     int start_time = 0, end_time = 0;
00269     TimeStamp restat_mtime = 0;
00270 
00271     start_time = atoi(start);
00272     start = end + 1;
00273 
00274     end = (char*)memchr(start, kFieldSeparator, line_end - start);
00275     if (!end)
00276       continue;
00277     *end = 0;
00278     end_time = atoi(start);
00279     start = end + 1;
00280 
00281     end = (char*)memchr(start, kFieldSeparator, line_end - start);
00282     if (!end)
00283       continue;
00284     *end = 0;
00285     restat_mtime = atol(start);
00286     start = end + 1;
00287 
00288     end = (char*)memchr(start, kFieldSeparator, line_end - start);
00289     if (!end)
00290       continue;
00291     string output = string(start, end - start);
00292 
00293     start = end + 1;
00294     end = line_end;
00295 
00296     LogEntry* entry;
00297     Entries::iterator i = entries_.find(output);
00298     if (i != entries_.end()) {
00299       entry = i->second;
00300     } else {
00301       entry = new LogEntry(output);
00302       entries_.insert(Entries::value_type(entry->output, entry));
00303       ++unique_entry_count;
00304     }
00305     ++total_entry_count;
00306 
00307     entry->start_time = start_time;
00308     entry->end_time = end_time;
00309     entry->restat_mtime = restat_mtime;
00310     if (log_version >= 5) {
00311       char c = *end; *end = '\0';
00312       entry->command_hash = (uint64_t)strtoull(start, NULL, 16);
00313       *end = c;
00314     } else {
00315       entry->command_hash = LogEntry::HashCommand(StringPiece(start,
00316                                                               end - start));
00317     }
00318   }
00319   fclose(file);
00320 
00321   if (!line_start) {
00322     return true; // file was empty
00323   }
00324 
00325   // Decide whether it's time to rebuild the log:
00326   // - if we're upgrading versions
00327   // - if it's getting large
00328   int kMinCompactionEntryCount = 100;
00329   int kCompactionRatio = 3;
00330   if (log_version < kCurrentVersion) {
00331     needs_recompaction_ = true;
00332   } else if (total_entry_count > kMinCompactionEntryCount &&
00333              total_entry_count > unique_entry_count * kCompactionRatio) {
00334     needs_recompaction_ = true;
00335   }
00336 
00337   return true;
00338 }
00339 
00340 BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) {
00341   Entries::iterator i = entries_.find(path);
00342   if (i != entries_.end())
00343     return i->second;
00344   return NULL;
00345 }
00346 
00347 bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) {
00348   return fprintf(f, "%d\t%d\t%d\t%s\t%" PRIx64 "\n",
00349           entry.start_time, entry.end_time, entry.restat_mtime,
00350           entry.output.c_str(), entry.command_hash) > 0;
00351 }
00352 
00353 bool BuildLog::Recompact(const string& path, string* err) {
00354   METRIC_RECORD(".ninja_log recompact");
00355   printf("Recompacting log...\n");
00356 
00357   Close();
00358   string temp_path = path + ".recompact";
00359   FILE* f = fopen(temp_path.c_str(), "wb");
00360   if (!f) {
00361     *err = strerror(errno);
00362     return false;
00363   }
00364 
00365   if (fprintf(f, kFileSignature, kCurrentVersion) < 0) {
00366     *err = strerror(errno);
00367     fclose(f);
00368     return false;
00369   }
00370 
00371   for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
00372     if (!WriteEntry(f, *i->second)) {
00373       *err = strerror(errno);
00374       fclose(f);
00375       return false;
00376     }
00377   }
00378 
00379   fclose(f);
00380   if (unlink(path.c_str()) < 0) {
00381     *err = strerror(errno);
00382     return false;
00383   }
00384 
00385   if (rename(temp_path.c_str(), path.c_str()) < 0) {
00386     *err = strerror(errno);
00387     return false;
00388   }
00389 
00390   return true;
00391 }