Ninja
build_log.cc
Go to the documentation of this file.
1 // Copyright 2011 Google Inc. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "build_log.h"
16 
17 #include <errno.h>
18 #include <stdlib.h>
19 #include <string.h>
20 
21 #ifndef _WIN32
22 #define __STDC_FORMAT_MACROS
23 #include <inttypes.h>
24 #include <unistd.h>
25 #endif
26 
27 #include "build.h"
28 #include "graph.h"
29 #include "metrics.h"
30 #include "util.h"
31 
32 // Implementation details:
33 // Each run's log appends to the log file.
34 // To load, we run through all log entries in series, throwing away
35 // older runs.
36 // Once the number of redundant entries exceeds a threshold, we write
37 // out a new file and replace the existing one with it.
38 
39 namespace {
40 
41 const char kFileSignature[] = "# ninja log v%d\n";
42 const int kOldestSupportedVersion = 4;
43 const int kCurrentVersion = 5;
44 
45 // 64bit MurmurHash2, by Austin Appleby
46 #if defined(_MSC_VER)
47 #define BIG_CONSTANT(x) (x)
48 #else // defined(_MSC_VER)
49 #define BIG_CONSTANT(x) (x##LLU)
50 #endif // !defined(_MSC_VER)
51 inline
52 uint64_t MurmurHash64A(const void* key, size_t len) {
53  static const uint64_t seed = 0xDECAFBADDECAFBADull;
54  const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
55  const int r = 47;
56  uint64_t h = seed ^ (len * m);
57  const unsigned char* data = (const unsigned char*)key;
58  while (len >= 8) {
59  uint64_t k;
60  memcpy(&k, data, sizeof k);
61  k *= m;
62  k ^= k >> r;
63  k *= m;
64  h ^= k;
65  h *= m;
66  data += 8;
67  len -= 8;
68  }
69  switch (len & 7)
70  {
71  case 7: h ^= uint64_t(data[6]) << 48;
72  case 6: h ^= uint64_t(data[5]) << 40;
73  case 5: h ^= uint64_t(data[4]) << 32;
74  case 4: h ^= uint64_t(data[3]) << 24;
75  case 3: h ^= uint64_t(data[2]) << 16;
76  case 2: h ^= uint64_t(data[1]) << 8;
77  case 1: h ^= uint64_t(data[0]);
78  h *= m;
79  };
80  h ^= h >> r;
81  h *= m;
82  h ^= h >> r;
83  return h;
84 }
85 #undef BIG_CONSTANT
86 
87 
88 } // namespace
89 
90 // static
92  return MurmurHash64A(command.str_, command.len_);
93 }
94 
95 BuildLog::LogEntry::LogEntry(const string& output)
96  : output(output) {}
97 
98 BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash,
99  int start_time, int end_time, TimeStamp restat_mtime)
100  : output(output), command_hash(command_hash),
101  start_time(start_time), end_time(end_time), restat_mtime(restat_mtime)
102 {}
103 
105  : log_file_(NULL), needs_recompaction_(false) {}
106 
108  Close();
109 }
110 
111 bool BuildLog::OpenForWrite(const string& path, const BuildLogUser& user,
112  string* err) {
113  if (needs_recompaction_) {
114  if (!Recompact(path, user, err))
115  return false;
116  }
117 
118  log_file_ = fopen(path.c_str(), "ab");
119  if (!log_file_) {
120  *err = strerror(errno);
121  return false;
122  }
123  setvbuf(log_file_, NULL, _IOLBF, BUFSIZ);
124  SetCloseOnExec(fileno(log_file_));
125 
126  // Opening a file in append mode doesn't set the file pointer to the file's
127  // end on Windows. Do that explicitly.
128  fseek(log_file_, 0, SEEK_END);
129 
130  if (ftell(log_file_) == 0) {
131  if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) {
132  *err = strerror(errno);
133  return false;
134  }
135  }
136 
137  return true;
138 }
139 
140 bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
141  TimeStamp restat_mtime) {
142  string command = edge->EvaluateCommand(true);
143  uint64_t command_hash = LogEntry::HashCommand(command);
144  for (vector<Node*>::iterator out = edge->outputs_.begin();
145  out != edge->outputs_.end(); ++out) {
146  const string& path = (*out)->path();
147  Entries::iterator i = entries_.find(path);
148  LogEntry* log_entry;
149  if (i != entries_.end()) {
150  log_entry = i->second;
151  } else {
152  log_entry = new LogEntry(path);
153  entries_.insert(Entries::value_type(log_entry->output, log_entry));
154  }
155  log_entry->command_hash = command_hash;
156  log_entry->start_time = start_time;
157  log_entry->end_time = end_time;
158  log_entry->restat_mtime = restat_mtime;
159 
160  if (log_file_) {
161  if (!WriteEntry(log_file_, *log_entry))
162  return false;
163  }
164  }
165  return true;
166 }
167 
169  if (log_file_)
170  fclose(log_file_);
171  log_file_ = NULL;
172 }
173 
174 struct LineReader {
175  explicit LineReader(FILE* file)
176  : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {
177  memset(buf_, 0, sizeof(buf_));
178  }
179 
180  // Reads a \n-terminated line from the file passed to the constructor.
181  // On return, *line_start points to the beginning of the next line, and
182  // *line_end points to the \n at the end of the line. If no newline is seen
183  // in a fixed buffer size, *line_end is set to NULL. Returns false on EOF.
184  bool ReadLine(char** line_start, char** line_end) {
185  if (line_start_ >= buf_end_ || !line_end_) {
186  // Buffer empty, refill.
187  size_t size_read = fread(buf_, 1, sizeof(buf_), file_);
188  if (!size_read)
189  return false;
190  line_start_ = buf_;
191  buf_end_ = buf_ + size_read;
192  } else {
193  // Advance to next line in buffer.
194  line_start_ = line_end_ + 1;
195  }
196 
197  line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
198  if (!line_end_) {
199  // No newline. Move rest of data to start of buffer, fill rest.
200  size_t already_consumed = line_start_ - buf_;
201  size_t size_rest = (buf_end_ - buf_) - already_consumed;
202  memmove(buf_, line_start_, size_rest);
203 
204  size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_);
205  buf_end_ = buf_ + size_rest + read;
206  line_start_ = buf_;
207  line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
208  }
209 
210  *line_start = line_start_;
211  *line_end = line_end_;
212  return true;
213  }
214 
215  private:
216  FILE* file_;
217  char buf_[256 << 10];
218  char* buf_end_; // Points one past the last valid byte in |buf_|.
219 
220  char* line_start_;
221  // Points at the next \n in buf_ after line_start, or NULL.
222  char* line_end_;
223 };
224 
225 bool BuildLog::Load(const string& path, string* err) {
226  METRIC_RECORD(".ninja_log load");
227  FILE* file = fopen(path.c_str(), "r");
228  if (!file) {
229  if (errno == ENOENT)
230  return true;
231  *err = strerror(errno);
232  return false;
233  }
234 
235  int log_version = 0;
236  int unique_entry_count = 0;
237  int total_entry_count = 0;
238 
239  LineReader reader(file);
240  char* line_start = 0;
241  char* line_end = 0;
242  while (reader.ReadLine(&line_start, &line_end)) {
243  if (!log_version) {
244  sscanf(line_start, kFileSignature, &log_version);
245 
246  if (log_version < kOldestSupportedVersion) {
247  *err = ("build log version invalid, perhaps due to being too old; "
248  "starting over");
249  fclose(file);
250  unlink(path.c_str());
251  // Don't report this as a failure. An empty build log will cause
252  // us to rebuild the outputs anyway.
253  return true;
254  }
255  }
256 
257  // If no newline was found in this chunk, read the next.
258  if (!line_end)
259  continue;
260 
261  const char kFieldSeparator = '\t';
262 
263  char* start = line_start;
264  char* end = (char*)memchr(start, kFieldSeparator, line_end - start);
265  if (!end)
266  continue;
267  *end = 0;
268 
269  int start_time = 0, end_time = 0;
270  TimeStamp restat_mtime = 0;
271 
272  start_time = atoi(start);
273  start = end + 1;
274 
275  end = (char*)memchr(start, kFieldSeparator, line_end - start);
276  if (!end)
277  continue;
278  *end = 0;
279  end_time = atoi(start);
280  start = end + 1;
281 
282  end = (char*)memchr(start, kFieldSeparator, line_end - start);
283  if (!end)
284  continue;
285  *end = 0;
286  restat_mtime = atol(start);
287  start = end + 1;
288 
289  end = (char*)memchr(start, kFieldSeparator, line_end - start);
290  if (!end)
291  continue;
292  string output = string(start, end - start);
293 
294  start = end + 1;
295  end = line_end;
296 
297  LogEntry* entry;
298  Entries::iterator i = entries_.find(output);
299  if (i != entries_.end()) {
300  entry = i->second;
301  } else {
302  entry = new LogEntry(output);
303  entries_.insert(Entries::value_type(entry->output, entry));
304  ++unique_entry_count;
305  }
306  ++total_entry_count;
307 
308  entry->start_time = start_time;
309  entry->end_time = end_time;
310  entry->restat_mtime = restat_mtime;
311  if (log_version >= 5) {
312  char c = *end; *end = '\0';
313  entry->command_hash = (uint64_t)strtoull(start, NULL, 16);
314  *end = c;
315  } else {
317  end - start));
318  }
319  }
320  fclose(file);
321 
322  if (!line_start) {
323  return true; // file was empty
324  }
325 
326  // Decide whether it's time to rebuild the log:
327  // - if we're upgrading versions
328  // - if it's getting large
329  int kMinCompactionEntryCount = 100;
330  int kCompactionRatio = 3;
331  if (log_version < kCurrentVersion) {
332  needs_recompaction_ = true;
333  } else if (total_entry_count > kMinCompactionEntryCount &&
334  total_entry_count > unique_entry_count * kCompactionRatio) {
335  needs_recompaction_ = true;
336  }
337 
338  return true;
339 }
340 
342  Entries::iterator i = entries_.find(path);
343  if (i != entries_.end())
344  return i->second;
345  return NULL;
346 }
347 
348 bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) {
349  return fprintf(f, "%d\t%d\t%d\t%s\t%" PRIx64 "\n",
350  entry.start_time, entry.end_time, entry.restat_mtime,
351  entry.output.c_str(), entry.command_hash) > 0;
352 }
353 
354 bool BuildLog::Recompact(const string& path, const BuildLogUser& user,
355  string* err) {
356  METRIC_RECORD(".ninja_log recompact");
357 
358  Close();
359  string temp_path = path + ".recompact";
360  FILE* f = fopen(temp_path.c_str(), "wb");
361  if (!f) {
362  *err = strerror(errno);
363  return false;
364  }
365 
366  if (fprintf(f, kFileSignature, kCurrentVersion) < 0) {
367  *err = strerror(errno);
368  fclose(f);
369  return false;
370  }
371 
372  vector<StringPiece> dead_outputs;
373  for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
374  if (user.IsPathDead(i->first)) {
375  dead_outputs.push_back(i->first);
376  continue;
377  }
378 
379  if (!WriteEntry(f, *i->second)) {
380  *err = strerror(errno);
381  fclose(f);
382  return false;
383  }
384  }
385 
386  for (size_t i = 0; i < dead_outputs.size(); ++i)
387  entries_.erase(dead_outputs[i]);
388 
389  fclose(f);
390  if (unlink(path.c_str()) < 0) {
391  *err = strerror(errno);
392  return false;
393  }
394 
395  if (rename(temp_path.c_str(), path.c_str()) < 0) {
396  *err = strerror(errno);
397  return false;
398  }
399 
400  return true;
401 }
const int kCurrentVersion
Definition: deps_log.cc:33
bool needs_recompaction_
Definition: build_log.h:90
const char * str_
Definition: string_piece.h:49
const char kFileSignature[]
Definition: deps_log.cc:32
TimeStamp restat_mtime
Definition: build_log.h:59
char buf_[256<< 10]
Definition: build_log.cc:217
bool Recompact(const string &path, const BuildLogUser &user, string *err)
Rewrite the known log entries, throwing away old data.
Definition: build_log.cc:354
StringPiece represents a slice of a string whose memory is managed externally.
Definition: string_piece.h:27
bool WriteEntry(FILE *f, const LogEntry &entry)
Serialize an entry into a log file.
Definition: build_log.cc:348
int TimeStamp
Definition: timestamp.h:22
void Close()
Definition: build_log.cc:168
An edge in the dependency graph; links between Nodes using Rules.
Definition: graph.h:124
string EvaluateCommand(bool incl_rsp_file=false)
Expand all variables in a command and return it as a string.
Definition: graph.cc:292
virtual bool IsPathDead(StringPiece s) const =0
Return if a given output no longer part of the build manifest.
void SetCloseOnExec(int fd)
Mark a file descriptor to not be inherited on exec()s.
Definition: util.cc:397
uint64_t command_hash
Definition: build_log.h:56
bool OpenForWrite(const string &path, const BuildLogUser &user, string *err)
Definition: build_log.cc:111
FILE * file_
Definition: build_log.cc:216
#define PRIx64
Definition: win32port.h:27
FILE * log_file_
Definition: build_log.h:89
#define BIG_CONSTANT(x)
Definition: build_log.cc:49
char * buf_end_
Definition: build_log.cc:218
LogEntry(const string &output)
Definition: build_log.cc:95
#define METRIC_RECORD(name)
The primary interface to metrics.
Definition: metrics.h:85
char * line_start_
Definition: build_log.cc:220
static uint64_t HashCommand(StringPiece command)
Definition: build_log.cc:91
LogEntry * LookupByOutput(const string &path)
Lookup a previously-run command by its output path.
Definition: build_log.cc:341
bool RecordCommand(Edge *edge, int start_time, int end_time, TimeStamp restat_mtime=0)
Definition: build_log.cc:140
size_t len_
Definition: string_piece.h:50
unsigned long long uint64_t
Definition: win32port.h:22
char * line_end_
Definition: build_log.cc:222
bool Load(const string &path, string *err)
Load the on-disk log.
Definition: build_log.cc:225
Entries entries_
Definition: build_log.h:88
bool ReadLine(char **line_start, char **line_end)
Definition: build_log.cc:184
Can answer questions about the manifest for the BuildLog.
Definition: build_log.h:29
LineReader(FILE *file)
Definition: build_log.cc:175
vector< Node * > outputs_
Definition: graph.h:151