1// Copyright 2011 Google Inc. All Rights Reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#include "build.h"
16
17#include <assert.h>
18#include <errno.h>
19#include <stdio.h>
20#include <stdlib.h>
21#include <functional>
22
23#if defined(__SVR4) && defined(__sun)
24#include <sys/termios.h>
25#endif
26
27#include "build_log.h"
28#include "clparser.h"
29#include "debug_flags.h"
30#include "depfile_parser.h"
31#include "deps_log.h"
32#include "disk_interface.h"
33#include "graph.h"
34#include "metrics.h"
35#include "state.h"
36#include "status.h"
37#include "subprocess.h"
38#include "util.h"
39
40using namespace std;
41
42namespace {
43
44/// A CommandRunner that doesn't actually run the commands.
45struct DryRunCommandRunner : public CommandRunner {
46 virtual ~DryRunCommandRunner() {}
47
48 // Overridden from CommandRunner:
49 virtual bool CanRunMore() const;
50 virtual bool StartCommand(Edge* edge);
51 virtual bool WaitForCommand(Result* result);
52
53 private:
54 queue<Edge*> finished_;
55};
56
57bool DryRunCommandRunner::CanRunMore() const {
58 return true;
59}
60
61bool DryRunCommandRunner::StartCommand(Edge* edge) {
62 finished_.push(edge);
63 return true;
64}
65
66bool DryRunCommandRunner::WaitForCommand(Result* result) {
67 if (finished_.empty())
68 return false;
69
70 result->status = ExitSuccess;
71 result->edge = finished_.front();
72 finished_.pop();
73 return true;
74}
75
76} // namespace
77
78Plan::Plan(Builder* builder)
79 : builder_(builder)
80 , command_edges_(0)
81 , wanted_edges_(0)
82{}
83
84void Plan::Reset() {
85 command_edges_ = 0;
86 wanted_edges_ = 0;
87 ready_.clear();
88 want_.clear();
89}
90
91bool Plan::AddTarget(const Node* target, string* err) {
92 return AddSubTarget(target, NULL, err, NULL);
93}
94
95bool Plan::AddSubTarget(const Node* node, const Node* dependent, string* err,
96 set<Edge*>* dyndep_walk) {
97 Edge* edge = node->in_edge();
98 if (!edge) { // Leaf node.
99 if (node->dirty()) {
100 string referenced;
101 if (dependent)
102 referenced = ", needed by '" + dependent->path() + "',";
103 *err = "'" + node->path() + "'" + referenced + " missing "
104 "and no known rule to make it";
105 }
106 return false;
107 }
108
109 if (edge->outputs_ready())
110 return false; // Don't need to do anything.
111
112 // If an entry in want_ does not already exist for edge, create an entry which
113 // maps to kWantNothing, indicating that we do not want to build this entry itself.
114 pair<map<Edge*, Want>::iterator, bool> want_ins =
115 want_.insert(make_pair(edge, kWantNothing));
116 Want& want = want_ins.first->second;
117
118 if (dyndep_walk && want == kWantToFinish)
119 return false; // Don't need to do anything with already-scheduled edge.
120
121 // If we do need to build edge and we haven't already marked it as wanted,
122 // mark it now.
123 if (node->dirty() && want == kWantNothing) {
124 want = kWantToStart;
125 EdgeWanted(edge);
126 if (!dyndep_walk && edge->AllInputsReady())
127 ScheduleWork(want_ins.first);
128 }
129
130 if (dyndep_walk)
131 dyndep_walk->insert(edge);
132
133 if (!want_ins.second)
134 return true; // We've already processed the inputs.
135
136 for (vector<Node*>::iterator i = edge->inputs_.begin();
137 i != edge->inputs_.end(); ++i) {
138 if (!AddSubTarget(*i, node, err, dyndep_walk) && !err->empty())
139 return false;
140 }
141
142 return true;
143}
144
145void Plan::EdgeWanted(const Edge* edge) {
146 ++wanted_edges_;
147 if (!edge->is_phony())
148 ++command_edges_;
149}
150
151Edge* Plan::FindWork() {
152 if (ready_.empty())
153 return NULL;
154 EdgeSet::iterator e = ready_.begin();
155 Edge* edge = *e;
156 ready_.erase(e);
157 return edge;
158}
159
160void Plan::ScheduleWork(map<Edge*, Want>::iterator want_e) {
161 if (want_e->second == kWantToFinish) {
162 // This edge has already been scheduled. We can get here again if an edge
163 // and one of its dependencies share an order-only input, or if a node
164 // duplicates an out edge (see https://github.com/ninja-build/ninja/pull/519).
165 // Avoid scheduling the work again.
166 return;
167 }
168 assert(want_e->second == kWantToStart);
169 want_e->second = kWantToFinish;
170
171 Edge* edge = want_e->first;
172 Pool* pool = edge->pool();
173 if (pool->ShouldDelayEdge()) {
174 pool->DelayEdge(edge);
175 pool->RetrieveReadyEdges(&ready_);
176 } else {
177 pool->EdgeScheduled(*edge);
178 ready_.insert(edge);
179 }
180}
181
182bool Plan::EdgeFinished(Edge* edge, EdgeResult result, string* err) {
183 map<Edge*, Want>::iterator e = want_.find(edge);
184 assert(e != want_.end());
185 bool directly_wanted = e->second != kWantNothing;
186
187 // See if this job frees up any delayed jobs.
188 if (directly_wanted)
189 edge->pool()->EdgeFinished(*edge);
190 edge->pool()->RetrieveReadyEdges(&ready_);
191
192 // The rest of this function only applies to successful commands.
193 if (result != kEdgeSucceeded)
194 return true;
195
196 if (directly_wanted)
197 --wanted_edges_;
198 want_.erase(e);
199 edge->outputs_ready_ = true;
200
201 // Check off any nodes we were waiting for with this edge.
202 for (vector<Node*>::iterator o = edge->outputs_.begin();
203 o != edge->outputs_.end(); ++o) {
204 if (!NodeFinished(*o, err))
205 return false;
206 }
207 return true;
208}
209
210bool Plan::NodeFinished(Node* node, string* err) {
211 // If this node provides dyndep info, load it now.
212 if (node->dyndep_pending()) {
213 assert(builder_ && "dyndep requires Plan to have a Builder");
214 // Load the now-clean dyndep file. This will also update the
215 // build plan and schedule any new work that is ready.
216 return builder_->LoadDyndeps(node, err);
217 }
218
219 // See if we we want any edges from this node.
220 for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
221 oe != node->out_edges().end(); ++oe) {
222 map<Edge*, Want>::iterator want_e = want_.find(*oe);
223 if (want_e == want_.end())
224 continue;
225
226 // See if the edge is now ready.
227 if (!EdgeMaybeReady(want_e, err))
228 return false;
229 }
230 return true;
231}
232
233bool Plan::EdgeMaybeReady(map<Edge*, Want>::iterator want_e, string* err) {
234 Edge* edge = want_e->first;
235 if (edge->AllInputsReady()) {
236 if (want_e->second != kWantNothing) {
237 ScheduleWork(want_e);
238 } else {
239 // We do not need to build this edge, but we might need to build one of
240 // its dependents.
241 if (!EdgeFinished(edge, kEdgeSucceeded, err))
242 return false;
243 }
244 }
245 return true;
246}
247
248bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) {
249 node->set_dirty(false);
250
251 for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
252 oe != node->out_edges().end(); ++oe) {
253 // Don't process edges that we don't actually want.
254 map<Edge*, Want>::iterator want_e = want_.find(*oe);
255 if (want_e == want_.end() || want_e->second == kWantNothing)
256 continue;
257
258 // Don't attempt to clean an edge if it failed to load deps.
259 if ((*oe)->deps_missing_)
260 continue;
261
262 // If all non-order-only inputs for this edge are now clean,
263 // we might have changed the dirty state of the outputs.
264 vector<Node*>::iterator
265 begin = (*oe)->inputs_.begin(),
266 end = (*oe)->inputs_.end() - (*oe)->order_only_deps_;
267#if __cplusplus < 201703L
268#define MEM_FN mem_fun
269#else
270#define MEM_FN mem_fn // mem_fun was removed in C++17.
271#endif
272 if (find_if(begin, end, MEM_FN(&Node::dirty)) == end) {
273 // Recompute most_recent_input.
274 Node* most_recent_input = NULL;
275 for (vector<Node*>::iterator i = begin; i != end; ++i) {
276 if (!most_recent_input || (*i)->mtime() > most_recent_input->mtime())
277 most_recent_input = *i;
278 }
279
280 // Now, this edge is dirty if any of the outputs are dirty.
281 // If the edge isn't dirty, clean the outputs and mark the edge as not
282 // wanted.
283 bool outputs_dirty = false;
284 if (!scan->RecomputeOutputsDirty(*oe, most_recent_input,
285 &outputs_dirty, err)) {
286 return false;
287 }
288 if (!outputs_dirty) {
289 for (vector<Node*>::iterator o = (*oe)->outputs_.begin();
290 o != (*oe)->outputs_.end(); ++o) {
291 if (!CleanNode(scan, *o, err))
292 return false;
293 }
294
295 want_e->second = kWantNothing;
296 --wanted_edges_;
297 if (!(*oe)->is_phony())
298 --command_edges_;
299 }
300 }
301 }
302 return true;
303}
304
305bool Plan::DyndepsLoaded(DependencyScan* scan, const Node* node,
306 const DyndepFile& ddf, string* err) {
307 // Recompute the dirty state of all our direct and indirect dependents now
308 // that our dyndep information has been loaded.
309 if (!RefreshDyndepDependents(scan, node, err))
310 return false;
311
312 // We loaded dyndep information for those out_edges of the dyndep node that
313 // specify the node in a dyndep binding, but they may not be in the plan.
314 // Starting with those already in the plan, walk newly-reachable portion
315 // of the graph through the dyndep-discovered dependencies.
316
317 // Find edges in the the build plan for which we have new dyndep info.
318 std::vector<DyndepFile::const_iterator> dyndep_roots;
319 for (DyndepFile::const_iterator oe = ddf.begin(); oe != ddf.end(); ++oe) {
320 Edge* edge = oe->first;
321
322 // If the edge outputs are ready we do not need to consider it here.
323 if (edge->outputs_ready())
324 continue;
325
326 map<Edge*, Want>::iterator want_e = want_.find(edge);
327
328 // If the edge has not been encountered before then nothing already in the
329 // plan depends on it so we do not need to consider the edge yet either.
330 if (want_e == want_.end())
331 continue;
332
333 // This edge is already in the plan so queue it for the walk.
334 dyndep_roots.push_back(oe);
335 }
336
337 // Walk dyndep-discovered portion of the graph to add it to the build plan.
338 std::set<Edge*> dyndep_walk;
339 for (std::vector<DyndepFile::const_iterator>::iterator
340 oei = dyndep_roots.begin(); oei != dyndep_roots.end(); ++oei) {
341 DyndepFile::const_iterator oe = *oei;
342 for (vector<Node*>::const_iterator i = oe->second.implicit_inputs_.begin();
343 i != oe->second.implicit_inputs_.end(); ++i) {
344 if (!AddSubTarget(*i, oe->first->outputs_[0], err, &dyndep_walk) &&
345 !err->empty())
346 return false;
347 }
348 }
349
350 // Add out edges from this node that are in the plan (just as
351 // Plan::NodeFinished would have without taking the dyndep code path).
352 for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
353 oe != node->out_edges().end(); ++oe) {
354 map<Edge*, Want>::iterator want_e = want_.find(*oe);
355 if (want_e == want_.end())
356 continue;
357 dyndep_walk.insert(want_e->first);
358 }
359
360 // See if any encountered edges are now ready.
361 for (set<Edge*>::iterator wi = dyndep_walk.begin();
362 wi != dyndep_walk.end(); ++wi) {
363 map<Edge*, Want>::iterator want_e = want_.find(*wi);
364 if (want_e == want_.end())
365 continue;
366 if (!EdgeMaybeReady(want_e, err))
367 return false;
368 }
369
370 return true;
371}
372
373bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node,
374 string* err) {
375 // Collect the transitive closure of dependents and mark their edges
376 // as not yet visited by RecomputeDirty.
377 set<Node*> dependents;
378 UnmarkDependents(node, &dependents);
379
380 // Update the dirty state of all dependents and check if their edges
381 // have become wanted.
382 for (set<Node*>::iterator i = dependents.begin();
383 i != dependents.end(); ++i) {
384 Node* n = *i;
385
386 // Check if this dependent node is now dirty. Also checks for new cycles.
387 std::vector<Node*> validation_nodes;
388 if (!scan->RecomputeDirty(n, &validation_nodes, err))
389 return false;
390
391 // Add any validation nodes found during RecomputeDirty as new top level
392 // targets.
393 for (std::vector<Node*>::iterator v = validation_nodes.begin();
394 v != validation_nodes.end(); ++v) {
395 if (Edge* in_edge = (*v)->in_edge()) {
396 if (!in_edge->outputs_ready() &&
397 !AddTarget(*v, err)) {
398 return false;
399 }
400 }
401 }
402 if (!n->dirty())
403 continue;
404
405 // This edge was encountered before. However, we may not have wanted to
406 // build it if the outputs were not known to be dirty. With dyndep
407 // information an output is now known to be dirty, so we want the edge.
408 Edge* edge = n->in_edge();
409 assert(edge && !edge->outputs_ready());
410 map<Edge*, Want>::iterator want_e = want_.find(edge);
411 assert(want_e != want_.end());
412 if (want_e->second == kWantNothing) {
413 want_e->second = kWantToStart;
414 EdgeWanted(edge);
415 }
416 }
417 return true;
418}
419
420void Plan::UnmarkDependents(const Node* node, set<Node*>* dependents) {
421 for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
422 oe != node->out_edges().end(); ++oe) {
423 Edge* edge = *oe;
424
425 map<Edge*, Want>::iterator want_e = want_.find(edge);
426 if (want_e == want_.end())
427 continue;
428
429 if (edge->mark_ != Edge::VisitNone) {
430 edge->mark_ = Edge::VisitNone;
431 for (vector<Node*>::iterator o = edge->outputs_.begin();
432 o != edge->outputs_.end(); ++o) {
433 if (dependents->insert(*o).second)
434 UnmarkDependents(*o, dependents);
435 }
436 }
437 }
438}
439
440void Plan::Dump() const {
441 printf("pending: %d\n", (int)want_.size());
442 for (map<Edge*, Want>::const_iterator e = want_.begin(); e != want_.end(); ++e) {
443 if (e->second != kWantNothing)
444 printf("want ");
445 e->first->Dump();
446 }
447 printf("ready: %d\n", (int)ready_.size());
448}
449
450struct RealCommandRunner : public CommandRunner {
451 explicit RealCommandRunner(const BuildConfig& config) : config_(config) {}
452 virtual ~RealCommandRunner() {}
453 virtual bool CanRunMore() const;
454 virtual bool StartCommand(Edge* edge);
455 virtual bool WaitForCommand(Result* result);
456 virtual vector<Edge*> GetActiveEdges();
457 virtual void Abort();
458
459 const BuildConfig& config_;
460 SubprocessSet subprocs_;
461 map<const Subprocess*, Edge*> subproc_to_edge_;
462};
463
464vector<Edge*> RealCommandRunner::GetActiveEdges() {
465 vector<Edge*> edges;
466 for (map<const Subprocess*, Edge*>::iterator e = subproc_to_edge_.begin();
467 e != subproc_to_edge_.end(); ++e)
468 edges.push_back(e->second);
469 return edges;
470}
471
472void RealCommandRunner::Abort() {
473 subprocs_.Clear();
474}
475
476bool RealCommandRunner::CanRunMore() const {
477 size_t subproc_number =
478 subprocs_.running_.size() + subprocs_.finished_.size();
479 return (int)subproc_number < config_.parallelism
480 && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f)
481 || GetLoadAverage() < config_.max_load_average);
482}
483
484bool RealCommandRunner::StartCommand(Edge* edge) {
485 string command = edge->EvaluateCommand();
486 Subprocess* subproc = subprocs_.Add(command, edge->use_console());
487 if (!subproc)
488 return false;
489 subproc_to_edge_.insert(make_pair(subproc, edge));
490
491 return true;
492}
493
494bool RealCommandRunner::WaitForCommand(Result* result) {
495 Subprocess* subproc;
496 while ((subproc = subprocs_.NextFinished()) == NULL) {
497 bool interrupted = subprocs_.DoWork();
498 if (interrupted)
499 return false;
500 }
501
502 result->status = subproc->Finish();
503 result->output = subproc->GetOutput();
504
505 map<const Subprocess*, Edge*>::iterator e = subproc_to_edge_.find(subproc);
506 result->edge = e->second;
507 subproc_to_edge_.erase(e);
508
509 delete subproc;
510 return true;
511}
512
513Builder::Builder(State* state, const BuildConfig& config,
514 BuildLog* build_log, DepsLog* deps_log,
515 DiskInterface* disk_interface, Status *status,
516 int64_t start_time_millis)
517 : state_(state), config_(config), plan_(this), status_(status),
518 start_time_millis_(start_time_millis), disk_interface_(disk_interface),
519 scan_(state, build_log, deps_log, disk_interface,
520 &config_.depfile_parser_options) {
521 lock_file_path_ = ".ninja_lock";
522 string build_dir = state_->bindings_.LookupVariable("builddir");
523 if (!build_dir.empty())
524 lock_file_path_ = build_dir + "/" + lock_file_path_;
525}
526
527Builder::~Builder() {
528 Cleanup();
529}
530
531void Builder::Cleanup() {
532 if (command_runner_.get()) {
533 vector<Edge*> active_edges = command_runner_->GetActiveEdges();
534 command_runner_->Abort();
535
536 for (vector<Edge*>::iterator e = active_edges.begin();
537 e != active_edges.end(); ++e) {
538 string depfile = (*e)->GetUnescapedDepfile();
539 for (vector<Node*>::iterator o = (*e)->outputs_.begin();
540 o != (*e)->outputs_.end(); ++o) {
541 // Only delete this output if it was actually modified. This is
542 // important for things like the generator where we don't want to
543 // delete the manifest file if we can avoid it. But if the rule
544 // uses a depfile, always delete. (Consider the case where we
545 // need to rebuild an output because of a modified header file
546 // mentioned in a depfile, and the command touches its depfile
547 // but is interrupted before it touches its output file.)
548 string err;
549 TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), &err);
550 if (new_mtime == -1) // Log and ignore Stat() errors.
551 status_->Error("%s", err.c_str());
552 if (!depfile.empty() || (*o)->mtime() != new_mtime)
553 disk_interface_->RemoveFile((*o)->path());
554 }
555 if (!depfile.empty())
556 disk_interface_->RemoveFile(depfile);
557 }
558 }
559
560 string err;
561 if (disk_interface_->Stat(lock_file_path_, &err) > 0)
562 disk_interface_->RemoveFile(lock_file_path_);
563}
564
565Node* Builder::AddTarget(const string& name, string* err) {
566 Node* node = state_->LookupNode(name);
567 if (!node) {
568 *err = "unknown target: '" + name + "'";
569 return NULL;
570 }
571 if (!AddTarget(node, err))
572 return NULL;
573 return node;
574}
575
576bool Builder::AddTarget(Node* target, string* err) {
577 std::vector<Node*> validation_nodes;
578 if (!scan_.RecomputeDirty(target, &validation_nodes, err))
579 return false;
580
581 Edge* in_edge = target->in_edge();
582 if (!in_edge || !in_edge->outputs_ready()) {
583 if (!plan_.AddTarget(target, err)) {
584 return false;
585 }
586 }
587
588 // Also add any validation nodes found during RecomputeDirty as top level
589 // targets.
590 for (std::vector<Node*>::iterator n = validation_nodes.begin();
591 n != validation_nodes.end(); ++n) {
592 if (Edge* validation_in_edge = (*n)->in_edge()) {
593 if (!validation_in_edge->outputs_ready() &&
594 !plan_.AddTarget(*n, err)) {
595 return false;
596 }
597 }
598 }
599
600 return true;
601}
602
603bool Builder::AlreadyUpToDate() const {
604 return !plan_.more_to_do();
605}
606
607bool Builder::Build(string* err) {
608 assert(!AlreadyUpToDate());
609
610 status_->PlanHasTotalEdges(plan_.command_edge_count());
611 int pending_commands = 0;
612 int failures_allowed = config_.failures_allowed;
613
614 // Set up the command runner if we haven't done so already.
615 if (!command_runner_.get()) {
616 if (config_.dry_run)
617 command_runner_.reset(new DryRunCommandRunner);
618 else
619 command_runner_.reset(new RealCommandRunner(config_));
620 }
621
622 // We are about to start the build process.
623 status_->BuildStarted();
624
625 // This main loop runs the entire build process.
626 // It is structured like this:
627 // First, we attempt to start as many commands as allowed by the
628 // command runner.
629 // Second, we attempt to wait for / reap the next finished command.
630 while (plan_.more_to_do()) {
631 // See if we can start any more commands.
632 if (failures_allowed && command_runner_->CanRunMore()) {
633 if (Edge* edge = plan_.FindWork()) {
634 if (edge->GetBindingBool("generator")) {
635 scan_.build_log()->Close();
636 }
637
638 if (!StartEdge(edge, err)) {
639 Cleanup();
640 status_->BuildFinished();
641 return false;
642 }
643
644 if (edge->is_phony()) {
645 if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err)) {
646 Cleanup();
647 status_->BuildFinished();
648 return false;
649 }
650 } else {
651 ++pending_commands;
652 }
653
654 // We made some progress; go back to the main loop.
655 continue;
656 }
657 }
658
659 // See if we can reap any finished commands.
660 if (pending_commands) {
661 CommandRunner::Result result;
662 if (!command_runner_->WaitForCommand(&result) ||
663 result.status == ExitInterrupted) {
664 Cleanup();
665 status_->BuildFinished();
666 *err = "interrupted by user";
667 return false;
668 }
669
670 --pending_commands;
671 if (!FinishCommand(&result, err)) {
672 Cleanup();
673 status_->BuildFinished();
674 return false;
675 }
676
677 if (!result.success()) {
678 if (failures_allowed)
679 failures_allowed--;
680 }
681
682 // We made some progress; start the main loop over.
683 continue;
684 }
685
686 // If we get here, we cannot make any more progress.
687 status_->BuildFinished();
688 if (failures_allowed == 0) {
689 if (config_.failures_allowed > 1)
690 *err = "subcommands failed";
691 else
692 *err = "subcommand failed";
693 } else if (failures_allowed < config_.failures_allowed)
694 *err = "cannot make progress due to previous errors";
695 else
696 *err = "stuck [this is a bug]";
697
698 return false;
699 }
700
701 status_->BuildFinished();
702 return true;
703}
704
705bool Builder::StartEdge(Edge* edge, string* err) {
706 METRIC_RECORD("StartEdge");
707 if (edge->is_phony())
708 return true;
709
710 int64_t start_time_millis = GetTimeMillis() - start_time_millis_;
711 running_edges_.insert(make_pair(edge, start_time_millis));
712
713 status_->BuildEdgeStarted(edge, start_time_millis);
714
715 TimeStamp build_start = -1;
716
717 // Create directories necessary for outputs and remember the current
718 // filesystem mtime to record later
719 // XXX: this will block; do we care?
720 for (vector<Node*>::iterator o = edge->outputs_.begin();
721 o != edge->outputs_.end(); ++o) {
722 if (!disk_interface_->MakeDirs((*o)->path()))
723 return false;
724 if (build_start == -1) {
725 disk_interface_->WriteFile(lock_file_path_, "");
726 build_start = disk_interface_->Stat(lock_file_path_, err);
727 if (build_start == -1)
728 build_start = 0;
729 }
730 }
731
732 edge->command_start_time_ = build_start;
733
734 // Create response file, if needed
735 // XXX: this may also block; do we care?
736 string rspfile = edge->GetUnescapedRspfile();
737 if (!rspfile.empty()) {
738 string content = edge->GetBinding("rspfile_content");
739 if (!disk_interface_->WriteFile(rspfile, content))
740 return false;
741 }
742
743 // start command computing and run it
744 if (!command_runner_->StartCommand(edge)) {
745 err->assign("command '" + edge->EvaluateCommand() + "' failed.");
746 return false;
747 }
748
749 return true;
750}
751
752bool Builder::FinishCommand(CommandRunner::Result* result, string* err) {
753 METRIC_RECORD("FinishCommand");
754
755 Edge* edge = result->edge;
756
757 // First try to extract dependencies from the result, if any.
758 // This must happen first as it filters the command output (we want
759 // to filter /showIncludes output, even on compile failure) and
760 // extraction itself can fail, which makes the command fail from a
761 // build perspective.
762 vector<Node*> deps_nodes;
763 string deps_type = edge->GetBinding("deps");
764 const string deps_prefix = edge->GetBinding("msvc_deps_prefix");
765 if (!deps_type.empty()) {
766 string extract_err;
767 if (!ExtractDeps(result, deps_type, deps_prefix, &deps_nodes,
768 &extract_err) &&
769 result->success()) {
770 if (!result->output.empty())
771 result->output.append("\n");
772 result->output.append(extract_err);
773 result->status = ExitFailure;
774 }
775 }
776
777 int64_t start_time_millis, end_time_millis;
778 RunningEdgeMap::iterator it = running_edges_.find(edge);
779 start_time_millis = it->second;
780 end_time_millis = GetTimeMillis() - start_time_millis_;
781 running_edges_.erase(it);
782
783 status_->BuildEdgeFinished(edge, end_time_millis, result->success(),
784 result->output);
785
786 // The rest of this function only applies to successful commands.
787 if (!result->success()) {
788 return plan_.EdgeFinished(edge, Plan::kEdgeFailed, err);
789 }
790
791 // Restat the edge outputs
792 TimeStamp record_mtime = 0;
793 if (!config_.dry_run) {
794 const bool restat = edge->GetBindingBool("restat");
795 const bool generator = edge->GetBindingBool("generator");
796 bool node_cleaned = false;
797 record_mtime = edge->command_start_time_;
798
799 // restat and generator rules must restat the outputs after the build
800 // has finished. if record_mtime == 0, then there was an error while
801 // attempting to touch/stat the temp file when the edge started and
802 // we should fall back to recording the outputs' current mtime in the
803 // log.
804 if (record_mtime == 0 || restat || generator) {
805 for (vector<Node*>::iterator o = edge->outputs_.begin();
806 o != edge->outputs_.end(); ++o) {
807 TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err);
808 if (new_mtime == -1)
809 return false;
810 if (new_mtime > record_mtime)
811 record_mtime = new_mtime;
812 if ((*o)->mtime() == new_mtime && restat) {
813 // The rule command did not change the output. Propagate the clean
814 // state through the build graph.
815 // Note that this also applies to nonexistent outputs (mtime == 0).
816 if (!plan_.CleanNode(&scan_, *o, err))
817 return false;
818 node_cleaned = true;
819 }
820 }
821 }
822 if (node_cleaned) {
823 record_mtime = edge->command_start_time_;
824
825 // The total number of edges in the plan may have changed as a result
826 // of a restat.
827 status_->PlanHasTotalEdges(plan_.command_edge_count());
828 }
829 }
830
831 if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err))
832 return false;
833
834 // Delete any left over response file.
835 string rspfile = edge->GetUnescapedRspfile();
836 if (!rspfile.empty() && !g_keep_rsp)
837 disk_interface_->RemoveFile(rspfile);
838
839 if (scan_.build_log()) {
840 if (!scan_.build_log()->RecordCommand(edge, start_time_millis,
841 end_time_millis, record_mtime)) {
842 *err = string("Error writing to build log: ") + strerror(errno);
843 return false;
844 }
845 }
846
847 if (!deps_type.empty() && !config_.dry_run) {
848 assert(!edge->outputs_.empty() && "should have been rejected by parser");
849 for (std::vector<Node*>::const_iterator o = edge->outputs_.begin();
850 o != edge->outputs_.end(); ++o) {
851 TimeStamp deps_mtime = disk_interface_->Stat((*o)->path(), err);
852 if (deps_mtime == -1)
853 return false;
854 if (!scan_.deps_log()->RecordDeps(*o, deps_mtime, deps_nodes)) {
855 *err = std::string("Error writing to deps log: ") + strerror(errno);
856 return false;
857 }
858 }
859 }
860 return true;
861}
862
863bool Builder::ExtractDeps(CommandRunner::Result* result,
864 const string& deps_type,
865 const string& deps_prefix,
866 vector<Node*>* deps_nodes,
867 string* err) {
868 if (deps_type == "msvc") {
869 CLParser parser;
870 string output;
871 if (!parser.Parse(result->output, deps_prefix, &output, err))
872 return false;
873 result->output = output;
874 for (set<string>::iterator i = parser.includes_.begin();
875 i != parser.includes_.end(); ++i) {
876 // ~0 is assuming that with MSVC-parsed headers, it's ok to always make
877 // all backslashes (as some of the slashes will certainly be backslashes
878 // anyway). This could be fixed if necessary with some additional
879 // complexity in IncludesNormalize::Relativize.
880 deps_nodes->push_back(state_->GetNode(*i, ~0u));
881 }
882 } else if (deps_type == "gcc") {
883 string depfile = result->edge->GetUnescapedDepfile();
884 if (depfile.empty()) {
885 *err = string("edge with deps=gcc but no depfile makes no sense");
886 return false;
887 }
888
889 // Read depfile content. Treat a missing depfile as empty.
890 string content;
891 switch (disk_interface_->ReadFile(depfile, &content, err)) {
892 case DiskInterface::Okay:
893 break;
894 case DiskInterface::NotFound:
895 err->clear();
896 break;
897 case DiskInterface::OtherError:
898 return false;
899 }
900 if (content.empty())
901 return true;
902
903 DepfileParser deps(config_.depfile_parser_options);
904 if (!deps.Parse(&content, err))
905 return false;
906
907 // XXX check depfile matches expected output.
908 deps_nodes->reserve(deps.ins_.size());
909 for (vector<StringPiece>::iterator i = deps.ins_.begin();
910 i != deps.ins_.end(); ++i) {
911 uint64_t slash_bits;
912 CanonicalizePath(const_cast<char*>(i->str_), &i->len_, &slash_bits);
913 deps_nodes->push_back(state_->GetNode(*i, slash_bits));
914 }
915
916 if (!g_keep_depfile) {
917 if (disk_interface_->RemoveFile(depfile) < 0) {
918 *err = string("deleting depfile: ") + strerror(errno) + string("\n");
919 return false;
920 }
921 }
922 } else {
923 Fatal("unknown deps type '%s'", deps_type.c_str());
924 }
925
926 return true;
927}
928
929bool Builder::LoadDyndeps(Node* node, string* err) {
930 status_->BuildLoadDyndeps();
931
932 // Load the dyndep information provided by this node.
933 DyndepFile ddf;
934 if (!scan_.LoadDyndeps(node, &ddf, err))
935 return false;
936
937 // Update the build plan to account for dyndep modifications to the graph.
938 if (!plan_.DyndepsLoaded(&scan_, node, ddf, err))
939 return false;
940
941 // New command edges may have been added to the plan.
942 status_->PlanHasTotalEdges(plan_.command_edge_count());
943
944 return true;
945}
946