1 | // Copyright 2011 Google Inc. All Rights Reserved. |
2 | // |
3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
4 | // you may not use this file except in compliance with the License. |
5 | // You may obtain a copy of the License at |
6 | // |
7 | // http://www.apache.org/licenses/LICENSE-2.0 |
8 | // |
9 | // Unless required by applicable law or agreed to in writing, software |
10 | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | // See the License for the specific language governing permissions and |
13 | // limitations under the License. |
14 | |
15 | #include "build.h" |
16 | |
17 | #include <assert.h> |
18 | #include <errno.h> |
19 | #include <stdio.h> |
20 | #include <stdlib.h> |
21 | #include <functional> |
22 | |
23 | #if defined(__SVR4) && defined(__sun) |
24 | #include <sys/termios.h> |
25 | #endif |
26 | |
27 | #include "build_log.h" |
28 | #include "clparser.h" |
29 | #include "debug_flags.h" |
30 | #include "depfile_parser.h" |
31 | #include "deps_log.h" |
32 | #include "disk_interface.h" |
33 | #include "graph.h" |
34 | #include "metrics.h" |
35 | #include "state.h" |
36 | #include "status.h" |
37 | #include "subprocess.h" |
38 | #include "util.h" |
39 | |
40 | using namespace std; |
41 | |
42 | namespace { |
43 | |
44 | /// A CommandRunner that doesn't actually run the commands. |
45 | struct DryRunCommandRunner : public CommandRunner { |
46 | virtual ~DryRunCommandRunner() {} |
47 | |
48 | // Overridden from CommandRunner: |
49 | virtual bool CanRunMore() const; |
50 | virtual bool StartCommand(Edge* edge); |
51 | virtual bool WaitForCommand(Result* result); |
52 | |
53 | private: |
54 | queue<Edge*> finished_; |
55 | }; |
56 | |
57 | bool DryRunCommandRunner::CanRunMore() const { |
58 | return true; |
59 | } |
60 | |
61 | bool DryRunCommandRunner::StartCommand(Edge* edge) { |
62 | finished_.push(edge); |
63 | return true; |
64 | } |
65 | |
66 | bool DryRunCommandRunner::WaitForCommand(Result* result) { |
67 | if (finished_.empty()) |
68 | return false; |
69 | |
70 | result->status = ExitSuccess; |
71 | result->edge = finished_.front(); |
72 | finished_.pop(); |
73 | return true; |
74 | } |
75 | |
76 | } // namespace |
77 | |
78 | Plan::Plan(Builder* builder) |
79 | : builder_(builder) |
80 | , command_edges_(0) |
81 | , wanted_edges_(0) |
82 | {} |
83 | |
84 | void Plan::Reset() { |
85 | command_edges_ = 0; |
86 | wanted_edges_ = 0; |
87 | ready_.clear(); |
88 | want_.clear(); |
89 | } |
90 | |
91 | bool Plan::AddTarget(const Node* target, string* err) { |
92 | return AddSubTarget(target, NULL, err, NULL); |
93 | } |
94 | |
95 | bool Plan::AddSubTarget(const Node* node, const Node* dependent, string* err, |
96 | set<Edge*>* dyndep_walk) { |
97 | Edge* edge = node->in_edge(); |
98 | if (!edge) { // Leaf node. |
99 | if (node->dirty()) { |
100 | string referenced; |
101 | if (dependent) |
102 | referenced = ", needed by '" + dependent->path() + "'," ; |
103 | *err = "'" + node->path() + "'" + referenced + " missing " |
104 | "and no known rule to make it" ; |
105 | } |
106 | return false; |
107 | } |
108 | |
109 | if (edge->outputs_ready()) |
110 | return false; // Don't need to do anything. |
111 | |
112 | // If an entry in want_ does not already exist for edge, create an entry which |
113 | // maps to kWantNothing, indicating that we do not want to build this entry itself. |
114 | pair<map<Edge*, Want>::iterator, bool> want_ins = |
115 | want_.insert(make_pair(edge, kWantNothing)); |
116 | Want& want = want_ins.first->second; |
117 | |
118 | if (dyndep_walk && want == kWantToFinish) |
119 | return false; // Don't need to do anything with already-scheduled edge. |
120 | |
121 | // If we do need to build edge and we haven't already marked it as wanted, |
122 | // mark it now. |
123 | if (node->dirty() && want == kWantNothing) { |
124 | want = kWantToStart; |
125 | EdgeWanted(edge); |
126 | if (!dyndep_walk && edge->AllInputsReady()) |
127 | ScheduleWork(want_ins.first); |
128 | } |
129 | |
130 | if (dyndep_walk) |
131 | dyndep_walk->insert(edge); |
132 | |
133 | if (!want_ins.second) |
134 | return true; // We've already processed the inputs. |
135 | |
136 | for (vector<Node*>::iterator i = edge->inputs_.begin(); |
137 | i != edge->inputs_.end(); ++i) { |
138 | if (!AddSubTarget(*i, node, err, dyndep_walk) && !err->empty()) |
139 | return false; |
140 | } |
141 | |
142 | return true; |
143 | } |
144 | |
145 | void Plan::EdgeWanted(const Edge* edge) { |
146 | ++wanted_edges_; |
147 | if (!edge->is_phony()) |
148 | ++command_edges_; |
149 | } |
150 | |
151 | Edge* Plan::FindWork() { |
152 | if (ready_.empty()) |
153 | return NULL; |
154 | EdgeSet::iterator e = ready_.begin(); |
155 | Edge* edge = *e; |
156 | ready_.erase(e); |
157 | return edge; |
158 | } |
159 | |
160 | void Plan::ScheduleWork(map<Edge*, Want>::iterator want_e) { |
161 | if (want_e->second == kWantToFinish) { |
162 | // This edge has already been scheduled. We can get here again if an edge |
163 | // and one of its dependencies share an order-only input, or if a node |
164 | // duplicates an out edge (see https://github.com/ninja-build/ninja/pull/519). |
165 | // Avoid scheduling the work again. |
166 | return; |
167 | } |
168 | assert(want_e->second == kWantToStart); |
169 | want_e->second = kWantToFinish; |
170 | |
171 | Edge* edge = want_e->first; |
172 | Pool* pool = edge->pool(); |
173 | if (pool->ShouldDelayEdge()) { |
174 | pool->DelayEdge(edge); |
175 | pool->RetrieveReadyEdges(&ready_); |
176 | } else { |
177 | pool->EdgeScheduled(*edge); |
178 | ready_.insert(edge); |
179 | } |
180 | } |
181 | |
182 | bool Plan::EdgeFinished(Edge* edge, EdgeResult result, string* err) { |
183 | map<Edge*, Want>::iterator e = want_.find(edge); |
184 | assert(e != want_.end()); |
185 | bool directly_wanted = e->second != kWantNothing; |
186 | |
187 | // See if this job frees up any delayed jobs. |
188 | if (directly_wanted) |
189 | edge->pool()->EdgeFinished(*edge); |
190 | edge->pool()->RetrieveReadyEdges(&ready_); |
191 | |
192 | // The rest of this function only applies to successful commands. |
193 | if (result != kEdgeSucceeded) |
194 | return true; |
195 | |
196 | if (directly_wanted) |
197 | --wanted_edges_; |
198 | want_.erase(e); |
199 | edge->outputs_ready_ = true; |
200 | |
201 | // Check off any nodes we were waiting for with this edge. |
202 | for (vector<Node*>::iterator o = edge->outputs_.begin(); |
203 | o != edge->outputs_.end(); ++o) { |
204 | if (!NodeFinished(*o, err)) |
205 | return false; |
206 | } |
207 | return true; |
208 | } |
209 | |
210 | bool Plan::NodeFinished(Node* node, string* err) { |
211 | // If this node provides dyndep info, load it now. |
212 | if (node->dyndep_pending()) { |
213 | assert(builder_ && "dyndep requires Plan to have a Builder" ); |
214 | // Load the now-clean dyndep file. This will also update the |
215 | // build plan and schedule any new work that is ready. |
216 | return builder_->LoadDyndeps(node, err); |
217 | } |
218 | |
219 | // See if we we want any edges from this node. |
220 | for (vector<Edge*>::const_iterator oe = node->out_edges().begin(); |
221 | oe != node->out_edges().end(); ++oe) { |
222 | map<Edge*, Want>::iterator want_e = want_.find(*oe); |
223 | if (want_e == want_.end()) |
224 | continue; |
225 | |
226 | // See if the edge is now ready. |
227 | if (!EdgeMaybeReady(want_e, err)) |
228 | return false; |
229 | } |
230 | return true; |
231 | } |
232 | |
233 | bool Plan::EdgeMaybeReady(map<Edge*, Want>::iterator want_e, string* err) { |
234 | Edge* edge = want_e->first; |
235 | if (edge->AllInputsReady()) { |
236 | if (want_e->second != kWantNothing) { |
237 | ScheduleWork(want_e); |
238 | } else { |
239 | // We do not need to build this edge, but we might need to build one of |
240 | // its dependents. |
241 | if (!EdgeFinished(edge, kEdgeSucceeded, err)) |
242 | return false; |
243 | } |
244 | } |
245 | return true; |
246 | } |
247 | |
248 | bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { |
249 | node->set_dirty(false); |
250 | |
251 | for (vector<Edge*>::const_iterator oe = node->out_edges().begin(); |
252 | oe != node->out_edges().end(); ++oe) { |
253 | // Don't process edges that we don't actually want. |
254 | map<Edge*, Want>::iterator want_e = want_.find(*oe); |
255 | if (want_e == want_.end() || want_e->second == kWantNothing) |
256 | continue; |
257 | |
258 | // Don't attempt to clean an edge if it failed to load deps. |
259 | if ((*oe)->deps_missing_) |
260 | continue; |
261 | |
262 | // If all non-order-only inputs for this edge are now clean, |
263 | // we might have changed the dirty state of the outputs. |
264 | vector<Node*>::iterator |
265 | begin = (*oe)->inputs_.begin(), |
266 | end = (*oe)->inputs_.end() - (*oe)->order_only_deps_; |
267 | #if __cplusplus < 201703L |
268 | #define MEM_FN mem_fun |
269 | #else |
270 | #define MEM_FN mem_fn // mem_fun was removed in C++17. |
271 | #endif |
272 | if (find_if(begin, end, MEM_FN(&Node::dirty)) == end) { |
273 | // Recompute most_recent_input. |
274 | Node* most_recent_input = NULL; |
275 | for (vector<Node*>::iterator i = begin; i != end; ++i) { |
276 | if (!most_recent_input || (*i)->mtime() > most_recent_input->mtime()) |
277 | most_recent_input = *i; |
278 | } |
279 | |
280 | // Now, this edge is dirty if any of the outputs are dirty. |
281 | // If the edge isn't dirty, clean the outputs and mark the edge as not |
282 | // wanted. |
283 | bool outputs_dirty = false; |
284 | if (!scan->RecomputeOutputsDirty(*oe, most_recent_input, |
285 | &outputs_dirty, err)) { |
286 | return false; |
287 | } |
288 | if (!outputs_dirty) { |
289 | for (vector<Node*>::iterator o = (*oe)->outputs_.begin(); |
290 | o != (*oe)->outputs_.end(); ++o) { |
291 | if (!CleanNode(scan, *o, err)) |
292 | return false; |
293 | } |
294 | |
295 | want_e->second = kWantNothing; |
296 | --wanted_edges_; |
297 | if (!(*oe)->is_phony()) |
298 | --command_edges_; |
299 | } |
300 | } |
301 | } |
302 | return true; |
303 | } |
304 | |
305 | bool Plan::DyndepsLoaded(DependencyScan* scan, const Node* node, |
306 | const DyndepFile& ddf, string* err) { |
307 | // Recompute the dirty state of all our direct and indirect dependents now |
308 | // that our dyndep information has been loaded. |
309 | if (!RefreshDyndepDependents(scan, node, err)) |
310 | return false; |
311 | |
312 | // We loaded dyndep information for those out_edges of the dyndep node that |
313 | // specify the node in a dyndep binding, but they may not be in the plan. |
314 | // Starting with those already in the plan, walk newly-reachable portion |
315 | // of the graph through the dyndep-discovered dependencies. |
316 | |
317 | // Find edges in the the build plan for which we have new dyndep info. |
318 | std::vector<DyndepFile::const_iterator> dyndep_roots; |
319 | for (DyndepFile::const_iterator oe = ddf.begin(); oe != ddf.end(); ++oe) { |
320 | Edge* edge = oe->first; |
321 | |
322 | // If the edge outputs are ready we do not need to consider it here. |
323 | if (edge->outputs_ready()) |
324 | continue; |
325 | |
326 | map<Edge*, Want>::iterator want_e = want_.find(edge); |
327 | |
328 | // If the edge has not been encountered before then nothing already in the |
329 | // plan depends on it so we do not need to consider the edge yet either. |
330 | if (want_e == want_.end()) |
331 | continue; |
332 | |
333 | // This edge is already in the plan so queue it for the walk. |
334 | dyndep_roots.push_back(oe); |
335 | } |
336 | |
337 | // Walk dyndep-discovered portion of the graph to add it to the build plan. |
338 | std::set<Edge*> dyndep_walk; |
339 | for (std::vector<DyndepFile::const_iterator>::iterator |
340 | oei = dyndep_roots.begin(); oei != dyndep_roots.end(); ++oei) { |
341 | DyndepFile::const_iterator oe = *oei; |
342 | for (vector<Node*>::const_iterator i = oe->second.implicit_inputs_.begin(); |
343 | i != oe->second.implicit_inputs_.end(); ++i) { |
344 | if (!AddSubTarget(*i, oe->first->outputs_[0], err, &dyndep_walk) && |
345 | !err->empty()) |
346 | return false; |
347 | } |
348 | } |
349 | |
350 | // Add out edges from this node that are in the plan (just as |
351 | // Plan::NodeFinished would have without taking the dyndep code path). |
352 | for (vector<Edge*>::const_iterator oe = node->out_edges().begin(); |
353 | oe != node->out_edges().end(); ++oe) { |
354 | map<Edge*, Want>::iterator want_e = want_.find(*oe); |
355 | if (want_e == want_.end()) |
356 | continue; |
357 | dyndep_walk.insert(want_e->first); |
358 | } |
359 | |
360 | // See if any encountered edges are now ready. |
361 | for (set<Edge*>::iterator wi = dyndep_walk.begin(); |
362 | wi != dyndep_walk.end(); ++wi) { |
363 | map<Edge*, Want>::iterator want_e = want_.find(*wi); |
364 | if (want_e == want_.end()) |
365 | continue; |
366 | if (!EdgeMaybeReady(want_e, err)) |
367 | return false; |
368 | } |
369 | |
370 | return true; |
371 | } |
372 | |
373 | bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node, |
374 | string* err) { |
375 | // Collect the transitive closure of dependents and mark their edges |
376 | // as not yet visited by RecomputeDirty. |
377 | set<Node*> dependents; |
378 | UnmarkDependents(node, &dependents); |
379 | |
380 | // Update the dirty state of all dependents and check if their edges |
381 | // have become wanted. |
382 | for (set<Node*>::iterator i = dependents.begin(); |
383 | i != dependents.end(); ++i) { |
384 | Node* n = *i; |
385 | |
386 | // Check if this dependent node is now dirty. Also checks for new cycles. |
387 | std::vector<Node*> validation_nodes; |
388 | if (!scan->RecomputeDirty(n, &validation_nodes, err)) |
389 | return false; |
390 | |
391 | // Add any validation nodes found during RecomputeDirty as new top level |
392 | // targets. |
393 | for (std::vector<Node*>::iterator v = validation_nodes.begin(); |
394 | v != validation_nodes.end(); ++v) { |
395 | if (Edge* in_edge = (*v)->in_edge()) { |
396 | if (!in_edge->outputs_ready() && |
397 | !AddTarget(*v, err)) { |
398 | return false; |
399 | } |
400 | } |
401 | } |
402 | if (!n->dirty()) |
403 | continue; |
404 | |
405 | // This edge was encountered before. However, we may not have wanted to |
406 | // build it if the outputs were not known to be dirty. With dyndep |
407 | // information an output is now known to be dirty, so we want the edge. |
408 | Edge* edge = n->in_edge(); |
409 | assert(edge && !edge->outputs_ready()); |
410 | map<Edge*, Want>::iterator want_e = want_.find(edge); |
411 | assert(want_e != want_.end()); |
412 | if (want_e->second == kWantNothing) { |
413 | want_e->second = kWantToStart; |
414 | EdgeWanted(edge); |
415 | } |
416 | } |
417 | return true; |
418 | } |
419 | |
420 | void Plan::UnmarkDependents(const Node* node, set<Node*>* dependents) { |
421 | for (vector<Edge*>::const_iterator oe = node->out_edges().begin(); |
422 | oe != node->out_edges().end(); ++oe) { |
423 | Edge* edge = *oe; |
424 | |
425 | map<Edge*, Want>::iterator want_e = want_.find(edge); |
426 | if (want_e == want_.end()) |
427 | continue; |
428 | |
429 | if (edge->mark_ != Edge::VisitNone) { |
430 | edge->mark_ = Edge::VisitNone; |
431 | for (vector<Node*>::iterator o = edge->outputs_.begin(); |
432 | o != edge->outputs_.end(); ++o) { |
433 | if (dependents->insert(*o).second) |
434 | UnmarkDependents(*o, dependents); |
435 | } |
436 | } |
437 | } |
438 | } |
439 | |
440 | void Plan::Dump() const { |
441 | printf("pending: %d\n" , (int)want_.size()); |
442 | for (map<Edge*, Want>::const_iterator e = want_.begin(); e != want_.end(); ++e) { |
443 | if (e->second != kWantNothing) |
444 | printf("want " ); |
445 | e->first->Dump(); |
446 | } |
447 | printf("ready: %d\n" , (int)ready_.size()); |
448 | } |
449 | |
450 | struct RealCommandRunner : public CommandRunner { |
451 | explicit RealCommandRunner(const BuildConfig& config) : config_(config) {} |
452 | virtual ~RealCommandRunner() {} |
453 | virtual bool CanRunMore() const; |
454 | virtual bool StartCommand(Edge* edge); |
455 | virtual bool WaitForCommand(Result* result); |
456 | virtual vector<Edge*> GetActiveEdges(); |
457 | virtual void Abort(); |
458 | |
459 | const BuildConfig& config_; |
460 | SubprocessSet subprocs_; |
461 | map<const Subprocess*, Edge*> subproc_to_edge_; |
462 | }; |
463 | |
464 | vector<Edge*> RealCommandRunner::GetActiveEdges() { |
465 | vector<Edge*> edges; |
466 | for (map<const Subprocess*, Edge*>::iterator e = subproc_to_edge_.begin(); |
467 | e != subproc_to_edge_.end(); ++e) |
468 | edges.push_back(e->second); |
469 | return edges; |
470 | } |
471 | |
472 | void RealCommandRunner::Abort() { |
473 | subprocs_.Clear(); |
474 | } |
475 | |
476 | bool RealCommandRunner::CanRunMore() const { |
477 | size_t subproc_number = |
478 | subprocs_.running_.size() + subprocs_.finished_.size(); |
479 | return (int)subproc_number < config_.parallelism |
480 | && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f) |
481 | || GetLoadAverage() < config_.max_load_average); |
482 | } |
483 | |
484 | bool RealCommandRunner::StartCommand(Edge* edge) { |
485 | string command = edge->EvaluateCommand(); |
486 | Subprocess* subproc = subprocs_.Add(command, edge->use_console()); |
487 | if (!subproc) |
488 | return false; |
489 | subproc_to_edge_.insert(make_pair(subproc, edge)); |
490 | |
491 | return true; |
492 | } |
493 | |
494 | bool RealCommandRunner::WaitForCommand(Result* result) { |
495 | Subprocess* subproc; |
496 | while ((subproc = subprocs_.NextFinished()) == NULL) { |
497 | bool interrupted = subprocs_.DoWork(); |
498 | if (interrupted) |
499 | return false; |
500 | } |
501 | |
502 | result->status = subproc->Finish(); |
503 | result->output = subproc->GetOutput(); |
504 | |
505 | map<const Subprocess*, Edge*>::iterator e = subproc_to_edge_.find(subproc); |
506 | result->edge = e->second; |
507 | subproc_to_edge_.erase(e); |
508 | |
509 | delete subproc; |
510 | return true; |
511 | } |
512 | |
513 | Builder::Builder(State* state, const BuildConfig& config, |
514 | BuildLog* build_log, DepsLog* deps_log, |
515 | DiskInterface* disk_interface, Status *status, |
516 | int64_t start_time_millis) |
517 | : state_(state), config_(config), plan_(this), status_(status), |
518 | start_time_millis_(start_time_millis), disk_interface_(disk_interface), |
519 | scan_(state, build_log, deps_log, disk_interface, |
520 | &config_.depfile_parser_options) { |
521 | lock_file_path_ = ".ninja_lock" ; |
522 | string build_dir = state_->bindings_.LookupVariable("builddir" ); |
523 | if (!build_dir.empty()) |
524 | lock_file_path_ = build_dir + "/" + lock_file_path_; |
525 | } |
526 | |
527 | Builder::~Builder() { |
528 | Cleanup(); |
529 | } |
530 | |
531 | void Builder::Cleanup() { |
532 | if (command_runner_.get()) { |
533 | vector<Edge*> active_edges = command_runner_->GetActiveEdges(); |
534 | command_runner_->Abort(); |
535 | |
536 | for (vector<Edge*>::iterator e = active_edges.begin(); |
537 | e != active_edges.end(); ++e) { |
538 | string depfile = (*e)->GetUnescapedDepfile(); |
539 | for (vector<Node*>::iterator o = (*e)->outputs_.begin(); |
540 | o != (*e)->outputs_.end(); ++o) { |
541 | // Only delete this output if it was actually modified. This is |
542 | // important for things like the generator where we don't want to |
543 | // delete the manifest file if we can avoid it. But if the rule |
544 | // uses a depfile, always delete. (Consider the case where we |
545 | // need to rebuild an output because of a modified header file |
546 | // mentioned in a depfile, and the command touches its depfile |
547 | // but is interrupted before it touches its output file.) |
548 | string err; |
549 | TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), &err); |
550 | if (new_mtime == -1) // Log and ignore Stat() errors. |
551 | status_->Error("%s" , err.c_str()); |
552 | if (!depfile.empty() || (*o)->mtime() != new_mtime) |
553 | disk_interface_->RemoveFile((*o)->path()); |
554 | } |
555 | if (!depfile.empty()) |
556 | disk_interface_->RemoveFile(depfile); |
557 | } |
558 | } |
559 | |
560 | string err; |
561 | if (disk_interface_->Stat(lock_file_path_, &err) > 0) |
562 | disk_interface_->RemoveFile(lock_file_path_); |
563 | } |
564 | |
565 | Node* Builder::AddTarget(const string& name, string* err) { |
566 | Node* node = state_->LookupNode(name); |
567 | if (!node) { |
568 | *err = "unknown target: '" + name + "'" ; |
569 | return NULL; |
570 | } |
571 | if (!AddTarget(node, err)) |
572 | return NULL; |
573 | return node; |
574 | } |
575 | |
576 | bool Builder::AddTarget(Node* target, string* err) { |
577 | std::vector<Node*> validation_nodes; |
578 | if (!scan_.RecomputeDirty(target, &validation_nodes, err)) |
579 | return false; |
580 | |
581 | Edge* in_edge = target->in_edge(); |
582 | if (!in_edge || !in_edge->outputs_ready()) { |
583 | if (!plan_.AddTarget(target, err)) { |
584 | return false; |
585 | } |
586 | } |
587 | |
588 | // Also add any validation nodes found during RecomputeDirty as top level |
589 | // targets. |
590 | for (std::vector<Node*>::iterator n = validation_nodes.begin(); |
591 | n != validation_nodes.end(); ++n) { |
592 | if (Edge* validation_in_edge = (*n)->in_edge()) { |
593 | if (!validation_in_edge->outputs_ready() && |
594 | !plan_.AddTarget(*n, err)) { |
595 | return false; |
596 | } |
597 | } |
598 | } |
599 | |
600 | return true; |
601 | } |
602 | |
603 | bool Builder::AlreadyUpToDate() const { |
604 | return !plan_.more_to_do(); |
605 | } |
606 | |
607 | bool Builder::Build(string* err) { |
608 | assert(!AlreadyUpToDate()); |
609 | |
610 | status_->PlanHasTotalEdges(plan_.command_edge_count()); |
611 | int pending_commands = 0; |
612 | int failures_allowed = config_.failures_allowed; |
613 | |
614 | // Set up the command runner if we haven't done so already. |
615 | if (!command_runner_.get()) { |
616 | if (config_.dry_run) |
617 | command_runner_.reset(new DryRunCommandRunner); |
618 | else |
619 | command_runner_.reset(new RealCommandRunner(config_)); |
620 | } |
621 | |
622 | // We are about to start the build process. |
623 | status_->BuildStarted(); |
624 | |
625 | // This main loop runs the entire build process. |
626 | // It is structured like this: |
627 | // First, we attempt to start as many commands as allowed by the |
628 | // command runner. |
629 | // Second, we attempt to wait for / reap the next finished command. |
630 | while (plan_.more_to_do()) { |
631 | // See if we can start any more commands. |
632 | if (failures_allowed && command_runner_->CanRunMore()) { |
633 | if (Edge* edge = plan_.FindWork()) { |
634 | if (edge->GetBindingBool("generator" )) { |
635 | scan_.build_log()->Close(); |
636 | } |
637 | |
638 | if (!StartEdge(edge, err)) { |
639 | Cleanup(); |
640 | status_->BuildFinished(); |
641 | return false; |
642 | } |
643 | |
644 | if (edge->is_phony()) { |
645 | if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err)) { |
646 | Cleanup(); |
647 | status_->BuildFinished(); |
648 | return false; |
649 | } |
650 | } else { |
651 | ++pending_commands; |
652 | } |
653 | |
654 | // We made some progress; go back to the main loop. |
655 | continue; |
656 | } |
657 | } |
658 | |
659 | // See if we can reap any finished commands. |
660 | if (pending_commands) { |
661 | CommandRunner::Result result; |
662 | if (!command_runner_->WaitForCommand(&result) || |
663 | result.status == ExitInterrupted) { |
664 | Cleanup(); |
665 | status_->BuildFinished(); |
666 | *err = "interrupted by user" ; |
667 | return false; |
668 | } |
669 | |
670 | --pending_commands; |
671 | if (!FinishCommand(&result, err)) { |
672 | Cleanup(); |
673 | status_->BuildFinished(); |
674 | return false; |
675 | } |
676 | |
677 | if (!result.success()) { |
678 | if (failures_allowed) |
679 | failures_allowed--; |
680 | } |
681 | |
682 | // We made some progress; start the main loop over. |
683 | continue; |
684 | } |
685 | |
686 | // If we get here, we cannot make any more progress. |
687 | status_->BuildFinished(); |
688 | if (failures_allowed == 0) { |
689 | if (config_.failures_allowed > 1) |
690 | *err = "subcommands failed" ; |
691 | else |
692 | *err = "subcommand failed" ; |
693 | } else if (failures_allowed < config_.failures_allowed) |
694 | *err = "cannot make progress due to previous errors" ; |
695 | else |
696 | *err = "stuck [this is a bug]" ; |
697 | |
698 | return false; |
699 | } |
700 | |
701 | status_->BuildFinished(); |
702 | return true; |
703 | } |
704 | |
705 | bool Builder::StartEdge(Edge* edge, string* err) { |
706 | METRIC_RECORD("StartEdge" ); |
707 | if (edge->is_phony()) |
708 | return true; |
709 | |
710 | int64_t start_time_millis = GetTimeMillis() - start_time_millis_; |
711 | running_edges_.insert(make_pair(edge, start_time_millis)); |
712 | |
713 | status_->BuildEdgeStarted(edge, start_time_millis); |
714 | |
715 | TimeStamp build_start = -1; |
716 | |
717 | // Create directories necessary for outputs and remember the current |
718 | // filesystem mtime to record later |
719 | // XXX: this will block; do we care? |
720 | for (vector<Node*>::iterator o = edge->outputs_.begin(); |
721 | o != edge->outputs_.end(); ++o) { |
722 | if (!disk_interface_->MakeDirs((*o)->path())) |
723 | return false; |
724 | if (build_start == -1) { |
725 | disk_interface_->WriteFile(lock_file_path_, "" ); |
726 | build_start = disk_interface_->Stat(lock_file_path_, err); |
727 | if (build_start == -1) |
728 | build_start = 0; |
729 | } |
730 | } |
731 | |
732 | edge->command_start_time_ = build_start; |
733 | |
734 | // Create response file, if needed |
735 | // XXX: this may also block; do we care? |
736 | string rspfile = edge->GetUnescapedRspfile(); |
737 | if (!rspfile.empty()) { |
738 | string content = edge->GetBinding("rspfile_content" ); |
739 | if (!disk_interface_->WriteFile(rspfile, content)) |
740 | return false; |
741 | } |
742 | |
743 | // start command computing and run it |
744 | if (!command_runner_->StartCommand(edge)) { |
745 | err->assign("command '" + edge->EvaluateCommand() + "' failed." ); |
746 | return false; |
747 | } |
748 | |
749 | return true; |
750 | } |
751 | |
752 | bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { |
753 | METRIC_RECORD("FinishCommand" ); |
754 | |
755 | Edge* edge = result->edge; |
756 | |
757 | // First try to extract dependencies from the result, if any. |
758 | // This must happen first as it filters the command output (we want |
759 | // to filter /showIncludes output, even on compile failure) and |
760 | // extraction itself can fail, which makes the command fail from a |
761 | // build perspective. |
762 | vector<Node*> deps_nodes; |
763 | string deps_type = edge->GetBinding("deps" ); |
764 | const string deps_prefix = edge->GetBinding("msvc_deps_prefix" ); |
765 | if (!deps_type.empty()) { |
766 | string ; |
767 | if (!ExtractDeps(result, deps_type, deps_prefix, &deps_nodes, |
768 | &extract_err) && |
769 | result->success()) { |
770 | if (!result->output.empty()) |
771 | result->output.append("\n" ); |
772 | result->output.append(extract_err); |
773 | result->status = ExitFailure; |
774 | } |
775 | } |
776 | |
777 | int64_t start_time_millis, end_time_millis; |
778 | RunningEdgeMap::iterator it = running_edges_.find(edge); |
779 | start_time_millis = it->second; |
780 | end_time_millis = GetTimeMillis() - start_time_millis_; |
781 | running_edges_.erase(it); |
782 | |
783 | status_->BuildEdgeFinished(edge, end_time_millis, result->success(), |
784 | result->output); |
785 | |
786 | // The rest of this function only applies to successful commands. |
787 | if (!result->success()) { |
788 | return plan_.EdgeFinished(edge, Plan::kEdgeFailed, err); |
789 | } |
790 | |
791 | // Restat the edge outputs |
792 | TimeStamp record_mtime = 0; |
793 | if (!config_.dry_run) { |
794 | const bool restat = edge->GetBindingBool("restat" ); |
795 | const bool generator = edge->GetBindingBool("generator" ); |
796 | bool node_cleaned = false; |
797 | record_mtime = edge->command_start_time_; |
798 | |
799 | // restat and generator rules must restat the outputs after the build |
800 | // has finished. if record_mtime == 0, then there was an error while |
801 | // attempting to touch/stat the temp file when the edge started and |
802 | // we should fall back to recording the outputs' current mtime in the |
803 | // log. |
804 | if (record_mtime == 0 || restat || generator) { |
805 | for (vector<Node*>::iterator o = edge->outputs_.begin(); |
806 | o != edge->outputs_.end(); ++o) { |
807 | TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err); |
808 | if (new_mtime == -1) |
809 | return false; |
810 | if (new_mtime > record_mtime) |
811 | record_mtime = new_mtime; |
812 | if ((*o)->mtime() == new_mtime && restat) { |
813 | // The rule command did not change the output. Propagate the clean |
814 | // state through the build graph. |
815 | // Note that this also applies to nonexistent outputs (mtime == 0). |
816 | if (!plan_.CleanNode(&scan_, *o, err)) |
817 | return false; |
818 | node_cleaned = true; |
819 | } |
820 | } |
821 | } |
822 | if (node_cleaned) { |
823 | record_mtime = edge->command_start_time_; |
824 | |
825 | // The total number of edges in the plan may have changed as a result |
826 | // of a restat. |
827 | status_->PlanHasTotalEdges(plan_.command_edge_count()); |
828 | } |
829 | } |
830 | |
831 | if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err)) |
832 | return false; |
833 | |
834 | // Delete any left over response file. |
835 | string rspfile = edge->GetUnescapedRspfile(); |
836 | if (!rspfile.empty() && !g_keep_rsp) |
837 | disk_interface_->RemoveFile(rspfile); |
838 | |
839 | if (scan_.build_log()) { |
840 | if (!scan_.build_log()->RecordCommand(edge, start_time_millis, |
841 | end_time_millis, record_mtime)) { |
842 | *err = string("Error writing to build log: " ) + strerror(errno); |
843 | return false; |
844 | } |
845 | } |
846 | |
847 | if (!deps_type.empty() && !config_.dry_run) { |
848 | assert(!edge->outputs_.empty() && "should have been rejected by parser" ); |
849 | for (std::vector<Node*>::const_iterator o = edge->outputs_.begin(); |
850 | o != edge->outputs_.end(); ++o) { |
851 | TimeStamp deps_mtime = disk_interface_->Stat((*o)->path(), err); |
852 | if (deps_mtime == -1) |
853 | return false; |
854 | if (!scan_.deps_log()->RecordDeps(*o, deps_mtime, deps_nodes)) { |
855 | *err = std::string("Error writing to deps log: " ) + strerror(errno); |
856 | return false; |
857 | } |
858 | } |
859 | } |
860 | return true; |
861 | } |
862 | |
863 | bool Builder::ExtractDeps(CommandRunner::Result* result, |
864 | const string& deps_type, |
865 | const string& deps_prefix, |
866 | vector<Node*>* deps_nodes, |
867 | string* err) { |
868 | if (deps_type == "msvc" ) { |
869 | CLParser parser; |
870 | string output; |
871 | if (!parser.Parse(result->output, deps_prefix, &output, err)) |
872 | return false; |
873 | result->output = output; |
874 | for (set<string>::iterator i = parser.includes_.begin(); |
875 | i != parser.includes_.end(); ++i) { |
876 | // ~0 is assuming that with MSVC-parsed headers, it's ok to always make |
877 | // all backslashes (as some of the slashes will certainly be backslashes |
878 | // anyway). This could be fixed if necessary with some additional |
879 | // complexity in IncludesNormalize::Relativize. |
880 | deps_nodes->push_back(state_->GetNode(*i, ~0u)); |
881 | } |
882 | } else if (deps_type == "gcc" ) { |
883 | string depfile = result->edge->GetUnescapedDepfile(); |
884 | if (depfile.empty()) { |
885 | *err = string("edge with deps=gcc but no depfile makes no sense" ); |
886 | return false; |
887 | } |
888 | |
889 | // Read depfile content. Treat a missing depfile as empty. |
890 | string content; |
891 | switch (disk_interface_->ReadFile(depfile, &content, err)) { |
892 | case DiskInterface::Okay: |
893 | break; |
894 | case DiskInterface::NotFound: |
895 | err->clear(); |
896 | break; |
897 | case DiskInterface::OtherError: |
898 | return false; |
899 | } |
900 | if (content.empty()) |
901 | return true; |
902 | |
903 | DepfileParser deps(config_.depfile_parser_options); |
904 | if (!deps.Parse(&content, err)) |
905 | return false; |
906 | |
907 | // XXX check depfile matches expected output. |
908 | deps_nodes->reserve(deps.ins_.size()); |
909 | for (vector<StringPiece>::iterator i = deps.ins_.begin(); |
910 | i != deps.ins_.end(); ++i) { |
911 | uint64_t slash_bits; |
912 | CanonicalizePath(const_cast<char*>(i->str_), &i->len_, &slash_bits); |
913 | deps_nodes->push_back(state_->GetNode(*i, slash_bits)); |
914 | } |
915 | |
916 | if (!g_keep_depfile) { |
917 | if (disk_interface_->RemoveFile(depfile) < 0) { |
918 | *err = string("deleting depfile: " ) + strerror(errno) + string("\n" ); |
919 | return false; |
920 | } |
921 | } |
922 | } else { |
923 | Fatal("unknown deps type '%s'" , deps_type.c_str()); |
924 | } |
925 | |
926 | return true; |
927 | } |
928 | |
929 | bool Builder::LoadDyndeps(Node* node, string* err) { |
930 | status_->BuildLoadDyndeps(); |
931 | |
932 | // Load the dyndep information provided by this node. |
933 | DyndepFile ddf; |
934 | if (!scan_.LoadDyndeps(node, &ddf, err)) |
935 | return false; |
936 | |
937 | // Update the build plan to account for dyndep modifications to the graph. |
938 | if (!plan_.DyndepsLoaded(&scan_, node, ddf, err)) |
939 | return false; |
940 | |
941 | // New command edges may have been added to the plan. |
942 | status_->PlanHasTotalEdges(plan_.command_edge_count()); |
943 | |
944 | return true; |
945 | } |
946 | |