1 | // Copyright 2012 Google Inc. All Rights Reserved. |
2 | // |
3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
4 | // you may not use this file except in compliance with the License. |
5 | // You may obtain a copy of the License at |
6 | // |
7 | // http://www.apache.org/licenses/LICENSE-2.0 |
8 | // |
9 | // Unless required by applicable law or agreed to in writing, software |
10 | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | // See the License for the specific language governing permissions and |
13 | // limitations under the License. |
14 | |
15 | #include "deps_log.h" |
16 | |
17 | #include <sys/stat.h> |
18 | #ifndef _WIN32 |
19 | #include <unistd.h> |
20 | #endif |
21 | |
22 | #include "graph.h" |
23 | #include "util.h" |
24 | #include "test.h" |
25 | |
26 | using namespace std; |
27 | |
28 | namespace { |
29 | |
30 | const char kTestFilename[] = "DepsLogTest-tempfile" ; |
31 | |
32 | struct DepsLogTest : public testing::Test { |
33 | virtual void SetUp() { |
34 | // In case a crashing test left a stale file behind. |
35 | unlink(kTestFilename); |
36 | } |
37 | virtual void TearDown() { |
38 | unlink(kTestFilename); |
39 | } |
40 | }; |
41 | |
42 | TEST_F(DepsLogTest, WriteRead) { |
43 | State state1; |
44 | DepsLog log1; |
45 | string err; |
46 | EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err)); |
47 | ASSERT_EQ("" , err); |
48 | |
49 | { |
50 | vector<Node*> deps; |
51 | deps.push_back(state1.GetNode("foo.h" , 0)); |
52 | deps.push_back(state1.GetNode("bar.h" , 0)); |
53 | log1.RecordDeps(state1.GetNode("out.o" , 0), 1, deps); |
54 | |
55 | deps.clear(); |
56 | deps.push_back(state1.GetNode("foo.h" , 0)); |
57 | deps.push_back(state1.GetNode("bar2.h" , 0)); |
58 | log1.RecordDeps(state1.GetNode("out2.o" , 0), 2, deps); |
59 | |
60 | DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o" , 0)); |
61 | ASSERT_TRUE(log_deps); |
62 | ASSERT_EQ(1, log_deps->mtime); |
63 | ASSERT_EQ(2, log_deps->node_count); |
64 | ASSERT_EQ("foo.h" , log_deps->nodes[0]->path()); |
65 | ASSERT_EQ("bar.h" , log_deps->nodes[1]->path()); |
66 | } |
67 | |
68 | log1.Close(); |
69 | |
70 | State state2; |
71 | DepsLog log2; |
72 | EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err)); |
73 | ASSERT_EQ("" , err); |
74 | |
75 | ASSERT_EQ(log1.nodes().size(), log2.nodes().size()); |
76 | for (int i = 0; i < (int)log1.nodes().size(); ++i) { |
77 | Node* node1 = log1.nodes()[i]; |
78 | Node* node2 = log2.nodes()[i]; |
79 | ASSERT_EQ(i, node1->id()); |
80 | ASSERT_EQ(node1->id(), node2->id()); |
81 | } |
82 | |
83 | // Spot-check the entries in log2. |
84 | DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o" , 0)); |
85 | ASSERT_TRUE(log_deps); |
86 | ASSERT_EQ(2, log_deps->mtime); |
87 | ASSERT_EQ(2, log_deps->node_count); |
88 | ASSERT_EQ("foo.h" , log_deps->nodes[0]->path()); |
89 | ASSERT_EQ("bar2.h" , log_deps->nodes[1]->path()); |
90 | } |
91 | |
92 | TEST_F(DepsLogTest, LotsOfDeps) { |
93 | const int kNumDeps = 100000; // More than 64k. |
94 | |
95 | State state1; |
96 | DepsLog log1; |
97 | string err; |
98 | EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err)); |
99 | ASSERT_EQ("" , err); |
100 | |
101 | { |
102 | vector<Node*> deps; |
103 | for (int i = 0; i < kNumDeps; ++i) { |
104 | char buf[32]; |
105 | sprintf(buf, "file%d.h" , i); |
106 | deps.push_back(state1.GetNode(buf, 0)); |
107 | } |
108 | log1.RecordDeps(state1.GetNode("out.o" , 0), 1, deps); |
109 | |
110 | DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o" , 0)); |
111 | ASSERT_EQ(kNumDeps, log_deps->node_count); |
112 | } |
113 | |
114 | log1.Close(); |
115 | |
116 | State state2; |
117 | DepsLog log2; |
118 | EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err)); |
119 | ASSERT_EQ("" , err); |
120 | |
121 | DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o" , 0)); |
122 | ASSERT_EQ(kNumDeps, log_deps->node_count); |
123 | } |
124 | |
125 | // Verify that adding the same deps twice doesn't grow the file. |
126 | TEST_F(DepsLogTest, DoubleEntry) { |
127 | // Write some deps to the file and grab its size. |
128 | int file_size; |
129 | { |
130 | State state; |
131 | DepsLog log; |
132 | string err; |
133 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |
134 | ASSERT_EQ("" , err); |
135 | |
136 | vector<Node*> deps; |
137 | deps.push_back(state.GetNode("foo.h" , 0)); |
138 | deps.push_back(state.GetNode("bar.h" , 0)); |
139 | log.RecordDeps(state.GetNode("out.o" , 0), 1, deps); |
140 | log.Close(); |
141 | #ifdef __USE_LARGEFILE64 |
142 | struct stat64 st; |
143 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |
144 | #else |
145 | struct stat st; |
146 | ASSERT_EQ(0, stat(kTestFilename, &st)); |
147 | #endif |
148 | file_size = (int)st.st_size; |
149 | ASSERT_GT(file_size, 0); |
150 | } |
151 | |
152 | // Now reload the file, and read the same deps. |
153 | { |
154 | State state; |
155 | DepsLog log; |
156 | string err; |
157 | EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); |
158 | |
159 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |
160 | ASSERT_EQ("" , err); |
161 | |
162 | vector<Node*> deps; |
163 | deps.push_back(state.GetNode("foo.h" , 0)); |
164 | deps.push_back(state.GetNode("bar.h" , 0)); |
165 | log.RecordDeps(state.GetNode("out.o" , 0), 1, deps); |
166 | log.Close(); |
167 | #ifdef __USE_LARGEFILE64 |
168 | struct stat64 st; |
169 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |
170 | #else |
171 | struct stat st; |
172 | ASSERT_EQ(0, stat(kTestFilename, &st)); |
173 | #endif |
174 | int file_size_2 = (int)st.st_size; |
175 | ASSERT_EQ(file_size, file_size_2); |
176 | } |
177 | } |
178 | |
179 | // Verify that adding the new deps works and can be compacted away. |
180 | TEST_F(DepsLogTest, Recompact) { |
181 | const char kManifest[] = |
182 | "rule cc\n" |
183 | " command = cc\n" |
184 | " deps = gcc\n" |
185 | "build out.o: cc\n" |
186 | "build other_out.o: cc\n" ; |
187 | |
188 | // Write some deps to the file and grab its size. |
189 | int file_size; |
190 | { |
191 | State state; |
192 | ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest)); |
193 | DepsLog log; |
194 | string err; |
195 | ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err)); |
196 | ASSERT_EQ("" , err); |
197 | |
198 | vector<Node*> deps; |
199 | deps.push_back(state.GetNode("foo.h" , 0)); |
200 | deps.push_back(state.GetNode("bar.h" , 0)); |
201 | log.RecordDeps(state.GetNode("out.o" , 0), 1, deps); |
202 | |
203 | deps.clear(); |
204 | deps.push_back(state.GetNode("foo.h" , 0)); |
205 | deps.push_back(state.GetNode("baz.h" , 0)); |
206 | log.RecordDeps(state.GetNode("other_out.o" , 0), 1, deps); |
207 | |
208 | log.Close(); |
209 | #ifdef __USE_LARGEFILE64 |
210 | struct stat64 st; |
211 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |
212 | #else |
213 | struct stat st; |
214 | ASSERT_EQ(0, stat(kTestFilename, &st)); |
215 | #endif |
216 | file_size = (int)st.st_size; |
217 | ASSERT_GT(file_size, 0); |
218 | } |
219 | |
220 | // Now reload the file, and add slightly different deps. |
221 | int file_size_2; |
222 | { |
223 | State state; |
224 | ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest)); |
225 | DepsLog log; |
226 | string err; |
227 | ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); |
228 | |
229 | ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err)); |
230 | ASSERT_EQ("" , err); |
231 | |
232 | vector<Node*> deps; |
233 | deps.push_back(state.GetNode("foo.h" , 0)); |
234 | log.RecordDeps(state.GetNode("out.o" , 0), 1, deps); |
235 | log.Close(); |
236 | |
237 | #ifdef __USE_LARGEFILE64 |
238 | struct stat64 st; |
239 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |
240 | #else |
241 | struct stat st; |
242 | ASSERT_EQ(0, stat(kTestFilename, &st)); |
243 | #endif |
244 | file_size_2 = (int)st.st_size; |
245 | // The file should grow to record the new deps. |
246 | ASSERT_GT(file_size_2, file_size); |
247 | } |
248 | |
249 | // Now reload the file, verify the new deps have replaced the old, then |
250 | // recompact. |
251 | int file_size_3; |
252 | { |
253 | State state; |
254 | ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest)); |
255 | DepsLog log; |
256 | string err; |
257 | ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); |
258 | |
259 | Node* out = state.GetNode("out.o" , 0); |
260 | DepsLog::Deps* deps = log.GetDeps(out); |
261 | ASSERT_TRUE(deps); |
262 | ASSERT_EQ(1, deps->mtime); |
263 | ASSERT_EQ(1, deps->node_count); |
264 | ASSERT_EQ("foo.h" , deps->nodes[0]->path()); |
265 | |
266 | Node* other_out = state.GetNode("other_out.o" , 0); |
267 | deps = log.GetDeps(other_out); |
268 | ASSERT_TRUE(deps); |
269 | ASSERT_EQ(1, deps->mtime); |
270 | ASSERT_EQ(2, deps->node_count); |
271 | ASSERT_EQ("foo.h" , deps->nodes[0]->path()); |
272 | ASSERT_EQ("baz.h" , deps->nodes[1]->path()); |
273 | |
274 | ASSERT_TRUE(log.Recompact(kTestFilename, &err)); |
275 | |
276 | // The in-memory deps graph should still be valid after recompaction. |
277 | deps = log.GetDeps(out); |
278 | ASSERT_TRUE(deps); |
279 | ASSERT_EQ(1, deps->mtime); |
280 | ASSERT_EQ(1, deps->node_count); |
281 | ASSERT_EQ("foo.h" , deps->nodes[0]->path()); |
282 | ASSERT_EQ(out, log.nodes()[out->id()]); |
283 | |
284 | deps = log.GetDeps(other_out); |
285 | ASSERT_TRUE(deps); |
286 | ASSERT_EQ(1, deps->mtime); |
287 | ASSERT_EQ(2, deps->node_count); |
288 | ASSERT_EQ("foo.h" , deps->nodes[0]->path()); |
289 | ASSERT_EQ("baz.h" , deps->nodes[1]->path()); |
290 | ASSERT_EQ(other_out, log.nodes()[other_out->id()]); |
291 | |
292 | // The file should have shrunk a bit for the smaller deps. |
293 | #ifdef __USE_LARGEFILE64 |
294 | struct stat64 st; |
295 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |
296 | #else |
297 | struct stat st; |
298 | ASSERT_EQ(0, stat(kTestFilename, &st)); |
299 | #endif |
300 | file_size_3 = (int)st.st_size; |
301 | ASSERT_LT(file_size_3, file_size_2); |
302 | } |
303 | |
304 | // Now reload the file and recompact with an empty manifest. The previous |
305 | // entries should be removed. |
306 | { |
307 | State state; |
308 | // Intentionally not parsing kManifest here. |
309 | DepsLog log; |
310 | string err; |
311 | ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); |
312 | |
313 | Node* out = state.GetNode("out.o" , 0); |
314 | DepsLog::Deps* deps = log.GetDeps(out); |
315 | ASSERT_TRUE(deps); |
316 | ASSERT_EQ(1, deps->mtime); |
317 | ASSERT_EQ(1, deps->node_count); |
318 | ASSERT_EQ("foo.h" , deps->nodes[0]->path()); |
319 | |
320 | Node* other_out = state.GetNode("other_out.o" , 0); |
321 | deps = log.GetDeps(other_out); |
322 | ASSERT_TRUE(deps); |
323 | ASSERT_EQ(1, deps->mtime); |
324 | ASSERT_EQ(2, deps->node_count); |
325 | ASSERT_EQ("foo.h" , deps->nodes[0]->path()); |
326 | ASSERT_EQ("baz.h" , deps->nodes[1]->path()); |
327 | |
328 | ASSERT_TRUE(log.Recompact(kTestFilename, &err)); |
329 | |
330 | // The previous entries should have been removed. |
331 | deps = log.GetDeps(out); |
332 | ASSERT_FALSE(deps); |
333 | |
334 | deps = log.GetDeps(other_out); |
335 | ASSERT_FALSE(deps); |
336 | |
337 | // The .h files pulled in via deps should no longer have ids either. |
338 | ASSERT_EQ(-1, state.LookupNode("foo.h" )->id()); |
339 | ASSERT_EQ(-1, state.LookupNode("baz.h" )->id()); |
340 | |
341 | // The file should have shrunk more. |
342 | #ifdef __USE_LARGEFILE64 |
343 | struct stat64 st; |
344 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |
345 | #else |
346 | struct stat st; |
347 | ASSERT_EQ(0, stat(kTestFilename, &st)); |
348 | #endif |
349 | int file_size_4 = (int)st.st_size; |
350 | ASSERT_LT(file_size_4, file_size_3); |
351 | } |
352 | } |
353 | |
354 | // Verify that invalid file headers cause a new build. |
355 | TEST_F(DepsLogTest, InvalidHeader) { |
356 | const char *[] = { |
357 | "" , // Empty file. |
358 | "# ninjad" , // Truncated first line. |
359 | "# ninjadeps\n" , // No version int. |
360 | "# ninjadeps\n\001\002" , // Truncated version int. |
361 | "# ninjadeps\n\001\002\003\004" // Invalid version int. |
362 | }; |
363 | for (size_t i = 0; i < sizeof(kInvalidHeaders) / sizeof(kInvalidHeaders[0]); |
364 | ++i) { |
365 | FILE* deps_log = fopen(kTestFilename, "wb" ); |
366 | ASSERT_TRUE(deps_log != NULL); |
367 | ASSERT_EQ( |
368 | strlen(kInvalidHeaders[i]), |
369 | fwrite(kInvalidHeaders[i], 1, strlen(kInvalidHeaders[i]), deps_log)); |
370 | ASSERT_EQ(0 ,fclose(deps_log)); |
371 | |
372 | string err; |
373 | DepsLog log; |
374 | State state; |
375 | ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); |
376 | EXPECT_EQ("bad deps log signature or version; starting over" , err); |
377 | } |
378 | } |
379 | |
380 | // Simulate what happens when loading a truncated log file. |
381 | TEST_F(DepsLogTest, Truncated) { |
382 | // Create a file with some entries. |
383 | { |
384 | State state; |
385 | DepsLog log; |
386 | string err; |
387 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |
388 | ASSERT_EQ("" , err); |
389 | |
390 | vector<Node*> deps; |
391 | deps.push_back(state.GetNode("foo.h" , 0)); |
392 | deps.push_back(state.GetNode("bar.h" , 0)); |
393 | log.RecordDeps(state.GetNode("out.o" , 0), 1, deps); |
394 | |
395 | deps.clear(); |
396 | deps.push_back(state.GetNode("foo.h" , 0)); |
397 | deps.push_back(state.GetNode("bar2.h" , 0)); |
398 | log.RecordDeps(state.GetNode("out2.o" , 0), 2, deps); |
399 | |
400 | log.Close(); |
401 | } |
402 | |
403 | // Get the file size. |
404 | #ifdef __USE_LARGEFILE64 |
405 | struct stat64 st; |
406 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |
407 | #else |
408 | struct stat st; |
409 | ASSERT_EQ(0, stat(kTestFilename, &st)); |
410 | #endif |
411 | |
412 | // Try reloading at truncated sizes. |
413 | // Track how many nodes/deps were found; they should decrease with |
414 | // smaller sizes. |
415 | int node_count = 5; |
416 | int deps_count = 2; |
417 | for (int size = (int)st.st_size; size > 0; --size) { |
418 | string err; |
419 | ASSERT_TRUE(Truncate(kTestFilename, size, &err)); |
420 | |
421 | State state; |
422 | DepsLog log; |
423 | EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); |
424 | if (!err.empty()) { |
425 | // At some point the log will be so short as to be unparsable. |
426 | break; |
427 | } |
428 | |
429 | ASSERT_GE(node_count, (int)log.nodes().size()); |
430 | node_count = log.nodes().size(); |
431 | |
432 | // Count how many non-NULL deps entries there are. |
433 | int new_deps_count = 0; |
434 | for (vector<DepsLog::Deps*>::const_iterator i = log.deps().begin(); |
435 | i != log.deps().end(); ++i) { |
436 | if (*i) |
437 | ++new_deps_count; |
438 | } |
439 | ASSERT_GE(deps_count, new_deps_count); |
440 | deps_count = new_deps_count; |
441 | } |
442 | } |
443 | |
444 | // Run the truncation-recovery logic. |
445 | TEST_F(DepsLogTest, TruncatedRecovery) { |
446 | // Create a file with some entries. |
447 | { |
448 | State state; |
449 | DepsLog log; |
450 | string err; |
451 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |
452 | ASSERT_EQ("" , err); |
453 | |
454 | vector<Node*> deps; |
455 | deps.push_back(state.GetNode("foo.h" , 0)); |
456 | deps.push_back(state.GetNode("bar.h" , 0)); |
457 | log.RecordDeps(state.GetNode("out.o" , 0), 1, deps); |
458 | |
459 | deps.clear(); |
460 | deps.push_back(state.GetNode("foo.h" , 0)); |
461 | deps.push_back(state.GetNode("bar2.h" , 0)); |
462 | log.RecordDeps(state.GetNode("out2.o" , 0), 2, deps); |
463 | |
464 | log.Close(); |
465 | } |
466 | |
467 | // Shorten the file, corrupting the last record. |
468 | { |
469 | #ifdef __USE_LARGEFILE64 |
470 | struct stat64 st; |
471 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |
472 | #else |
473 | struct stat st; |
474 | ASSERT_EQ(0, stat(kTestFilename, &st)); |
475 | #endif |
476 | string err; |
477 | ASSERT_TRUE(Truncate(kTestFilename, st.st_size - 2, &err)); |
478 | } |
479 | |
480 | // Load the file again, add an entry. |
481 | { |
482 | State state; |
483 | DepsLog log; |
484 | string err; |
485 | EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); |
486 | ASSERT_EQ("premature end of file; recovering" , err); |
487 | err.clear(); |
488 | |
489 | // The truncated entry should've been discarded. |
490 | EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o" , 0))); |
491 | |
492 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |
493 | ASSERT_EQ("" , err); |
494 | |
495 | // Add a new entry. |
496 | vector<Node*> deps; |
497 | deps.push_back(state.GetNode("foo.h" , 0)); |
498 | deps.push_back(state.GetNode("bar2.h" , 0)); |
499 | log.RecordDeps(state.GetNode("out2.o" , 0), 3, deps); |
500 | |
501 | log.Close(); |
502 | } |
503 | |
504 | // Load the file a third time to verify appending after a mangled |
505 | // entry doesn't break things. |
506 | { |
507 | State state; |
508 | DepsLog log; |
509 | string err; |
510 | EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); |
511 | |
512 | // The truncated entry should exist. |
513 | DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o" , 0)); |
514 | ASSERT_TRUE(deps); |
515 | } |
516 | } |
517 | |
518 | TEST_F(DepsLogTest, ReverseDepsNodes) { |
519 | State state; |
520 | DepsLog log; |
521 | string err; |
522 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |
523 | ASSERT_EQ("" , err); |
524 | |
525 | vector<Node*> deps; |
526 | deps.push_back(state.GetNode("foo.h" , 0)); |
527 | deps.push_back(state.GetNode("bar.h" , 0)); |
528 | log.RecordDeps(state.GetNode("out.o" , 0), 1, deps); |
529 | |
530 | deps.clear(); |
531 | deps.push_back(state.GetNode("foo.h" , 0)); |
532 | deps.push_back(state.GetNode("bar2.h" , 0)); |
533 | log.RecordDeps(state.GetNode("out2.o" , 0), 2, deps); |
534 | |
535 | log.Close(); |
536 | |
537 | Node* rev_deps = log.GetFirstReverseDepsNode(state.GetNode("foo.h" , 0)); |
538 | EXPECT_TRUE(rev_deps == state.GetNode("out.o" , 0) || |
539 | rev_deps == state.GetNode("out2.o" , 0)); |
540 | |
541 | rev_deps = log.GetFirstReverseDepsNode(state.GetNode("bar.h" , 0)); |
542 | EXPECT_TRUE(rev_deps == state.GetNode("out.o" , 0)); |
543 | } |
544 | |
545 | } // anonymous namespace |
546 | |