1 | // Copyright 2012 Google Inc. All Rights Reserved. |
---|---|

2 | // |

3 | // Licensed under the Apache License, Version 2.0 (the "License"); |

4 | // you may not use this file except in compliance with the License. |

5 | // You may obtain a copy of the License at |

6 | // |

7 | // http://www.apache.org/licenses/LICENSE-2.0 |

8 | // |

9 | // Unless required by applicable law or agreed to in writing, software |

10 | // distributed under the License is distributed on an "AS IS" BASIS, |

11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |

12 | // See the License for the specific language governing permissions and |

13 | // limitations under the License. |

14 | |

15 | #include "deps_log.h" |

16 | |

17 | #include <sys/stat.h> |

18 | #ifndef _WIN32 |

19 | #include <unistd.h> |

20 | #endif |

21 | |

22 | #include "graph.h" |

23 | #include "util.h" |

24 | #include "test.h" |

25 | |

26 | using namespace std; |

27 | |

28 | namespace { |

29 | |

30 | const char kTestFilename[] = "DepsLogTest-tempfile"; |

31 | |

32 | struct DepsLogTest : public testing::Test { |

33 | virtual void SetUp() { |

34 | // In case a crashing test left a stale file behind. |

35 | unlink(kTestFilename); |

36 | } |

37 | virtual void TearDown() { |

38 | unlink(kTestFilename); |

39 | } |

40 | }; |

41 | |

42 | TEST_F(DepsLogTest, WriteRead) { |

43 | State state1; |

44 | DepsLog log1; |

45 | string err; |

46 | EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err)); |

47 | ASSERT_EQ("", err); |

48 | |

49 | { |

50 | vector<Node*> deps; |

51 | deps.push_back(state1.GetNode("foo.h", 0)); |

52 | deps.push_back(state1.GetNode("bar.h", 0)); |

53 | log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps); |

54 | |

55 | deps.clear(); |

56 | deps.push_back(state1.GetNode("foo.h", 0)); |

57 | deps.push_back(state1.GetNode("bar2.h", 0)); |

58 | log1.RecordDeps(state1.GetNode("out2.o", 0), 2, deps); |

59 | |

60 | DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0)); |

61 | ASSERT_TRUE(log_deps); |

62 | ASSERT_EQ(1, log_deps->mtime); |

63 | ASSERT_EQ(2, log_deps->node_count); |

64 | ASSERT_EQ("foo.h", log_deps->nodes[ 0]->path()); |

65 | ASSERT_EQ("bar.h", log_deps->nodes[ 1]->path()); |

66 | } |

67 | |

68 | log1.Close(); |

69 | |

70 | State state2; |

71 | DepsLog log2; |

72 | EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err)); |

73 | ASSERT_EQ("", err); |

74 | |

75 | ASSERT_EQ(log1.nodes().size(), log2.nodes().size()); |

76 | for (int i = 0; i < (int)log1.nodes().size(); ++i) { |

77 | Node* node1 = log1.nodes()[i]; |

78 | Node* node2 = log2.nodes()[i]; |

79 | ASSERT_EQ(i, node1->id()); |

80 | ASSERT_EQ(node1->id(), node2->id()); |

81 | } |

82 | |

83 | // Spot-check the entries in log2. |

84 | DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o", 0)); |

85 | ASSERT_TRUE(log_deps); |

86 | ASSERT_EQ(2, log_deps->mtime); |

87 | ASSERT_EQ(2, log_deps->node_count); |

88 | ASSERT_EQ("foo.h", log_deps->nodes[ 0]->path()); |

89 | ASSERT_EQ("bar2.h", log_deps->nodes[ 1]->path()); |

90 | } |

91 | |

92 | TEST_F(DepsLogTest, LotsOfDeps) { |

93 | const int kNumDeps = 100000; // More than 64k. |

94 | |

95 | State state1; |

96 | DepsLog log1; |

97 | string err; |

98 | EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err)); |

99 | ASSERT_EQ("", err); |

100 | |

101 | { |

102 | vector<Node*> deps; |

103 | for (int i = 0; i < kNumDeps; ++i) { |

104 | char buf[32]; |

105 | sprintf(buf, "file%d.h", i); |

106 | deps.push_back(state1.GetNode(buf, 0)); |

107 | } |

108 | log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps); |

109 | |

110 | DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0)); |

111 | ASSERT_EQ(kNumDeps, log_deps->node_count); |

112 | } |

113 | |

114 | log1.Close(); |

115 | |

116 | State state2; |

117 | DepsLog log2; |

118 | EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err)); |

119 | ASSERT_EQ("", err); |

120 | |

121 | DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o", 0)); |

122 | ASSERT_EQ(kNumDeps, log_deps->node_count); |

123 | } |

124 | |

125 | // Verify that adding the same deps twice doesn't grow the file. |

126 | TEST_F(DepsLogTest, DoubleEntry) { |

127 | // Write some deps to the file and grab its size. |

128 | int file_size; |

129 | { |

130 | State state; |

131 | DepsLog log; |

132 | string err; |

133 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |

134 | ASSERT_EQ("", err); |

135 | |

136 | vector<Node*> deps; |

137 | deps.push_back(state.GetNode("foo.h", 0)); |

138 | deps.push_back(state.GetNode("bar.h", 0)); |

139 | log.RecordDeps(state.GetNode("out.o", 0), 1, deps); |

140 | log.Close(); |

141 | #ifdef __USE_LARGEFILE64 |

142 | struct stat64 st; |

143 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |

144 | #else |

145 | struct stat st; |

146 | ASSERT_EQ(0, stat(kTestFilename, &st)); |

147 | #endif |

148 | file_size = (int)st.st_size; |

149 | ASSERT_GT(file_size, 0); |

150 | } |

151 | |

152 | // Now reload the file, and read the same deps. |

153 | { |

154 | State state; |

155 | DepsLog log; |

156 | string err; |

157 | EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); |

158 | |

159 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |

160 | ASSERT_EQ("", err); |

161 | |

162 | vector<Node*> deps; |

163 | deps.push_back(state.GetNode("foo.h", 0)); |

164 | deps.push_back(state.GetNode("bar.h", 0)); |

165 | log.RecordDeps(state.GetNode("out.o", 0), 1, deps); |

166 | log.Close(); |

167 | #ifdef __USE_LARGEFILE64 |

168 | struct stat64 st; |

169 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |

170 | #else |

171 | struct stat st; |

172 | ASSERT_EQ(0, stat(kTestFilename, &st)); |

173 | #endif |

174 | int file_size_2 = (int)st.st_size; |

175 | ASSERT_EQ(file_size, file_size_2); |

176 | } |

177 | } |

178 | |

179 | // Verify that adding the new deps works and can be compacted away. |

180 | TEST_F(DepsLogTest, Recompact) { |

181 | const char kManifest[] = |

182 | "rule cc\n" |

183 | " command = cc\n" |

184 | " deps = gcc\n" |

185 | "build out.o: cc\n" |

186 | "build other_out.o: cc\n"; |

187 | |

188 | // Write some deps to the file and grab its size. |

189 | int file_size; |

190 | { |

191 | State state; |

192 | ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest)); |

193 | DepsLog log; |

194 | string err; |

195 | ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err)); |

196 | ASSERT_EQ("", err); |

197 | |

198 | vector<Node*> deps; |

199 | deps.push_back(state.GetNode("foo.h", 0)); |

200 | deps.push_back(state.GetNode("bar.h", 0)); |

201 | log.RecordDeps(state.GetNode("out.o", 0), 1, deps); |

202 | |

203 | deps.clear(); |

204 | deps.push_back(state.GetNode("foo.h", 0)); |

205 | deps.push_back(state.GetNode("baz.h", 0)); |

206 | log.RecordDeps(state.GetNode("other_out.o", 0), 1, deps); |

207 | |

208 | log.Close(); |

209 | #ifdef __USE_LARGEFILE64 |

210 | struct stat64 st; |

211 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |

212 | #else |

213 | struct stat st; |

214 | ASSERT_EQ(0, stat(kTestFilename, &st)); |

215 | #endif |

216 | file_size = (int)st.st_size; |

217 | ASSERT_GT(file_size, 0); |

218 | } |

219 | |

220 | // Now reload the file, and add slightly different deps. |

221 | int file_size_2; |

222 | { |

223 | State state; |

224 | ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest)); |

225 | DepsLog log; |

226 | string err; |

227 | ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); |

228 | |

229 | ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err)); |

230 | ASSERT_EQ("", err); |

231 | |

232 | vector<Node*> deps; |

233 | deps.push_back(state.GetNode("foo.h", 0)); |

234 | log.RecordDeps(state.GetNode("out.o", 0), 1, deps); |

235 | log.Close(); |

236 | |

237 | #ifdef __USE_LARGEFILE64 |

238 | struct stat64 st; |

239 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |

240 | #else |

241 | struct stat st; |

242 | ASSERT_EQ(0, stat(kTestFilename, &st)); |

243 | #endif |

244 | file_size_2 = (int)st.st_size; |

245 | // The file should grow to record the new deps. |

246 | ASSERT_GT(file_size_2, file_size); |

247 | } |

248 | |

249 | // Now reload the file, verify the new deps have replaced the old, then |

250 | // recompact. |

251 | int file_size_3; |

252 | { |

253 | State state; |

254 | ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest)); |

255 | DepsLog log; |

256 | string err; |

257 | ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); |

258 | |

259 | Node* out = state.GetNode("out.o", 0); |

260 | DepsLog::Deps* deps = log.GetDeps(out); |

261 | ASSERT_TRUE(deps); |

262 | ASSERT_EQ(1, deps->mtime); |

263 | ASSERT_EQ(1, deps->node_count); |

264 | ASSERT_EQ("foo.h", deps->nodes[ 0]->path()); |

265 | |

266 | Node* other_out = state.GetNode("other_out.o", 0); |

267 | deps = log.GetDeps(other_out); |

268 | ASSERT_TRUE(deps); |

269 | ASSERT_EQ(1, deps->mtime); |

270 | ASSERT_EQ(2, deps->node_count); |

271 | ASSERT_EQ("foo.h", deps->nodes[ 0]->path()); |

272 | ASSERT_EQ("baz.h", deps->nodes[ 1]->path()); |

273 | |

274 | ASSERT_TRUE(log.Recompact(kTestFilename, &err)); |

275 | |

276 | // The in-memory deps graph should still be valid after recompaction. |

277 | deps = log.GetDeps(out); |

278 | ASSERT_TRUE(deps); |

279 | ASSERT_EQ(1, deps->mtime); |

280 | ASSERT_EQ(1, deps->node_count); |

281 | ASSERT_EQ("foo.h", deps->nodes[ 0]->path()); |

282 | ASSERT_EQ(out, log.nodes()[out->id()]); |

283 | |

284 | deps = log.GetDeps(other_out); |

285 | ASSERT_TRUE(deps); |

286 | ASSERT_EQ(1, deps->mtime); |

287 | ASSERT_EQ(2, deps->node_count); |

288 | ASSERT_EQ("foo.h", deps->nodes[ 0]->path()); |

289 | ASSERT_EQ("baz.h", deps->nodes[ 1]->path()); |

290 | ASSERT_EQ(other_out, log.nodes()[other_out->id()]); |

291 | |

292 | // The file should have shrunk a bit for the smaller deps. |

293 | #ifdef __USE_LARGEFILE64 |

294 | struct stat64 st; |

295 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |

296 | #else |

297 | struct stat st; |

298 | ASSERT_EQ(0, stat(kTestFilename, &st)); |

299 | #endif |

300 | file_size_3 = (int)st.st_size; |

301 | ASSERT_LT(file_size_3, file_size_2); |

302 | } |

303 | |

304 | // Now reload the file and recompact with an empty manifest. The previous |

305 | // entries should be removed. |

306 | { |

307 | State state; |

308 | // Intentionally not parsing kManifest here. |

309 | DepsLog log; |

310 | string err; |

311 | ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); |

312 | |

313 | Node* out = state.GetNode("out.o", 0); |

314 | DepsLog::Deps* deps = log.GetDeps(out); |

315 | ASSERT_TRUE(deps); |

316 | ASSERT_EQ(1, deps->mtime); |

317 | ASSERT_EQ(1, deps->node_count); |

318 | ASSERT_EQ("foo.h", deps->nodes[ 0]->path()); |

319 | |

320 | Node* other_out = state.GetNode("other_out.o", 0); |

321 | deps = log.GetDeps(other_out); |

322 | ASSERT_TRUE(deps); |

323 | ASSERT_EQ(1, deps->mtime); |

324 | ASSERT_EQ(2, deps->node_count); |

325 | ASSERT_EQ("foo.h", deps->nodes[ 0]->path()); |

326 | ASSERT_EQ("baz.h", deps->nodes[ 1]->path()); |

327 | |

328 | ASSERT_TRUE(log.Recompact(kTestFilename, &err)); |

329 | |

330 | // The previous entries should have been removed. |

331 | deps = log.GetDeps(out); |

332 | ASSERT_FALSE(deps); |

333 | |

334 | deps = log.GetDeps(other_out); |

335 | ASSERT_FALSE(deps); |

336 | |

337 | // The .h files pulled in via deps should no longer have ids either. |

338 | ASSERT_EQ(-1, state.LookupNode("foo.h")->id()); |

339 | ASSERT_EQ(-1, state.LookupNode("baz.h")->id()); |

340 | |

341 | // The file should have shrunk more. |

342 | #ifdef __USE_LARGEFILE64 |

343 | struct stat64 st; |

344 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |

345 | #else |

346 | struct stat st; |

347 | ASSERT_EQ(0, stat(kTestFilename, &st)); |

348 | #endif |

349 | int file_size_4 = (int)st.st_size; |

350 | ASSERT_LT(file_size_4, file_size_3); |

351 | } |

352 | } |

353 | |

354 | // Verify that invalid file headers cause a new build. |

355 | TEST_F(DepsLogTest, InvalidHeader) { |

356 | const char *kInvalidHeaders[] = { |

357 | "", // Empty file. |

358 | "# ninjad", // Truncated first line. |

359 | "# ninjadeps\n", // No version int. |

360 | "# ninjadeps\n\001\002", // Truncated version int. |

361 | "# ninjadeps\n\001\002\003\004" // Invalid version int. |

362 | }; |

363 | for (size_t i = 0; i < sizeof(kInvalidHeaders) / sizeof(kInvalidHeaders[0]); |

364 | ++i) { |

365 | FILE* deps_log = fopen(kTestFilename, "wb"); |

366 | ASSERT_TRUE(deps_log != NULL); |

367 | ASSERT_EQ( |

368 | strlen(kInvalidHeaders[i]), |

369 | fwrite(kInvalidHeaders[i], 1, strlen(kInvalidHeaders[i]), deps_log)); |

370 | ASSERT_EQ(0 ,fclose(deps_log)); |

371 | |

372 | string err; |

373 | DepsLog log; |

374 | State state; |

375 | ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); |

376 | EXPECT_EQ("bad deps log signature or version; starting over", err); |

377 | } |

378 | } |

379 | |

380 | // Simulate what happens when loading a truncated log file. |

381 | TEST_F(DepsLogTest, Truncated) { |

382 | // Create a file with some entries. |

383 | { |

384 | State state; |

385 | DepsLog log; |

386 | string err; |

387 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |

388 | ASSERT_EQ("", err); |

389 | |

390 | vector<Node*> deps; |

391 | deps.push_back(state.GetNode("foo.h", 0)); |

392 | deps.push_back(state.GetNode("bar.h", 0)); |

393 | log.RecordDeps(state.GetNode("out.o", 0), 1, deps); |

394 | |

395 | deps.clear(); |

396 | deps.push_back(state.GetNode("foo.h", 0)); |

397 | deps.push_back(state.GetNode("bar2.h", 0)); |

398 | log.RecordDeps(state.GetNode("out2.o", 0), 2, deps); |

399 | |

400 | log.Close(); |

401 | } |

402 | |

403 | // Get the file size. |

404 | #ifdef __USE_LARGEFILE64 |

405 | struct stat64 st; |

406 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |

407 | #else |

408 | struct stat st; |

409 | ASSERT_EQ(0, stat(kTestFilename, &st)); |

410 | #endif |

411 | |

412 | // Try reloading at truncated sizes. |

413 | // Track how many nodes/deps were found; they should decrease with |

414 | // smaller sizes. |

415 | int node_count = 5; |

416 | int deps_count = 2; |

417 | for (int size = (int)st.st_size; size > 0; --size) { |

418 | string err; |

419 | ASSERT_TRUE(Truncate(kTestFilename, size, &err)); |

420 | |

421 | State state; |

422 | DepsLog log; |

423 | EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); |

424 | if (!err.empty()) { |

425 | // At some point the log will be so short as to be unparsable. |

426 | break; |

427 | } |

428 | |

429 | ASSERT_GE(node_count, (int)log.nodes().size()); |

430 | node_count = log.nodes().size(); |

431 | |

432 | // Count how many non-NULL deps entries there are. |

433 | int new_deps_count = 0; |

434 | for (vector<DepsLog::Deps*>::const_iterator i = log.deps().begin(); |

435 | i != log.deps().end(); ++i) { |

436 | if (*i) |

437 | ++new_deps_count; |

438 | } |

439 | ASSERT_GE(deps_count, new_deps_count); |

440 | deps_count = new_deps_count; |

441 | } |

442 | } |

443 | |

444 | // Run the truncation-recovery logic. |

445 | TEST_F(DepsLogTest, TruncatedRecovery) { |

446 | // Create a file with some entries. |

447 | { |

448 | State state; |

449 | DepsLog log; |

450 | string err; |

451 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |

452 | ASSERT_EQ("", err); |

453 | |

454 | vector<Node*> deps; |

455 | deps.push_back(state.GetNode("foo.h", 0)); |

456 | deps.push_back(state.GetNode("bar.h", 0)); |

457 | log.RecordDeps(state.GetNode("out.o", 0), 1, deps); |

458 | |

459 | deps.clear(); |

460 | deps.push_back(state.GetNode("foo.h", 0)); |

461 | deps.push_back(state.GetNode("bar2.h", 0)); |

462 | log.RecordDeps(state.GetNode("out2.o", 0), 2, deps); |

463 | |

464 | log.Close(); |

465 | } |

466 | |

467 | // Shorten the file, corrupting the last record. |

468 | { |

469 | #ifdef __USE_LARGEFILE64 |

470 | struct stat64 st; |

471 | ASSERT_EQ(0, stat64(kTestFilename, &st)); |

472 | #else |

473 | struct stat st; |

474 | ASSERT_EQ(0, stat(kTestFilename, &st)); |

475 | #endif |

476 | string err; |

477 | ASSERT_TRUE(Truncate(kTestFilename, st.st_size - 2, &err)); |

478 | } |

479 | |

480 | // Load the file again, add an entry. |

481 | { |

482 | State state; |

483 | DepsLog log; |

484 | string err; |

485 | EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); |

486 | ASSERT_EQ("premature end of file; recovering", err); |

487 | err.clear(); |

488 | |

489 | // The truncated entry should've been discarded. |

490 | EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o", 0))); |

491 | |

492 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |

493 | ASSERT_EQ("", err); |

494 | |

495 | // Add a new entry. |

496 | vector<Node*> deps; |

497 | deps.push_back(state.GetNode("foo.h", 0)); |

498 | deps.push_back(state.GetNode("bar2.h", 0)); |

499 | log.RecordDeps(state.GetNode("out2.o", 0), 3, deps); |

500 | |

501 | log.Close(); |

502 | } |

503 | |

504 | // Load the file a third time to verify appending after a mangled |

505 | // entry doesn't break things. |

506 | { |

507 | State state; |

508 | DepsLog log; |

509 | string err; |

510 | EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); |

511 | |

512 | // The truncated entry should exist. |

513 | DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o", 0)); |

514 | ASSERT_TRUE(deps); |

515 | } |

516 | } |

517 | |

518 | TEST_F(DepsLogTest, ReverseDepsNodes) { |

519 | State state; |

520 | DepsLog log; |

521 | string err; |

522 | EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); |

523 | ASSERT_EQ("", err); |

524 | |

525 | vector<Node*> deps; |

526 | deps.push_back(state.GetNode("foo.h", 0)); |

527 | deps.push_back(state.GetNode("bar.h", 0)); |

528 | log.RecordDeps(state.GetNode("out.o", 0), 1, deps); |

529 | |

530 | deps.clear(); |

531 | deps.push_back(state.GetNode("foo.h", 0)); |

532 | deps.push_back(state.GetNode("bar2.h", 0)); |

533 | log.RecordDeps(state.GetNode("out2.o", 0), 2, deps); |

534 | |

535 | log.Close(); |

536 | |

537 | Node* rev_deps = log.GetFirstReverseDepsNode(state.GetNode("foo.h", 0)); |

538 | EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0) || |

539 | rev_deps == state.GetNode("out2.o", 0)); |

540 | |

541 | rev_deps = log.GetFirstReverseDepsNode(state.GetNode("bar.h", 0)); |

542 | EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0)); |

543 | } |

544 | |

545 | } // anonymous namespace |

546 |