THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Git Source Code Mirror - This is a publish-only repository and all pull requests are ignored. Please follow Documentation/SubmittingPatches procedure for any of your improvements.
git
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1296 lines
34KB

  1. #include "cache.h"
  2. #include "config.h"
  3. #include "dir.h"
  4. #include "git-compat-util.h"
  5. #include "lockfile.h"
  6. #include "pack.h"
  7. #include "packfile.h"
  8. #include "commit.h"
  9. #include "object.h"
  10. #include "refs.h"
  11. #include "revision.h"
  12. #include "sha1-lookup.h"
  13. #include "commit-graph.h"
  14. #include "object-store.h"
  15. #include "alloc.h"
  16. #include "hashmap.h"
  17. #include "replace-object.h"
  18. #include "progress.h"
  19. #define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
  20. #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
  21. #define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
  22. #define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
  23. #define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */
  24. #define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16)
  25. #define GRAPH_VERSION_1 0x1
  26. #define GRAPH_VERSION GRAPH_VERSION_1
  27. #define GRAPH_EXTRA_EDGES_NEEDED 0x80000000
  28. #define GRAPH_EDGE_LAST_MASK 0x7fffffff
  29. #define GRAPH_PARENT_NONE 0x70000000
  30. #define GRAPH_LAST_EDGE 0x80000000
  31. #define GRAPH_HEADER_SIZE 8
  32. #define GRAPH_FANOUT_SIZE (4 * 256)
  33. #define GRAPH_CHUNKLOOKUP_WIDTH 12
  34. #define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * GRAPH_CHUNKLOOKUP_WIDTH \
  35. + GRAPH_FANOUT_SIZE + the_hash_algo->rawsz)
  36. char *get_commit_graph_filename(const char *obj_dir)
  37. {
  38. return xstrfmt("%s/info/commit-graph", obj_dir);
  39. }
  40. static uint8_t oid_version(void)
  41. {
  42. return 1;
  43. }
  44. static struct commit_graph *alloc_commit_graph(void)
  45. {
  46. struct commit_graph *g = xcalloc(1, sizeof(*g));
  47. g->graph_fd = -1;
  48. return g;
  49. }
  50. extern int read_replace_refs;
  51. static int commit_graph_compatible(struct repository *r)
  52. {
  53. if (!r->gitdir)
  54. return 0;
  55. if (read_replace_refs) {
  56. prepare_replace_object(r);
  57. if (hashmap_get_size(&r->objects->replace_map->map))
  58. return 0;
  59. }
  60. prepare_commit_graft(r);
  61. if (r->parsed_objects && r->parsed_objects->grafts_nr)
  62. return 0;
  63. if (is_repository_shallow(r))
  64. return 0;
  65. return 1;
  66. }
  67. int open_commit_graph(const char *graph_file, int *fd, struct stat *st)
  68. {
  69. *fd = git_open(graph_file);
  70. if (*fd < 0)
  71. return 0;
  72. if (fstat(*fd, st)) {
  73. close(*fd);
  74. return 0;
  75. }
  76. return 1;
  77. }
  78. struct commit_graph *load_commit_graph_one_fd_st(const char *graph_file,
  79. int fd, struct stat *st)
  80. {
  81. void *graph_map;
  82. size_t graph_size;
  83. struct commit_graph *ret;
  84. graph_size = xsize_t(st->st_size);
  85. if (graph_size < GRAPH_MIN_SIZE) {
  86. close(fd);
  87. error(_("graph file %s is too small"), graph_file);
  88. return NULL;
  89. }
  90. graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0);
  91. ret = parse_commit_graph(graph_map, fd, graph_size);
  92. if (!ret) {
  93. munmap(graph_map, graph_size);
  94. close(fd);
  95. }
  96. return ret;
  97. }
  98. static int verify_commit_graph_lite(struct commit_graph *g)
  99. {
  100. /*
  101. * Basic validation shared between parse_commit_graph()
  102. * which'll be called every time the graph is used, and the
  103. * much more expensive verify_commit_graph() used by
  104. * "commit-graph verify".
  105. *
  106. * There should only be very basic checks here to ensure that
  107. * we don't e.g. segfault in fill_commit_in_graph(), but
  108. * because this is a very hot codepath nothing that e.g. loops
  109. * over g->num_commits, or runs a checksum on the commit-graph
  110. * itself.
  111. */
  112. if (!g->chunk_oid_fanout) {
  113. error("commit-graph is missing the OID Fanout chunk");
  114. return 1;
  115. }
  116. if (!g->chunk_oid_lookup) {
  117. error("commit-graph is missing the OID Lookup chunk");
  118. return 1;
  119. }
  120. if (!g->chunk_commit_data) {
  121. error("commit-graph is missing the Commit Data chunk");
  122. return 1;
  123. }
  124. return 0;
  125. }
  126. struct commit_graph *parse_commit_graph(void *graph_map, int fd,
  127. size_t graph_size)
  128. {
  129. const unsigned char *data, *chunk_lookup;
  130. uint32_t i;
  131. struct commit_graph *graph;
  132. uint64_t last_chunk_offset;
  133. uint32_t last_chunk_id;
  134. uint32_t graph_signature;
  135. unsigned char graph_version, hash_version;
  136. if (!graph_map)
  137. return NULL;
  138. if (graph_size < GRAPH_MIN_SIZE)
  139. return NULL;
  140. data = (const unsigned char *)graph_map;
  141. graph_signature = get_be32(data);
  142. if (graph_signature != GRAPH_SIGNATURE) {
  143. error(_("graph signature %X does not match signature %X"),
  144. graph_signature, GRAPH_SIGNATURE);
  145. return NULL;
  146. }
  147. graph_version = *(unsigned char*)(data + 4);
  148. if (graph_version != GRAPH_VERSION) {
  149. error(_("graph version %X does not match version %X"),
  150. graph_version, GRAPH_VERSION);
  151. return NULL;
  152. }
  153. hash_version = *(unsigned char*)(data + 5);
  154. if (hash_version != oid_version()) {
  155. error(_("hash version %X does not match version %X"),
  156. hash_version, oid_version());
  157. return NULL;
  158. }
  159. graph = alloc_commit_graph();
  160. graph->hash_len = the_hash_algo->rawsz;
  161. graph->num_chunks = *(unsigned char*)(data + 6);
  162. graph->graph_fd = fd;
  163. graph->data = graph_map;
  164. graph->data_len = graph_size;
  165. last_chunk_id = 0;
  166. last_chunk_offset = 8;
  167. chunk_lookup = data + 8;
  168. for (i = 0; i < graph->num_chunks; i++) {
  169. uint32_t chunk_id;
  170. uint64_t chunk_offset;
  171. int chunk_repeated = 0;
  172. if (data + graph_size - chunk_lookup <
  173. GRAPH_CHUNKLOOKUP_WIDTH) {
  174. error(_("chunk lookup table entry missing; graph file may be incomplete"));
  175. free(graph);
  176. return NULL;
  177. }
  178. chunk_id = get_be32(chunk_lookup + 0);
  179. chunk_offset = get_be64(chunk_lookup + 4);
  180. chunk_lookup += GRAPH_CHUNKLOOKUP_WIDTH;
  181. if (chunk_offset > graph_size - the_hash_algo->rawsz) {
  182. error(_("improper chunk offset %08x%08x"), (uint32_t)(chunk_offset >> 32),
  183. (uint32_t)chunk_offset);
  184. free(graph);
  185. return NULL;
  186. }
  187. switch (chunk_id) {
  188. case GRAPH_CHUNKID_OIDFANOUT:
  189. if (graph->chunk_oid_fanout)
  190. chunk_repeated = 1;
  191. else
  192. graph->chunk_oid_fanout = (uint32_t*)(data + chunk_offset);
  193. break;
  194. case GRAPH_CHUNKID_OIDLOOKUP:
  195. if (graph->chunk_oid_lookup)
  196. chunk_repeated = 1;
  197. else
  198. graph->chunk_oid_lookup = data + chunk_offset;
  199. break;
  200. case GRAPH_CHUNKID_DATA:
  201. if (graph->chunk_commit_data)
  202. chunk_repeated = 1;
  203. else
  204. graph->chunk_commit_data = data + chunk_offset;
  205. break;
  206. case GRAPH_CHUNKID_EXTRAEDGES:
  207. if (graph->chunk_extra_edges)
  208. chunk_repeated = 1;
  209. else
  210. graph->chunk_extra_edges = data + chunk_offset;
  211. break;
  212. }
  213. if (chunk_repeated) {
  214. error(_("chunk id %08x appears multiple times"), chunk_id);
  215. free(graph);
  216. return NULL;
  217. }
  218. if (last_chunk_id == GRAPH_CHUNKID_OIDLOOKUP)
  219. {
  220. graph->num_commits = (chunk_offset - last_chunk_offset)
  221. / graph->hash_len;
  222. }
  223. last_chunk_id = chunk_id;
  224. last_chunk_offset = chunk_offset;
  225. }
  226. if (verify_commit_graph_lite(graph))
  227. return NULL;
  228. return graph;
  229. }
  230. static struct commit_graph *load_commit_graph_one(const char *graph_file)
  231. {
  232. struct stat st;
  233. int fd;
  234. int open_ok = open_commit_graph(graph_file, &fd, &st);
  235. if (!open_ok)
  236. return NULL;
  237. return load_commit_graph_one_fd_st(graph_file, fd, &st);
  238. }
  239. static void prepare_commit_graph_one(struct repository *r, const char *obj_dir)
  240. {
  241. char *graph_name;
  242. if (r->objects->commit_graph)
  243. return;
  244. graph_name = get_commit_graph_filename(obj_dir);
  245. r->objects->commit_graph =
  246. load_commit_graph_one(graph_name);
  247. FREE_AND_NULL(graph_name);
  248. }
  249. /*
  250. * Return 1 if commit_graph is non-NULL, and 0 otherwise.
  251. *
  252. * On the first invocation, this function attemps to load the commit
  253. * graph if the_repository is configured to have one.
  254. */
  255. static int prepare_commit_graph(struct repository *r)
  256. {
  257. struct object_directory *odb;
  258. int config_value;
  259. if (r->objects->commit_graph_attempted)
  260. return !!r->objects->commit_graph;
  261. r->objects->commit_graph_attempted = 1;
  262. if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) &&
  263. (repo_config_get_bool(r, "core.commitgraph", &config_value) ||
  264. !config_value))
  265. /*
  266. * This repository is not configured to use commit graphs, so
  267. * do not load one. (But report commit_graph_attempted anyway
  268. * so that commit graph loading is not attempted again for this
  269. * repository.)
  270. */
  271. return 0;
  272. if (!commit_graph_compatible(r))
  273. return 0;
  274. prepare_alt_odb(r);
  275. for (odb = r->objects->odb;
  276. !r->objects->commit_graph && odb;
  277. odb = odb->next)
  278. prepare_commit_graph_one(r, odb->path);
  279. return !!r->objects->commit_graph;
  280. }
  281. int generation_numbers_enabled(struct repository *r)
  282. {
  283. uint32_t first_generation;
  284. struct commit_graph *g;
  285. if (!prepare_commit_graph(r))
  286. return 0;
  287. g = r->objects->commit_graph;
  288. if (!g->num_commits)
  289. return 0;
  290. first_generation = get_be32(g->chunk_commit_data +
  291. g->hash_len + 8) >> 2;
  292. return !!first_generation;
  293. }
  294. void close_commit_graph(struct repository *r)
  295. {
  296. free_commit_graph(r->objects->commit_graph);
  297. r->objects->commit_graph = NULL;
  298. }
  299. static int bsearch_graph(struct commit_graph *g, struct object_id *oid, uint32_t *pos)
  300. {
  301. return bsearch_hash(oid->hash, g->chunk_oid_fanout,
  302. g->chunk_oid_lookup, g->hash_len, pos);
  303. }
  304. static struct commit_list **insert_parent_or_die(struct repository *r,
  305. struct commit_graph *g,
  306. uint64_t pos,
  307. struct commit_list **pptr)
  308. {
  309. struct commit *c;
  310. struct object_id oid;
  311. if (pos >= g->num_commits)
  312. die("invalid parent position %"PRIu64, pos);
  313. hashcpy(oid.hash, g->chunk_oid_lookup + g->hash_len * pos);
  314. c = lookup_commit(r, &oid);
  315. if (!c)
  316. die(_("could not find commit %s"), oid_to_hex(&oid));
  317. c->graph_pos = pos;
  318. return &commit_list_insert(c, pptr)->next;
  319. }
  320. static void fill_commit_graph_info(struct commit *item, struct commit_graph *g, uint32_t pos)
  321. {
  322. const unsigned char *commit_data = g->chunk_commit_data + GRAPH_DATA_WIDTH * pos;
  323. item->graph_pos = pos;
  324. item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
  325. }
  326. static int fill_commit_in_graph(struct repository *r,
  327. struct commit *item,
  328. struct commit_graph *g, uint32_t pos)
  329. {
  330. uint32_t edge_value;
  331. uint32_t *parent_data_ptr;
  332. uint64_t date_low, date_high;
  333. struct commit_list **pptr;
  334. const unsigned char *commit_data = g->chunk_commit_data + (g->hash_len + 16) * pos;
  335. item->object.parsed = 1;
  336. item->graph_pos = pos;
  337. item->maybe_tree = NULL;
  338. date_high = get_be32(commit_data + g->hash_len + 8) & 0x3;
  339. date_low = get_be32(commit_data + g->hash_len + 12);
  340. item->date = (timestamp_t)((date_high << 32) | date_low);
  341. item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
  342. pptr = &item->parents;
  343. edge_value = get_be32(commit_data + g->hash_len);
  344. if (edge_value == GRAPH_PARENT_NONE)
  345. return 1;
  346. pptr = insert_parent_or_die(r, g, edge_value, pptr);
  347. edge_value = get_be32(commit_data + g->hash_len + 4);
  348. if (edge_value == GRAPH_PARENT_NONE)
  349. return 1;
  350. if (!(edge_value & GRAPH_EXTRA_EDGES_NEEDED)) {
  351. pptr = insert_parent_or_die(r, g, edge_value, pptr);
  352. return 1;
  353. }
  354. parent_data_ptr = (uint32_t*)(g->chunk_extra_edges +
  355. 4 * (uint64_t)(edge_value & GRAPH_EDGE_LAST_MASK));
  356. do {
  357. edge_value = get_be32(parent_data_ptr);
  358. pptr = insert_parent_or_die(r, g,
  359. edge_value & GRAPH_EDGE_LAST_MASK,
  360. pptr);
  361. parent_data_ptr++;
  362. } while (!(edge_value & GRAPH_LAST_EDGE));
  363. return 1;
  364. }
  365. static int find_commit_in_graph(struct commit *item, struct commit_graph *g, uint32_t *pos)
  366. {
  367. if (item->graph_pos != COMMIT_NOT_FROM_GRAPH) {
  368. *pos = item->graph_pos;
  369. return 1;
  370. } else {
  371. return bsearch_graph(g, &(item->object.oid), pos);
  372. }
  373. }
  374. static int parse_commit_in_graph_one(struct repository *r,
  375. struct commit_graph *g,
  376. struct commit *item)
  377. {
  378. uint32_t pos;
  379. if (item->object.parsed)
  380. return 1;
  381. if (find_commit_in_graph(item, g, &pos))
  382. return fill_commit_in_graph(r, item, g, pos);
  383. return 0;
  384. }
  385. int parse_commit_in_graph(struct repository *r, struct commit *item)
  386. {
  387. if (!prepare_commit_graph(r))
  388. return 0;
  389. return parse_commit_in_graph_one(r, r->objects->commit_graph, item);
  390. }
  391. void load_commit_graph_info(struct repository *r, struct commit *item)
  392. {
  393. uint32_t pos;
  394. if (!prepare_commit_graph(r))
  395. return;
  396. if (find_commit_in_graph(item, r->objects->commit_graph, &pos))
  397. fill_commit_graph_info(item, r->objects->commit_graph, pos);
  398. }
  399. static struct tree *load_tree_for_commit(struct repository *r,
  400. struct commit_graph *g,
  401. struct commit *c)
  402. {
  403. struct object_id oid;
  404. const unsigned char *commit_data = g->chunk_commit_data +
  405. GRAPH_DATA_WIDTH * (c->graph_pos);
  406. hashcpy(oid.hash, commit_data);
  407. c->maybe_tree = lookup_tree(r, &oid);
  408. return c->maybe_tree;
  409. }
  410. static struct tree *get_commit_tree_in_graph_one(struct repository *r,
  411. struct commit_graph *g,
  412. const struct commit *c)
  413. {
  414. if (c->maybe_tree)
  415. return c->maybe_tree;
  416. if (c->graph_pos == COMMIT_NOT_FROM_GRAPH)
  417. BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
  418. return load_tree_for_commit(r, g, (struct commit *)c);
  419. }
  420. struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit *c)
  421. {
  422. return get_commit_tree_in_graph_one(r, r->objects->commit_graph, c);
  423. }
  424. static void write_graph_chunk_fanout(struct hashfile *f,
  425. struct commit **commits,
  426. int nr_commits,
  427. struct progress *progress,
  428. uint64_t *progress_cnt)
  429. {
  430. int i, count = 0;
  431. struct commit **list = commits;
  432. /*
  433. * Write the first-level table (the list is sorted,
  434. * but we use a 256-entry lookup to be able to avoid
  435. * having to do eight extra binary search iterations).
  436. */
  437. for (i = 0; i < 256; i++) {
  438. while (count < nr_commits) {
  439. if ((*list)->object.oid.hash[0] != i)
  440. break;
  441. display_progress(progress, ++*progress_cnt);
  442. count++;
  443. list++;
  444. }
  445. hashwrite_be32(f, count);
  446. }
  447. }
  448. static void write_graph_chunk_oids(struct hashfile *f, int hash_len,
  449. struct commit **commits, int nr_commits,
  450. struct progress *progress,
  451. uint64_t *progress_cnt)
  452. {
  453. struct commit **list = commits;
  454. int count;
  455. for (count = 0; count < nr_commits; count++, list++) {
  456. display_progress(progress, ++*progress_cnt);
  457. hashwrite(f, (*list)->object.oid.hash, (int)hash_len);
  458. }
  459. }
  460. static const unsigned char *commit_to_sha1(size_t index, void *table)
  461. {
  462. struct commit **commits = table;
  463. return commits[index]->object.oid.hash;
  464. }
  465. static void write_graph_chunk_data(struct hashfile *f, int hash_len,
  466. struct commit **commits, int nr_commits,
  467. struct progress *progress,
  468. uint64_t *progress_cnt)
  469. {
  470. struct commit **list = commits;
  471. struct commit **last = commits + nr_commits;
  472. uint32_t num_extra_edges = 0;
  473. while (list < last) {
  474. struct commit_list *parent;
  475. int edge_value;
  476. uint32_t packedDate[2];
  477. display_progress(progress, ++*progress_cnt);
  478. parse_commit(*list);
  479. hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len);
  480. parent = (*list)->parents;
  481. if (!parent)
  482. edge_value = GRAPH_PARENT_NONE;
  483. else {
  484. edge_value = sha1_pos(parent->item->object.oid.hash,
  485. commits,
  486. nr_commits,
  487. commit_to_sha1);
  488. if (edge_value < 0)
  489. BUG("missing parent %s for commit %s",
  490. oid_to_hex(&parent->item->object.oid),
  491. oid_to_hex(&(*list)->object.oid));
  492. }
  493. hashwrite_be32(f, edge_value);
  494. if (parent)
  495. parent = parent->next;
  496. if (!parent)
  497. edge_value = GRAPH_PARENT_NONE;
  498. else if (parent->next)
  499. edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges;
  500. else {
  501. edge_value = sha1_pos(parent->item->object.oid.hash,
  502. commits,
  503. nr_commits,
  504. commit_to_sha1);
  505. if (edge_value < 0)
  506. BUG("missing parent %s for commit %s",
  507. oid_to_hex(&parent->item->object.oid),
  508. oid_to_hex(&(*list)->object.oid));
  509. }
  510. hashwrite_be32(f, edge_value);
  511. if (edge_value & GRAPH_EXTRA_EDGES_NEEDED) {
  512. do {
  513. num_extra_edges++;
  514. parent = parent->next;
  515. } while (parent);
  516. }
  517. if (sizeof((*list)->date) > 4)
  518. packedDate[0] = htonl(((*list)->date >> 32) & 0x3);
  519. else
  520. packedDate[0] = 0;
  521. packedDate[0] |= htonl((*list)->generation << 2);
  522. packedDate[1] = htonl((*list)->date);
  523. hashwrite(f, packedDate, 8);
  524. list++;
  525. }
  526. }
  527. static void write_graph_chunk_extra_edges(struct hashfile *f,
  528. struct commit **commits,
  529. int nr_commits,
  530. struct progress *progress,
  531. uint64_t *progress_cnt)
  532. {
  533. struct commit **list = commits;
  534. struct commit **last = commits + nr_commits;
  535. struct commit_list *parent;
  536. while (list < last) {
  537. int num_parents = 0;
  538. display_progress(progress, ++*progress_cnt);
  539. for (parent = (*list)->parents; num_parents < 3 && parent;
  540. parent = parent->next)
  541. num_parents++;
  542. if (num_parents <= 2) {
  543. list++;
  544. continue;
  545. }
  546. /* Since num_parents > 2, this initializer is safe. */
  547. for (parent = (*list)->parents->next; parent; parent = parent->next) {
  548. int edge_value = sha1_pos(parent->item->object.oid.hash,
  549. commits,
  550. nr_commits,
  551. commit_to_sha1);
  552. if (edge_value < 0)
  553. BUG("missing parent %s for commit %s",
  554. oid_to_hex(&parent->item->object.oid),
  555. oid_to_hex(&(*list)->object.oid));
  556. else if (!parent->next)
  557. edge_value |= GRAPH_LAST_EDGE;
  558. hashwrite_be32(f, edge_value);
  559. }
  560. list++;
  561. }
  562. }
  563. static int commit_compare(const void *_a, const void *_b)
  564. {
  565. const struct object_id *a = (const struct object_id *)_a;
  566. const struct object_id *b = (const struct object_id *)_b;
  567. return oidcmp(a, b);
  568. }
  569. struct packed_commit_list {
  570. struct commit **list;
  571. int nr;
  572. int alloc;
  573. };
  574. struct packed_oid_list {
  575. struct object_id *list;
  576. int nr;
  577. int alloc;
  578. struct progress *progress;
  579. int progress_done;
  580. };
  581. static int add_packed_commits(const struct object_id *oid,
  582. struct packed_git *pack,
  583. uint32_t pos,
  584. void *data)
  585. {
  586. struct packed_oid_list *list = (struct packed_oid_list*)data;
  587. enum object_type type;
  588. off_t offset = nth_packed_object_offset(pack, pos);
  589. struct object_info oi = OBJECT_INFO_INIT;
  590. if (list->progress)
  591. display_progress(list->progress, ++list->progress_done);
  592. oi.typep = &type;
  593. if (packed_object_info(the_repository, pack, offset, &oi) < 0)
  594. die(_("unable to get type of object %s"), oid_to_hex(oid));
  595. if (type != OBJ_COMMIT)
  596. return 0;
  597. ALLOC_GROW(list->list, list->nr + 1, list->alloc);
  598. oidcpy(&(list->list[list->nr]), oid);
  599. list->nr++;
  600. return 0;
  601. }
  602. static void add_missing_parents(struct packed_oid_list *oids, struct commit *commit)
  603. {
  604. struct commit_list *parent;
  605. for (parent = commit->parents; parent; parent = parent->next) {
  606. if (!(parent->item->object.flags & UNINTERESTING)) {
  607. ALLOC_GROW(oids->list, oids->nr + 1, oids->alloc);
  608. oidcpy(&oids->list[oids->nr], &(parent->item->object.oid));
  609. oids->nr++;
  610. parent->item->object.flags |= UNINTERESTING;
  611. }
  612. }
  613. }
  614. static void close_reachable(struct packed_oid_list *oids, int report_progress)
  615. {
  616. int i;
  617. struct commit *commit;
  618. struct progress *progress = NULL;
  619. if (report_progress)
  620. progress = start_delayed_progress(
  621. _("Loading known commits in commit graph"), oids->nr);
  622. for (i = 0; i < oids->nr; i++) {
  623. display_progress(progress, i + 1);
  624. commit = lookup_commit(the_repository, &oids->list[i]);
  625. if (commit)
  626. commit->object.flags |= UNINTERESTING;
  627. }
  628. stop_progress(&progress);
  629. /*
  630. * As this loop runs, oids->nr may grow, but not more
  631. * than the number of missing commits in the reachable
  632. * closure.
  633. */
  634. if (report_progress)
  635. progress = start_delayed_progress(
  636. _("Expanding reachable commits in commit graph"), oids->nr);
  637. for (i = 0; i < oids->nr; i++) {
  638. display_progress(progress, i + 1);
  639. commit = lookup_commit(the_repository, &oids->list[i]);
  640. if (commit && !parse_commit(commit))
  641. add_missing_parents(oids, commit);
  642. }
  643. stop_progress(&progress);
  644. if (report_progress)
  645. progress = start_delayed_progress(
  646. _("Clearing commit marks in commit graph"), oids->nr);
  647. for (i = 0; i < oids->nr; i++) {
  648. display_progress(progress, i + 1);
  649. commit = lookup_commit(the_repository, &oids->list[i]);
  650. if (commit)
  651. commit->object.flags &= ~UNINTERESTING;
  652. }
  653. stop_progress(&progress);
  654. }
  655. static void compute_generation_numbers(struct packed_commit_list* commits,
  656. int report_progress)
  657. {
  658. int i;
  659. struct commit_list *list = NULL;
  660. struct progress *progress = NULL;
  661. if (report_progress)
  662. progress = start_progress(
  663. _("Computing commit graph generation numbers"),
  664. commits->nr);
  665. for (i = 0; i < commits->nr; i++) {
  666. display_progress(progress, i + 1);
  667. if (commits->list[i]->generation != GENERATION_NUMBER_INFINITY &&
  668. commits->list[i]->generation != GENERATION_NUMBER_ZERO)
  669. continue;
  670. commit_list_insert(commits->list[i], &list);
  671. while (list) {
  672. struct commit *current = list->item;
  673. struct commit_list *parent;
  674. int all_parents_computed = 1;
  675. uint32_t max_generation = 0;
  676. for (parent = current->parents; parent; parent = parent->next) {
  677. if (parent->item->generation == GENERATION_NUMBER_INFINITY ||
  678. parent->item->generation == GENERATION_NUMBER_ZERO) {
  679. all_parents_computed = 0;
  680. commit_list_insert(parent->item, &list);
  681. break;
  682. } else if (parent->item->generation > max_generation) {
  683. max_generation = parent->item->generation;
  684. }
  685. }
  686. if (all_parents_computed) {
  687. current->generation = max_generation + 1;
  688. pop_commit(&list);
  689. if (current->generation > GENERATION_NUMBER_MAX)
  690. current->generation = GENERATION_NUMBER_MAX;
  691. }
  692. }
  693. }
  694. stop_progress(&progress);
  695. }
  696. static int add_ref_to_list(const char *refname,
  697. const struct object_id *oid,
  698. int flags, void *cb_data)
  699. {
  700. struct string_list *list = (struct string_list *)cb_data;
  701. string_list_append(list, oid_to_hex(oid));
  702. return 0;
  703. }
  704. void write_commit_graph_reachable(const char *obj_dir, int append,
  705. int report_progress)
  706. {
  707. struct string_list list = STRING_LIST_INIT_DUP;
  708. for_each_ref(add_ref_to_list, &list);
  709. write_commit_graph(obj_dir, NULL, &list, append, report_progress);
  710. string_list_clear(&list, 0);
  711. }
  712. void write_commit_graph(const char *obj_dir,
  713. struct string_list *pack_indexes,
  714. struct string_list *commit_hex,
  715. int append, int report_progress)
  716. {
  717. struct packed_oid_list oids;
  718. struct packed_commit_list commits;
  719. struct hashfile *f;
  720. uint32_t i, count_distinct = 0;
  721. char *graph_name;
  722. struct lock_file lk = LOCK_INIT;
  723. uint32_t chunk_ids[5];
  724. uint64_t chunk_offsets[5];
  725. int num_chunks;
  726. int num_extra_edges;
  727. struct commit_list *parent;
  728. struct progress *progress = NULL;
  729. const unsigned hashsz = the_hash_algo->rawsz;
  730. uint64_t progress_cnt = 0;
  731. struct strbuf progress_title = STRBUF_INIT;
  732. unsigned long approx_nr_objects;
  733. if (!commit_graph_compatible(the_repository))
  734. return;
  735. oids.nr = 0;
  736. approx_nr_objects = approximate_object_count();
  737. oids.alloc = approx_nr_objects / 32;
  738. oids.progress = NULL;
  739. oids.progress_done = 0;
  740. if (append) {
  741. prepare_commit_graph_one(the_repository, obj_dir);
  742. if (the_repository->objects->commit_graph)
  743. oids.alloc += the_repository->objects->commit_graph->num_commits;
  744. }
  745. if (oids.alloc < 1024)
  746. oids.alloc = 1024;
  747. ALLOC_ARRAY(oids.list, oids.alloc);
  748. if (append && the_repository->objects->commit_graph) {
  749. struct commit_graph *commit_graph =
  750. the_repository->objects->commit_graph;
  751. for (i = 0; i < commit_graph->num_commits; i++) {
  752. const unsigned char *hash = commit_graph->chunk_oid_lookup +
  753. commit_graph->hash_len * i;
  754. hashcpy(oids.list[oids.nr++].hash, hash);
  755. }
  756. }
  757. if (pack_indexes) {
  758. struct strbuf packname = STRBUF_INIT;
  759. int dirlen;
  760. strbuf_addf(&packname, "%s/pack/", obj_dir);
  761. dirlen = packname.len;
  762. if (report_progress) {
  763. strbuf_addf(&progress_title,
  764. Q_("Finding commits for commit graph in %d pack",
  765. "Finding commits for commit graph in %d packs",
  766. pack_indexes->nr),
  767. pack_indexes->nr);
  768. oids.progress = start_delayed_progress(progress_title.buf, 0);
  769. oids.progress_done = 0;
  770. }
  771. for (i = 0; i < pack_indexes->nr; i++) {
  772. struct packed_git *p;
  773. strbuf_setlen(&packname, dirlen);
  774. strbuf_addstr(&packname, pack_indexes->items[i].string);
  775. p = add_packed_git(packname.buf, packname.len, 1);
  776. if (!p)
  777. die(_("error adding pack %s"), packname.buf);
  778. if (open_pack_index(p))
  779. die(_("error opening index for %s"), packname.buf);
  780. for_each_object_in_pack(p, add_packed_commits, &oids,
  781. FOR_EACH_OBJECT_PACK_ORDER);
  782. close_pack(p);
  783. free(p);
  784. }
  785. stop_progress(&oids.progress);
  786. strbuf_reset(&progress_title);
  787. strbuf_release(&packname);
  788. }
  789. if (commit_hex) {
  790. if (report_progress) {
  791. strbuf_addf(&progress_title,
  792. Q_("Finding commits for commit graph from %d ref",
  793. "Finding commits for commit graph from %d refs",
  794. commit_hex->nr),
  795. commit_hex->nr);
  796. progress = start_delayed_progress(progress_title.buf,
  797. commit_hex->nr);
  798. }
  799. for (i = 0; i < commit_hex->nr; i++) {
  800. const char *end;
  801. struct object_id oid;
  802. struct commit *result;
  803. display_progress(progress, i + 1);
  804. if (commit_hex->items[i].string &&
  805. parse_oid_hex(commit_hex->items[i].string, &oid, &end))
  806. continue;
  807. result = lookup_commit_reference_gently(the_repository, &oid, 1);
  808. if (result) {
  809. ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc);
  810. oidcpy(&oids.list[oids.nr], &(result->object.oid));
  811. oids.nr++;
  812. }
  813. }
  814. stop_progress(&progress);
  815. strbuf_reset(&progress_title);
  816. }
  817. if (!pack_indexes && !commit_hex) {
  818. if (report_progress)
  819. oids.progress = start_delayed_progress(
  820. _("Finding commits for commit graph among packed objects"),
  821. approx_nr_objects);
  822. for_each_packed_object(add_packed_commits, &oids,
  823. FOR_EACH_OBJECT_PACK_ORDER);
  824. if (oids.progress_done < approx_nr_objects)
  825. display_progress(oids.progress, approx_nr_objects);
  826. stop_progress(&oids.progress);
  827. }
  828. close_reachable(&oids, report_progress);
  829. if (report_progress)
  830. progress = start_delayed_progress(
  831. _("Counting distinct commits in commit graph"),
  832. oids.nr);
  833. display_progress(progress, 0); /* TODO: Measure QSORT() progress */
  834. QSORT(oids.list, oids.nr, commit_compare);
  835. count_distinct = 1;
  836. for (i = 1; i < oids.nr; i++) {
  837. display_progress(progress, i + 1);
  838. if (!oideq(&oids.list[i - 1], &oids.list[i]))
  839. count_distinct++;
  840. }
  841. stop_progress(&progress);
  842. if (count_distinct >= GRAPH_EDGE_LAST_MASK)
  843. die(_("the commit graph format cannot write %d commits"), count_distinct);
  844. commits.nr = 0;
  845. commits.alloc = count_distinct;
  846. ALLOC_ARRAY(commits.list, commits.alloc);
  847. num_extra_edges = 0;
  848. if (report_progress)
  849. progress = start_delayed_progress(
  850. _("Finding extra edges in commit graph"),
  851. oids.nr);
  852. for (i = 0; i < oids.nr; i++) {
  853. int num_parents = 0;
  854. display_progress(progress, i + 1);
  855. if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i]))
  856. continue;
  857. commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]);
  858. parse_commit(commits.list[commits.nr]);
  859. for (parent = commits.list[commits.nr]->parents;
  860. parent; parent = parent->next)
  861. num_parents++;
  862. if (num_parents > 2)
  863. num_extra_edges += num_parents - 1;
  864. commits.nr++;
  865. }
  866. num_chunks = num_extra_edges ? 4 : 3;
  867. stop_progress(&progress);
  868. if (commits.nr >= GRAPH_EDGE_LAST_MASK)
  869. die(_("too many commits to write graph"));
  870. compute_generation_numbers(&commits, report_progress);
  871. graph_name = get_commit_graph_filename(obj_dir);
  872. if (safe_create_leading_directories(graph_name)) {
  873. UNLEAK(graph_name);
  874. die_errno(_("unable to create leading directories of %s"),
  875. graph_name);
  876. }
  877. hold_lock_file_for_update(&lk, graph_name, LOCK_DIE_ON_ERROR);
  878. f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);
  879. hashwrite_be32(f, GRAPH_SIGNATURE);
  880. hashwrite_u8(f, GRAPH_VERSION);
  881. hashwrite_u8(f, oid_version());
  882. hashwrite_u8(f, num_chunks);
  883. hashwrite_u8(f, 0); /* unused padding byte */
  884. chunk_ids[0] = GRAPH_CHUNKID_OIDFANOUT;
  885. chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP;
  886. chunk_ids[2] = GRAPH_CHUNKID_DATA;
  887. if (num_extra_edges)
  888. chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES;
  889. else
  890. chunk_ids[3] = 0;
  891. chunk_ids[4] = 0;
  892. chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH;
  893. chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE;
  894. chunk_offsets[2] = chunk_offsets[1] + hashsz * commits.nr;
  895. chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * commits.nr;
  896. chunk_offsets[4] = chunk_offsets[3] + 4 * num_extra_edges;
  897. for (i = 0; i <= num_chunks; i++) {
  898. uint32_t chunk_write[3];
  899. chunk_write[0] = htonl(chunk_ids[i]);
  900. chunk_write[1] = htonl(chunk_offsets[i] >> 32);
  901. chunk_write[2] = htonl(chunk_offsets[i] & 0xffffffff);
  902. hashwrite(f, chunk_write, 12);
  903. }
  904. if (report_progress) {
  905. strbuf_addf(&progress_title,
  906. Q_("Writing out commit graph in %d pass",
  907. "Writing out commit graph in %d passes",
  908. num_chunks),
  909. num_chunks);
  910. progress = start_delayed_progress(
  911. progress_title.buf,
  912. num_chunks * commits.nr);
  913. }
  914. write_graph_chunk_fanout(f, commits.list, commits.nr, progress, &progress_cnt);
  915. write_graph_chunk_oids(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
  916. write_graph_chunk_data(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
  917. if (num_extra_edges)
  918. write_graph_chunk_extra_edges(f, commits.list, commits.nr, progress, &progress_cnt);
  919. stop_progress(&progress);
  920. strbuf_release(&progress_title);
  921. close_commit_graph(the_repository);
  922. finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC);
  923. commit_lock_file(&lk);
  924. free(graph_name);
  925. free(commits.list);
  926. free(oids.list);
  927. }
  928. #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2
  929. static int verify_commit_graph_error;
  930. static void graph_report(const char *fmt, ...)
  931. {
  932. va_list ap;
  933. verify_commit_graph_error = 1;
  934. va_start(ap, fmt);
  935. vfprintf(stderr, fmt, ap);
  936. fprintf(stderr, "\n");
  937. va_end(ap);
  938. }
  939. #define GENERATION_ZERO_EXISTS 1
  940. #define GENERATION_NUMBER_EXISTS 2
  941. int verify_commit_graph(struct repository *r, struct commit_graph *g)
  942. {
  943. uint32_t i, cur_fanout_pos = 0;
  944. struct object_id prev_oid, cur_oid, checksum;
  945. int generation_zero = 0;
  946. struct hashfile *f;
  947. int devnull;
  948. struct progress *progress = NULL;
  949. if (!g) {
  950. graph_report("no commit-graph file loaded");
  951. return 1;
  952. }
  953. verify_commit_graph_error = verify_commit_graph_lite(g);
  954. if (verify_commit_graph_error)
  955. return verify_commit_graph_error;
  956. devnull = open("/dev/null", O_WRONLY);
  957. f = hashfd(devnull, NULL);
  958. hashwrite(f, g->data, g->data_len - g->hash_len);
  959. finalize_hashfile(f, checksum.hash, CSUM_CLOSE);
  960. if (!hasheq(checksum.hash, g->data + g->data_len - g->hash_len)) {
  961. graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt"));
  962. verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH;
  963. }
  964. for (i = 0; i < g->num_commits; i++) {
  965. struct commit *graph_commit;
  966. hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
  967. if (i && oidcmp(&prev_oid, &cur_oid) >= 0)
  968. graph_report("commit-graph has incorrect OID order: %s then %s",
  969. oid_to_hex(&prev_oid),
  970. oid_to_hex(&cur_oid));
  971. oidcpy(&prev_oid, &cur_oid);
  972. while (cur_oid.hash[0] > cur_fanout_pos) {
  973. uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
  974. if (i != fanout_value)
  975. graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
  976. cur_fanout_pos, fanout_value, i);
  977. cur_fanout_pos++;
  978. }
  979. graph_commit = lookup_commit(r, &cur_oid);
  980. if (!parse_commit_in_graph_one(r, g, graph_commit))
  981. graph_report("failed to parse %s from commit-graph",
  982. oid_to_hex(&cur_oid));
  983. }
  984. while (cur_fanout_pos < 256) {
  985. uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
  986. if (g->num_commits != fanout_value)
  987. graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
  988. cur_fanout_pos, fanout_value, i);
  989. cur_fanout_pos++;
  990. }
  991. if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH)
  992. return verify_commit_graph_error;
  993. progress = start_progress(_("Verifying commits in commit graph"),
  994. g->num_commits);
  995. for (i = 0; i < g->num_commits; i++) {
  996. struct commit *graph_commit, *odb_commit;
  997. struct commit_list *graph_parents, *odb_parents;
  998. uint32_t max_generation = 0;
  999. display_progress(progress, i + 1);
  1000. hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
  1001. graph_commit = lookup_commit(r, &cur_oid);
  1002. odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r));
  1003. if (parse_commit_internal(odb_commit, 0, 0)) {
  1004. graph_report("failed to parse %s from object database",
  1005. oid_to_hex(&cur_oid));
  1006. continue;
  1007. }
  1008. if (!oideq(&get_commit_tree_in_graph_one(r, g, graph_commit)->object.oid,
  1009. get_commit_tree_oid(odb_commit)))
  1010. graph_report("root tree OID for commit %s in commit-graph is %s != %s",
  1011. oid_to_hex(&cur_oid),
  1012. oid_to_hex(get_commit_tree_oid(graph_commit)),
  1013. oid_to_hex(get_commit_tree_oid(odb_commit)));
  1014. graph_parents = graph_commit->parents;
  1015. odb_parents = odb_commit->parents;
  1016. while (graph_parents) {
  1017. if (odb_parents == NULL) {
  1018. graph_report("commit-graph parent list for commit %s is too long",
  1019. oid_to_hex(&cur_oid));
  1020. break;
  1021. }
  1022. if (!oideq(&graph_parents->item->object.oid, &odb_parents->item->object.oid))
  1023. graph_report("commit-graph parent for %s is %s != %s",
  1024. oid_to_hex(&cur_oid),
  1025. oid_to_hex(&graph_parents->item->object.oid),
  1026. oid_to_hex(&odb_parents->item->object.oid));
  1027. if (graph_parents->item->generation > max_generation)
  1028. max_generation = graph_parents->item->generation;
  1029. graph_parents = graph_parents->next;
  1030. odb_parents = odb_parents->next;
  1031. }
  1032. if (odb_parents != NULL)
  1033. graph_report("commit-graph parent list for commit %s terminates early",
  1034. oid_to_hex(&cur_oid));
  1035. if (!graph_commit->generation) {
  1036. if (generation_zero == GENERATION_NUMBER_EXISTS)
  1037. graph_report("commit-graph has generation number zero for commit %s, but non-zero elsewhere",
  1038. oid_to_hex(&cur_oid));
  1039. generation_zero = GENERATION_ZERO_EXISTS;
  1040. } else if (generation_zero == GENERATION_ZERO_EXISTS)
  1041. graph_report("commit-graph has non-zero generation number for commit %s, but zero elsewhere",
  1042. oid_to_hex(&cur_oid));
  1043. if (generation_zero == GENERATION_ZERO_EXISTS)
  1044. continue;
  1045. /*
  1046. * If one of our parents has generation GENERATION_NUMBER_MAX, then
  1047. * our generation is also GENERATION_NUMBER_MAX. Decrement to avoid
  1048. * extra logic in the following condition.
  1049. */
  1050. if (max_generation == GENERATION_NUMBER_MAX)
  1051. max_generation--;
  1052. if (graph_commit->generation != max_generation + 1)
  1053. graph_report("commit-graph generation for commit %s is %u != %u",
  1054. oid_to_hex(&cur_oid),
  1055. graph_commit->generation,
  1056. max_generation + 1);
  1057. if (graph_commit->date != odb_commit->date)
  1058. graph_report("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime,
  1059. oid_to_hex(&cur_oid),
  1060. graph_commit->date,
  1061. odb_commit->date);
  1062. }
  1063. stop_progress(&progress);
  1064. return verify_commit_graph_error;
  1065. }
  1066. void free_commit_graph(struct commit_graph *g)
  1067. {
  1068. if (!g)
  1069. return;
  1070. if (g->graph_fd >= 0) {
  1071. munmap((void *)g->data, g->data_len);
  1072. g->data = NULL;
  1073. close(g->graph_fd);
  1074. }
  1075. free(g);
  1076. }