THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Git Source Code Mirror - This is a publish-only repository and all pull requests are ignored. Please follow Documentation/SubmittingPatches procedure for any of your improvements.
git
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1301 lines
34KB

  1. #include "cache.h"
  2. #include "config.h"
  3. #include "dir.h"
  4. #include "git-compat-util.h"
  5. #include "lockfile.h"
  6. #include "pack.h"
  7. #include "packfile.h"
  8. #include "commit.h"
  9. #include "object.h"
  10. #include "refs.h"
  11. #include "revision.h"
  12. #include "sha1-lookup.h"
  13. #include "commit-graph.h"
  14. #include "object-store.h"
  15. #include "alloc.h"
  16. #include "hashmap.h"
  17. #include "replace-object.h"
  18. #include "progress.h"
  19. #define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
  20. #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
  21. #define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
  22. #define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
  23. #define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */
  24. #define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16)
  25. #define GRAPH_VERSION_1 0x1
  26. #define GRAPH_VERSION GRAPH_VERSION_1
  27. #define GRAPH_EXTRA_EDGES_NEEDED 0x80000000
  28. #define GRAPH_EDGE_LAST_MASK 0x7fffffff
  29. #define GRAPH_PARENT_NONE 0x70000000
  30. #define GRAPH_LAST_EDGE 0x80000000
  31. #define GRAPH_HEADER_SIZE 8
  32. #define GRAPH_FANOUT_SIZE (4 * 256)
  33. #define GRAPH_CHUNKLOOKUP_WIDTH 12
  34. #define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * GRAPH_CHUNKLOOKUP_WIDTH \
  35. + GRAPH_FANOUT_SIZE + the_hash_algo->rawsz)
  36. char *get_commit_graph_filename(const char *obj_dir)
  37. {
  38. return xstrfmt("%s/info/commit-graph", obj_dir);
  39. }
  40. static uint8_t oid_version(void)
  41. {
  42. return 1;
  43. }
  44. static struct commit_graph *alloc_commit_graph(void)
  45. {
  46. struct commit_graph *g = xcalloc(1, sizeof(*g));
  47. g->graph_fd = -1;
  48. return g;
  49. }
  50. extern int read_replace_refs;
  51. static int commit_graph_compatible(struct repository *r)
  52. {
  53. if (!r->gitdir)
  54. return 0;
  55. if (read_replace_refs) {
  56. prepare_replace_object(r);
  57. if (hashmap_get_size(&r->objects->replace_map->map))
  58. return 0;
  59. }
  60. prepare_commit_graft(r);
  61. if (r->parsed_objects && r->parsed_objects->grafts_nr)
  62. return 0;
  63. if (is_repository_shallow(r))
  64. return 0;
  65. return 1;
  66. }
  67. int open_commit_graph(const char *graph_file, int *fd, struct stat *st)
  68. {
  69. *fd = git_open(graph_file);
  70. if (*fd < 0)
  71. return 0;
  72. if (fstat(*fd, st)) {
  73. close(*fd);
  74. return 0;
  75. }
  76. return 1;
  77. }
  78. struct commit_graph *load_commit_graph_one_fd_st(int fd, struct stat *st)
  79. {
  80. void *graph_map;
  81. size_t graph_size;
  82. struct commit_graph *ret;
  83. graph_size = xsize_t(st->st_size);
  84. if (graph_size < GRAPH_MIN_SIZE) {
  85. close(fd);
  86. error(_("commit-graph file is too small"));
  87. return NULL;
  88. }
  89. graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0);
  90. ret = parse_commit_graph(graph_map, fd, graph_size);
  91. if (!ret) {
  92. munmap(graph_map, graph_size);
  93. close(fd);
  94. }
  95. return ret;
  96. }
  97. static int verify_commit_graph_lite(struct commit_graph *g)
  98. {
  99. /*
  100. * Basic validation shared between parse_commit_graph()
  101. * which'll be called every time the graph is used, and the
  102. * much more expensive verify_commit_graph() used by
  103. * "commit-graph verify".
  104. *
  105. * There should only be very basic checks here to ensure that
  106. * we don't e.g. segfault in fill_commit_in_graph(), but
  107. * because this is a very hot codepath nothing that e.g. loops
  108. * over g->num_commits, or runs a checksum on the commit-graph
  109. * itself.
  110. */
  111. if (!g->chunk_oid_fanout) {
  112. error("commit-graph is missing the OID Fanout chunk");
  113. return 1;
  114. }
  115. if (!g->chunk_oid_lookup) {
  116. error("commit-graph is missing the OID Lookup chunk");
  117. return 1;
  118. }
  119. if (!g->chunk_commit_data) {
  120. error("commit-graph is missing the Commit Data chunk");
  121. return 1;
  122. }
  123. return 0;
  124. }
  125. struct commit_graph *parse_commit_graph(void *graph_map, int fd,
  126. size_t graph_size)
  127. {
  128. const unsigned char *data, *chunk_lookup;
  129. uint32_t i;
  130. struct commit_graph *graph;
  131. uint64_t last_chunk_offset;
  132. uint32_t last_chunk_id;
  133. uint32_t graph_signature;
  134. unsigned char graph_version, hash_version;
  135. if (!graph_map)
  136. return NULL;
  137. if (graph_size < GRAPH_MIN_SIZE)
  138. return NULL;
  139. data = (const unsigned char *)graph_map;
  140. graph_signature = get_be32(data);
  141. if (graph_signature != GRAPH_SIGNATURE) {
  142. error(_("commit-graph signature %X does not match signature %X"),
  143. graph_signature, GRAPH_SIGNATURE);
  144. return NULL;
  145. }
  146. graph_version = *(unsigned char*)(data + 4);
  147. if (graph_version != GRAPH_VERSION) {
  148. error(_("commit-graph version %X does not match version %X"),
  149. graph_version, GRAPH_VERSION);
  150. return NULL;
  151. }
  152. hash_version = *(unsigned char*)(data + 5);
  153. if (hash_version != oid_version()) {
  154. error(_("commit-graph hash version %X does not match version %X"),
  155. hash_version, oid_version());
  156. return NULL;
  157. }
  158. graph = alloc_commit_graph();
  159. graph->hash_len = the_hash_algo->rawsz;
  160. graph->num_chunks = *(unsigned char*)(data + 6);
  161. graph->graph_fd = fd;
  162. graph->data = graph_map;
  163. graph->data_len = graph_size;
  164. last_chunk_id = 0;
  165. last_chunk_offset = 8;
  166. chunk_lookup = data + 8;
  167. for (i = 0; i < graph->num_chunks; i++) {
  168. uint32_t chunk_id;
  169. uint64_t chunk_offset;
  170. int chunk_repeated = 0;
  171. if (data + graph_size - chunk_lookup <
  172. GRAPH_CHUNKLOOKUP_WIDTH) {
  173. error(_("commit-graph chunk lookup table entry missing; file may be incomplete"));
  174. free(graph);
  175. return NULL;
  176. }
  177. chunk_id = get_be32(chunk_lookup + 0);
  178. chunk_offset = get_be64(chunk_lookup + 4);
  179. chunk_lookup += GRAPH_CHUNKLOOKUP_WIDTH;
  180. if (chunk_offset > graph_size - the_hash_algo->rawsz) {
  181. error(_("commit-graph improper chunk offset %08x%08x"), (uint32_t)(chunk_offset >> 32),
  182. (uint32_t)chunk_offset);
  183. free(graph);
  184. return NULL;
  185. }
  186. switch (chunk_id) {
  187. case GRAPH_CHUNKID_OIDFANOUT:
  188. if (graph->chunk_oid_fanout)
  189. chunk_repeated = 1;
  190. else
  191. graph->chunk_oid_fanout = (uint32_t*)(data + chunk_offset);
  192. break;
  193. case GRAPH_CHUNKID_OIDLOOKUP:
  194. if (graph->chunk_oid_lookup)
  195. chunk_repeated = 1;
  196. else
  197. graph->chunk_oid_lookup = data + chunk_offset;
  198. break;
  199. case GRAPH_CHUNKID_DATA:
  200. if (graph->chunk_commit_data)
  201. chunk_repeated = 1;
  202. else
  203. graph->chunk_commit_data = data + chunk_offset;
  204. break;
  205. case GRAPH_CHUNKID_EXTRAEDGES:
  206. if (graph->chunk_extra_edges)
  207. chunk_repeated = 1;
  208. else
  209. graph->chunk_extra_edges = data + chunk_offset;
  210. break;
  211. }
  212. if (chunk_repeated) {
  213. error(_("commit-graph chunk id %08x appears multiple times"), chunk_id);
  214. free(graph);
  215. return NULL;
  216. }
  217. if (last_chunk_id == GRAPH_CHUNKID_OIDLOOKUP)
  218. {
  219. graph->num_commits = (chunk_offset - last_chunk_offset)
  220. / graph->hash_len;
  221. }
  222. last_chunk_id = chunk_id;
  223. last_chunk_offset = chunk_offset;
  224. }
  225. if (verify_commit_graph_lite(graph)) {
  226. free(graph);
  227. return NULL;
  228. }
  229. return graph;
  230. }
  231. static struct commit_graph *load_commit_graph_one(const char *graph_file)
  232. {
  233. struct stat st;
  234. int fd;
  235. int open_ok = open_commit_graph(graph_file, &fd, &st);
  236. if (!open_ok)
  237. return NULL;
  238. return load_commit_graph_one_fd_st(fd, &st);
  239. }
  240. static void prepare_commit_graph_one(struct repository *r, const char *obj_dir)
  241. {
  242. char *graph_name;
  243. if (r->objects->commit_graph)
  244. return;
  245. graph_name = get_commit_graph_filename(obj_dir);
  246. r->objects->commit_graph =
  247. load_commit_graph_one(graph_name);
  248. FREE_AND_NULL(graph_name);
  249. }
  250. /*
  251. * Return 1 if commit_graph is non-NULL, and 0 otherwise.
  252. *
  253. * On the first invocation, this function attemps to load the commit
  254. * graph if the_repository is configured to have one.
  255. */
  256. static int prepare_commit_graph(struct repository *r)
  257. {
  258. struct object_directory *odb;
  259. int config_value;
  260. if (git_env_bool(GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD, 0))
  261. die("dying as requested by the '%s' variable on commit-graph load!",
  262. GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD);
  263. if (r->objects->commit_graph_attempted)
  264. return !!r->objects->commit_graph;
  265. r->objects->commit_graph_attempted = 1;
  266. if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) &&
  267. (repo_config_get_bool(r, "core.commitgraph", &config_value) ||
  268. !config_value))
  269. /*
  270. * This repository is not configured to use commit graphs, so
  271. * do not load one. (But report commit_graph_attempted anyway
  272. * so that commit graph loading is not attempted again for this
  273. * repository.)
  274. */
  275. return 0;
  276. if (!commit_graph_compatible(r))
  277. return 0;
  278. prepare_alt_odb(r);
  279. for (odb = r->objects->odb;
  280. !r->objects->commit_graph && odb;
  281. odb = odb->next)
  282. prepare_commit_graph_one(r, odb->path);
  283. return !!r->objects->commit_graph;
  284. }
  285. int generation_numbers_enabled(struct repository *r)
  286. {
  287. uint32_t first_generation;
  288. struct commit_graph *g;
  289. if (!prepare_commit_graph(r))
  290. return 0;
  291. g = r->objects->commit_graph;
  292. if (!g->num_commits)
  293. return 0;
  294. first_generation = get_be32(g->chunk_commit_data +
  295. g->hash_len + 8) >> 2;
  296. return !!first_generation;
  297. }
  298. void close_commit_graph(struct repository *r)
  299. {
  300. free_commit_graph(r->objects->commit_graph);
  301. r->objects->commit_graph = NULL;
  302. }
  303. static int bsearch_graph(struct commit_graph *g, struct object_id *oid, uint32_t *pos)
  304. {
  305. return bsearch_hash(oid->hash, g->chunk_oid_fanout,
  306. g->chunk_oid_lookup, g->hash_len, pos);
  307. }
  308. static struct commit_list **insert_parent_or_die(struct repository *r,
  309. struct commit_graph *g,
  310. uint64_t pos,
  311. struct commit_list **pptr)
  312. {
  313. struct commit *c;
  314. struct object_id oid;
  315. if (pos >= g->num_commits)
  316. die("invalid parent position %"PRIu64, pos);
  317. hashcpy(oid.hash, g->chunk_oid_lookup + g->hash_len * pos);
  318. c = lookup_commit(r, &oid);
  319. if (!c)
  320. die(_("could not find commit %s"), oid_to_hex(&oid));
  321. c->graph_pos = pos;
  322. return &commit_list_insert(c, pptr)->next;
  323. }
  324. static void fill_commit_graph_info(struct commit *item, struct commit_graph *g, uint32_t pos)
  325. {
  326. const unsigned char *commit_data = g->chunk_commit_data + GRAPH_DATA_WIDTH * pos;
  327. item->graph_pos = pos;
  328. item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
  329. }
  330. static int fill_commit_in_graph(struct repository *r,
  331. struct commit *item,
  332. struct commit_graph *g, uint32_t pos)
  333. {
  334. uint32_t edge_value;
  335. uint32_t *parent_data_ptr;
  336. uint64_t date_low, date_high;
  337. struct commit_list **pptr;
  338. const unsigned char *commit_data = g->chunk_commit_data + (g->hash_len + 16) * pos;
  339. item->object.parsed = 1;
  340. item->graph_pos = pos;
  341. item->maybe_tree = NULL;
  342. date_high = get_be32(commit_data + g->hash_len + 8) & 0x3;
  343. date_low = get_be32(commit_data + g->hash_len + 12);
  344. item->date = (timestamp_t)((date_high << 32) | date_low);
  345. item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
  346. pptr = &item->parents;
  347. edge_value = get_be32(commit_data + g->hash_len);
  348. if (edge_value == GRAPH_PARENT_NONE)
  349. return 1;
  350. pptr = insert_parent_or_die(r, g, edge_value, pptr);
  351. edge_value = get_be32(commit_data + g->hash_len + 4);
  352. if (edge_value == GRAPH_PARENT_NONE)
  353. return 1;
  354. if (!(edge_value & GRAPH_EXTRA_EDGES_NEEDED)) {
  355. pptr = insert_parent_or_die(r, g, edge_value, pptr);
  356. return 1;
  357. }
  358. parent_data_ptr = (uint32_t*)(g->chunk_extra_edges +
  359. 4 * (uint64_t)(edge_value & GRAPH_EDGE_LAST_MASK));
  360. do {
  361. edge_value = get_be32(parent_data_ptr);
  362. pptr = insert_parent_or_die(r, g,
  363. edge_value & GRAPH_EDGE_LAST_MASK,
  364. pptr);
  365. parent_data_ptr++;
  366. } while (!(edge_value & GRAPH_LAST_EDGE));
  367. return 1;
  368. }
  369. static int find_commit_in_graph(struct commit *item, struct commit_graph *g, uint32_t *pos)
  370. {
  371. if (item->graph_pos != COMMIT_NOT_FROM_GRAPH) {
  372. *pos = item->graph_pos;
  373. return 1;
  374. } else {
  375. return bsearch_graph(g, &(item->object.oid), pos);
  376. }
  377. }
  378. static int parse_commit_in_graph_one(struct repository *r,
  379. struct commit_graph *g,
  380. struct commit *item)
  381. {
  382. uint32_t pos;
  383. if (item->object.parsed)
  384. return 1;
  385. if (find_commit_in_graph(item, g, &pos))
  386. return fill_commit_in_graph(r, item, g, pos);
  387. return 0;
  388. }
  389. int parse_commit_in_graph(struct repository *r, struct commit *item)
  390. {
  391. if (!prepare_commit_graph(r))
  392. return 0;
  393. return parse_commit_in_graph_one(r, r->objects->commit_graph, item);
  394. }
  395. void load_commit_graph_info(struct repository *r, struct commit *item)
  396. {
  397. uint32_t pos;
  398. if (!prepare_commit_graph(r))
  399. return;
  400. if (find_commit_in_graph(item, r->objects->commit_graph, &pos))
  401. fill_commit_graph_info(item, r->objects->commit_graph, pos);
  402. }
  403. static struct tree *load_tree_for_commit(struct repository *r,
  404. struct commit_graph *g,
  405. struct commit *c)
  406. {
  407. struct object_id oid;
  408. const unsigned char *commit_data = g->chunk_commit_data +
  409. GRAPH_DATA_WIDTH * (c->graph_pos);
  410. hashcpy(oid.hash, commit_data);
  411. c->maybe_tree = lookup_tree(r, &oid);
  412. return c->maybe_tree;
  413. }
  414. static struct tree *get_commit_tree_in_graph_one(struct repository *r,
  415. struct commit_graph *g,
  416. const struct commit *c)
  417. {
  418. if (c->maybe_tree)
  419. return c->maybe_tree;
  420. if (c->graph_pos == COMMIT_NOT_FROM_GRAPH)
  421. BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
  422. return load_tree_for_commit(r, g, (struct commit *)c);
  423. }
  424. struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit *c)
  425. {
  426. return get_commit_tree_in_graph_one(r, r->objects->commit_graph, c);
  427. }
  428. static void write_graph_chunk_fanout(struct hashfile *f,
  429. struct commit **commits,
  430. int nr_commits,
  431. struct progress *progress,
  432. uint64_t *progress_cnt)
  433. {
  434. int i, count = 0;
  435. struct commit **list = commits;
  436. /*
  437. * Write the first-level table (the list is sorted,
  438. * but we use a 256-entry lookup to be able to avoid
  439. * having to do eight extra binary search iterations).
  440. */
  441. for (i = 0; i < 256; i++) {
  442. while (count < nr_commits) {
  443. if ((*list)->object.oid.hash[0] != i)
  444. break;
  445. display_progress(progress, ++*progress_cnt);
  446. count++;
  447. list++;
  448. }
  449. hashwrite_be32(f, count);
  450. }
  451. }
  452. static void write_graph_chunk_oids(struct hashfile *f, int hash_len,
  453. struct commit **commits, int nr_commits,
  454. struct progress *progress,
  455. uint64_t *progress_cnt)
  456. {
  457. struct commit **list = commits;
  458. int count;
  459. for (count = 0; count < nr_commits; count++, list++) {
  460. display_progress(progress, ++*progress_cnt);
  461. hashwrite(f, (*list)->object.oid.hash, (int)hash_len);
  462. }
  463. }
  464. static const unsigned char *commit_to_sha1(size_t index, void *table)
  465. {
  466. struct commit **commits = table;
  467. return commits[index]->object.oid.hash;
  468. }
  469. static void write_graph_chunk_data(struct hashfile *f, int hash_len,
  470. struct commit **commits, int nr_commits,
  471. struct progress *progress,
  472. uint64_t *progress_cnt)
  473. {
  474. struct commit **list = commits;
  475. struct commit **last = commits + nr_commits;
  476. uint32_t num_extra_edges = 0;
  477. while (list < last) {
  478. struct commit_list *parent;
  479. int edge_value;
  480. uint32_t packedDate[2];
  481. display_progress(progress, ++*progress_cnt);
  482. parse_commit_no_graph(*list);
  483. hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len);
  484. parent = (*list)->parents;
  485. if (!parent)
  486. edge_value = GRAPH_PARENT_NONE;
  487. else {
  488. edge_value = sha1_pos(parent->item->object.oid.hash,
  489. commits,
  490. nr_commits,
  491. commit_to_sha1);
  492. if (edge_value < 0)
  493. BUG("missing parent %s for commit %s",
  494. oid_to_hex(&parent->item->object.oid),
  495. oid_to_hex(&(*list)->object.oid));
  496. }
  497. hashwrite_be32(f, edge_value);
  498. if (parent)
  499. parent = parent->next;
  500. if (!parent)
  501. edge_value = GRAPH_PARENT_NONE;
  502. else if (parent->next)
  503. edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges;
  504. else {
  505. edge_value = sha1_pos(parent->item->object.oid.hash,
  506. commits,
  507. nr_commits,
  508. commit_to_sha1);
  509. if (edge_value < 0)
  510. BUG("missing parent %s for commit %s",
  511. oid_to_hex(&parent->item->object.oid),
  512. oid_to_hex(&(*list)->object.oid));
  513. }
  514. hashwrite_be32(f, edge_value);
  515. if (edge_value & GRAPH_EXTRA_EDGES_NEEDED) {
  516. do {
  517. num_extra_edges++;
  518. parent = parent->next;
  519. } while (parent);
  520. }
  521. if (sizeof((*list)->date) > 4)
  522. packedDate[0] = htonl(((*list)->date >> 32) & 0x3);
  523. else
  524. packedDate[0] = 0;
  525. packedDate[0] |= htonl((*list)->generation << 2);
  526. packedDate[1] = htonl((*list)->date);
  527. hashwrite(f, packedDate, 8);
  528. list++;
  529. }
  530. }
  531. static void write_graph_chunk_extra_edges(struct hashfile *f,
  532. struct commit **commits,
  533. int nr_commits,
  534. struct progress *progress,
  535. uint64_t *progress_cnt)
  536. {
  537. struct commit **list = commits;
  538. struct commit **last = commits + nr_commits;
  539. struct commit_list *parent;
  540. while (list < last) {
  541. int num_parents = 0;
  542. display_progress(progress, ++*progress_cnt);
  543. for (parent = (*list)->parents; num_parents < 3 && parent;
  544. parent = parent->next)
  545. num_parents++;
  546. if (num_parents <= 2) {
  547. list++;
  548. continue;
  549. }
  550. /* Since num_parents > 2, this initializer is safe. */
  551. for (parent = (*list)->parents->next; parent; parent = parent->next) {
  552. int edge_value = sha1_pos(parent->item->object.oid.hash,
  553. commits,
  554. nr_commits,
  555. commit_to_sha1);
  556. if (edge_value < 0)
  557. BUG("missing parent %s for commit %s",
  558. oid_to_hex(&parent->item->object.oid),
  559. oid_to_hex(&(*list)->object.oid));
  560. else if (!parent->next)
  561. edge_value |= GRAPH_LAST_EDGE;
  562. hashwrite_be32(f, edge_value);
  563. }
  564. list++;
  565. }
  566. }
  567. static int commit_compare(const void *_a, const void *_b)
  568. {
  569. const struct object_id *a = (const struct object_id *)_a;
  570. const struct object_id *b = (const struct object_id *)_b;
  571. return oidcmp(a, b);
  572. }
  573. struct packed_commit_list {
  574. struct commit **list;
  575. int nr;
  576. int alloc;
  577. };
  578. struct packed_oid_list {
  579. struct object_id *list;
  580. int nr;
  581. int alloc;
  582. struct progress *progress;
  583. int progress_done;
  584. };
  585. static int add_packed_commits(const struct object_id *oid,
  586. struct packed_git *pack,
  587. uint32_t pos,
  588. void *data)
  589. {
  590. struct packed_oid_list *list = (struct packed_oid_list*)data;
  591. enum object_type type;
  592. off_t offset = nth_packed_object_offset(pack, pos);
  593. struct object_info oi = OBJECT_INFO_INIT;
  594. if (list->progress)
  595. display_progress(list->progress, ++list->progress_done);
  596. oi.typep = &type;
  597. if (packed_object_info(the_repository, pack, offset, &oi) < 0)
  598. die(_("unable to get type of object %s"), oid_to_hex(oid));
  599. if (type != OBJ_COMMIT)
  600. return 0;
  601. ALLOC_GROW(list->list, list->nr + 1, list->alloc);
  602. oidcpy(&(list->list[list->nr]), oid);
  603. list->nr++;
  604. return 0;
  605. }
  606. static void add_missing_parents(struct packed_oid_list *oids, struct commit *commit)
  607. {
  608. struct commit_list *parent;
  609. for (parent = commit->parents; parent; parent = parent->next) {
  610. if (!(parent->item->object.flags & UNINTERESTING)) {
  611. ALLOC_GROW(oids->list, oids->nr + 1, oids->alloc);
  612. oidcpy(&oids->list[oids->nr], &(parent->item->object.oid));
  613. oids->nr++;
  614. parent->item->object.flags |= UNINTERESTING;
  615. }
  616. }
  617. }
  618. static void close_reachable(struct packed_oid_list *oids, int report_progress)
  619. {
  620. int i;
  621. struct commit *commit;
  622. struct progress *progress = NULL;
  623. if (report_progress)
  624. progress = start_delayed_progress(
  625. _("Loading known commits in commit graph"), oids->nr);
  626. for (i = 0; i < oids->nr; i++) {
  627. display_progress(progress, i + 1);
  628. commit = lookup_commit(the_repository, &oids->list[i]);
  629. if (commit)
  630. commit->object.flags |= UNINTERESTING;
  631. }
  632. stop_progress(&progress);
  633. /*
  634. * As this loop runs, oids->nr may grow, but not more
  635. * than the number of missing commits in the reachable
  636. * closure.
  637. */
  638. if (report_progress)
  639. progress = start_delayed_progress(
  640. _("Expanding reachable commits in commit graph"), oids->nr);
  641. for (i = 0; i < oids->nr; i++) {
  642. display_progress(progress, i + 1);
  643. commit = lookup_commit(the_repository, &oids->list[i]);
  644. if (commit && !parse_commit_no_graph(commit))
  645. add_missing_parents(oids, commit);
  646. }
  647. stop_progress(&progress);
  648. if (report_progress)
  649. progress = start_delayed_progress(
  650. _("Clearing commit marks in commit graph"), oids->nr);
  651. for (i = 0; i < oids->nr; i++) {
  652. display_progress(progress, i + 1);
  653. commit = lookup_commit(the_repository, &oids->list[i]);
  654. if (commit)
  655. commit->object.flags &= ~UNINTERESTING;
  656. }
  657. stop_progress(&progress);
  658. }
  659. static void compute_generation_numbers(struct packed_commit_list* commits,
  660. int report_progress)
  661. {
  662. int i;
  663. struct commit_list *list = NULL;
  664. struct progress *progress = NULL;
  665. if (report_progress)
  666. progress = start_progress(
  667. _("Computing commit graph generation numbers"),
  668. commits->nr);
  669. for (i = 0; i < commits->nr; i++) {
  670. display_progress(progress, i + 1);
  671. if (commits->list[i]->generation != GENERATION_NUMBER_INFINITY &&
  672. commits->list[i]->generation != GENERATION_NUMBER_ZERO)
  673. continue;
  674. commit_list_insert(commits->list[i], &list);
  675. while (list) {
  676. struct commit *current = list->item;
  677. struct commit_list *parent;
  678. int all_parents_computed = 1;
  679. uint32_t max_generation = 0;
  680. for (parent = current->parents; parent; parent = parent->next) {
  681. if (parent->item->generation == GENERATION_NUMBER_INFINITY ||
  682. parent->item->generation == GENERATION_NUMBER_ZERO) {
  683. all_parents_computed = 0;
  684. commit_list_insert(parent->item, &list);
  685. break;
  686. } else if (parent->item->generation > max_generation) {
  687. max_generation = parent->item->generation;
  688. }
  689. }
  690. if (all_parents_computed) {
  691. current->generation = max_generation + 1;
  692. pop_commit(&list);
  693. if (current->generation > GENERATION_NUMBER_MAX)
  694. current->generation = GENERATION_NUMBER_MAX;
  695. }
  696. }
  697. }
  698. stop_progress(&progress);
  699. }
  700. static int add_ref_to_list(const char *refname,
  701. const struct object_id *oid,
  702. int flags, void *cb_data)
  703. {
  704. struct string_list *list = (struct string_list *)cb_data;
  705. string_list_append(list, oid_to_hex(oid));
  706. return 0;
  707. }
  708. void write_commit_graph_reachable(const char *obj_dir, int append,
  709. int report_progress)
  710. {
  711. struct string_list list = STRING_LIST_INIT_DUP;
  712. for_each_ref(add_ref_to_list, &list);
  713. write_commit_graph(obj_dir, NULL, &list, append, report_progress);
  714. string_list_clear(&list, 0);
  715. }
  716. void write_commit_graph(const char *obj_dir,
  717. struct string_list *pack_indexes,
  718. struct string_list *commit_hex,
  719. int append, int report_progress)
  720. {
  721. struct packed_oid_list oids;
  722. struct packed_commit_list commits;
  723. struct hashfile *f;
  724. uint32_t i, count_distinct = 0;
  725. char *graph_name;
  726. struct lock_file lk = LOCK_INIT;
  727. uint32_t chunk_ids[5];
  728. uint64_t chunk_offsets[5];
  729. int num_chunks;
  730. int num_extra_edges;
  731. struct commit_list *parent;
  732. struct progress *progress = NULL;
  733. const unsigned hashsz = the_hash_algo->rawsz;
  734. uint64_t progress_cnt = 0;
  735. struct strbuf progress_title = STRBUF_INIT;
  736. unsigned long approx_nr_objects;
  737. if (!commit_graph_compatible(the_repository))
  738. return;
  739. oids.nr = 0;
  740. approx_nr_objects = approximate_object_count();
  741. oids.alloc = approx_nr_objects / 32;
  742. oids.progress = NULL;
  743. oids.progress_done = 0;
  744. if (append) {
  745. prepare_commit_graph_one(the_repository, obj_dir);
  746. if (the_repository->objects->commit_graph)
  747. oids.alloc += the_repository->objects->commit_graph->num_commits;
  748. }
  749. if (oids.alloc < 1024)
  750. oids.alloc = 1024;
  751. ALLOC_ARRAY(oids.list, oids.alloc);
  752. if (append && the_repository->objects->commit_graph) {
  753. struct commit_graph *commit_graph =
  754. the_repository->objects->commit_graph;
  755. for (i = 0; i < commit_graph->num_commits; i++) {
  756. const unsigned char *hash = commit_graph->chunk_oid_lookup +
  757. commit_graph->hash_len * i;
  758. hashcpy(oids.list[oids.nr++].hash, hash);
  759. }
  760. }
  761. if (pack_indexes) {
  762. struct strbuf packname = STRBUF_INIT;
  763. int dirlen;
  764. strbuf_addf(&packname, "%s/pack/", obj_dir);
  765. dirlen = packname.len;
  766. if (report_progress) {
  767. strbuf_addf(&progress_title,
  768. Q_("Finding commits for commit graph in %d pack",
  769. "Finding commits for commit graph in %d packs",
  770. pack_indexes->nr),
  771. pack_indexes->nr);
  772. oids.progress = start_delayed_progress(progress_title.buf, 0);
  773. oids.progress_done = 0;
  774. }
  775. for (i = 0; i < pack_indexes->nr; i++) {
  776. struct packed_git *p;
  777. strbuf_setlen(&packname, dirlen);
  778. strbuf_addstr(&packname, pack_indexes->items[i].string);
  779. p = add_packed_git(packname.buf, packname.len, 1);
  780. if (!p)
  781. die(_("error adding pack %s"), packname.buf);
  782. if (open_pack_index(p))
  783. die(_("error opening index for %s"), packname.buf);
  784. for_each_object_in_pack(p, add_packed_commits, &oids,
  785. FOR_EACH_OBJECT_PACK_ORDER);
  786. close_pack(p);
  787. free(p);
  788. }
  789. stop_progress(&oids.progress);
  790. strbuf_reset(&progress_title);
  791. strbuf_release(&packname);
  792. }
  793. if (commit_hex) {
  794. if (report_progress) {
  795. strbuf_addf(&progress_title,
  796. Q_("Finding commits for commit graph from %d ref",
  797. "Finding commits for commit graph from %d refs",
  798. commit_hex->nr),
  799. commit_hex->nr);
  800. progress = start_delayed_progress(progress_title.buf,
  801. commit_hex->nr);
  802. }
  803. for (i = 0; i < commit_hex->nr; i++) {
  804. const char *end;
  805. struct object_id oid;
  806. struct commit *result;
  807. display_progress(progress, i + 1);
  808. if (commit_hex->items[i].string &&
  809. parse_oid_hex(commit_hex->items[i].string, &oid, &end))
  810. continue;
  811. result = lookup_commit_reference_gently(the_repository, &oid, 1);
  812. if (result) {
  813. ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc);
  814. oidcpy(&oids.list[oids.nr], &(result->object.oid));
  815. oids.nr++;
  816. }
  817. }
  818. stop_progress(&progress);
  819. strbuf_reset(&progress_title);
  820. }
  821. if (!pack_indexes && !commit_hex) {
  822. if (report_progress)
  823. oids.progress = start_delayed_progress(
  824. _("Finding commits for commit graph among packed objects"),
  825. approx_nr_objects);
  826. for_each_packed_object(add_packed_commits, &oids,
  827. FOR_EACH_OBJECT_PACK_ORDER);
  828. if (oids.progress_done < approx_nr_objects)
  829. display_progress(oids.progress, approx_nr_objects);
  830. stop_progress(&oids.progress);
  831. }
  832. close_reachable(&oids, report_progress);
  833. if (report_progress)
  834. progress = start_delayed_progress(
  835. _("Counting distinct commits in commit graph"),
  836. oids.nr);
  837. display_progress(progress, 0); /* TODO: Measure QSORT() progress */
  838. QSORT(oids.list, oids.nr, commit_compare);
  839. count_distinct = 1;
  840. for (i = 1; i < oids.nr; i++) {
  841. display_progress(progress, i + 1);
  842. if (!oideq(&oids.list[i - 1], &oids.list[i]))
  843. count_distinct++;
  844. }
  845. stop_progress(&progress);
  846. if (count_distinct >= GRAPH_EDGE_LAST_MASK)
  847. die(_("the commit graph format cannot write %d commits"), count_distinct);
  848. commits.nr = 0;
  849. commits.alloc = count_distinct;
  850. ALLOC_ARRAY(commits.list, commits.alloc);
  851. num_extra_edges = 0;
  852. if (report_progress)
  853. progress = start_delayed_progress(
  854. _("Finding extra edges in commit graph"),
  855. oids.nr);
  856. for (i = 0; i < oids.nr; i++) {
  857. int num_parents = 0;
  858. display_progress(progress, i + 1);
  859. if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i]))
  860. continue;
  861. commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]);
  862. parse_commit_no_graph(commits.list[commits.nr]);
  863. for (parent = commits.list[commits.nr]->parents;
  864. parent; parent = parent->next)
  865. num_parents++;
  866. if (num_parents > 2)
  867. num_extra_edges += num_parents - 1;
  868. commits.nr++;
  869. }
  870. num_chunks = num_extra_edges ? 4 : 3;
  871. stop_progress(&progress);
  872. if (commits.nr >= GRAPH_EDGE_LAST_MASK)
  873. die(_("too many commits to write graph"));
  874. compute_generation_numbers(&commits, report_progress);
  875. graph_name = get_commit_graph_filename(obj_dir);
  876. if (safe_create_leading_directories(graph_name)) {
  877. UNLEAK(graph_name);
  878. die_errno(_("unable to create leading directories of %s"),
  879. graph_name);
  880. }
  881. hold_lock_file_for_update(&lk, graph_name, LOCK_DIE_ON_ERROR);
  882. f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);
  883. hashwrite_be32(f, GRAPH_SIGNATURE);
  884. hashwrite_u8(f, GRAPH_VERSION);
  885. hashwrite_u8(f, oid_version());
  886. hashwrite_u8(f, num_chunks);
  887. hashwrite_u8(f, 0); /* unused padding byte */
  888. chunk_ids[0] = GRAPH_CHUNKID_OIDFANOUT;
  889. chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP;
  890. chunk_ids[2] = GRAPH_CHUNKID_DATA;
  891. if (num_extra_edges)
  892. chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES;
  893. else
  894. chunk_ids[3] = 0;
  895. chunk_ids[4] = 0;
  896. chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH;
  897. chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE;
  898. chunk_offsets[2] = chunk_offsets[1] + hashsz * commits.nr;
  899. chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * commits.nr;
  900. chunk_offsets[4] = chunk_offsets[3] + 4 * num_extra_edges;
  901. for (i = 0; i <= num_chunks; i++) {
  902. uint32_t chunk_write[3];
  903. chunk_write[0] = htonl(chunk_ids[i]);
  904. chunk_write[1] = htonl(chunk_offsets[i] >> 32);
  905. chunk_write[2] = htonl(chunk_offsets[i] & 0xffffffff);
  906. hashwrite(f, chunk_write, 12);
  907. }
  908. if (report_progress) {
  909. strbuf_addf(&progress_title,
  910. Q_("Writing out commit graph in %d pass",
  911. "Writing out commit graph in %d passes",
  912. num_chunks),
  913. num_chunks);
  914. progress = start_delayed_progress(
  915. progress_title.buf,
  916. num_chunks * commits.nr);
  917. }
  918. write_graph_chunk_fanout(f, commits.list, commits.nr, progress, &progress_cnt);
  919. write_graph_chunk_oids(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
  920. write_graph_chunk_data(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
  921. if (num_extra_edges)
  922. write_graph_chunk_extra_edges(f, commits.list, commits.nr, progress, &progress_cnt);
  923. stop_progress(&progress);
  924. strbuf_release(&progress_title);
  925. close_commit_graph(the_repository);
  926. finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC);
  927. commit_lock_file(&lk);
  928. free(graph_name);
  929. free(commits.list);
  930. free(oids.list);
  931. }
  932. #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2
  933. static int verify_commit_graph_error;
  934. static void graph_report(const char *fmt, ...)
  935. {
  936. va_list ap;
  937. verify_commit_graph_error = 1;
  938. va_start(ap, fmt);
  939. vfprintf(stderr, fmt, ap);
  940. fprintf(stderr, "\n");
  941. va_end(ap);
  942. }
  943. #define GENERATION_ZERO_EXISTS 1
  944. #define GENERATION_NUMBER_EXISTS 2
  945. int verify_commit_graph(struct repository *r, struct commit_graph *g)
  946. {
  947. uint32_t i, cur_fanout_pos = 0;
  948. struct object_id prev_oid, cur_oid, checksum;
  949. int generation_zero = 0;
  950. struct hashfile *f;
  951. int devnull;
  952. struct progress *progress = NULL;
  953. if (!g) {
  954. graph_report("no commit-graph file loaded");
  955. return 1;
  956. }
  957. verify_commit_graph_error = verify_commit_graph_lite(g);
  958. if (verify_commit_graph_error)
  959. return verify_commit_graph_error;
  960. devnull = open("/dev/null", O_WRONLY);
  961. f = hashfd(devnull, NULL);
  962. hashwrite(f, g->data, g->data_len - g->hash_len);
  963. finalize_hashfile(f, checksum.hash, CSUM_CLOSE);
  964. if (!hasheq(checksum.hash, g->data + g->data_len - g->hash_len)) {
  965. graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt"));
  966. verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH;
  967. }
  968. for (i = 0; i < g->num_commits; i++) {
  969. struct commit *graph_commit;
  970. hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
  971. if (i && oidcmp(&prev_oid, &cur_oid) >= 0)
  972. graph_report(_("commit-graph has incorrect OID order: %s then %s"),
  973. oid_to_hex(&prev_oid),
  974. oid_to_hex(&cur_oid));
  975. oidcpy(&prev_oid, &cur_oid);
  976. while (cur_oid.hash[0] > cur_fanout_pos) {
  977. uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
  978. if (i != fanout_value)
  979. graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"),
  980. cur_fanout_pos, fanout_value, i);
  981. cur_fanout_pos++;
  982. }
  983. graph_commit = lookup_commit(r, &cur_oid);
  984. if (!parse_commit_in_graph_one(r, g, graph_commit))
  985. graph_report(_("failed to parse commit %s from commit-graph"),
  986. oid_to_hex(&cur_oid));
  987. }
  988. while (cur_fanout_pos < 256) {
  989. uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
  990. if (g->num_commits != fanout_value)
  991. graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"),
  992. cur_fanout_pos, fanout_value, i);
  993. cur_fanout_pos++;
  994. }
  995. if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH)
  996. return verify_commit_graph_error;
  997. progress = start_progress(_("Verifying commits in commit graph"),
  998. g->num_commits);
  999. for (i = 0; i < g->num_commits; i++) {
  1000. struct commit *graph_commit, *odb_commit;
  1001. struct commit_list *graph_parents, *odb_parents;
  1002. uint32_t max_generation = 0;
  1003. display_progress(progress, i + 1);
  1004. hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
  1005. graph_commit = lookup_commit(r, &cur_oid);
  1006. odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r));
  1007. if (parse_commit_internal(odb_commit, 0, 0)) {
  1008. graph_report(_("failed to parse commit %s from object database for commit-graph"),
  1009. oid_to_hex(&cur_oid));
  1010. continue;
  1011. }
  1012. if (!oideq(&get_commit_tree_in_graph_one(r, g, graph_commit)->object.oid,
  1013. get_commit_tree_oid(odb_commit)))
  1014. graph_report(_("root tree OID for commit %s in commit-graph is %s != %s"),
  1015. oid_to_hex(&cur_oid),
  1016. oid_to_hex(get_commit_tree_oid(graph_commit)),
  1017. oid_to_hex(get_commit_tree_oid(odb_commit)));
  1018. graph_parents = graph_commit->parents;
  1019. odb_parents = odb_commit->parents;
  1020. while (graph_parents) {
  1021. if (odb_parents == NULL) {
  1022. graph_report(_("commit-graph parent list for commit %s is too long"),
  1023. oid_to_hex(&cur_oid));
  1024. break;
  1025. }
  1026. if (!oideq(&graph_parents->item->object.oid, &odb_parents->item->object.oid))
  1027. graph_report(_("commit-graph parent for %s is %s != %s"),
  1028. oid_to_hex(&cur_oid),
  1029. oid_to_hex(&graph_parents->item->object.oid),
  1030. oid_to_hex(&odb_parents->item->object.oid));
  1031. if (graph_parents->item->generation > max_generation)
  1032. max_generation = graph_parents->item->generation;
  1033. graph_parents = graph_parents->next;
  1034. odb_parents = odb_parents->next;
  1035. }
  1036. if (odb_parents != NULL)
  1037. graph_report(_("commit-graph parent list for commit %s terminates early"),
  1038. oid_to_hex(&cur_oid));
  1039. if (!graph_commit->generation) {
  1040. if (generation_zero == GENERATION_NUMBER_EXISTS)
  1041. graph_report(_("commit-graph has generation number zero for commit %s, but non-zero elsewhere"),
  1042. oid_to_hex(&cur_oid));
  1043. generation_zero = GENERATION_ZERO_EXISTS;
  1044. } else if (generation_zero == GENERATION_ZERO_EXISTS)
  1045. graph_report(_("commit-graph has non-zero generation number for commit %s, but zero elsewhere"),
  1046. oid_to_hex(&cur_oid));
  1047. if (generation_zero == GENERATION_ZERO_EXISTS)
  1048. continue;
  1049. /*
  1050. * If one of our parents has generation GENERATION_NUMBER_MAX, then
  1051. * our generation is also GENERATION_NUMBER_MAX. Decrement to avoid
  1052. * extra logic in the following condition.
  1053. */
  1054. if (max_generation == GENERATION_NUMBER_MAX)
  1055. max_generation--;
  1056. if (graph_commit->generation != max_generation + 1)
  1057. graph_report(_("commit-graph generation for commit %s is %u != %u"),
  1058. oid_to_hex(&cur_oid),
  1059. graph_commit->generation,
  1060. max_generation + 1);
  1061. if (graph_commit->date != odb_commit->date)
  1062. graph_report(_("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime),
  1063. oid_to_hex(&cur_oid),
  1064. graph_commit->date,
  1065. odb_commit->date);
  1066. }
  1067. stop_progress(&progress);
  1068. return verify_commit_graph_error;
  1069. }
  1070. void free_commit_graph(struct commit_graph *g)
  1071. {
  1072. if (!g)
  1073. return;
  1074. if (g->graph_fd >= 0) {
  1075. munmap((void *)g->data, g->data_len);
  1076. g->data = NULL;
  1077. close(g->graph_fd);
  1078. }
  1079. free(g);
  1080. }