THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Git Source Code Mirror - This is a publish-only repository and all pull requests are ignored. Please follow Documentation/SubmittingPatches procedure for any of your improvements.
git
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1044 lines
27KB

  1. #include "cache.h"
  2. #include "config.h"
  3. #include "dir.h"
  4. #include "git-compat-util.h"
  5. #include "lockfile.h"
  6. #include "pack.h"
  7. #include "packfile.h"
  8. #include "commit.h"
  9. #include "object.h"
  10. #include "refs.h"
  11. #include "revision.h"
  12. #include "sha1-lookup.h"
  13. #include "commit-graph.h"
  14. #include "object-store.h"
  15. #include "alloc.h"
  16. #define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
  17. #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
  18. #define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
  19. #define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
  20. #define GRAPH_CHUNKID_LARGEEDGES 0x45444745 /* "EDGE" */
  21. #define GRAPH_DATA_WIDTH 36
  22. #define GRAPH_VERSION_1 0x1
  23. #define GRAPH_VERSION GRAPH_VERSION_1
  24. #define GRAPH_OID_VERSION_SHA1 1
  25. #define GRAPH_OID_LEN_SHA1 GIT_SHA1_RAWSZ
  26. #define GRAPH_OID_VERSION GRAPH_OID_VERSION_SHA1
  27. #define GRAPH_OID_LEN GRAPH_OID_LEN_SHA1
  28. #define GRAPH_OCTOPUS_EDGES_NEEDED 0x80000000
  29. #define GRAPH_PARENT_MISSING 0x7fffffff
  30. #define GRAPH_EDGE_LAST_MASK 0x7fffffff
  31. #define GRAPH_PARENT_NONE 0x70000000
  32. #define GRAPH_LAST_EDGE 0x80000000
  33. #define GRAPH_HEADER_SIZE 8
  34. #define GRAPH_FANOUT_SIZE (4 * 256)
  35. #define GRAPH_CHUNKLOOKUP_WIDTH 12
  36. #define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * GRAPH_CHUNKLOOKUP_WIDTH \
  37. + GRAPH_FANOUT_SIZE + GRAPH_OID_LEN)
  38. char *get_commit_graph_filename(const char *obj_dir)
  39. {
  40. return xstrfmt("%s/info/commit-graph", obj_dir);
  41. }
  42. static struct commit_graph *alloc_commit_graph(void)
  43. {
  44. struct commit_graph *g = xcalloc(1, sizeof(*g));
  45. g->graph_fd = -1;
  46. return g;
  47. }
  48. struct commit_graph *load_commit_graph_one(const char *graph_file)
  49. {
  50. void *graph_map;
  51. const unsigned char *data, *chunk_lookup;
  52. size_t graph_size;
  53. struct stat st;
  54. uint32_t i;
  55. struct commit_graph *graph;
  56. int fd = git_open(graph_file);
  57. uint64_t last_chunk_offset;
  58. uint32_t last_chunk_id;
  59. uint32_t graph_signature;
  60. unsigned char graph_version, hash_version;
  61. if (fd < 0)
  62. return NULL;
  63. if (fstat(fd, &st)) {
  64. close(fd);
  65. return NULL;
  66. }
  67. graph_size = xsize_t(st.st_size);
  68. if (graph_size < GRAPH_MIN_SIZE) {
  69. close(fd);
  70. die(_("graph file %s is too small"), graph_file);
  71. }
  72. graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0);
  73. data = (const unsigned char *)graph_map;
  74. graph_signature = get_be32(data);
  75. if (graph_signature != GRAPH_SIGNATURE) {
  76. error(_("graph signature %X does not match signature %X"),
  77. graph_signature, GRAPH_SIGNATURE);
  78. goto cleanup_fail;
  79. }
  80. graph_version = *(unsigned char*)(data + 4);
  81. if (graph_version != GRAPH_VERSION) {
  82. error(_("graph version %X does not match version %X"),
  83. graph_version, GRAPH_VERSION);
  84. goto cleanup_fail;
  85. }
  86. hash_version = *(unsigned char*)(data + 5);
  87. if (hash_version != GRAPH_OID_VERSION) {
  88. error(_("hash version %X does not match version %X"),
  89. hash_version, GRAPH_OID_VERSION);
  90. goto cleanup_fail;
  91. }
  92. graph = alloc_commit_graph();
  93. graph->hash_len = GRAPH_OID_LEN;
  94. graph->num_chunks = *(unsigned char*)(data + 6);
  95. graph->graph_fd = fd;
  96. graph->data = graph_map;
  97. graph->data_len = graph_size;
  98. last_chunk_id = 0;
  99. last_chunk_offset = 8;
  100. chunk_lookup = data + 8;
  101. for (i = 0; i < graph->num_chunks; i++) {
  102. uint32_t chunk_id = get_be32(chunk_lookup + 0);
  103. uint64_t chunk_offset = get_be64(chunk_lookup + 4);
  104. int chunk_repeated = 0;
  105. chunk_lookup += GRAPH_CHUNKLOOKUP_WIDTH;
  106. if (chunk_offset > graph_size - GIT_MAX_RAWSZ) {
  107. error(_("improper chunk offset %08x%08x"), (uint32_t)(chunk_offset >> 32),
  108. (uint32_t)chunk_offset);
  109. goto cleanup_fail;
  110. }
  111. switch (chunk_id) {
  112. case GRAPH_CHUNKID_OIDFANOUT:
  113. if (graph->chunk_oid_fanout)
  114. chunk_repeated = 1;
  115. else
  116. graph->chunk_oid_fanout = (uint32_t*)(data + chunk_offset);
  117. break;
  118. case GRAPH_CHUNKID_OIDLOOKUP:
  119. if (graph->chunk_oid_lookup)
  120. chunk_repeated = 1;
  121. else
  122. graph->chunk_oid_lookup = data + chunk_offset;
  123. break;
  124. case GRAPH_CHUNKID_DATA:
  125. if (graph->chunk_commit_data)
  126. chunk_repeated = 1;
  127. else
  128. graph->chunk_commit_data = data + chunk_offset;
  129. break;
  130. case GRAPH_CHUNKID_LARGEEDGES:
  131. if (graph->chunk_large_edges)
  132. chunk_repeated = 1;
  133. else
  134. graph->chunk_large_edges = data + chunk_offset;
  135. break;
  136. }
  137. if (chunk_repeated) {
  138. error(_("chunk id %08x appears multiple times"), chunk_id);
  139. goto cleanup_fail;
  140. }
  141. if (last_chunk_id == GRAPH_CHUNKID_OIDLOOKUP)
  142. {
  143. graph->num_commits = (chunk_offset - last_chunk_offset)
  144. / graph->hash_len;
  145. }
  146. last_chunk_id = chunk_id;
  147. last_chunk_offset = chunk_offset;
  148. }
  149. return graph;
  150. cleanup_fail:
  151. munmap(graph_map, graph_size);
  152. close(fd);
  153. exit(1);
  154. }
  155. static void prepare_commit_graph_one(struct repository *r, const char *obj_dir)
  156. {
  157. char *graph_name;
  158. if (r->objects->commit_graph)
  159. return;
  160. graph_name = get_commit_graph_filename(obj_dir);
  161. r->objects->commit_graph =
  162. load_commit_graph_one(graph_name);
  163. FREE_AND_NULL(graph_name);
  164. }
  165. /*
  166. * Return 1 if commit_graph is non-NULL, and 0 otherwise.
  167. *
  168. * On the first invocation, this function attemps to load the commit
  169. * graph if the_repository is configured to have one.
  170. */
  171. static int prepare_commit_graph(struct repository *r)
  172. {
  173. struct alternate_object_database *alt;
  174. char *obj_dir;
  175. int config_value;
  176. if (r->objects->commit_graph_attempted)
  177. return !!r->objects->commit_graph;
  178. r->objects->commit_graph_attempted = 1;
  179. if (repo_config_get_bool(r, "core.commitgraph", &config_value) ||
  180. !config_value)
  181. /*
  182. * This repository is not configured to use commit graphs, so
  183. * do not load one. (But report commit_graph_attempted anyway
  184. * so that commit graph loading is not attempted again for this
  185. * repository.)
  186. */
  187. return 0;
  188. obj_dir = r->objects->objectdir;
  189. prepare_commit_graph_one(r, obj_dir);
  190. prepare_alt_odb(r);
  191. for (alt = r->objects->alt_odb_list;
  192. !r->objects->commit_graph && alt;
  193. alt = alt->next)
  194. prepare_commit_graph_one(r, alt->path);
  195. return !!r->objects->commit_graph;
  196. }
  197. static void close_commit_graph(void)
  198. {
  199. free_commit_graph(the_repository->objects->commit_graph);
  200. the_repository->objects->commit_graph = NULL;
  201. }
  202. static int bsearch_graph(struct commit_graph *g, struct object_id *oid, uint32_t *pos)
  203. {
  204. return bsearch_hash(oid->hash, g->chunk_oid_fanout,
  205. g->chunk_oid_lookup, g->hash_len, pos);
  206. }
  207. static struct commit_list **insert_parent_or_die(struct commit_graph *g,
  208. uint64_t pos,
  209. struct commit_list **pptr)
  210. {
  211. struct commit *c;
  212. struct object_id oid;
  213. if (pos >= g->num_commits)
  214. die("invalid parent position %"PRIu64, pos);
  215. hashcpy(oid.hash, g->chunk_oid_lookup + g->hash_len * pos);
  216. c = lookup_commit(the_repository, &oid);
  217. if (!c)
  218. die(_("could not find commit %s"), oid_to_hex(&oid));
  219. c->graph_pos = pos;
  220. return &commit_list_insert(c, pptr)->next;
  221. }
  222. static void fill_commit_graph_info(struct commit *item, struct commit_graph *g, uint32_t pos)
  223. {
  224. const unsigned char *commit_data = g->chunk_commit_data + GRAPH_DATA_WIDTH * pos;
  225. item->graph_pos = pos;
  226. item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
  227. }
  228. static int fill_commit_in_graph(struct commit *item, struct commit_graph *g, uint32_t pos)
  229. {
  230. uint32_t edge_value;
  231. uint32_t *parent_data_ptr;
  232. uint64_t date_low, date_high;
  233. struct commit_list **pptr;
  234. const unsigned char *commit_data = g->chunk_commit_data + (g->hash_len + 16) * pos;
  235. item->object.parsed = 1;
  236. item->graph_pos = pos;
  237. item->maybe_tree = NULL;
  238. date_high = get_be32(commit_data + g->hash_len + 8) & 0x3;
  239. date_low = get_be32(commit_data + g->hash_len + 12);
  240. item->date = (timestamp_t)((date_high << 32) | date_low);
  241. item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
  242. pptr = &item->parents;
  243. edge_value = get_be32(commit_data + g->hash_len);
  244. if (edge_value == GRAPH_PARENT_NONE)
  245. return 1;
  246. pptr = insert_parent_or_die(g, edge_value, pptr);
  247. edge_value = get_be32(commit_data + g->hash_len + 4);
  248. if (edge_value == GRAPH_PARENT_NONE)
  249. return 1;
  250. if (!(edge_value & GRAPH_OCTOPUS_EDGES_NEEDED)) {
  251. pptr = insert_parent_or_die(g, edge_value, pptr);
  252. return 1;
  253. }
  254. parent_data_ptr = (uint32_t*)(g->chunk_large_edges +
  255. 4 * (uint64_t)(edge_value & GRAPH_EDGE_LAST_MASK));
  256. do {
  257. edge_value = get_be32(parent_data_ptr);
  258. pptr = insert_parent_or_die(g,
  259. edge_value & GRAPH_EDGE_LAST_MASK,
  260. pptr);
  261. parent_data_ptr++;
  262. } while (!(edge_value & GRAPH_LAST_EDGE));
  263. return 1;
  264. }
  265. static int find_commit_in_graph(struct commit *item, struct commit_graph *g, uint32_t *pos)
  266. {
  267. if (item->graph_pos != COMMIT_NOT_FROM_GRAPH) {
  268. *pos = item->graph_pos;
  269. return 1;
  270. } else {
  271. return bsearch_graph(g, &(item->object.oid), pos);
  272. }
  273. }
  274. static int parse_commit_in_graph_one(struct commit_graph *g, struct commit *item)
  275. {
  276. uint32_t pos;
  277. if (item->object.parsed)
  278. return 1;
  279. if (find_commit_in_graph(item, g, &pos))
  280. return fill_commit_in_graph(item, g, pos);
  281. return 0;
  282. }
  283. int parse_commit_in_graph(struct repository *r, struct commit *item)
  284. {
  285. if (!prepare_commit_graph(r))
  286. return 0;
  287. return parse_commit_in_graph_one(r->objects->commit_graph, item);
  288. }
  289. void load_commit_graph_info(struct repository *r, struct commit *item)
  290. {
  291. uint32_t pos;
  292. if (!prepare_commit_graph(r))
  293. return;
  294. if (find_commit_in_graph(item, r->objects->commit_graph, &pos))
  295. fill_commit_graph_info(item, r->objects->commit_graph, pos);
  296. }
  297. static struct tree *load_tree_for_commit(struct commit_graph *g, struct commit *c)
  298. {
  299. struct object_id oid;
  300. const unsigned char *commit_data = g->chunk_commit_data +
  301. GRAPH_DATA_WIDTH * (c->graph_pos);
  302. hashcpy(oid.hash, commit_data);
  303. c->maybe_tree = lookup_tree(the_repository, &oid);
  304. return c->maybe_tree;
  305. }
  306. static struct tree *get_commit_tree_in_graph_one(struct commit_graph *g,
  307. const struct commit *c)
  308. {
  309. if (c->maybe_tree)
  310. return c->maybe_tree;
  311. if (c->graph_pos == COMMIT_NOT_FROM_GRAPH)
  312. BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
  313. return load_tree_for_commit(g, (struct commit *)c);
  314. }
  315. struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit *c)
  316. {
  317. return get_commit_tree_in_graph_one(r->objects->commit_graph, c);
  318. }
  319. static void write_graph_chunk_fanout(struct hashfile *f,
  320. struct commit **commits,
  321. int nr_commits)
  322. {
  323. int i, count = 0;
  324. struct commit **list = commits;
  325. /*
  326. * Write the first-level table (the list is sorted,
  327. * but we use a 256-entry lookup to be able to avoid
  328. * having to do eight extra binary search iterations).
  329. */
  330. for (i = 0; i < 256; i++) {
  331. while (count < nr_commits) {
  332. if ((*list)->object.oid.hash[0] != i)
  333. break;
  334. count++;
  335. list++;
  336. }
  337. hashwrite_be32(f, count);
  338. }
  339. }
  340. static void write_graph_chunk_oids(struct hashfile *f, int hash_len,
  341. struct commit **commits, int nr_commits)
  342. {
  343. struct commit **list = commits;
  344. int count;
  345. for (count = 0; count < nr_commits; count++, list++)
  346. hashwrite(f, (*list)->object.oid.hash, (int)hash_len);
  347. }
  348. static const unsigned char *commit_to_sha1(size_t index, void *table)
  349. {
  350. struct commit **commits = table;
  351. return commits[index]->object.oid.hash;
  352. }
  353. static void write_graph_chunk_data(struct hashfile *f, int hash_len,
  354. struct commit **commits, int nr_commits)
  355. {
  356. struct commit **list = commits;
  357. struct commit **last = commits + nr_commits;
  358. uint32_t num_extra_edges = 0;
  359. while (list < last) {
  360. struct commit_list *parent;
  361. int edge_value;
  362. uint32_t packedDate[2];
  363. parse_commit(*list);
  364. hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len);
  365. parent = (*list)->parents;
  366. if (!parent)
  367. edge_value = GRAPH_PARENT_NONE;
  368. else {
  369. edge_value = sha1_pos(parent->item->object.oid.hash,
  370. commits,
  371. nr_commits,
  372. commit_to_sha1);
  373. if (edge_value < 0)
  374. edge_value = GRAPH_PARENT_MISSING;
  375. }
  376. hashwrite_be32(f, edge_value);
  377. if (parent)
  378. parent = parent->next;
  379. if (!parent)
  380. edge_value = GRAPH_PARENT_NONE;
  381. else if (parent->next)
  382. edge_value = GRAPH_OCTOPUS_EDGES_NEEDED | num_extra_edges;
  383. else {
  384. edge_value = sha1_pos(parent->item->object.oid.hash,
  385. commits,
  386. nr_commits,
  387. commit_to_sha1);
  388. if (edge_value < 0)
  389. edge_value = GRAPH_PARENT_MISSING;
  390. }
  391. hashwrite_be32(f, edge_value);
  392. if (edge_value & GRAPH_OCTOPUS_EDGES_NEEDED) {
  393. do {
  394. num_extra_edges++;
  395. parent = parent->next;
  396. } while (parent);
  397. }
  398. if (sizeof((*list)->date) > 4)
  399. packedDate[0] = htonl(((*list)->date >> 32) & 0x3);
  400. else
  401. packedDate[0] = 0;
  402. packedDate[0] |= htonl((*list)->generation << 2);
  403. packedDate[1] = htonl((*list)->date);
  404. hashwrite(f, packedDate, 8);
  405. list++;
  406. }
  407. }
  408. static void write_graph_chunk_large_edges(struct hashfile *f,
  409. struct commit **commits,
  410. int nr_commits)
  411. {
  412. struct commit **list = commits;
  413. struct commit **last = commits + nr_commits;
  414. struct commit_list *parent;
  415. while (list < last) {
  416. int num_parents = 0;
  417. for (parent = (*list)->parents; num_parents < 3 && parent;
  418. parent = parent->next)
  419. num_parents++;
  420. if (num_parents <= 2) {
  421. list++;
  422. continue;
  423. }
  424. /* Since num_parents > 2, this initializer is safe. */
  425. for (parent = (*list)->parents->next; parent; parent = parent->next) {
  426. int edge_value = sha1_pos(parent->item->object.oid.hash,
  427. commits,
  428. nr_commits,
  429. commit_to_sha1);
  430. if (edge_value < 0)
  431. edge_value = GRAPH_PARENT_MISSING;
  432. else if (!parent->next)
  433. edge_value |= GRAPH_LAST_EDGE;
  434. hashwrite_be32(f, edge_value);
  435. }
  436. list++;
  437. }
  438. }
  439. static int commit_compare(const void *_a, const void *_b)
  440. {
  441. const struct object_id *a = (const struct object_id *)_a;
  442. const struct object_id *b = (const struct object_id *)_b;
  443. return oidcmp(a, b);
  444. }
  445. struct packed_commit_list {
  446. struct commit **list;
  447. int nr;
  448. int alloc;
  449. };
  450. struct packed_oid_list {
  451. struct object_id *list;
  452. int nr;
  453. int alloc;
  454. };
  455. static int add_packed_commits(const struct object_id *oid,
  456. struct packed_git *pack,
  457. uint32_t pos,
  458. void *data)
  459. {
  460. struct packed_oid_list *list = (struct packed_oid_list*)data;
  461. enum object_type type;
  462. off_t offset = nth_packed_object_offset(pack, pos);
  463. struct object_info oi = OBJECT_INFO_INIT;
  464. oi.typep = &type;
  465. if (packed_object_info(the_repository, pack, offset, &oi) < 0)
  466. die(_("unable to get type of object %s"), oid_to_hex(oid));
  467. if (type != OBJ_COMMIT)
  468. return 0;
  469. ALLOC_GROW(list->list, list->nr + 1, list->alloc);
  470. oidcpy(&(list->list[list->nr]), oid);
  471. list->nr++;
  472. return 0;
  473. }
  474. static void add_missing_parents(struct packed_oid_list *oids, struct commit *commit)
  475. {
  476. struct commit_list *parent;
  477. for (parent = commit->parents; parent; parent = parent->next) {
  478. if (!(parent->item->object.flags & UNINTERESTING)) {
  479. ALLOC_GROW(oids->list, oids->nr + 1, oids->alloc);
  480. oidcpy(&oids->list[oids->nr], &(parent->item->object.oid));
  481. oids->nr++;
  482. parent->item->object.flags |= UNINTERESTING;
  483. }
  484. }
  485. }
  486. static void close_reachable(struct packed_oid_list *oids)
  487. {
  488. int i;
  489. struct commit *commit;
  490. for (i = 0; i < oids->nr; i++) {
  491. commit = lookup_commit(the_repository, &oids->list[i]);
  492. if (commit)
  493. commit->object.flags |= UNINTERESTING;
  494. }
  495. /*
  496. * As this loop runs, oids->nr may grow, but not more
  497. * than the number of missing commits in the reachable
  498. * closure.
  499. */
  500. for (i = 0; i < oids->nr; i++) {
  501. commit = lookup_commit(the_repository, &oids->list[i]);
  502. if (commit && !parse_commit(commit))
  503. add_missing_parents(oids, commit);
  504. }
  505. for (i = 0; i < oids->nr; i++) {
  506. commit = lookup_commit(the_repository, &oids->list[i]);
  507. if (commit)
  508. commit->object.flags &= ~UNINTERESTING;
  509. }
  510. }
  511. static void compute_generation_numbers(struct packed_commit_list* commits)
  512. {
  513. int i;
  514. struct commit_list *list = NULL;
  515. for (i = 0; i < commits->nr; i++) {
  516. if (commits->list[i]->generation != GENERATION_NUMBER_INFINITY &&
  517. commits->list[i]->generation != GENERATION_NUMBER_ZERO)
  518. continue;
  519. commit_list_insert(commits->list[i], &list);
  520. while (list) {
  521. struct commit *current = list->item;
  522. struct commit_list *parent;
  523. int all_parents_computed = 1;
  524. uint32_t max_generation = 0;
  525. for (parent = current->parents; parent; parent = parent->next) {
  526. if (parent->item->generation == GENERATION_NUMBER_INFINITY ||
  527. parent->item->generation == GENERATION_NUMBER_ZERO) {
  528. all_parents_computed = 0;
  529. commit_list_insert(parent->item, &list);
  530. break;
  531. } else if (parent->item->generation > max_generation) {
  532. max_generation = parent->item->generation;
  533. }
  534. }
  535. if (all_parents_computed) {
  536. current->generation = max_generation + 1;
  537. pop_commit(&list);
  538. if (current->generation > GENERATION_NUMBER_MAX)
  539. current->generation = GENERATION_NUMBER_MAX;
  540. }
  541. }
  542. }
  543. }
  544. static int add_ref_to_list(const char *refname,
  545. const struct object_id *oid,
  546. int flags, void *cb_data)
  547. {
  548. struct string_list *list = (struct string_list *)cb_data;
  549. string_list_append(list, oid_to_hex(oid));
  550. return 0;
  551. }
  552. void write_commit_graph_reachable(const char *obj_dir, int append)
  553. {
  554. struct string_list list;
  555. string_list_init(&list, 1);
  556. for_each_ref(add_ref_to_list, &list);
  557. write_commit_graph(obj_dir, NULL, &list, append);
  558. }
  559. void write_commit_graph(const char *obj_dir,
  560. struct string_list *pack_indexes,
  561. struct string_list *commit_hex,
  562. int append)
  563. {
  564. struct packed_oid_list oids;
  565. struct packed_commit_list commits;
  566. struct hashfile *f;
  567. uint32_t i, count_distinct = 0;
  568. char *graph_name;
  569. struct lock_file lk = LOCK_INIT;
  570. uint32_t chunk_ids[5];
  571. uint64_t chunk_offsets[5];
  572. int num_chunks;
  573. int num_extra_edges;
  574. struct commit_list *parent;
  575. oids.nr = 0;
  576. oids.alloc = approximate_object_count() / 4;
  577. if (append) {
  578. prepare_commit_graph_one(the_repository, obj_dir);
  579. if (the_repository->objects->commit_graph)
  580. oids.alloc += the_repository->objects->commit_graph->num_commits;
  581. }
  582. if (oids.alloc < 1024)
  583. oids.alloc = 1024;
  584. ALLOC_ARRAY(oids.list, oids.alloc);
  585. if (append && the_repository->objects->commit_graph) {
  586. struct commit_graph *commit_graph =
  587. the_repository->objects->commit_graph;
  588. for (i = 0; i < commit_graph->num_commits; i++) {
  589. const unsigned char *hash = commit_graph->chunk_oid_lookup +
  590. commit_graph->hash_len * i;
  591. hashcpy(oids.list[oids.nr++].hash, hash);
  592. }
  593. }
  594. if (pack_indexes) {
  595. struct strbuf packname = STRBUF_INIT;
  596. int dirlen;
  597. strbuf_addf(&packname, "%s/pack/", obj_dir);
  598. dirlen = packname.len;
  599. for (i = 0; i < pack_indexes->nr; i++) {
  600. struct packed_git *p;
  601. strbuf_setlen(&packname, dirlen);
  602. strbuf_addstr(&packname, pack_indexes->items[i].string);
  603. p = add_packed_git(packname.buf, packname.len, 1);
  604. if (!p)
  605. die(_("error adding pack %s"), packname.buf);
  606. if (open_pack_index(p))
  607. die(_("error opening index for %s"), packname.buf);
  608. for_each_object_in_pack(p, add_packed_commits, &oids, 0);
  609. close_pack(p);
  610. }
  611. strbuf_release(&packname);
  612. }
  613. if (commit_hex) {
  614. for (i = 0; i < commit_hex->nr; i++) {
  615. const char *end;
  616. struct object_id oid;
  617. struct commit *result;
  618. if (commit_hex->items[i].string &&
  619. parse_oid_hex(commit_hex->items[i].string, &oid, &end))
  620. continue;
  621. result = lookup_commit_reference_gently(the_repository, &oid, 1);
  622. if (result) {
  623. ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc);
  624. oidcpy(&oids.list[oids.nr], &(result->object.oid));
  625. oids.nr++;
  626. }
  627. }
  628. }
  629. if (!pack_indexes && !commit_hex)
  630. for_each_packed_object(add_packed_commits, &oids, 0);
  631. close_reachable(&oids);
  632. QSORT(oids.list, oids.nr, commit_compare);
  633. count_distinct = 1;
  634. for (i = 1; i < oids.nr; i++) {
  635. if (!oideq(&oids.list[i - 1], &oids.list[i]))
  636. count_distinct++;
  637. }
  638. if (count_distinct >= GRAPH_PARENT_MISSING)
  639. die(_("the commit graph format cannot write %d commits"), count_distinct);
  640. commits.nr = 0;
  641. commits.alloc = count_distinct;
  642. ALLOC_ARRAY(commits.list, commits.alloc);
  643. num_extra_edges = 0;
  644. for (i = 0; i < oids.nr; i++) {
  645. int num_parents = 0;
  646. if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i]))
  647. continue;
  648. commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]);
  649. parse_commit(commits.list[commits.nr]);
  650. for (parent = commits.list[commits.nr]->parents;
  651. parent; parent = parent->next)
  652. num_parents++;
  653. if (num_parents > 2)
  654. num_extra_edges += num_parents - 1;
  655. commits.nr++;
  656. }
  657. num_chunks = num_extra_edges ? 4 : 3;
  658. if (commits.nr >= GRAPH_PARENT_MISSING)
  659. die(_("too many commits to write graph"));
  660. compute_generation_numbers(&commits);
  661. graph_name = get_commit_graph_filename(obj_dir);
  662. if (safe_create_leading_directories(graph_name))
  663. die_errno(_("unable to create leading directories of %s"),
  664. graph_name);
  665. hold_lock_file_for_update(&lk, graph_name, LOCK_DIE_ON_ERROR);
  666. f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);
  667. hashwrite_be32(f, GRAPH_SIGNATURE);
  668. hashwrite_u8(f, GRAPH_VERSION);
  669. hashwrite_u8(f, GRAPH_OID_VERSION);
  670. hashwrite_u8(f, num_chunks);
  671. hashwrite_u8(f, 0); /* unused padding byte */
  672. chunk_ids[0] = GRAPH_CHUNKID_OIDFANOUT;
  673. chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP;
  674. chunk_ids[2] = GRAPH_CHUNKID_DATA;
  675. if (num_extra_edges)
  676. chunk_ids[3] = GRAPH_CHUNKID_LARGEEDGES;
  677. else
  678. chunk_ids[3] = 0;
  679. chunk_ids[4] = 0;
  680. chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH;
  681. chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE;
  682. chunk_offsets[2] = chunk_offsets[1] + GRAPH_OID_LEN * commits.nr;
  683. chunk_offsets[3] = chunk_offsets[2] + (GRAPH_OID_LEN + 16) * commits.nr;
  684. chunk_offsets[4] = chunk_offsets[3] + 4 * num_extra_edges;
  685. for (i = 0; i <= num_chunks; i++) {
  686. uint32_t chunk_write[3];
  687. chunk_write[0] = htonl(chunk_ids[i]);
  688. chunk_write[1] = htonl(chunk_offsets[i] >> 32);
  689. chunk_write[2] = htonl(chunk_offsets[i] & 0xffffffff);
  690. hashwrite(f, chunk_write, 12);
  691. }
  692. write_graph_chunk_fanout(f, commits.list, commits.nr);
  693. write_graph_chunk_oids(f, GRAPH_OID_LEN, commits.list, commits.nr);
  694. write_graph_chunk_data(f, GRAPH_OID_LEN, commits.list, commits.nr);
  695. write_graph_chunk_large_edges(f, commits.list, commits.nr);
  696. close_commit_graph();
  697. finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC);
  698. commit_lock_file(&lk);
  699. free(oids.list);
  700. oids.alloc = 0;
  701. oids.nr = 0;
  702. }
  703. #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2
  704. static int verify_commit_graph_error;
  705. static void graph_report(const char *fmt, ...)
  706. {
  707. va_list ap;
  708. verify_commit_graph_error = 1;
  709. va_start(ap, fmt);
  710. vfprintf(stderr, fmt, ap);
  711. fprintf(stderr, "\n");
  712. va_end(ap);
  713. }
  714. #define GENERATION_ZERO_EXISTS 1
  715. #define GENERATION_NUMBER_EXISTS 2
  716. int verify_commit_graph(struct repository *r, struct commit_graph *g)
  717. {
  718. uint32_t i, cur_fanout_pos = 0;
  719. struct object_id prev_oid, cur_oid, checksum;
  720. int generation_zero = 0;
  721. struct hashfile *f;
  722. int devnull;
  723. if (!g) {
  724. graph_report("no commit-graph file loaded");
  725. return 1;
  726. }
  727. verify_commit_graph_error = 0;
  728. if (!g->chunk_oid_fanout)
  729. graph_report("commit-graph is missing the OID Fanout chunk");
  730. if (!g->chunk_oid_lookup)
  731. graph_report("commit-graph is missing the OID Lookup chunk");
  732. if (!g->chunk_commit_data)
  733. graph_report("commit-graph is missing the Commit Data chunk");
  734. if (verify_commit_graph_error)
  735. return verify_commit_graph_error;
  736. devnull = open("/dev/null", O_WRONLY);
  737. f = hashfd(devnull, NULL);
  738. hashwrite(f, g->data, g->data_len - g->hash_len);
  739. finalize_hashfile(f, checksum.hash, CSUM_CLOSE);
  740. if (!hasheq(checksum.hash, g->data + g->data_len - g->hash_len)) {
  741. graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt"));
  742. verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH;
  743. }
  744. for (i = 0; i < g->num_commits; i++) {
  745. struct commit *graph_commit;
  746. hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
  747. if (i && oidcmp(&prev_oid, &cur_oid) >= 0)
  748. graph_report("commit-graph has incorrect OID order: %s then %s",
  749. oid_to_hex(&prev_oid),
  750. oid_to_hex(&cur_oid));
  751. oidcpy(&prev_oid, &cur_oid);
  752. while (cur_oid.hash[0] > cur_fanout_pos) {
  753. uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
  754. if (i != fanout_value)
  755. graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
  756. cur_fanout_pos, fanout_value, i);
  757. cur_fanout_pos++;
  758. }
  759. graph_commit = lookup_commit(r, &cur_oid);
  760. if (!parse_commit_in_graph_one(g, graph_commit))
  761. graph_report("failed to parse %s from commit-graph",
  762. oid_to_hex(&cur_oid));
  763. }
  764. while (cur_fanout_pos < 256) {
  765. uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
  766. if (g->num_commits != fanout_value)
  767. graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
  768. cur_fanout_pos, fanout_value, i);
  769. cur_fanout_pos++;
  770. }
  771. if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH)
  772. return verify_commit_graph_error;
  773. for (i = 0; i < g->num_commits; i++) {
  774. struct commit *graph_commit, *odb_commit;
  775. struct commit_list *graph_parents, *odb_parents;
  776. uint32_t max_generation = 0;
  777. hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
  778. graph_commit = lookup_commit(r, &cur_oid);
  779. odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r));
  780. if (parse_commit_internal(odb_commit, 0, 0)) {
  781. graph_report("failed to parse %s from object database",
  782. oid_to_hex(&cur_oid));
  783. continue;
  784. }
  785. if (!oideq(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid,
  786. get_commit_tree_oid(odb_commit)))
  787. graph_report("root tree OID for commit %s in commit-graph is %s != %s",
  788. oid_to_hex(&cur_oid),
  789. oid_to_hex(get_commit_tree_oid(graph_commit)),
  790. oid_to_hex(get_commit_tree_oid(odb_commit)));
  791. graph_parents = graph_commit->parents;
  792. odb_parents = odb_commit->parents;
  793. while (graph_parents) {
  794. if (odb_parents == NULL) {
  795. graph_report("commit-graph parent list for commit %s is too long",
  796. oid_to_hex(&cur_oid));
  797. break;
  798. }
  799. if (!oideq(&graph_parents->item->object.oid, &odb_parents->item->object.oid))
  800. graph_report("commit-graph parent for %s is %s != %s",
  801. oid_to_hex(&cur_oid),
  802. oid_to_hex(&graph_parents->item->object.oid),
  803. oid_to_hex(&odb_parents->item->object.oid));
  804. if (graph_parents->item->generation > max_generation)
  805. max_generation = graph_parents->item->generation;
  806. graph_parents = graph_parents->next;
  807. odb_parents = odb_parents->next;
  808. }
  809. if (odb_parents != NULL)
  810. graph_report("commit-graph parent list for commit %s terminates early",
  811. oid_to_hex(&cur_oid));
  812. if (!graph_commit->generation) {
  813. if (generation_zero == GENERATION_NUMBER_EXISTS)
  814. graph_report("commit-graph has generation number zero for commit %s, but non-zero elsewhere",
  815. oid_to_hex(&cur_oid));
  816. generation_zero = GENERATION_ZERO_EXISTS;
  817. } else if (generation_zero == GENERATION_ZERO_EXISTS)
  818. graph_report("commit-graph has non-zero generation number for commit %s, but zero elsewhere",
  819. oid_to_hex(&cur_oid));
  820. if (generation_zero == GENERATION_ZERO_EXISTS)
  821. continue;
  822. /*
  823. * If one of our parents has generation GENERATION_NUMBER_MAX, then
  824. * our generation is also GENERATION_NUMBER_MAX. Decrement to avoid
  825. * extra logic in the following condition.
  826. */
  827. if (max_generation == GENERATION_NUMBER_MAX)
  828. max_generation--;
  829. if (graph_commit->generation != max_generation + 1)
  830. graph_report("commit-graph generation for commit %s is %u != %u",
  831. oid_to_hex(&cur_oid),
  832. graph_commit->generation,
  833. max_generation + 1);
  834. if (graph_commit->date != odb_commit->date)
  835. graph_report("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime,
  836. oid_to_hex(&cur_oid),
  837. graph_commit->date,
  838. odb_commit->date);
  839. }
  840. return verify_commit_graph_error;
  841. }
  842. void free_commit_graph(struct commit_graph *g)
  843. {
  844. if (!g)
  845. return;
  846. if (g->graph_fd >= 0) {
  847. munmap((void *)g->data, g->data_len);
  848. g->data = NULL;
  849. close(g->graph_fd);
  850. }
  851. free(g);
  852. }