THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Git Source Code Mirror - This is a publish-only repository and all pull requests are ignored. Please follow Documentation/SubmittingPatches procedure for any of your improvements.
git
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1382 lines
35KB

  1. #include "cache.h"
  2. #include "config.h"
  3. #include "csum-file.h"
  4. #include "dir.h"
  5. #include "lockfile.h"
  6. #include "packfile.h"
  7. #include "object-store.h"
  8. #include "sha1-lookup.h"
  9. #include "midx.h"
  10. #include "progress.h"
  11. #include "trace2.h"
  12. #include "run-command.h"
  13. #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
  14. #define MIDX_VERSION 1
  15. #define MIDX_BYTE_FILE_VERSION 4
  16. #define MIDX_BYTE_HASH_VERSION 5
  17. #define MIDX_BYTE_NUM_CHUNKS 6
  18. #define MIDX_BYTE_NUM_PACKS 8
  19. #define MIDX_HASH_VERSION 1
  20. #define MIDX_HEADER_SIZE 12
  21. #define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + the_hash_algo->rawsz)
  22. #define MIDX_MAX_CHUNKS 5
  23. #define MIDX_CHUNK_ALIGNMENT 4
  24. #define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */
  25. #define MIDX_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
  26. #define MIDX_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
  27. #define MIDX_CHUNKID_OBJECTOFFSETS 0x4f4f4646 /* "OOFF" */
  28. #define MIDX_CHUNKID_LARGEOFFSETS 0x4c4f4646 /* "LOFF" */
  29. #define MIDX_CHUNKLOOKUP_WIDTH (sizeof(uint32_t) + sizeof(uint64_t))
  30. #define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256)
  31. #define MIDX_CHUNK_OFFSET_WIDTH (2 * sizeof(uint32_t))
  32. #define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t))
  33. #define MIDX_LARGE_OFFSET_NEEDED 0x80000000
  34. #define PACK_EXPIRED UINT_MAX
  35. static char *get_midx_filename(const char *object_dir)
  36. {
  37. return xstrfmt("%s/pack/multi-pack-index", object_dir);
  38. }
  39. struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local)
  40. {
  41. struct multi_pack_index *m = NULL;
  42. int fd;
  43. struct stat st;
  44. size_t midx_size;
  45. void *midx_map = NULL;
  46. uint32_t hash_version;
  47. char *midx_name = get_midx_filename(object_dir);
  48. uint32_t i;
  49. const char *cur_pack_name;
  50. fd = git_open(midx_name);
  51. if (fd < 0)
  52. goto cleanup_fail;
  53. if (fstat(fd, &st)) {
  54. error_errno(_("failed to read %s"), midx_name);
  55. goto cleanup_fail;
  56. }
  57. midx_size = xsize_t(st.st_size);
  58. if (midx_size < MIDX_MIN_SIZE) {
  59. error(_("multi-pack-index file %s is too small"), midx_name);
  60. goto cleanup_fail;
  61. }
  62. FREE_AND_NULL(midx_name);
  63. midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0);
  64. FLEX_ALLOC_STR(m, object_dir, object_dir);
  65. m->fd = fd;
  66. m->data = midx_map;
  67. m->data_len = midx_size;
  68. m->local = local;
  69. m->signature = get_be32(m->data);
  70. if (m->signature != MIDX_SIGNATURE)
  71. die(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
  72. m->signature, MIDX_SIGNATURE);
  73. m->version = m->data[MIDX_BYTE_FILE_VERSION];
  74. if (m->version != MIDX_VERSION)
  75. die(_("multi-pack-index version %d not recognized"),
  76. m->version);
  77. hash_version = m->data[MIDX_BYTE_HASH_VERSION];
  78. if (hash_version != MIDX_HASH_VERSION)
  79. die(_("hash version %u does not match"), hash_version);
  80. m->hash_len = the_hash_algo->rawsz;
  81. m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS];
  82. m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS);
  83. for (i = 0; i < m->num_chunks; i++) {
  84. uint32_t chunk_id = get_be32(m->data + MIDX_HEADER_SIZE +
  85. MIDX_CHUNKLOOKUP_WIDTH * i);
  86. uint64_t chunk_offset = get_be64(m->data + MIDX_HEADER_SIZE + 4 +
  87. MIDX_CHUNKLOOKUP_WIDTH * i);
  88. if (chunk_offset >= m->data_len)
  89. die(_("invalid chunk offset (too large)"));
  90. switch (chunk_id) {
  91. case MIDX_CHUNKID_PACKNAMES:
  92. m->chunk_pack_names = m->data + chunk_offset;
  93. break;
  94. case MIDX_CHUNKID_OIDFANOUT:
  95. m->chunk_oid_fanout = (uint32_t *)(m->data + chunk_offset);
  96. break;
  97. case MIDX_CHUNKID_OIDLOOKUP:
  98. m->chunk_oid_lookup = m->data + chunk_offset;
  99. break;
  100. case MIDX_CHUNKID_OBJECTOFFSETS:
  101. m->chunk_object_offsets = m->data + chunk_offset;
  102. break;
  103. case MIDX_CHUNKID_LARGEOFFSETS:
  104. m->chunk_large_offsets = m->data + chunk_offset;
  105. break;
  106. case 0:
  107. die(_("terminating multi-pack-index chunk id appears earlier than expected"));
  108. break;
  109. default:
  110. /*
  111. * Do nothing on unrecognized chunks, allowing future
  112. * extensions to add optional chunks.
  113. */
  114. break;
  115. }
  116. }
  117. if (!m->chunk_pack_names)
  118. die(_("multi-pack-index missing required pack-name chunk"));
  119. if (!m->chunk_oid_fanout)
  120. die(_("multi-pack-index missing required OID fanout chunk"));
  121. if (!m->chunk_oid_lookup)
  122. die(_("multi-pack-index missing required OID lookup chunk"));
  123. if (!m->chunk_object_offsets)
  124. die(_("multi-pack-index missing required object offsets chunk"));
  125. m->num_objects = ntohl(m->chunk_oid_fanout[255]);
  126. m->pack_names = xcalloc(m->num_packs, sizeof(*m->pack_names));
  127. m->packs = xcalloc(m->num_packs, sizeof(*m->packs));
  128. cur_pack_name = (const char *)m->chunk_pack_names;
  129. for (i = 0; i < m->num_packs; i++) {
  130. m->pack_names[i] = cur_pack_name;
  131. cur_pack_name += strlen(cur_pack_name) + 1;
  132. if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0)
  133. die(_("multi-pack-index pack names out of order: '%s' before '%s'"),
  134. m->pack_names[i - 1],
  135. m->pack_names[i]);
  136. }
  137. trace2_data_intmax("midx", the_repository, "load/num_packs", m->num_packs);
  138. trace2_data_intmax("midx", the_repository, "load/num_objects", m->num_objects);
  139. return m;
  140. cleanup_fail:
  141. free(m);
  142. free(midx_name);
  143. if (midx_map)
  144. munmap(midx_map, midx_size);
  145. if (0 <= fd)
  146. close(fd);
  147. return NULL;
  148. }
  149. void close_midx(struct multi_pack_index *m)
  150. {
  151. uint32_t i;
  152. if (!m)
  153. return;
  154. munmap((unsigned char *)m->data, m->data_len);
  155. close(m->fd);
  156. m->fd = -1;
  157. for (i = 0; i < m->num_packs; i++) {
  158. if (m->packs[i])
  159. m->packs[i]->multi_pack_index = 0;
  160. }
  161. FREE_AND_NULL(m->packs);
  162. FREE_AND_NULL(m->pack_names);
  163. }
  164. int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id)
  165. {
  166. struct strbuf pack_name = STRBUF_INIT;
  167. struct packed_git *p;
  168. if (pack_int_id >= m->num_packs)
  169. die(_("bad pack-int-id: %u (%u total packs)"),
  170. pack_int_id, m->num_packs);
  171. if (m->packs[pack_int_id])
  172. return 0;
  173. strbuf_addf(&pack_name, "%s/pack/%s", m->object_dir,
  174. m->pack_names[pack_int_id]);
  175. p = add_packed_git(pack_name.buf, pack_name.len, m->local);
  176. strbuf_release(&pack_name);
  177. if (!p)
  178. return 1;
  179. p->multi_pack_index = 1;
  180. m->packs[pack_int_id] = p;
  181. install_packed_git(r, p);
  182. list_add_tail(&p->mru, &r->objects->packed_git_mru);
  183. return 0;
  184. }
  185. int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result)
  186. {
  187. return bsearch_hash(oid->hash, m->chunk_oid_fanout, m->chunk_oid_lookup,
  188. the_hash_algo->rawsz, result);
  189. }
  190. struct object_id *nth_midxed_object_oid(struct object_id *oid,
  191. struct multi_pack_index *m,
  192. uint32_t n)
  193. {
  194. if (n >= m->num_objects)
  195. return NULL;
  196. hashcpy(oid->hash, m->chunk_oid_lookup + m->hash_len * n);
  197. return oid;
  198. }
  199. static off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos)
  200. {
  201. const unsigned char *offset_data;
  202. uint32_t offset32;
  203. offset_data = m->chunk_object_offsets + pos * MIDX_CHUNK_OFFSET_WIDTH;
  204. offset32 = get_be32(offset_data + sizeof(uint32_t));
  205. if (m->chunk_large_offsets && offset32 & MIDX_LARGE_OFFSET_NEEDED) {
  206. if (sizeof(off_t) < sizeof(uint64_t))
  207. die(_("multi-pack-index stores a 64-bit offset, but off_t is too small"));
  208. offset32 ^= MIDX_LARGE_OFFSET_NEEDED;
  209. return get_be64(m->chunk_large_offsets + sizeof(uint64_t) * offset32);
  210. }
  211. return offset32;
  212. }
  213. static uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos)
  214. {
  215. return get_be32(m->chunk_object_offsets + pos * MIDX_CHUNK_OFFSET_WIDTH);
  216. }
  217. static int nth_midxed_pack_entry(struct repository *r,
  218. struct multi_pack_index *m,
  219. struct pack_entry *e,
  220. uint32_t pos)
  221. {
  222. uint32_t pack_int_id;
  223. struct packed_git *p;
  224. if (pos >= m->num_objects)
  225. return 0;
  226. pack_int_id = nth_midxed_pack_int_id(m, pos);
  227. if (prepare_midx_pack(r, m, pack_int_id))
  228. die(_("error preparing packfile from multi-pack-index"));
  229. p = m->packs[pack_int_id];
  230. /*
  231. * We are about to tell the caller where they can locate the
  232. * requested object. We better make sure the packfile is
  233. * still here and can be accessed before supplying that
  234. * answer, as it may have been deleted since the MIDX was
  235. * loaded!
  236. */
  237. if (!is_pack_valid(p))
  238. return 0;
  239. if (p->num_bad_objects) {
  240. uint32_t i;
  241. struct object_id oid;
  242. nth_midxed_object_oid(&oid, m, pos);
  243. for (i = 0; i < p->num_bad_objects; i++)
  244. if (hasheq(oid.hash,
  245. p->bad_object_sha1 + the_hash_algo->rawsz * i))
  246. return 0;
  247. }
  248. e->offset = nth_midxed_offset(m, pos);
  249. e->p = p;
  250. return 1;
  251. }
  252. int fill_midx_entry(struct repository * r,
  253. const struct object_id *oid,
  254. struct pack_entry *e,
  255. struct multi_pack_index *m)
  256. {
  257. uint32_t pos;
  258. if (!bsearch_midx(oid, m, &pos))
  259. return 0;
  260. return nth_midxed_pack_entry(r, m, e, pos);
  261. }
  262. /* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */
  263. static int cmp_idx_or_pack_name(const char *idx_or_pack_name,
  264. const char *idx_name)
  265. {
  266. /* Skip past any initial matching prefix. */
  267. while (*idx_name && *idx_name == *idx_or_pack_name) {
  268. idx_name++;
  269. idx_or_pack_name++;
  270. }
  271. /*
  272. * If we didn't match completely, we may have matched "pack-1234." and
  273. * be left with "idx" and "pack" respectively, which is also OK. We do
  274. * not have to check for "idx" and "idx", because that would have been
  275. * a complete match (and in that case these strcmps will be false, but
  276. * we'll correctly return 0 from the final strcmp() below.
  277. *
  278. * Technically this matches "fooidx" and "foopack", but we'd never have
  279. * such names in the first place.
  280. */
  281. if (!strcmp(idx_name, "idx") && !strcmp(idx_or_pack_name, "pack"))
  282. return 0;
  283. /*
  284. * This not only checks for a complete match, but also orders based on
  285. * the first non-identical character, which means our ordering will
  286. * match a raw strcmp(). That makes it OK to use this to binary search
  287. * a naively-sorted list.
  288. */
  289. return strcmp(idx_or_pack_name, idx_name);
  290. }
  291. int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name)
  292. {
  293. uint32_t first = 0, last = m->num_packs;
  294. while (first < last) {
  295. uint32_t mid = first + (last - first) / 2;
  296. const char *current;
  297. int cmp;
  298. current = m->pack_names[mid];
  299. cmp = cmp_idx_or_pack_name(idx_or_pack_name, current);
  300. if (!cmp)
  301. return 1;
  302. if (cmp > 0) {
  303. first = mid + 1;
  304. continue;
  305. }
  306. last = mid;
  307. }
  308. return 0;
  309. }
  310. int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local)
  311. {
  312. struct multi_pack_index *m;
  313. struct multi_pack_index *m_search;
  314. int config_value;
  315. static int env_value = -1;
  316. if (env_value < 0)
  317. env_value = git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0);
  318. if (!env_value &&
  319. (repo_config_get_bool(r, "core.multipackindex", &config_value) ||
  320. !config_value))
  321. return 0;
  322. for (m_search = r->objects->multi_pack_index; m_search; m_search = m_search->next)
  323. if (!strcmp(object_dir, m_search->object_dir))
  324. return 1;
  325. m = load_multi_pack_index(object_dir, local);
  326. if (m) {
  327. m->next = r->objects->multi_pack_index;
  328. r->objects->multi_pack_index = m;
  329. return 1;
  330. }
  331. return 0;
  332. }
  333. static size_t write_midx_header(struct hashfile *f,
  334. unsigned char num_chunks,
  335. uint32_t num_packs)
  336. {
  337. unsigned char byte_values[4];
  338. hashwrite_be32(f, MIDX_SIGNATURE);
  339. byte_values[0] = MIDX_VERSION;
  340. byte_values[1] = MIDX_HASH_VERSION;
  341. byte_values[2] = num_chunks;
  342. byte_values[3] = 0; /* unused */
  343. hashwrite(f, byte_values, sizeof(byte_values));
  344. hashwrite_be32(f, num_packs);
  345. return MIDX_HEADER_SIZE;
  346. }
  347. struct pack_info {
  348. uint32_t orig_pack_int_id;
  349. char *pack_name;
  350. struct packed_git *p;
  351. unsigned expired : 1;
  352. };
  353. static int pack_info_compare(const void *_a, const void *_b)
  354. {
  355. struct pack_info *a = (struct pack_info *)_a;
  356. struct pack_info *b = (struct pack_info *)_b;
  357. return strcmp(a->pack_name, b->pack_name);
  358. }
  359. struct pack_list {
  360. struct pack_info *info;
  361. uint32_t nr;
  362. uint32_t alloc;
  363. struct multi_pack_index *m;
  364. };
  365. static void add_pack_to_midx(const char *full_path, size_t full_path_len,
  366. const char *file_name, void *data)
  367. {
  368. struct pack_list *packs = (struct pack_list *)data;
  369. if (ends_with(file_name, ".idx")) {
  370. if (packs->m && midx_contains_pack(packs->m, file_name))
  371. return;
  372. ALLOC_GROW(packs->info, packs->nr + 1, packs->alloc);
  373. packs->info[packs->nr].p = add_packed_git(full_path,
  374. full_path_len,
  375. 0);
  376. if (!packs->info[packs->nr].p) {
  377. warning(_("failed to add packfile '%s'"),
  378. full_path);
  379. return;
  380. }
  381. if (open_pack_index(packs->info[packs->nr].p)) {
  382. warning(_("failed to open pack-index '%s'"),
  383. full_path);
  384. close_pack(packs->info[packs->nr].p);
  385. FREE_AND_NULL(packs->info[packs->nr].p);
  386. return;
  387. }
  388. packs->info[packs->nr].pack_name = xstrdup(file_name);
  389. packs->info[packs->nr].orig_pack_int_id = packs->nr;
  390. packs->info[packs->nr].expired = 0;
  391. packs->nr++;
  392. }
  393. }
  394. struct pack_midx_entry {
  395. struct object_id oid;
  396. uint32_t pack_int_id;
  397. time_t pack_mtime;
  398. uint64_t offset;
  399. };
  400. static int midx_oid_compare(const void *_a, const void *_b)
  401. {
  402. const struct pack_midx_entry *a = (const struct pack_midx_entry *)_a;
  403. const struct pack_midx_entry *b = (const struct pack_midx_entry *)_b;
  404. int cmp = oidcmp(&a->oid, &b->oid);
  405. if (cmp)
  406. return cmp;
  407. if (a->pack_mtime > b->pack_mtime)
  408. return -1;
  409. else if (a->pack_mtime < b->pack_mtime)
  410. return 1;
  411. return a->pack_int_id - b->pack_int_id;
  412. }
  413. static int nth_midxed_pack_midx_entry(struct multi_pack_index *m,
  414. struct pack_midx_entry *e,
  415. uint32_t pos)
  416. {
  417. if (pos >= m->num_objects)
  418. return 1;
  419. nth_midxed_object_oid(&e->oid, m, pos);
  420. e->pack_int_id = nth_midxed_pack_int_id(m, pos);
  421. e->offset = nth_midxed_offset(m, pos);
  422. /* consider objects in midx to be from "old" packs */
  423. e->pack_mtime = 0;
  424. return 0;
  425. }
  426. static void fill_pack_entry(uint32_t pack_int_id,
  427. struct packed_git *p,
  428. uint32_t cur_object,
  429. struct pack_midx_entry *entry)
  430. {
  431. if (!nth_packed_object_oid(&entry->oid, p, cur_object))
  432. die(_("failed to locate object %d in packfile"), cur_object);
  433. entry->pack_int_id = pack_int_id;
  434. entry->pack_mtime = p->mtime;
  435. entry->offset = nth_packed_object_offset(p, cur_object);
  436. }
  437. /*
  438. * It is possible to artificially get into a state where there are many
  439. * duplicate copies of objects. That can create high memory pressure if
  440. * we are to create a list of all objects before de-duplication. To reduce
  441. * this memory pressure without a significant performance drop, automatically
  442. * group objects by the first byte of their object id. Use the IDX fanout
  443. * tables to group the data, copy to a local array, then sort.
  444. *
  445. * Copy only the de-duplicated entries (selected by most-recent modified time
  446. * of a packfile containing the object).
  447. */
  448. static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m,
  449. struct pack_info *info,
  450. uint32_t nr_packs,
  451. uint32_t *nr_objects)
  452. {
  453. uint32_t cur_fanout, cur_pack, cur_object;
  454. uint32_t alloc_fanout, alloc_objects, total_objects = 0;
  455. struct pack_midx_entry *entries_by_fanout = NULL;
  456. struct pack_midx_entry *deduplicated_entries = NULL;
  457. uint32_t start_pack = m ? m->num_packs : 0;
  458. for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++)
  459. total_objects += info[cur_pack].p->num_objects;
  460. /*
  461. * As we de-duplicate by fanout value, we expect the fanout
  462. * slices to be evenly distributed, with some noise. Hence,
  463. * allocate slightly more than one 256th.
  464. */
  465. alloc_objects = alloc_fanout = total_objects > 3200 ? total_objects / 200 : 16;
  466. ALLOC_ARRAY(entries_by_fanout, alloc_fanout);
  467. ALLOC_ARRAY(deduplicated_entries, alloc_objects);
  468. *nr_objects = 0;
  469. for (cur_fanout = 0; cur_fanout < 256; cur_fanout++) {
  470. uint32_t nr_fanout = 0;
  471. if (m) {
  472. uint32_t start = 0, end;
  473. if (cur_fanout)
  474. start = ntohl(m->chunk_oid_fanout[cur_fanout - 1]);
  475. end = ntohl(m->chunk_oid_fanout[cur_fanout]);
  476. for (cur_object = start; cur_object < end; cur_object++) {
  477. ALLOC_GROW(entries_by_fanout, nr_fanout + 1, alloc_fanout);
  478. nth_midxed_pack_midx_entry(m,
  479. &entries_by_fanout[nr_fanout],
  480. cur_object);
  481. nr_fanout++;
  482. }
  483. }
  484. for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++) {
  485. uint32_t start = 0, end;
  486. if (cur_fanout)
  487. start = get_pack_fanout(info[cur_pack].p, cur_fanout - 1);
  488. end = get_pack_fanout(info[cur_pack].p, cur_fanout);
  489. for (cur_object = start; cur_object < end; cur_object++) {
  490. ALLOC_GROW(entries_by_fanout, nr_fanout + 1, alloc_fanout);
  491. fill_pack_entry(cur_pack, info[cur_pack].p, cur_object, &entries_by_fanout[nr_fanout]);
  492. nr_fanout++;
  493. }
  494. }
  495. QSORT(entries_by_fanout, nr_fanout, midx_oid_compare);
  496. /*
  497. * The batch is now sorted by OID and then mtime (descending).
  498. * Take only the first duplicate.
  499. */
  500. for (cur_object = 0; cur_object < nr_fanout; cur_object++) {
  501. if (cur_object && oideq(&entries_by_fanout[cur_object - 1].oid,
  502. &entries_by_fanout[cur_object].oid))
  503. continue;
  504. ALLOC_GROW(deduplicated_entries, *nr_objects + 1, alloc_objects);
  505. memcpy(&deduplicated_entries[*nr_objects],
  506. &entries_by_fanout[cur_object],
  507. sizeof(struct pack_midx_entry));
  508. (*nr_objects)++;
  509. }
  510. }
  511. free(entries_by_fanout);
  512. return deduplicated_entries;
  513. }
  514. static size_t write_midx_pack_names(struct hashfile *f,
  515. struct pack_info *info,
  516. uint32_t num_packs)
  517. {
  518. uint32_t i;
  519. unsigned char padding[MIDX_CHUNK_ALIGNMENT];
  520. size_t written = 0;
  521. for (i = 0; i < num_packs; i++) {
  522. size_t writelen;
  523. if (info[i].expired)
  524. continue;
  525. if (i && strcmp(info[i].pack_name, info[i - 1].pack_name) <= 0)
  526. BUG("incorrect pack-file order: %s before %s",
  527. info[i - 1].pack_name,
  528. info[i].pack_name);
  529. writelen = strlen(info[i].pack_name) + 1;
  530. hashwrite(f, info[i].pack_name, writelen);
  531. written += writelen;
  532. }
  533. /* add padding to be aligned */
  534. i = MIDX_CHUNK_ALIGNMENT - (written % MIDX_CHUNK_ALIGNMENT);
  535. if (i < MIDX_CHUNK_ALIGNMENT) {
  536. memset(padding, 0, sizeof(padding));
  537. hashwrite(f, padding, i);
  538. written += i;
  539. }
  540. return written;
  541. }
  542. static size_t write_midx_oid_fanout(struct hashfile *f,
  543. struct pack_midx_entry *objects,
  544. uint32_t nr_objects)
  545. {
  546. struct pack_midx_entry *list = objects;
  547. struct pack_midx_entry *last = objects + nr_objects;
  548. uint32_t count = 0;
  549. uint32_t i;
  550. /*
  551. * Write the first-level table (the list is sorted,
  552. * but we use a 256-entry lookup to be able to avoid
  553. * having to do eight extra binary search iterations).
  554. */
  555. for (i = 0; i < 256; i++) {
  556. struct pack_midx_entry *next = list;
  557. while (next < last && next->oid.hash[0] == i) {
  558. count++;
  559. next++;
  560. }
  561. hashwrite_be32(f, count);
  562. list = next;
  563. }
  564. return MIDX_CHUNK_FANOUT_SIZE;
  565. }
  566. static size_t write_midx_oid_lookup(struct hashfile *f, unsigned char hash_len,
  567. struct pack_midx_entry *objects,
  568. uint32_t nr_objects)
  569. {
  570. struct pack_midx_entry *list = objects;
  571. uint32_t i;
  572. size_t written = 0;
  573. for (i = 0; i < nr_objects; i++) {
  574. struct pack_midx_entry *obj = list++;
  575. if (i < nr_objects - 1) {
  576. struct pack_midx_entry *next = list;
  577. if (oidcmp(&obj->oid, &next->oid) >= 0)
  578. BUG("OIDs not in order: %s >= %s",
  579. oid_to_hex(&obj->oid),
  580. oid_to_hex(&next->oid));
  581. }
  582. hashwrite(f, obj->oid.hash, (int)hash_len);
  583. written += hash_len;
  584. }
  585. return written;
  586. }
  587. static size_t write_midx_object_offsets(struct hashfile *f, int large_offset_needed,
  588. uint32_t *perm,
  589. struct pack_midx_entry *objects, uint32_t nr_objects)
  590. {
  591. struct pack_midx_entry *list = objects;
  592. uint32_t i, nr_large_offset = 0;
  593. size_t written = 0;
  594. for (i = 0; i < nr_objects; i++) {
  595. struct pack_midx_entry *obj = list++;
  596. if (perm[obj->pack_int_id] == PACK_EXPIRED)
  597. BUG("object %s is in an expired pack with int-id %d",
  598. oid_to_hex(&obj->oid),
  599. obj->pack_int_id);
  600. hashwrite_be32(f, perm[obj->pack_int_id]);
  601. if (large_offset_needed && obj->offset >> 31)
  602. hashwrite_be32(f, MIDX_LARGE_OFFSET_NEEDED | nr_large_offset++);
  603. else if (!large_offset_needed && obj->offset >> 32)
  604. BUG("object %s requires a large offset (%"PRIx64") but the MIDX is not writing large offsets!",
  605. oid_to_hex(&obj->oid),
  606. obj->offset);
  607. else
  608. hashwrite_be32(f, (uint32_t)obj->offset);
  609. written += MIDX_CHUNK_OFFSET_WIDTH;
  610. }
  611. return written;
  612. }
  613. static size_t write_midx_large_offsets(struct hashfile *f, uint32_t nr_large_offset,
  614. struct pack_midx_entry *objects, uint32_t nr_objects)
  615. {
  616. struct pack_midx_entry *list = objects, *end = objects + nr_objects;
  617. size_t written = 0;
  618. while (nr_large_offset) {
  619. struct pack_midx_entry *obj;
  620. uint64_t offset;
  621. if (list >= end)
  622. BUG("too many large-offset objects");
  623. obj = list++;
  624. offset = obj->offset;
  625. if (!(offset >> 31))
  626. continue;
  627. hashwrite_be32(f, offset >> 32);
  628. hashwrite_be32(f, offset & 0xffffffffUL);
  629. written += 2 * sizeof(uint32_t);
  630. nr_large_offset--;
  631. }
  632. return written;
  633. }
  634. static int write_midx_internal(const char *object_dir, struct multi_pack_index *m,
  635. struct string_list *packs_to_drop)
  636. {
  637. unsigned char cur_chunk, num_chunks = 0;
  638. char *midx_name;
  639. uint32_t i;
  640. struct hashfile *f = NULL;
  641. struct lock_file lk;
  642. struct pack_list packs;
  643. uint32_t *pack_perm = NULL;
  644. uint64_t written = 0;
  645. uint32_t chunk_ids[MIDX_MAX_CHUNKS + 1];
  646. uint64_t chunk_offsets[MIDX_MAX_CHUNKS + 1];
  647. uint32_t nr_entries, num_large_offsets = 0;
  648. struct pack_midx_entry *entries = NULL;
  649. int large_offsets_needed = 0;
  650. int pack_name_concat_len = 0;
  651. int dropped_packs = 0;
  652. int result = 0;
  653. midx_name = get_midx_filename(object_dir);
  654. if (safe_create_leading_directories(midx_name)) {
  655. UNLEAK(midx_name);
  656. die_errno(_("unable to create leading directories of %s"),
  657. midx_name);
  658. }
  659. if (m)
  660. packs.m = m;
  661. else
  662. packs.m = load_multi_pack_index(object_dir, 1);
  663. packs.nr = 0;
  664. packs.alloc = packs.m ? packs.m->num_packs : 16;
  665. packs.info = NULL;
  666. ALLOC_ARRAY(packs.info, packs.alloc);
  667. if (packs.m) {
  668. for (i = 0; i < packs.m->num_packs; i++) {
  669. ALLOC_GROW(packs.info, packs.nr + 1, packs.alloc);
  670. packs.info[packs.nr].orig_pack_int_id = i;
  671. packs.info[packs.nr].pack_name = xstrdup(packs.m->pack_names[i]);
  672. packs.info[packs.nr].p = NULL;
  673. packs.info[packs.nr].expired = 0;
  674. packs.nr++;
  675. }
  676. }
  677. for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &packs);
  678. if (packs.m && packs.nr == packs.m->num_packs && !packs_to_drop)
  679. goto cleanup;
  680. entries = get_sorted_entries(packs.m, packs.info, packs.nr, &nr_entries);
  681. for (i = 0; i < nr_entries; i++) {
  682. if (entries[i].offset > 0x7fffffff)
  683. num_large_offsets++;
  684. if (entries[i].offset > 0xffffffff)
  685. large_offsets_needed = 1;
  686. }
  687. QSORT(packs.info, packs.nr, pack_info_compare);
  688. if (packs_to_drop && packs_to_drop->nr) {
  689. int drop_index = 0;
  690. int missing_drops = 0;
  691. for (i = 0; i < packs.nr && drop_index < packs_to_drop->nr; i++) {
  692. int cmp = strcmp(packs.info[i].pack_name,
  693. packs_to_drop->items[drop_index].string);
  694. if (!cmp) {
  695. drop_index++;
  696. packs.info[i].expired = 1;
  697. } else if (cmp > 0) {
  698. error(_("did not see pack-file %s to drop"),
  699. packs_to_drop->items[drop_index].string);
  700. drop_index++;
  701. missing_drops++;
  702. i--;
  703. } else {
  704. packs.info[i].expired = 0;
  705. }
  706. }
  707. if (missing_drops) {
  708. result = 1;
  709. goto cleanup;
  710. }
  711. }
  712. /*
  713. * pack_perm stores a permutation between pack-int-ids from the
  714. * previous multi-pack-index to the new one we are writing:
  715. *
  716. * pack_perm[old_id] = new_id
  717. */
  718. ALLOC_ARRAY(pack_perm, packs.nr);
  719. for (i = 0; i < packs.nr; i++) {
  720. if (packs.info[i].expired) {
  721. dropped_packs++;
  722. pack_perm[packs.info[i].orig_pack_int_id] = PACK_EXPIRED;
  723. } else {
  724. pack_perm[packs.info[i].orig_pack_int_id] = i - dropped_packs;
  725. }
  726. }
  727. for (i = 0; i < packs.nr; i++) {
  728. if (!packs.info[i].expired)
  729. pack_name_concat_len += strlen(packs.info[i].pack_name) + 1;
  730. }
  731. if (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT)
  732. pack_name_concat_len += MIDX_CHUNK_ALIGNMENT -
  733. (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT);
  734. hold_lock_file_for_update(&lk, midx_name, LOCK_DIE_ON_ERROR);
  735. f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);
  736. FREE_AND_NULL(midx_name);
  737. if (packs.m)
  738. close_midx(packs.m);
  739. cur_chunk = 0;
  740. num_chunks = large_offsets_needed ? 5 : 4;
  741. written = write_midx_header(f, num_chunks, packs.nr - dropped_packs);
  742. chunk_ids[cur_chunk] = MIDX_CHUNKID_PACKNAMES;
  743. chunk_offsets[cur_chunk] = written + (num_chunks + 1) * MIDX_CHUNKLOOKUP_WIDTH;
  744. cur_chunk++;
  745. chunk_ids[cur_chunk] = MIDX_CHUNKID_OIDFANOUT;
  746. chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + pack_name_concat_len;
  747. cur_chunk++;
  748. chunk_ids[cur_chunk] = MIDX_CHUNKID_OIDLOOKUP;
  749. chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + MIDX_CHUNK_FANOUT_SIZE;
  750. cur_chunk++;
  751. chunk_ids[cur_chunk] = MIDX_CHUNKID_OBJECTOFFSETS;
  752. chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + nr_entries * the_hash_algo->rawsz;
  753. cur_chunk++;
  754. chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + nr_entries * MIDX_CHUNK_OFFSET_WIDTH;
  755. if (large_offsets_needed) {
  756. chunk_ids[cur_chunk] = MIDX_CHUNKID_LARGEOFFSETS;
  757. cur_chunk++;
  758. chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] +
  759. num_large_offsets * MIDX_CHUNK_LARGE_OFFSET_WIDTH;
  760. }
  761. chunk_ids[cur_chunk] = 0;
  762. for (i = 0; i <= num_chunks; i++) {
  763. if (i && chunk_offsets[i] < chunk_offsets[i - 1])
  764. BUG("incorrect chunk offsets: %"PRIu64" before %"PRIu64,
  765. chunk_offsets[i - 1],
  766. chunk_offsets[i]);
  767. if (chunk_offsets[i] % MIDX_CHUNK_ALIGNMENT)
  768. BUG("chunk offset %"PRIu64" is not properly aligned",
  769. chunk_offsets[i]);
  770. hashwrite_be32(f, chunk_ids[i]);
  771. hashwrite_be32(f, chunk_offsets[i] >> 32);
  772. hashwrite_be32(f, chunk_offsets[i]);
  773. written += MIDX_CHUNKLOOKUP_WIDTH;
  774. }
  775. for (i = 0; i < num_chunks; i++) {
  776. if (written != chunk_offsets[i])
  777. BUG("incorrect chunk offset (%"PRIu64" != %"PRIu64") for chunk id %"PRIx32,
  778. chunk_offsets[i],
  779. written,
  780. chunk_ids[i]);
  781. switch (chunk_ids[i]) {
  782. case MIDX_CHUNKID_PACKNAMES:
  783. written += write_midx_pack_names(f, packs.info, packs.nr);
  784. break;
  785. case MIDX_CHUNKID_OIDFANOUT:
  786. written += write_midx_oid_fanout(f, entries, nr_entries);
  787. break;
  788. case MIDX_CHUNKID_OIDLOOKUP:
  789. written += write_midx_oid_lookup(f, the_hash_algo->rawsz, entries, nr_entries);
  790. break;
  791. case MIDX_CHUNKID_OBJECTOFFSETS:
  792. written += write_midx_object_offsets(f, large_offsets_needed, pack_perm, entries, nr_entries);
  793. break;
  794. case MIDX_CHUNKID_LARGEOFFSETS:
  795. written += write_midx_large_offsets(f, num_large_offsets, entries, nr_entries);
  796. break;
  797. default:
  798. BUG("trying to write unknown chunk id %"PRIx32,
  799. chunk_ids[i]);
  800. }
  801. }
  802. if (written != chunk_offsets[num_chunks])
  803. BUG("incorrect final offset %"PRIu64" != %"PRIu64,
  804. written,
  805. chunk_offsets[num_chunks]);
  806. finalize_hashfile(f, NULL, CSUM_FSYNC | CSUM_HASH_IN_STREAM);
  807. commit_lock_file(&lk);
  808. cleanup:
  809. for (i = 0; i < packs.nr; i++) {
  810. if (packs.info[i].p) {
  811. close_pack(packs.info[i].p);
  812. free(packs.info[i].p);
  813. }
  814. free(packs.info[i].pack_name);
  815. }
  816. free(packs.info);
  817. free(entries);
  818. free(pack_perm);
  819. free(midx_name);
  820. return result;
  821. }
  822. int write_midx_file(const char *object_dir)
  823. {
  824. return write_midx_internal(object_dir, NULL, NULL);
  825. }
  826. void clear_midx_file(struct repository *r)
  827. {
  828. char *midx = get_midx_filename(r->objects->odb->path);
  829. if (r->objects && r->objects->multi_pack_index) {
  830. close_midx(r->objects->multi_pack_index);
  831. r->objects->multi_pack_index = NULL;
  832. }
  833. if (remove_path(midx)) {
  834. UNLEAK(midx);
  835. die(_("failed to clear multi-pack-index at %s"), midx);
  836. }
  837. free(midx);
  838. }
  839. static int verify_midx_error;
  840. static void midx_report(const char *fmt, ...)
  841. {
  842. va_list ap;
  843. verify_midx_error = 1;
  844. va_start(ap, fmt);
  845. vfprintf(stderr, fmt, ap);
  846. fprintf(stderr, "\n");
  847. va_end(ap);
  848. }
  849. struct pair_pos_vs_id
  850. {
  851. uint32_t pos;
  852. uint32_t pack_int_id;
  853. };
  854. static int compare_pair_pos_vs_id(const void *_a, const void *_b)
  855. {
  856. struct pair_pos_vs_id *a = (struct pair_pos_vs_id *)_a;
  857. struct pair_pos_vs_id *b = (struct pair_pos_vs_id *)_b;
  858. return b->pack_int_id - a->pack_int_id;
  859. }
  860. /*
  861. * Limit calls to display_progress() for performance reasons.
  862. * The interval here was arbitrarily chosen.
  863. */
  864. #define SPARSE_PROGRESS_INTERVAL (1 << 12)
  865. #define midx_display_sparse_progress(progress, n) \
  866. do { \
  867. uint64_t _n = (n); \
  868. if ((_n & (SPARSE_PROGRESS_INTERVAL - 1)) == 0) \
  869. display_progress(progress, _n); \
  870. } while (0)
  871. int verify_midx_file(struct repository *r, const char *object_dir)
  872. {
  873. struct pair_pos_vs_id *pairs = NULL;
  874. uint32_t i;
  875. struct progress *progress;
  876. struct multi_pack_index *m = load_multi_pack_index(object_dir, 1);
  877. verify_midx_error = 0;
  878. if (!m)
  879. return 0;
  880. progress = start_progress(_("Looking for referenced packfiles"),
  881. m->num_packs);
  882. for (i = 0; i < m->num_packs; i++) {
  883. if (prepare_midx_pack(r, m, i))
  884. midx_report("failed to load pack in position %d", i);
  885. display_progress(progress, i + 1);
  886. }
  887. stop_progress(&progress);
  888. for (i = 0; i < 255; i++) {
  889. uint32_t oid_fanout1 = ntohl(m->chunk_oid_fanout[i]);
  890. uint32_t oid_fanout2 = ntohl(m->chunk_oid_fanout[i + 1]);
  891. if (oid_fanout1 > oid_fanout2)
  892. midx_report(_("oid fanout out of order: fanout[%d] = %"PRIx32" > %"PRIx32" = fanout[%d]"),
  893. i, oid_fanout1, oid_fanout2, i + 1);
  894. }
  895. progress = start_sparse_progress(_("Verifying OID order in MIDX"),
  896. m->num_objects - 1);
  897. for (i = 0; i < m->num_objects - 1; i++) {
  898. struct object_id oid1, oid2;
  899. nth_midxed_object_oid(&oid1, m, i);
  900. nth_midxed_object_oid(&oid2, m, i + 1);
  901. if (oidcmp(&oid1, &oid2) >= 0)
  902. midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"),
  903. i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1);
  904. midx_display_sparse_progress(progress, i + 1);
  905. }
  906. stop_progress(&progress);
  907. /*
  908. * Create an array mapping each object to its packfile id. Sort it
  909. * to group the objects by packfile. Use this permutation to visit
  910. * each of the objects and only require 1 packfile to be open at a
  911. * time.
  912. */
  913. ALLOC_ARRAY(pairs, m->num_objects);
  914. for (i = 0; i < m->num_objects; i++) {
  915. pairs[i].pos = i;
  916. pairs[i].pack_int_id = nth_midxed_pack_int_id(m, i);
  917. }
  918. progress = start_sparse_progress(_("Sorting objects by packfile"),
  919. m->num_objects);
  920. display_progress(progress, 0); /* TODO: Measure QSORT() progress */
  921. QSORT(pairs, m->num_objects, compare_pair_pos_vs_id);
  922. stop_progress(&progress);
  923. progress = start_sparse_progress(_("Verifying object offsets"), m->num_objects);
  924. for (i = 0; i < m->num_objects; i++) {
  925. struct object_id oid;
  926. struct pack_entry e;
  927. off_t m_offset, p_offset;
  928. if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id &&
  929. m->packs[pairs[i-1].pack_int_id])
  930. {
  931. close_pack_fd(m->packs[pairs[i-1].pack_int_id]);
  932. close_pack_index(m->packs[pairs[i-1].pack_int_id]);
  933. }
  934. nth_midxed_object_oid(&oid, m, pairs[i].pos);
  935. if (!fill_midx_entry(r, &oid, &e, m)) {
  936. midx_report(_("failed to load pack entry for oid[%d] = %s"),
  937. pairs[i].pos, oid_to_hex(&oid));
  938. continue;
  939. }
  940. if (open_pack_index(e.p)) {
  941. midx_report(_("failed to load pack-index for packfile %s"),
  942. e.p->pack_name);
  943. break;
  944. }
  945. m_offset = e.offset;
  946. p_offset = find_pack_entry_one(oid.hash, e.p);
  947. if (m_offset != p_offset)
  948. midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64),
  949. pairs[i].pos, oid_to_hex(&oid), m_offset, p_offset);
  950. midx_display_sparse_progress(progress, i + 1);
  951. }
  952. stop_progress(&progress);
  953. free(pairs);
  954. return verify_midx_error;
  955. }
  956. int expire_midx_packs(struct repository *r, const char *object_dir)
  957. {
  958. uint32_t i, *count, result = 0;
  959. struct string_list packs_to_drop = STRING_LIST_INIT_DUP;
  960. struct multi_pack_index *m = load_multi_pack_index(object_dir, 1);
  961. if (!m)
  962. return 0;
  963. count = xcalloc(m->num_packs, sizeof(uint32_t));
  964. for (i = 0; i < m->num_objects; i++) {
  965. int pack_int_id = nth_midxed_pack_int_id(m, i);
  966. count[pack_int_id]++;
  967. }
  968. for (i = 0; i < m->num_packs; i++) {
  969. char *pack_name;
  970. if (count[i])
  971. continue;
  972. if (prepare_midx_pack(r, m, i))
  973. continue;
  974. if (m->packs[i]->pack_keep)
  975. continue;
  976. pack_name = xstrdup(m->packs[i]->pack_name);
  977. close_pack(m->packs[i]);
  978. string_list_insert(&packs_to_drop, m->pack_names[i]);
  979. unlink_pack_path(pack_name, 0);
  980. free(pack_name);
  981. }
  982. free(count);
  983. if (packs_to_drop.nr)
  984. result = write_midx_internal(object_dir, m, &packs_to_drop);
  985. string_list_clear(&packs_to_drop, 0);
  986. return result;
  987. }
  988. struct repack_info {
  989. timestamp_t mtime;
  990. uint32_t referenced_objects;
  991. uint32_t pack_int_id;
  992. };
  993. static int compare_by_mtime(const void *a_, const void *b_)
  994. {
  995. const struct repack_info *a, *b;
  996. a = (const struct repack_info *)a_;
  997. b = (const struct repack_info *)b_;
  998. if (a->mtime < b->mtime)
  999. return -1;
  1000. if (a->mtime > b->mtime)
  1001. return 1;
  1002. return 0;
  1003. }
  1004. static int fill_included_packs_all(struct multi_pack_index *m,
  1005. unsigned char *include_pack)
  1006. {
  1007. uint32_t i;
  1008. for (i = 0; i < m->num_packs; i++)
  1009. include_pack[i] = 1;
  1010. return m->num_packs < 2;
  1011. }
  1012. static int fill_included_packs_batch(struct repository *r,
  1013. struct multi_pack_index *m,
  1014. unsigned char *include_pack,
  1015. size_t batch_size)
  1016. {
  1017. uint32_t i, packs_to_repack;
  1018. size_t total_size;
  1019. struct repack_info *pack_info = xcalloc(m->num_packs, sizeof(struct repack_info));
  1020. for (i = 0; i < m->num_packs; i++) {
  1021. pack_info[i].pack_int_id = i;
  1022. if (prepare_midx_pack(r, m, i))
  1023. continue;
  1024. pack_info[i].mtime = m->packs[i]->mtime;
  1025. }
  1026. for (i = 0; batch_size && i < m->num_objects; i++) {
  1027. uint32_t pack_int_id = nth_midxed_pack_int_id(m, i);
  1028. pack_info[pack_int_id].referenced_objects++;
  1029. }
  1030. QSORT(pack_info, m->num_packs, compare_by_mtime);
  1031. total_size = 0;
  1032. packs_to_repack = 0;
  1033. for (i = 0; total_size < batch_size && i < m->num_packs; i++) {
  1034. int pack_int_id = pack_info[i].pack_int_id;
  1035. struct packed_git *p = m->packs[pack_int_id];
  1036. size_t expected_size;
  1037. if (!p)
  1038. continue;
  1039. if (open_pack_index(p) || !p->num_objects)
  1040. continue;
  1041. expected_size = (size_t)(p->pack_size
  1042. * pack_info[i].referenced_objects);
  1043. expected_size /= p->num_objects;
  1044. if (expected_size >= batch_size)
  1045. continue;
  1046. packs_to_repack++;
  1047. total_size += expected_size;
  1048. include_pack[pack_int_id] = 1;
  1049. }
  1050. free(pack_info);
  1051. if (total_size < batch_size || packs_to_repack < 2)
  1052. return 1;
  1053. return 0;
  1054. }
  1055. int midx_repack(struct repository *r, const char *object_dir, size_t batch_size)
  1056. {
  1057. int result = 0;
  1058. uint32_t i;
  1059. unsigned char *include_pack;
  1060. struct child_process cmd = CHILD_PROCESS_INIT;
  1061. struct strbuf base_name = STRBUF_INIT;
  1062. struct multi_pack_index *m = load_multi_pack_index(object_dir, 1);
  1063. if (!m)
  1064. return 0;
  1065. include_pack = xcalloc(m->num_packs, sizeof(unsigned char));
  1066. if (batch_size) {
  1067. if (fill_included_packs_batch(r, m, include_pack, batch_size))
  1068. goto cleanup;
  1069. } else if (fill_included_packs_all(m, include_pack))
  1070. goto cleanup;
  1071. argv_array_push(&cmd.args, "pack-objects");
  1072. strbuf_addstr(&base_name, object_dir);
  1073. strbuf_addstr(&base_name, "/pack/pack");
  1074. argv_array_push(&cmd.args, base_name.buf);
  1075. strbuf_release(&base_name);
  1076. cmd.git_cmd = 1;
  1077. cmd.in = cmd.out = -1;
  1078. if (start_command(&cmd)) {
  1079. error(_("could not start pack-objects"));
  1080. result = 1;
  1081. goto cleanup;
  1082. }
  1083. for (i = 0; i < m->num_objects; i++) {
  1084. struct object_id oid;
  1085. uint32_t pack_int_id = nth_midxed_pack_int_id(m, i);
  1086. if (!include_pack[pack_int_id])
  1087. continue;
  1088. nth_midxed_object_oid(&oid, m, i);
  1089. xwrite(cmd.in, oid_to_hex(&oid), the_hash_algo->hexsz);
  1090. xwrite(cmd.in, "\n", 1);
  1091. }
  1092. close(cmd.in);
  1093. if (finish_command(&cmd)) {
  1094. error(_("could not finish pack-objects"));
  1095. result = 1;
  1096. goto cleanup;
  1097. }
  1098. result = write_midx_internal(object_dir, m, NULL);
  1099. m = NULL;
  1100. cleanup:
  1101. if (m)
  1102. close_midx(m);
  1103. free(include_pack);
  1104. return result;
  1105. }