THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Git Source Code Mirror - This is a publish-only repository and all pull requests are ignored. Please follow Documentation/SubmittingPatches procedure for any of your improvements.
git
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

380 lines
11KB

  1. #include "cache.h"
  2. #include "pack.h"
  3. #include "csum-file.h"
  4. void reset_pack_idx_option(struct pack_idx_option *opts)
  5. {
  6. memset(opts, 0, sizeof(*opts));
  7. opts->version = 2;
  8. opts->off32_limit = 0x7fffffff;
  9. }
  10. static int sha1_compare(const void *_a, const void *_b)
  11. {
  12. struct pack_idx_entry *a = *(struct pack_idx_entry **)_a;
  13. struct pack_idx_entry *b = *(struct pack_idx_entry **)_b;
  14. return oidcmp(&a->oid, &b->oid);
  15. }
  16. static int cmp_uint32(const void *a_, const void *b_)
  17. {
  18. uint32_t a = *((uint32_t *)a_);
  19. uint32_t b = *((uint32_t *)b_);
  20. return (a < b) ? -1 : (a != b);
  21. }
  22. static int need_large_offset(off_t offset, const struct pack_idx_option *opts)
  23. {
  24. uint32_t ofsval;
  25. if ((offset >> 31) || (opts->off32_limit < offset))
  26. return 1;
  27. if (!opts->anomaly_nr)
  28. return 0;
  29. ofsval = offset;
  30. return !!bsearch(&ofsval, opts->anomaly, opts->anomaly_nr,
  31. sizeof(ofsval), cmp_uint32);
  32. }
  33. /*
  34. * On entry *sha1 contains the pack content SHA1 hash, on exit it is
  35. * the SHA1 hash of sorted object names. The objects array passed in
  36. * will be sorted by SHA1 on exit.
  37. */
  38. const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects,
  39. int nr_objects, const struct pack_idx_option *opts,
  40. const unsigned char *sha1)
  41. {
  42. struct hashfile *f;
  43. struct pack_idx_entry **sorted_by_sha, **list, **last;
  44. off_t last_obj_offset = 0;
  45. uint32_t array[256];
  46. int i, fd;
  47. uint32_t index_version;
  48. if (nr_objects) {
  49. sorted_by_sha = objects;
  50. list = sorted_by_sha;
  51. last = sorted_by_sha + nr_objects;
  52. for (i = 0; i < nr_objects; ++i) {
  53. if (objects[i]->offset > last_obj_offset)
  54. last_obj_offset = objects[i]->offset;
  55. }
  56. QSORT(sorted_by_sha, nr_objects, sha1_compare);
  57. }
  58. else
  59. sorted_by_sha = list = last = NULL;
  60. if (opts->flags & WRITE_IDX_VERIFY) {
  61. assert(index_name);
  62. f = hashfd_check(index_name);
  63. } else {
  64. if (!index_name) {
  65. struct strbuf tmp_file = STRBUF_INIT;
  66. fd = odb_mkstemp(&tmp_file, "pack/tmp_idx_XXXXXX");
  67. index_name = strbuf_detach(&tmp_file, NULL);
  68. } else {
  69. unlink(index_name);
  70. fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
  71. if (fd < 0)
  72. die_errno("unable to create '%s'", index_name);
  73. }
  74. f = hashfd(fd, index_name);
  75. }
  76. /* if last object's offset is >= 2^31 we should use index V2 */
  77. index_version = need_large_offset(last_obj_offset, opts) ? 2 : opts->version;
  78. /* index versions 2 and above need a header */
  79. if (index_version >= 2) {
  80. struct pack_idx_header hdr;
  81. hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
  82. hdr.idx_version = htonl(index_version);
  83. hashwrite(f, &hdr, sizeof(hdr));
  84. }
  85. /*
  86. * Write the first-level table (the list is sorted,
  87. * but we use a 256-entry lookup to be able to avoid
  88. * having to do eight extra binary search iterations).
  89. */
  90. for (i = 0; i < 256; i++) {
  91. struct pack_idx_entry **next = list;
  92. while (next < last) {
  93. struct pack_idx_entry *obj = *next;
  94. if (obj->oid.hash[0] != i)
  95. break;
  96. next++;
  97. }
  98. array[i] = htonl(next - sorted_by_sha);
  99. list = next;
  100. }
  101. hashwrite(f, array, 256 * 4);
  102. /*
  103. * Write the actual SHA1 entries..
  104. */
  105. list = sorted_by_sha;
  106. for (i = 0; i < nr_objects; i++) {
  107. struct pack_idx_entry *obj = *list++;
  108. if (index_version < 2) {
  109. uint32_t offset = htonl(obj->offset);
  110. hashwrite(f, &offset, 4);
  111. }
  112. hashwrite(f, obj->oid.hash, the_hash_algo->rawsz);
  113. if ((opts->flags & WRITE_IDX_STRICT) &&
  114. (i && oideq(&list[-2]->oid, &obj->oid)))
  115. die("The same object %s appears twice in the pack",
  116. oid_to_hex(&obj->oid));
  117. }
  118. if (index_version >= 2) {
  119. unsigned int nr_large_offset = 0;
  120. /* write the crc32 table */
  121. list = sorted_by_sha;
  122. for (i = 0; i < nr_objects; i++) {
  123. struct pack_idx_entry *obj = *list++;
  124. uint32_t crc32_val = htonl(obj->crc32);
  125. hashwrite(f, &crc32_val, 4);
  126. }
  127. /* write the 32-bit offset table */
  128. list = sorted_by_sha;
  129. for (i = 0; i < nr_objects; i++) {
  130. struct pack_idx_entry *obj = *list++;
  131. uint32_t offset;
  132. offset = (need_large_offset(obj->offset, opts)
  133. ? (0x80000000 | nr_large_offset++)
  134. : obj->offset);
  135. offset = htonl(offset);
  136. hashwrite(f, &offset, 4);
  137. }
  138. /* write the large offset table */
  139. list = sorted_by_sha;
  140. while (nr_large_offset) {
  141. struct pack_idx_entry *obj = *list++;
  142. uint64_t offset = obj->offset;
  143. uint32_t split[2];
  144. if (!need_large_offset(offset, opts))
  145. continue;
  146. split[0] = htonl(offset >> 32);
  147. split[1] = htonl(offset & 0xffffffff);
  148. hashwrite(f, split, 8);
  149. nr_large_offset--;
  150. }
  151. }
  152. hashwrite(f, sha1, the_hash_algo->rawsz);
  153. finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_CLOSE |
  154. ((opts->flags & WRITE_IDX_VERIFY)
  155. ? 0 : CSUM_FSYNC));
  156. return index_name;
  157. }
  158. off_t write_pack_header(struct hashfile *f, uint32_t nr_entries)
  159. {
  160. struct pack_header hdr;
  161. hdr.hdr_signature = htonl(PACK_SIGNATURE);
  162. hdr.hdr_version = htonl(PACK_VERSION);
  163. hdr.hdr_entries = htonl(nr_entries);
  164. hashwrite(f, &hdr, sizeof(hdr));
  165. return sizeof(hdr);
  166. }
  167. /*
  168. * Update pack header with object_count and compute new SHA1 for pack data
  169. * associated to pack_fd, and write that SHA1 at the end. That new SHA1
  170. * is also returned in new_pack_sha1.
  171. *
  172. * If partial_pack_sha1 is non null, then the SHA1 of the existing pack
  173. * (without the header update) is computed and validated against the
  174. * one provided in partial_pack_sha1. The validation is performed at
  175. * partial_pack_offset bytes in the pack file. The SHA1 of the remaining
  176. * data (i.e. from partial_pack_offset to the end) is then computed and
  177. * returned in partial_pack_sha1.
  178. *
  179. * Note that new_pack_sha1 is updated last, so both new_pack_sha1 and
  180. * partial_pack_sha1 can refer to the same buffer if the caller is not
  181. * interested in the resulting SHA1 of pack data above partial_pack_offset.
  182. */
  183. void fixup_pack_header_footer(int pack_fd,
  184. unsigned char *new_pack_hash,
  185. const char *pack_name,
  186. uint32_t object_count,
  187. unsigned char *partial_pack_hash,
  188. off_t partial_pack_offset)
  189. {
  190. int aligned_sz, buf_sz = 8 * 1024;
  191. git_hash_ctx old_hash_ctx, new_hash_ctx;
  192. struct pack_header hdr;
  193. char *buf;
  194. ssize_t read_result;
  195. the_hash_algo->init_fn(&old_hash_ctx);
  196. the_hash_algo->init_fn(&new_hash_ctx);
  197. if (lseek(pack_fd, 0, SEEK_SET) != 0)
  198. die_errno("Failed seeking to start of '%s'", pack_name);
  199. read_result = read_in_full(pack_fd, &hdr, sizeof(hdr));
  200. if (read_result < 0)
  201. die_errno("Unable to reread header of '%s'", pack_name);
  202. else if (read_result != sizeof(hdr))
  203. die_errno("Unexpected short read for header of '%s'",
  204. pack_name);
  205. if (lseek(pack_fd, 0, SEEK_SET) != 0)
  206. die_errno("Failed seeking to start of '%s'", pack_name);
  207. the_hash_algo->update_fn(&old_hash_ctx, &hdr, sizeof(hdr));
  208. hdr.hdr_entries = htonl(object_count);
  209. the_hash_algo->update_fn(&new_hash_ctx, &hdr, sizeof(hdr));
  210. write_or_die(pack_fd, &hdr, sizeof(hdr));
  211. partial_pack_offset -= sizeof(hdr);
  212. buf = xmalloc(buf_sz);
  213. aligned_sz = buf_sz - sizeof(hdr);
  214. for (;;) {
  215. ssize_t m, n;
  216. m = (partial_pack_hash && partial_pack_offset < aligned_sz) ?
  217. partial_pack_offset : aligned_sz;
  218. n = xread(pack_fd, buf, m);
  219. if (!n)
  220. break;
  221. if (n < 0)
  222. die_errno("Failed to checksum '%s'", pack_name);
  223. the_hash_algo->update_fn(&new_hash_ctx, buf, n);
  224. aligned_sz -= n;
  225. if (!aligned_sz)
  226. aligned_sz = buf_sz;
  227. if (!partial_pack_hash)
  228. continue;
  229. the_hash_algo->update_fn(&old_hash_ctx, buf, n);
  230. partial_pack_offset -= n;
  231. if (partial_pack_offset == 0) {
  232. unsigned char hash[GIT_MAX_RAWSZ];
  233. the_hash_algo->final_fn(hash, &old_hash_ctx);
  234. if (!hasheq(hash, partial_pack_hash))
  235. die("Unexpected checksum for %s "
  236. "(disk corruption?)", pack_name);
  237. /*
  238. * Now let's compute the SHA1 of the remainder of the
  239. * pack, which also means making partial_pack_offset
  240. * big enough not to matter anymore.
  241. */
  242. the_hash_algo->init_fn(&old_hash_ctx);
  243. partial_pack_offset = ~partial_pack_offset;
  244. partial_pack_offset -= MSB(partial_pack_offset, 1);
  245. }
  246. }
  247. free(buf);
  248. if (partial_pack_hash)
  249. the_hash_algo->final_fn(partial_pack_hash, &old_hash_ctx);
  250. the_hash_algo->final_fn(new_pack_hash, &new_hash_ctx);
  251. write_or_die(pack_fd, new_pack_hash, the_hash_algo->rawsz);
  252. fsync_or_die(pack_fd, pack_name);
  253. }
  254. char *index_pack_lockfile(int ip_out)
  255. {
  256. char packname[GIT_MAX_HEXSZ + 6];
  257. const int len = the_hash_algo->hexsz + 6;
  258. /*
  259. * The first thing we expect from index-pack's output
  260. * is "pack\t%40s\n" or "keep\t%40s\n" (46 bytes) where
  261. * %40s is the newly created pack SHA1 name. In the "keep"
  262. * case, we need it to remove the corresponding .keep file
  263. * later on. If we don't get that then tough luck with it.
  264. */
  265. if (read_in_full(ip_out, packname, len) == len && packname[len-1] == '\n') {
  266. const char *name;
  267. packname[len-1] = 0;
  268. if (skip_prefix(packname, "keep\t", &name))
  269. return xstrfmt("%s/pack/pack-%s.keep",
  270. get_object_directory(), name);
  271. }
  272. return NULL;
  273. }
  274. /*
  275. * The per-object header is a pretty dense thing, which is
  276. * - first byte: low four bits are "size", then three bits of "type",
  277. * and the high bit is "size continues".
  278. * - each byte afterwards: low seven bits are size continuation,
  279. * with the high bit being "size continues"
  280. */
  281. int encode_in_pack_object_header(unsigned char *hdr, int hdr_len,
  282. enum object_type type, uintmax_t size)
  283. {
  284. int n = 1;
  285. unsigned char c;
  286. if (type < OBJ_COMMIT || type > OBJ_REF_DELTA)
  287. die("bad type %d", type);
  288. c = (type << 4) | (size & 15);
  289. size >>= 4;
  290. while (size) {
  291. if (n == hdr_len)
  292. die("object size is too enormous to format");
  293. *hdr++ = c | 0x80;
  294. c = size & 0x7f;
  295. size >>= 7;
  296. n++;
  297. }
  298. *hdr = c;
  299. return n;
  300. }
  301. struct hashfile *create_tmp_packfile(char **pack_tmp_name)
  302. {
  303. struct strbuf tmpname = STRBUF_INIT;
  304. int fd;
  305. fd = odb_mkstemp(&tmpname, "pack/tmp_pack_XXXXXX");
  306. *pack_tmp_name = strbuf_detach(&tmpname, NULL);
  307. return hashfd(fd, *pack_tmp_name);
  308. }
  309. void finish_tmp_packfile(struct strbuf *name_buffer,
  310. const char *pack_tmp_name,
  311. struct pack_idx_entry **written_list,
  312. uint32_t nr_written,
  313. struct pack_idx_option *pack_idx_opts,
  314. unsigned char hash[])
  315. {
  316. const char *idx_tmp_name;
  317. int basename_len = name_buffer->len;
  318. if (adjust_shared_perm(pack_tmp_name))
  319. die_errno("unable to make temporary pack file readable");
  320. idx_tmp_name = write_idx_file(NULL, written_list, nr_written,
  321. pack_idx_opts, hash);
  322. if (adjust_shared_perm(idx_tmp_name))
  323. die_errno("unable to make temporary index file readable");
  324. strbuf_addf(name_buffer, "%s.pack", hash_to_hex(hash));
  325. if (rename(pack_tmp_name, name_buffer->buf))
  326. die_errno("unable to rename temporary pack file");
  327. strbuf_setlen(name_buffer, basename_len);
  328. strbuf_addf(name_buffer, "%s.idx", hash_to_hex(hash));
  329. if (rename(idx_tmp_name, name_buffer->buf))
  330. die_errno("unable to rename temporary index file");
  331. strbuf_setlen(name_buffer, basename_len);
  332. free((void *)idx_tmp_name);
  333. }