THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Git Source Code Mirror - This is a publish-only repository and all pull requests are ignored. Please follow Documentation/SubmittingPatches procedure for any of your improvements.
git
Vous ne pouvez pas sélectionner plus de 25 sujets Les noms de sujets doivent commencer par une lettre ou un nombre, peuvent contenir des tirets ('-') et peuvent comporter jusqu'à 35 caractères.

2469 lignes
61KB

  1. /*
  2. * GIT - The information manager from hell
  3. *
  4. * Copyright (C) Linus Torvalds, 2005
  5. *
  6. * This handles basic git sha1 object files - packing, unpacking,
  7. * creation etc.
  8. */
  9. #include "cache.h"
  10. #include "config.h"
  11. #include "string-list.h"
  12. #include "lockfile.h"
  13. #include "delta.h"
  14. #include "pack.h"
  15. #include "blob.h"
  16. #include "commit.h"
  17. #include "run-command.h"
  18. #include "tag.h"
  19. #include "tree.h"
  20. #include "tree-walk.h"
  21. #include "refs.h"
  22. #include "pack-revindex.h"
  23. #include "sha1-lookup.h"
  24. #include "bulk-checkin.h"
  25. #include "repository.h"
  26. #include "replace-object.h"
  27. #include "streaming.h"
  28. #include "dir.h"
  29. #include "list.h"
  30. #include "mergesort.h"
  31. #include "quote.h"
  32. #include "packfile.h"
  33. #include "object-store.h"
  34. #include "promisor-remote.h"
  35. /* The maximum size for an object header. */
  36. #define MAX_HEADER_LEN 32
  37. #define EMPTY_TREE_SHA1_BIN_LITERAL \
  38. "\x4b\x82\x5d\xc6\x42\xcb\x6e\xb9\xa0\x60" \
  39. "\xe5\x4b\xf8\xd6\x92\x88\xfb\xee\x49\x04"
  40. #define EMPTY_TREE_SHA256_BIN_LITERAL \
  41. "\x6e\xf1\x9b\x41\x22\x5c\x53\x69\xf1\xc1" \
  42. "\x04\xd4\x5d\x8d\x85\xef\xa9\xb0\x57\xb5" \
  43. "\x3b\x14\xb4\xb9\xb9\x39\xdd\x74\xde\xcc" \
  44. "\x53\x21"
  45. #define EMPTY_BLOB_SHA1_BIN_LITERAL \
  46. "\xe6\x9d\xe2\x9b\xb2\xd1\xd6\x43\x4b\x8b" \
  47. "\x29\xae\x77\x5a\xd8\xc2\xe4\x8c\x53\x91"
  48. #define EMPTY_BLOB_SHA256_BIN_LITERAL \
  49. "\x47\x3a\x0f\x4c\x3b\xe8\xa9\x36\x81\xa2" \
  50. "\x67\xe3\xb1\xe9\xa7\xdc\xda\x11\x85\x43" \
  51. "\x6f\xe1\x41\xf7\x74\x91\x20\xa3\x03\x72" \
  52. "\x18\x13"
  53. const struct object_id null_oid;
  54. static const struct object_id empty_tree_oid = {
  55. EMPTY_TREE_SHA1_BIN_LITERAL
  56. };
  57. static const struct object_id empty_blob_oid = {
  58. EMPTY_BLOB_SHA1_BIN_LITERAL
  59. };
  60. static const struct object_id empty_tree_oid_sha256 = {
  61. EMPTY_TREE_SHA256_BIN_LITERAL
  62. };
  63. static const struct object_id empty_blob_oid_sha256 = {
  64. EMPTY_BLOB_SHA256_BIN_LITERAL
  65. };
  66. static void git_hash_sha1_init(git_hash_ctx *ctx)
  67. {
  68. git_SHA1_Init(&ctx->sha1);
  69. }
  70. static void git_hash_sha1_update(git_hash_ctx *ctx, const void *data, size_t len)
  71. {
  72. git_SHA1_Update(&ctx->sha1, data, len);
  73. }
  74. static void git_hash_sha1_final(unsigned char *hash, git_hash_ctx *ctx)
  75. {
  76. git_SHA1_Final(hash, &ctx->sha1);
  77. }
  78. static void git_hash_sha256_init(git_hash_ctx *ctx)
  79. {
  80. git_SHA256_Init(&ctx->sha256);
  81. }
  82. static void git_hash_sha256_update(git_hash_ctx *ctx, const void *data, size_t len)
  83. {
  84. git_SHA256_Update(&ctx->sha256, data, len);
  85. }
  86. static void git_hash_sha256_final(unsigned char *hash, git_hash_ctx *ctx)
  87. {
  88. git_SHA256_Final(hash, &ctx->sha256);
  89. }
  90. static void git_hash_unknown_init(git_hash_ctx *ctx)
  91. {
  92. BUG("trying to init unknown hash");
  93. }
  94. static void git_hash_unknown_update(git_hash_ctx *ctx, const void *data, size_t len)
  95. {
  96. BUG("trying to update unknown hash");
  97. }
  98. static void git_hash_unknown_final(unsigned char *hash, git_hash_ctx *ctx)
  99. {
  100. BUG("trying to finalize unknown hash");
  101. }
  102. const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
  103. {
  104. NULL,
  105. 0x00000000,
  106. 0,
  107. 0,
  108. 0,
  109. git_hash_unknown_init,
  110. git_hash_unknown_update,
  111. git_hash_unknown_final,
  112. NULL,
  113. NULL,
  114. },
  115. {
  116. "sha1",
  117. /* "sha1", big-endian */
  118. 0x73686131,
  119. GIT_SHA1_RAWSZ,
  120. GIT_SHA1_HEXSZ,
  121. GIT_SHA1_BLKSZ,
  122. git_hash_sha1_init,
  123. git_hash_sha1_update,
  124. git_hash_sha1_final,
  125. &empty_tree_oid,
  126. &empty_blob_oid,
  127. },
  128. {
  129. "sha256",
  130. /* "s256", big-endian */
  131. 0x73323536,
  132. GIT_SHA256_RAWSZ,
  133. GIT_SHA256_HEXSZ,
  134. GIT_SHA256_BLKSZ,
  135. git_hash_sha256_init,
  136. git_hash_sha256_update,
  137. git_hash_sha256_final,
  138. &empty_tree_oid_sha256,
  139. &empty_blob_oid_sha256,
  140. }
  141. };
  142. const char *empty_tree_oid_hex(void)
  143. {
  144. static char buf[GIT_MAX_HEXSZ + 1];
  145. return oid_to_hex_r(buf, the_hash_algo->empty_tree);
  146. }
  147. const char *empty_blob_oid_hex(void)
  148. {
  149. static char buf[GIT_MAX_HEXSZ + 1];
  150. return oid_to_hex_r(buf, the_hash_algo->empty_blob);
  151. }
  152. int hash_algo_by_name(const char *name)
  153. {
  154. int i;
  155. if (!name)
  156. return GIT_HASH_UNKNOWN;
  157. for (i = 1; i < GIT_HASH_NALGOS; i++)
  158. if (!strcmp(name, hash_algos[i].name))
  159. return i;
  160. return GIT_HASH_UNKNOWN;
  161. }
  162. int hash_algo_by_id(uint32_t format_id)
  163. {
  164. int i;
  165. for (i = 1; i < GIT_HASH_NALGOS; i++)
  166. if (format_id == hash_algos[i].format_id)
  167. return i;
  168. return GIT_HASH_UNKNOWN;
  169. }
  170. int hash_algo_by_length(int len)
  171. {
  172. int i;
  173. for (i = 1; i < GIT_HASH_NALGOS; i++)
  174. if (len == hash_algos[i].rawsz)
  175. return i;
  176. return GIT_HASH_UNKNOWN;
  177. }
  178. /*
  179. * This is meant to hold a *small* number of objects that you would
  180. * want read_object_file() to be able to return, but yet you do not want
  181. * to write them into the object store (e.g. a browse-only
  182. * application).
  183. */
  184. static struct cached_object {
  185. struct object_id oid;
  186. enum object_type type;
  187. void *buf;
  188. unsigned long size;
  189. } *cached_objects;
  190. static int cached_object_nr, cached_object_alloc;
  191. static struct cached_object empty_tree = {
  192. { EMPTY_TREE_SHA1_BIN_LITERAL },
  193. OBJ_TREE,
  194. "",
  195. 0
  196. };
  197. static struct cached_object *find_cached_object(const struct object_id *oid)
  198. {
  199. int i;
  200. struct cached_object *co = cached_objects;
  201. for (i = 0; i < cached_object_nr; i++, co++) {
  202. if (oideq(&co->oid, oid))
  203. return co;
  204. }
  205. if (oideq(oid, the_hash_algo->empty_tree))
  206. return &empty_tree;
  207. return NULL;
  208. }
  209. static int get_conv_flags(unsigned flags)
  210. {
  211. if (flags & HASH_RENORMALIZE)
  212. return CONV_EOL_RENORMALIZE;
  213. else if (flags & HASH_WRITE_OBJECT)
  214. return global_conv_flags_eol | CONV_WRITE_OBJECT;
  215. else
  216. return 0;
  217. }
  218. int mkdir_in_gitdir(const char *path)
  219. {
  220. if (mkdir(path, 0777)) {
  221. int saved_errno = errno;
  222. struct stat st;
  223. struct strbuf sb = STRBUF_INIT;
  224. if (errno != EEXIST)
  225. return -1;
  226. /*
  227. * Are we looking at a path in a symlinked worktree
  228. * whose original repository does not yet have it?
  229. * e.g. .git/rr-cache pointing at its original
  230. * repository in which the user hasn't performed any
  231. * conflict resolution yet?
  232. */
  233. if (lstat(path, &st) || !S_ISLNK(st.st_mode) ||
  234. strbuf_readlink(&sb, path, st.st_size) ||
  235. !is_absolute_path(sb.buf) ||
  236. mkdir(sb.buf, 0777)) {
  237. strbuf_release(&sb);
  238. errno = saved_errno;
  239. return -1;
  240. }
  241. strbuf_release(&sb);
  242. }
  243. return adjust_shared_perm(path);
  244. }
  245. enum scld_error safe_create_leading_directories(char *path)
  246. {
  247. char *next_component = path + offset_1st_component(path);
  248. enum scld_error ret = SCLD_OK;
  249. while (ret == SCLD_OK && next_component) {
  250. struct stat st;
  251. char *slash = next_component, slash_character;
  252. while (*slash && !is_dir_sep(*slash))
  253. slash++;
  254. if (!*slash)
  255. break;
  256. next_component = slash + 1;
  257. while (is_dir_sep(*next_component))
  258. next_component++;
  259. if (!*next_component)
  260. break;
  261. slash_character = *slash;
  262. *slash = '\0';
  263. if (!stat(path, &st)) {
  264. /* path exists */
  265. if (!S_ISDIR(st.st_mode)) {
  266. errno = ENOTDIR;
  267. ret = SCLD_EXISTS;
  268. }
  269. } else if (mkdir(path, 0777)) {
  270. if (errno == EEXIST &&
  271. !stat(path, &st) && S_ISDIR(st.st_mode))
  272. ; /* somebody created it since we checked */
  273. else if (errno == ENOENT)
  274. /*
  275. * Either mkdir() failed because
  276. * somebody just pruned the containing
  277. * directory, or stat() failed because
  278. * the file that was in our way was
  279. * just removed. Either way, inform
  280. * the caller that it might be worth
  281. * trying again:
  282. */
  283. ret = SCLD_VANISHED;
  284. else
  285. ret = SCLD_FAILED;
  286. } else if (adjust_shared_perm(path)) {
  287. ret = SCLD_PERMS;
  288. }
  289. *slash = slash_character;
  290. }
  291. return ret;
  292. }
  293. enum scld_error safe_create_leading_directories_const(const char *path)
  294. {
  295. int save_errno;
  296. /* path points to cache entries, so xstrdup before messing with it */
  297. char *buf = xstrdup(path);
  298. enum scld_error result = safe_create_leading_directories(buf);
  299. save_errno = errno;
  300. free(buf);
  301. errno = save_errno;
  302. return result;
  303. }
  304. int raceproof_create_file(const char *path, create_file_fn fn, void *cb)
  305. {
  306. /*
  307. * The number of times we will try to remove empty directories
  308. * in the way of path. This is only 1 because if another
  309. * process is racily creating directories that conflict with
  310. * us, we don't want to fight against them.
  311. */
  312. int remove_directories_remaining = 1;
  313. /*
  314. * The number of times that we will try to create the
  315. * directories containing path. We are willing to attempt this
  316. * more than once, because another process could be trying to
  317. * clean up empty directories at the same time as we are
  318. * trying to create them.
  319. */
  320. int create_directories_remaining = 3;
  321. /* A scratch copy of path, filled lazily if we need it: */
  322. struct strbuf path_copy = STRBUF_INIT;
  323. int ret, save_errno;
  324. /* Sanity check: */
  325. assert(*path);
  326. retry_fn:
  327. ret = fn(path, cb);
  328. save_errno = errno;
  329. if (!ret)
  330. goto out;
  331. if (errno == EISDIR && remove_directories_remaining-- > 0) {
  332. /*
  333. * A directory is in the way. Maybe it is empty; try
  334. * to remove it:
  335. */
  336. if (!path_copy.len)
  337. strbuf_addstr(&path_copy, path);
  338. if (!remove_dir_recursively(&path_copy, REMOVE_DIR_EMPTY_ONLY))
  339. goto retry_fn;
  340. } else if (errno == ENOENT && create_directories_remaining-- > 0) {
  341. /*
  342. * Maybe the containing directory didn't exist, or
  343. * maybe it was just deleted by a process that is
  344. * racing with us to clean up empty directories. Try
  345. * to create it:
  346. */
  347. enum scld_error scld_result;
  348. if (!path_copy.len)
  349. strbuf_addstr(&path_copy, path);
  350. do {
  351. scld_result = safe_create_leading_directories(path_copy.buf);
  352. if (scld_result == SCLD_OK)
  353. goto retry_fn;
  354. } while (scld_result == SCLD_VANISHED && create_directories_remaining-- > 0);
  355. }
  356. out:
  357. strbuf_release(&path_copy);
  358. errno = save_errno;
  359. return ret;
  360. }
  361. static void fill_loose_path(struct strbuf *buf, const struct object_id *oid)
  362. {
  363. int i;
  364. for (i = 0; i < the_hash_algo->rawsz; i++) {
  365. static char hex[] = "0123456789abcdef";
  366. unsigned int val = oid->hash[i];
  367. strbuf_addch(buf, hex[val >> 4]);
  368. strbuf_addch(buf, hex[val & 0xf]);
  369. if (!i)
  370. strbuf_addch(buf, '/');
  371. }
  372. }
  373. static const char *odb_loose_path(struct object_directory *odb,
  374. struct strbuf *buf,
  375. const struct object_id *oid)
  376. {
  377. strbuf_reset(buf);
  378. strbuf_addstr(buf, odb->path);
  379. strbuf_addch(buf, '/');
  380. fill_loose_path(buf, oid);
  381. return buf->buf;
  382. }
  383. const char *loose_object_path(struct repository *r, struct strbuf *buf,
  384. const struct object_id *oid)
  385. {
  386. return odb_loose_path(r->objects->odb, buf, oid);
  387. }
  388. /*
  389. * Return non-zero iff the path is usable as an alternate object database.
  390. */
  391. static int alt_odb_usable(struct raw_object_store *o,
  392. struct strbuf *path,
  393. const char *normalized_objdir)
  394. {
  395. struct object_directory *odb;
  396. /* Detect cases where alternate disappeared */
  397. if (!is_directory(path->buf)) {
  398. error(_("object directory %s does not exist; "
  399. "check .git/objects/info/alternates"),
  400. path->buf);
  401. return 0;
  402. }
  403. /*
  404. * Prevent the common mistake of listing the same
  405. * thing twice, or object directory itself.
  406. */
  407. for (odb = o->odb; odb; odb = odb->next) {
  408. if (!fspathcmp(path->buf, odb->path))
  409. return 0;
  410. }
  411. if (!fspathcmp(path->buf, normalized_objdir))
  412. return 0;
  413. return 1;
  414. }
  415. /*
  416. * Prepare alternate object database registry.
  417. *
  418. * The variable alt_odb_list points at the list of struct
  419. * object_directory. The elements on this list come from
  420. * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
  421. * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
  422. * whose contents is similar to that environment variable but can be
  423. * LF separated. Its base points at a statically allocated buffer that
  424. * contains "/the/directory/corresponding/to/.git/objects/...", while
  425. * its name points just after the slash at the end of ".git/objects/"
  426. * in the example above, and has enough space to hold 40-byte hex
  427. * SHA1, an extra slash for the first level indirection, and the
  428. * terminating NUL.
  429. */
  430. static void read_info_alternates(struct repository *r,
  431. const char *relative_base,
  432. int depth);
  433. static int link_alt_odb_entry(struct repository *r, const char *entry,
  434. const char *relative_base, int depth, const char *normalized_objdir)
  435. {
  436. struct object_directory *ent;
  437. struct strbuf pathbuf = STRBUF_INIT;
  438. if (!is_absolute_path(entry) && relative_base) {
  439. strbuf_realpath(&pathbuf, relative_base, 1);
  440. strbuf_addch(&pathbuf, '/');
  441. }
  442. strbuf_addstr(&pathbuf, entry);
  443. if (strbuf_normalize_path(&pathbuf) < 0 && relative_base) {
  444. error(_("unable to normalize alternate object path: %s"),
  445. pathbuf.buf);
  446. strbuf_release(&pathbuf);
  447. return -1;
  448. }
  449. /*
  450. * The trailing slash after the directory name is given by
  451. * this function at the end. Remove duplicates.
  452. */
  453. while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
  454. strbuf_setlen(&pathbuf, pathbuf.len - 1);
  455. if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir)) {
  456. strbuf_release(&pathbuf);
  457. return -1;
  458. }
  459. ent = xcalloc(1, sizeof(*ent));
  460. ent->path = xstrdup(pathbuf.buf);
  461. /* add the alternate entry */
  462. *r->objects->odb_tail = ent;
  463. r->objects->odb_tail = &(ent->next);
  464. ent->next = NULL;
  465. /* recursively add alternates */
  466. read_info_alternates(r, pathbuf.buf, depth + 1);
  467. strbuf_release(&pathbuf);
  468. return 0;
  469. }
  470. static const char *parse_alt_odb_entry(const char *string,
  471. int sep,
  472. struct strbuf *out)
  473. {
  474. const char *end;
  475. strbuf_reset(out);
  476. if (*string == '#') {
  477. /* comment; consume up to next separator */
  478. end = strchrnul(string, sep);
  479. } else if (*string == '"' && !unquote_c_style(out, string, &end)) {
  480. /*
  481. * quoted path; unquote_c_style has copied the
  482. * data for us and set "end". Broken quoting (e.g.,
  483. * an entry that doesn't end with a quote) falls
  484. * back to the unquoted case below.
  485. */
  486. } else {
  487. /* normal, unquoted path */
  488. end = strchrnul(string, sep);
  489. strbuf_add(out, string, end - string);
  490. }
  491. if (*end)
  492. end++;
  493. return end;
  494. }
  495. static void link_alt_odb_entries(struct repository *r, const char *alt,
  496. int sep, const char *relative_base, int depth)
  497. {
  498. struct strbuf objdirbuf = STRBUF_INIT;
  499. struct strbuf entry = STRBUF_INIT;
  500. if (!alt || !*alt)
  501. return;
  502. if (depth > 5) {
  503. error(_("%s: ignoring alternate object stores, nesting too deep"),
  504. relative_base);
  505. return;
  506. }
  507. strbuf_add_absolute_path(&objdirbuf, r->objects->odb->path);
  508. if (strbuf_normalize_path(&objdirbuf) < 0)
  509. die(_("unable to normalize object directory: %s"),
  510. objdirbuf.buf);
  511. while (*alt) {
  512. alt = parse_alt_odb_entry(alt, sep, &entry);
  513. if (!entry.len)
  514. continue;
  515. link_alt_odb_entry(r, entry.buf,
  516. relative_base, depth, objdirbuf.buf);
  517. }
  518. strbuf_release(&entry);
  519. strbuf_release(&objdirbuf);
  520. }
  521. static void read_info_alternates(struct repository *r,
  522. const char *relative_base,
  523. int depth)
  524. {
  525. char *path;
  526. struct strbuf buf = STRBUF_INIT;
  527. path = xstrfmt("%s/info/alternates", relative_base);
  528. if (strbuf_read_file(&buf, path, 1024) < 0) {
  529. warn_on_fopen_errors(path);
  530. free(path);
  531. return;
  532. }
  533. link_alt_odb_entries(r, buf.buf, '\n', relative_base, depth);
  534. strbuf_release(&buf);
  535. free(path);
  536. }
  537. void add_to_alternates_file(const char *reference)
  538. {
  539. struct lock_file lock = LOCK_INIT;
  540. char *alts = git_pathdup("objects/info/alternates");
  541. FILE *in, *out;
  542. int found = 0;
  543. hold_lock_file_for_update(&lock, alts, LOCK_DIE_ON_ERROR);
  544. out = fdopen_lock_file(&lock, "w");
  545. if (!out)
  546. die_errno(_("unable to fdopen alternates lockfile"));
  547. in = fopen(alts, "r");
  548. if (in) {
  549. struct strbuf line = STRBUF_INIT;
  550. while (strbuf_getline(&line, in) != EOF) {
  551. if (!strcmp(reference, line.buf)) {
  552. found = 1;
  553. break;
  554. }
  555. fprintf_or_die(out, "%s\n", line.buf);
  556. }
  557. strbuf_release(&line);
  558. fclose(in);
  559. }
  560. else if (errno != ENOENT)
  561. die_errno(_("unable to read alternates file"));
  562. if (found) {
  563. rollback_lock_file(&lock);
  564. } else {
  565. fprintf_or_die(out, "%s\n", reference);
  566. if (commit_lock_file(&lock))
  567. die_errno(_("unable to move new alternates file into place"));
  568. if (the_repository->objects->loaded_alternates)
  569. link_alt_odb_entries(the_repository, reference,
  570. '\n', NULL, 0);
  571. }
  572. free(alts);
  573. }
  574. void add_to_alternates_memory(const char *reference)
  575. {
  576. /*
  577. * Make sure alternates are initialized, or else our entry may be
  578. * overwritten when they are.
  579. */
  580. prepare_alt_odb(the_repository);
  581. link_alt_odb_entries(the_repository, reference,
  582. '\n', NULL, 0);
  583. }
  584. /*
  585. * Compute the exact path an alternate is at and returns it. In case of
  586. * error NULL is returned and the human readable error is added to `err`
  587. * `path` may be relative and should point to $GIT_DIR.
  588. * `err` must not be null.
  589. */
  590. char *compute_alternate_path(const char *path, struct strbuf *err)
  591. {
  592. char *ref_git = NULL;
  593. const char *repo, *ref_git_s;
  594. int seen_error = 0;
  595. ref_git_s = real_path_if_valid(path);
  596. if (!ref_git_s) {
  597. seen_error = 1;
  598. strbuf_addf(err, _("path '%s' does not exist"), path);
  599. goto out;
  600. } else
  601. /*
  602. * Beware: read_gitfile(), real_path() and mkpath()
  603. * return static buffer
  604. */
  605. ref_git = xstrdup(ref_git_s);
  606. repo = read_gitfile(ref_git);
  607. if (!repo)
  608. repo = read_gitfile(mkpath("%s/.git", ref_git));
  609. if (repo) {
  610. free(ref_git);
  611. ref_git = xstrdup(repo);
  612. }
  613. if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) {
  614. char *ref_git_git = mkpathdup("%s/.git", ref_git);
  615. free(ref_git);
  616. ref_git = ref_git_git;
  617. } else if (!is_directory(mkpath("%s/objects", ref_git))) {
  618. struct strbuf sb = STRBUF_INIT;
  619. seen_error = 1;
  620. if (get_common_dir(&sb, ref_git)) {
  621. strbuf_addf(err,
  622. _("reference repository '%s' as a linked "
  623. "checkout is not supported yet."),
  624. path);
  625. goto out;
  626. }
  627. strbuf_addf(err, _("reference repository '%s' is not a "
  628. "local repository."), path);
  629. goto out;
  630. }
  631. if (!access(mkpath("%s/shallow", ref_git), F_OK)) {
  632. strbuf_addf(err, _("reference repository '%s' is shallow"),
  633. path);
  634. seen_error = 1;
  635. goto out;
  636. }
  637. if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) {
  638. strbuf_addf(err,
  639. _("reference repository '%s' is grafted"),
  640. path);
  641. seen_error = 1;
  642. goto out;
  643. }
  644. out:
  645. if (seen_error) {
  646. FREE_AND_NULL(ref_git);
  647. }
  648. return ref_git;
  649. }
  650. static void fill_alternate_refs_command(struct child_process *cmd,
  651. const char *repo_path)
  652. {
  653. const char *value;
  654. if (!git_config_get_value("core.alternateRefsCommand", &value)) {
  655. cmd->use_shell = 1;
  656. argv_array_push(&cmd->args, value);
  657. argv_array_push(&cmd->args, repo_path);
  658. } else {
  659. cmd->git_cmd = 1;
  660. argv_array_pushf(&cmd->args, "--git-dir=%s", repo_path);
  661. argv_array_push(&cmd->args, "for-each-ref");
  662. argv_array_push(&cmd->args, "--format=%(objectname)");
  663. if (!git_config_get_value("core.alternateRefsPrefixes", &value)) {
  664. argv_array_push(&cmd->args, "--");
  665. argv_array_split(&cmd->args, value);
  666. }
  667. }
  668. cmd->env = local_repo_env;
  669. cmd->out = -1;
  670. }
  671. static void read_alternate_refs(const char *path,
  672. alternate_ref_fn *cb,
  673. void *data)
  674. {
  675. struct child_process cmd = CHILD_PROCESS_INIT;
  676. struct strbuf line = STRBUF_INIT;
  677. FILE *fh;
  678. fill_alternate_refs_command(&cmd, path);
  679. if (start_command(&cmd))
  680. return;
  681. fh = xfdopen(cmd.out, "r");
  682. while (strbuf_getline_lf(&line, fh) != EOF) {
  683. struct object_id oid;
  684. const char *p;
  685. if (parse_oid_hex(line.buf, &oid, &p) || *p) {
  686. warning(_("invalid line while parsing alternate refs: %s"),
  687. line.buf);
  688. break;
  689. }
  690. cb(&oid, data);
  691. }
  692. fclose(fh);
  693. finish_command(&cmd);
  694. strbuf_release(&line);
  695. }
  696. struct alternate_refs_data {
  697. alternate_ref_fn *fn;
  698. void *data;
  699. };
  700. static int refs_from_alternate_cb(struct object_directory *e,
  701. void *data)
  702. {
  703. struct strbuf path = STRBUF_INIT;
  704. size_t base_len;
  705. struct alternate_refs_data *cb = data;
  706. if (!strbuf_realpath(&path, e->path, 0))
  707. goto out;
  708. if (!strbuf_strip_suffix(&path, "/objects"))
  709. goto out;
  710. base_len = path.len;
  711. /* Is this a git repository with refs? */
  712. strbuf_addstr(&path, "/refs");
  713. if (!is_directory(path.buf))
  714. goto out;
  715. strbuf_setlen(&path, base_len);
  716. read_alternate_refs(path.buf, cb->fn, cb->data);
  717. out:
  718. strbuf_release(&path);
  719. return 0;
  720. }
  721. void for_each_alternate_ref(alternate_ref_fn fn, void *data)
  722. {
  723. struct alternate_refs_data cb;
  724. cb.fn = fn;
  725. cb.data = data;
  726. foreach_alt_odb(refs_from_alternate_cb, &cb);
  727. }
  728. int foreach_alt_odb(alt_odb_fn fn, void *cb)
  729. {
  730. struct object_directory *ent;
  731. int r = 0;
  732. prepare_alt_odb(the_repository);
  733. for (ent = the_repository->objects->odb->next; ent; ent = ent->next) {
  734. r = fn(ent, cb);
  735. if (r)
  736. break;
  737. }
  738. return r;
  739. }
  740. void prepare_alt_odb(struct repository *r)
  741. {
  742. if (r->objects->loaded_alternates)
  743. return;
  744. link_alt_odb_entries(r, r->objects->alternate_db, PATH_SEP, NULL, 0);
  745. read_info_alternates(r, r->objects->odb->path, 0);
  746. r->objects->loaded_alternates = 1;
  747. }
  748. /* Returns 1 if we have successfully freshened the file, 0 otherwise. */
  749. static int freshen_file(const char *fn)
  750. {
  751. struct utimbuf t;
  752. t.actime = t.modtime = time(NULL);
  753. return !utime(fn, &t);
  754. }
  755. /*
  756. * All of the check_and_freshen functions return 1 if the file exists and was
  757. * freshened (if freshening was requested), 0 otherwise. If they return
  758. * 0, you should not assume that it is safe to skip a write of the object (it
  759. * either does not exist on disk, or has a stale mtime and may be subject to
  760. * pruning).
  761. */
  762. int check_and_freshen_file(const char *fn, int freshen)
  763. {
  764. if (access(fn, F_OK))
  765. return 0;
  766. if (freshen && !freshen_file(fn))
  767. return 0;
  768. return 1;
  769. }
  770. static int check_and_freshen_odb(struct object_directory *odb,
  771. const struct object_id *oid,
  772. int freshen)
  773. {
  774. static struct strbuf path = STRBUF_INIT;
  775. odb_loose_path(odb, &path, oid);
  776. return check_and_freshen_file(path.buf, freshen);
  777. }
  778. static int check_and_freshen_local(const struct object_id *oid, int freshen)
  779. {
  780. return check_and_freshen_odb(the_repository->objects->odb, oid, freshen);
  781. }
  782. static int check_and_freshen_nonlocal(const struct object_id *oid, int freshen)
  783. {
  784. struct object_directory *odb;
  785. prepare_alt_odb(the_repository);
  786. for (odb = the_repository->objects->odb->next; odb; odb = odb->next) {
  787. if (check_and_freshen_odb(odb, oid, freshen))
  788. return 1;
  789. }
  790. return 0;
  791. }
  792. static int check_and_freshen(const struct object_id *oid, int freshen)
  793. {
  794. return check_and_freshen_local(oid, freshen) ||
  795. check_and_freshen_nonlocal(oid, freshen);
  796. }
  797. int has_loose_object_nonlocal(const struct object_id *oid)
  798. {
  799. return check_and_freshen_nonlocal(oid, 0);
  800. }
  801. static int has_loose_object(const struct object_id *oid)
  802. {
  803. return check_and_freshen(oid, 0);
  804. }
  805. static void mmap_limit_check(size_t length)
  806. {
  807. static size_t limit = 0;
  808. if (!limit) {
  809. limit = git_env_ulong("GIT_MMAP_LIMIT", 0);
  810. if (!limit)
  811. limit = SIZE_MAX;
  812. }
  813. if (length > limit)
  814. die(_("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX),
  815. (uintmax_t)length, (uintmax_t)limit);
  816. }
  817. void *xmmap_gently(void *start, size_t length,
  818. int prot, int flags, int fd, off_t offset)
  819. {
  820. void *ret;
  821. mmap_limit_check(length);
  822. ret = mmap(start, length, prot, flags, fd, offset);
  823. if (ret == MAP_FAILED && !length)
  824. ret = NULL;
  825. return ret;
  826. }
  827. void *xmmap(void *start, size_t length,
  828. int prot, int flags, int fd, off_t offset)
  829. {
  830. void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
  831. if (ret == MAP_FAILED)
  832. die_errno(_("mmap failed"));
  833. return ret;
  834. }
  835. /*
  836. * With an in-core object data in "map", rehash it to make sure the
  837. * object name actually matches "oid" to detect object corruption.
  838. * With "map" == NULL, try reading the object named with "oid" using
  839. * the streaming interface and rehash it to do the same.
  840. */
  841. int check_object_signature(const struct object_id *oid, void *map,
  842. unsigned long size, const char *type)
  843. {
  844. struct object_id real_oid;
  845. enum object_type obj_type;
  846. struct git_istream *st;
  847. git_hash_ctx c;
  848. char hdr[MAX_HEADER_LEN];
  849. int hdrlen;
  850. if (map) {
  851. hash_object_file(map, size, type, &real_oid);
  852. return !oideq(oid, &real_oid) ? -1 : 0;
  853. }
  854. st = open_istream(oid, &obj_type, &size, NULL);
  855. if (!st)
  856. return -1;
  857. /* Generate the header */
  858. hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(obj_type), (uintmax_t)size) + 1;
  859. /* Sha1.. */
  860. the_hash_algo->init_fn(&c);
  861. the_hash_algo->update_fn(&c, hdr, hdrlen);
  862. for (;;) {
  863. char buf[1024 * 16];
  864. ssize_t readlen = read_istream(st, buf, sizeof(buf));
  865. if (readlen < 0) {
  866. close_istream(st);
  867. return -1;
  868. }
  869. if (!readlen)
  870. break;
  871. the_hash_algo->update_fn(&c, buf, readlen);
  872. }
  873. the_hash_algo->final_fn(real_oid.hash, &c);
  874. close_istream(st);
  875. return !oideq(oid, &real_oid) ? -1 : 0;
  876. }
  877. int git_open_cloexec(const char *name, int flags)
  878. {
  879. int fd;
  880. static int o_cloexec = O_CLOEXEC;
  881. fd = open(name, flags | o_cloexec);
  882. if ((o_cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) {
  883. /* Try again w/o O_CLOEXEC: the kernel might not support it */
  884. o_cloexec &= ~O_CLOEXEC;
  885. fd = open(name, flags | o_cloexec);
  886. }
  887. #if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC)
  888. {
  889. static int fd_cloexec = FD_CLOEXEC;
  890. if (!o_cloexec && 0 <= fd && fd_cloexec) {
  891. /* Opened w/o O_CLOEXEC? try with fcntl(2) to add it */
  892. int flags = fcntl(fd, F_GETFD);
  893. if (fcntl(fd, F_SETFD, flags | fd_cloexec))
  894. fd_cloexec = 0;
  895. }
  896. }
  897. #endif
  898. return fd;
  899. }
  900. /*
  901. * Find "oid" as a loose object in the local repository or in an alternate.
  902. * Returns 0 on success, negative on failure.
  903. *
  904. * The "path" out-parameter will give the path of the object we found (if any).
  905. * Note that it may point to static storage and is only valid until another
  906. * call to stat_loose_object().
  907. */
  908. static int stat_loose_object(struct repository *r, const struct object_id *oid,
  909. struct stat *st, const char **path)
  910. {
  911. struct object_directory *odb;
  912. static struct strbuf buf = STRBUF_INIT;
  913. prepare_alt_odb(r);
  914. for (odb = r->objects->odb; odb; odb = odb->next) {
  915. *path = odb_loose_path(odb, &buf, oid);
  916. if (!lstat(*path, st))
  917. return 0;
  918. }
  919. return -1;
  920. }
  921. /*
  922. * Like stat_loose_object(), but actually open the object and return the
  923. * descriptor. See the caveats on the "path" parameter above.
  924. */
  925. static int open_loose_object(struct repository *r,
  926. const struct object_id *oid, const char **path)
  927. {
  928. int fd;
  929. struct object_directory *odb;
  930. int most_interesting_errno = ENOENT;
  931. static struct strbuf buf = STRBUF_INIT;
  932. prepare_alt_odb(r);
  933. for (odb = r->objects->odb; odb; odb = odb->next) {
  934. *path = odb_loose_path(odb, &buf, oid);
  935. fd = git_open(*path);
  936. if (fd >= 0)
  937. return fd;
  938. if (most_interesting_errno == ENOENT)
  939. most_interesting_errno = errno;
  940. }
  941. errno = most_interesting_errno;
  942. return -1;
  943. }
  944. static int quick_has_loose(struct repository *r,
  945. const struct object_id *oid)
  946. {
  947. struct object_directory *odb;
  948. prepare_alt_odb(r);
  949. for (odb = r->objects->odb; odb; odb = odb->next) {
  950. if (oid_array_lookup(odb_loose_cache(odb, oid), oid) >= 0)
  951. return 1;
  952. }
  953. return 0;
  954. }
  955. /*
  956. * Map the loose object at "path" if it is not NULL, or the path found by
  957. * searching for a loose object named "oid".
  958. */
  959. static void *map_loose_object_1(struct repository *r, const char *path,
  960. const struct object_id *oid, unsigned long *size)
  961. {
  962. void *map;
  963. int fd;
  964. if (path)
  965. fd = git_open(path);
  966. else
  967. fd = open_loose_object(r, oid, &path);
  968. map = NULL;
  969. if (fd >= 0) {
  970. struct stat st;
  971. if (!fstat(fd, &st)) {
  972. *size = xsize_t(st.st_size);
  973. if (!*size) {
  974. /* mmap() is forbidden on empty files */
  975. error(_("object file %s is empty"), path);
  976. close(fd);
  977. return NULL;
  978. }
  979. map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
  980. }
  981. close(fd);
  982. }
  983. return map;
  984. }
  985. void *map_loose_object(struct repository *r,
  986. const struct object_id *oid,
  987. unsigned long *size)
  988. {
  989. return map_loose_object_1(r, NULL, oid, size);
  990. }
  991. static int unpack_loose_short_header(git_zstream *stream,
  992. unsigned char *map, unsigned long mapsize,
  993. void *buffer, unsigned long bufsiz)
  994. {
  995. /* Get the data stream */
  996. memset(stream, 0, sizeof(*stream));
  997. stream->next_in = map;
  998. stream->avail_in = mapsize;
  999. stream->next_out = buffer;
  1000. stream->avail_out = bufsiz;
  1001. git_inflate_init(stream);
  1002. return git_inflate(stream, 0);
  1003. }
  1004. int unpack_loose_header(git_zstream *stream,
  1005. unsigned char *map, unsigned long mapsize,
  1006. void *buffer, unsigned long bufsiz)
  1007. {
  1008. int status = unpack_loose_short_header(stream, map, mapsize,
  1009. buffer, bufsiz);
  1010. if (status < Z_OK)
  1011. return status;
  1012. /* Make sure we have the terminating NUL */
  1013. if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
  1014. return -1;
  1015. return 0;
  1016. }
  1017. static int unpack_loose_header_to_strbuf(git_zstream *stream, unsigned char *map,
  1018. unsigned long mapsize, void *buffer,
  1019. unsigned long bufsiz, struct strbuf *header)
  1020. {
  1021. int status;
  1022. status = unpack_loose_short_header(stream, map, mapsize, buffer, bufsiz);
  1023. if (status < Z_OK)
  1024. return -1;
  1025. /*
  1026. * Check if entire header is unpacked in the first iteration.
  1027. */
  1028. if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
  1029. return 0;
  1030. /*
  1031. * buffer[0..bufsiz] was not large enough. Copy the partial
  1032. * result out to header, and then append the result of further
  1033. * reading the stream.
  1034. */
  1035. strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
  1036. stream->next_out = buffer;
  1037. stream->avail_out = bufsiz;
  1038. do {
  1039. status = git_inflate(stream, 0);
  1040. strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
  1041. if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
  1042. return 0;
  1043. stream->next_out = buffer;
  1044. stream->avail_out = bufsiz;
  1045. } while (status != Z_STREAM_END);
  1046. return -1;
  1047. }
  1048. static void *unpack_loose_rest(git_zstream *stream,
  1049. void *buffer, unsigned long size,
  1050. const struct object_id *oid)
  1051. {
  1052. int bytes = strlen(buffer) + 1;
  1053. unsigned char *buf = xmallocz(size);
  1054. unsigned long n;
  1055. int status = Z_OK;
  1056. n = stream->total_out - bytes;
  1057. if (n > size)
  1058. n = size;
  1059. memcpy(buf, (char *) buffer + bytes, n);
  1060. bytes = n;
  1061. if (bytes <= size) {
  1062. /*
  1063. * The above condition must be (bytes <= size), not
  1064. * (bytes < size). In other words, even though we
  1065. * expect no more output and set avail_out to zero,
  1066. * the input zlib stream may have bytes that express
  1067. * "this concludes the stream", and we *do* want to
  1068. * eat that input.
  1069. *
  1070. * Otherwise we would not be able to test that we
  1071. * consumed all the input to reach the expected size;
  1072. * we also want to check that zlib tells us that all
  1073. * went well with status == Z_STREAM_END at the end.
  1074. */
  1075. stream->next_out = buf + bytes;
  1076. stream->avail_out = size - bytes;
  1077. while (status == Z_OK)
  1078. status = git_inflate(stream, Z_FINISH);
  1079. }
  1080. if (status == Z_STREAM_END && !stream->avail_in) {
  1081. git_inflate_end(stream);
  1082. return buf;
  1083. }
  1084. if (status < 0)
  1085. error(_("corrupt loose object '%s'"), oid_to_hex(oid));
  1086. else if (stream->avail_in)
  1087. error(_("garbage at end of loose object '%s'"),
  1088. oid_to_hex(oid));
  1089. free(buf);
  1090. return NULL;
  1091. }
  1092. /*
  1093. * We used to just use "sscanf()", but that's actually way
  1094. * too permissive for what we want to check. So do an anal
  1095. * object header parse by hand.
  1096. */
  1097. static int parse_loose_header_extended(const char *hdr, struct object_info *oi,
  1098. unsigned int flags)
  1099. {
  1100. const char *type_buf = hdr;
  1101. unsigned long size;
  1102. int type, type_len = 0;
  1103. /*
  1104. * The type can be of any size but is followed by
  1105. * a space.
  1106. */
  1107. for (;;) {
  1108. char c = *hdr++;
  1109. if (!c)
  1110. return -1;
  1111. if (c == ' ')
  1112. break;
  1113. type_len++;
  1114. }
  1115. type = type_from_string_gently(type_buf, type_len, 1);
  1116. if (oi->type_name)
  1117. strbuf_add(oi->type_name, type_buf, type_len);
  1118. /*
  1119. * Set type to 0 if its an unknown object and
  1120. * we're obtaining the type using '--allow-unknown-type'
  1121. * option.
  1122. */
  1123. if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE) && (type < 0))
  1124. type = 0;
  1125. else if (type < 0)
  1126. die(_("invalid object type"));
  1127. if (oi->typep)
  1128. *oi->typep = type;
  1129. /*
  1130. * The length must follow immediately, and be in canonical
  1131. * decimal format (ie "010" is not valid).
  1132. */
  1133. size = *hdr++ - '0';
  1134. if (size > 9)
  1135. return -1;
  1136. if (size) {
  1137. for (;;) {
  1138. unsigned long c = *hdr - '0';
  1139. if (c > 9)
  1140. break;
  1141. hdr++;
  1142. size = size * 10 + c;
  1143. }
  1144. }
  1145. if (oi->sizep)
  1146. *oi->sizep = size;
  1147. /*
  1148. * The length must be followed by a zero byte
  1149. */
  1150. return *hdr ? -1 : type;
  1151. }
  1152. int parse_loose_header(const char *hdr, unsigned long *sizep)
  1153. {
  1154. struct object_info oi = OBJECT_INFO_INIT;
  1155. oi.sizep = sizep;
  1156. return parse_loose_header_extended(hdr, &oi, 0);
  1157. }
  1158. static int loose_object_info(struct repository *r,
  1159. const struct object_id *oid,
  1160. struct object_info *oi, int flags)
  1161. {
  1162. int status = 0;
  1163. unsigned long mapsize;
  1164. void *map;
  1165. git_zstream stream;
  1166. char hdr[MAX_HEADER_LEN];
  1167. struct strbuf hdrbuf = STRBUF_INIT;
  1168. unsigned long size_scratch;
  1169. if (oi->delta_base_sha1)
  1170. hashclr(oi->delta_base_sha1);
  1171. /*
  1172. * If we don't care about type or size, then we don't
  1173. * need to look inside the object at all. Note that we
  1174. * do not optimize out the stat call, even if the
  1175. * caller doesn't care about the disk-size, since our
  1176. * return value implicitly indicates whether the
  1177. * object even exists.
  1178. */
  1179. if (!oi->typep && !oi->type_name && !oi->sizep && !oi->contentp) {
  1180. const char *path;
  1181. struct stat st;
  1182. if (!oi->disk_sizep && (flags & OBJECT_INFO_QUICK))
  1183. return quick_has_loose(r, oid) ? 0 : -1;
  1184. if (stat_loose_object(r, oid, &st, &path) < 0)
  1185. return -1;
  1186. if (oi->disk_sizep)
  1187. *oi->disk_sizep = st.st_size;
  1188. return 0;
  1189. }
  1190. map = map_loose_object(r, oid, &mapsize);
  1191. if (!map)
  1192. return -1;
  1193. if (!oi->sizep)
  1194. oi->sizep = &size_scratch;
  1195. if (oi->disk_sizep)
  1196. *oi->disk_sizep = mapsize;
  1197. if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE)) {
  1198. if (unpack_loose_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
  1199. status = error(_("unable to unpack %s header with --allow-unknown-type"),
  1200. oid_to_hex(oid));
  1201. } else if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
  1202. status = error(_("unable to unpack %s header"),
  1203. oid_to_hex(oid));
  1204. if (status < 0)
  1205. ; /* Do nothing */
  1206. else if (hdrbuf.len) {
  1207. if ((status = parse_loose_header_extended(hdrbuf.buf, oi, flags)) < 0)
  1208. status = error(_("unable to parse %s header with --allow-unknown-type"),
  1209. oid_to_hex(oid));
  1210. } else if ((status = parse_loose_header_extended(hdr, oi, flags)) < 0)
  1211. status = error(_("unable to parse %s header"), oid_to_hex(oid));
  1212. if (status >= 0 && oi->contentp) {
  1213. *oi->contentp = unpack_loose_rest(&stream, hdr,
  1214. *oi->sizep, oid);
  1215. if (!*oi->contentp) {
  1216. git_inflate_end(&stream);
  1217. status = -1;
  1218. }
  1219. } else
  1220. git_inflate_end(&stream);
  1221. munmap(map, mapsize);
  1222. if (status && oi->typep)
  1223. *oi->typep = status;
  1224. if (oi->sizep == &size_scratch)
  1225. oi->sizep = NULL;
  1226. strbuf_release(&hdrbuf);
  1227. oi->whence = OI_LOOSE;
  1228. return (status < 0) ? status : 0;
  1229. }
  1230. int fetch_if_missing = 1;
  1231. int oid_object_info_extended(struct repository *r, const struct object_id *oid,
  1232. struct object_info *oi, unsigned flags)
  1233. {
  1234. static struct object_info blank_oi = OBJECT_INFO_INIT;
  1235. struct pack_entry e;
  1236. int rtype;
  1237. const struct object_id *real = oid;
  1238. int already_retried = 0;
  1239. if (flags & OBJECT_INFO_LOOKUP_REPLACE)
  1240. real = lookup_replace_object(r, oid);
  1241. if (is_null_oid(real))
  1242. return -1;
  1243. if (!oi)
  1244. oi = &blank_oi;
  1245. if (!(flags & OBJECT_INFO_SKIP_CACHED)) {
  1246. struct cached_object *co = find_cached_object(real);
  1247. if (co) {
  1248. if (oi->typep)
  1249. *(oi->typep) = co->type;
  1250. if (oi->sizep)
  1251. *(oi->sizep) = co->size;
  1252. if (oi->disk_sizep)
  1253. *(oi->disk_sizep) = 0;
  1254. if (oi->delta_base_sha1)
  1255. hashclr(oi->delta_base_sha1);
  1256. if (oi->type_name)
  1257. strbuf_addstr(oi->type_name, type_name(co->type));
  1258. if (oi->contentp)
  1259. *oi->contentp = xmemdupz(co->buf, co->size);
  1260. oi->whence = OI_CACHED;
  1261. return 0;
  1262. }
  1263. }
  1264. while (1) {
  1265. if (find_pack_entry(r, real, &e))
  1266. break;
  1267. if (flags & OBJECT_INFO_IGNORE_LOOSE)
  1268. return -1;
  1269. /* Most likely it's a loose object. */
  1270. if (!loose_object_info(r, real, oi, flags))
  1271. return 0;
  1272. /* Not a loose object; someone else may have just packed it. */
  1273. if (!(flags & OBJECT_INFO_QUICK)) {
  1274. reprepare_packed_git(r);
  1275. if (find_pack_entry(r, real, &e))
  1276. break;
  1277. }
  1278. /* Check if it is a missing object */
  1279. if (fetch_if_missing && has_promisor_remote() &&
  1280. !already_retried && r == the_repository &&
  1281. !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) {
  1282. /*
  1283. * TODO Investigate checking promisor_remote_get_direct()
  1284. * TODO return value and stopping on error here.
  1285. * TODO Pass a repository struct through
  1286. * promisor_remote_get_direct(), such that arbitrary
  1287. * repositories work.
  1288. */
  1289. promisor_remote_get_direct(r, real, 1);
  1290. already_retried = 1;
  1291. continue;
  1292. }
  1293. return -1;
  1294. }
  1295. if (oi == &blank_oi)
  1296. /*
  1297. * We know that the caller doesn't actually need the
  1298. * information below, so return early.
  1299. */
  1300. return 0;
  1301. rtype = packed_object_info(r, e.p, e.offset, oi);
  1302. if (rtype < 0) {
  1303. mark_bad_packed_object(e.p, real->hash);
  1304. return oid_object_info_extended(r, real, oi, 0);
  1305. } else if (oi->whence == OI_PACKED) {
  1306. oi->u.packed.offset = e.offset;
  1307. oi->u.packed.pack = e.p;
  1308. oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
  1309. rtype == OBJ_OFS_DELTA);
  1310. }
  1311. return 0;
  1312. }
  1313. /* returns enum object_type or negative */
  1314. int oid_object_info(struct repository *r,
  1315. const struct object_id *oid,
  1316. unsigned long *sizep)
  1317. {
  1318. enum object_type type;
  1319. struct object_info oi = OBJECT_INFO_INIT;
  1320. oi.typep = &type;
  1321. oi.sizep = sizep;
  1322. if (oid_object_info_extended(r, oid, &oi,
  1323. OBJECT_INFO_LOOKUP_REPLACE) < 0)
  1324. return -1;
  1325. return type;
  1326. }
  1327. static void *read_object(struct repository *r,
  1328. const struct object_id *oid, enum object_type *type,
  1329. unsigned long *size)
  1330. {
  1331. struct object_info oi = OBJECT_INFO_INIT;
  1332. void *content;
  1333. oi.typep = type;
  1334. oi.sizep = size;
  1335. oi.contentp = &content;
  1336. if (oid_object_info_extended(r, oid, &oi, 0) < 0)
  1337. return NULL;
  1338. return content;
  1339. }
  1340. int pretend_object_file(void *buf, unsigned long len, enum object_type type,
  1341. struct object_id *oid)
  1342. {
  1343. struct cached_object *co;
  1344. hash_object_file(buf, len, type_name(type), oid);
  1345. if (has_object_file(oid) || find_cached_object(oid))
  1346. return 0;
  1347. ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc);
  1348. co = &cached_objects[cached_object_nr++];
  1349. co->size = len;
  1350. co->type = type;
  1351. co->buf = xmalloc(len);
  1352. memcpy(co->buf, buf, len);
  1353. oidcpy(&co->oid, oid);
  1354. return 0;
  1355. }
  1356. /*
  1357. * This function dies on corrupt objects; the callers who want to
  1358. * deal with them should arrange to call read_object() and give error
  1359. * messages themselves.
  1360. */
  1361. void *read_object_file_extended(struct repository *r,
  1362. const struct object_id *oid,
  1363. enum object_type *type,
  1364. unsigned long *size,
  1365. int lookup_replace)
  1366. {
  1367. void *data;
  1368. const struct packed_git *p;
  1369. const char *path;
  1370. struct stat st;
  1371. const struct object_id *repl = lookup_replace ?
  1372. lookup_replace_object(r, oid) : oid;
  1373. errno = 0;
  1374. data = read_object(r, repl, type, size);
  1375. if (data)
  1376. return data;
  1377. if (errno && errno != ENOENT)
  1378. die_errno(_("failed to read object %s"), oid_to_hex(oid));
  1379. /* die if we replaced an object with one that does not exist */
  1380. if (repl != oid)
  1381. die(_("replacement %s not found for %s"),
  1382. oid_to_hex(repl), oid_to_hex(oid));
  1383. if (!stat_loose_object(r, repl, &st, &path))
  1384. die(_("loose object %s (stored in %s) is corrupt"),
  1385. oid_to_hex(repl), path);
  1386. if ((p = has_packed_and_bad(r, repl->hash)) != NULL)
  1387. die(_("packed object %s (stored in %s) is corrupt"),
  1388. oid_to_hex(repl), p->pack_name);
  1389. return NULL;
  1390. }
  1391. void *read_object_with_reference(struct repository *r,
  1392. const struct object_id *oid,
  1393. const char *required_type_name,
  1394. unsigned long *size,
  1395. struct object_id *actual_oid_return)
  1396. {
  1397. enum object_type type, required_type;
  1398. void *buffer;
  1399. unsigned long isize;
  1400. struct object_id actual_oid;
  1401. required_type = type_from_string(required_type_name);
  1402. oidcpy(&actual_oid, oid);
  1403. while (1) {
  1404. int ref_length = -1;
  1405. const char *ref_type = NULL;
  1406. buffer = repo_read_object_file(r, &actual_oid, &type, &isize);
  1407. if (!buffer)
  1408. return NULL;
  1409. if (type == required_type) {
  1410. *size = isize;
  1411. if (actual_oid_return)
  1412. oidcpy(actual_oid_return, &actual_oid);
  1413. return buffer;
  1414. }
  1415. /* Handle references */
  1416. else if (type == OBJ_COMMIT)
  1417. ref_type = "tree ";
  1418. else if (type == OBJ_TAG)
  1419. ref_type = "object ";
  1420. else {
  1421. free(buffer);
  1422. return NULL;
  1423. }
  1424. ref_length = strlen(ref_type);
  1425. if (ref_length + the_hash_algo->hexsz > isize ||
  1426. memcmp(buffer, ref_type, ref_length) ||
  1427. get_oid_hex((char *) buffer + ref_length, &actual_oid)) {
  1428. free(buffer);
  1429. return NULL;
  1430. }
  1431. free(buffer);
  1432. /* Now we have the ID of the referred-to object in
  1433. * actual_oid. Check again. */
  1434. }
  1435. }
  1436. static void write_object_file_prepare(const void *buf, unsigned long len,
  1437. const char *type, struct object_id *oid,
  1438. char *hdr, int *hdrlen)
  1439. {
  1440. git_hash_ctx c;
  1441. /* Generate the header */
  1442. *hdrlen = xsnprintf(hdr, *hdrlen, "%s %"PRIuMAX , type, (uintmax_t)len)+1;
  1443. /* Sha1.. */
  1444. the_hash_algo->init_fn(&c);
  1445. the_hash_algo->update_fn(&c, hdr, *hdrlen);
  1446. the_hash_algo->update_fn(&c, buf, len);
  1447. the_hash_algo->final_fn(oid->hash, &c);
  1448. }
  1449. /*
  1450. * Move the just written object into its final resting place.
  1451. */
  1452. int finalize_object_file(const char *tmpfile, const char *filename)
  1453. {
  1454. int ret = 0;
  1455. if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
  1456. goto try_rename;
  1457. else if (link(tmpfile, filename))
  1458. ret = errno;
  1459. /*
  1460. * Coda hack - coda doesn't like cross-directory links,
  1461. * so we fall back to a rename, which will mean that it
  1462. * won't be able to check collisions, but that's not a
  1463. * big deal.
  1464. *
  1465. * The same holds for FAT formatted media.
  1466. *
  1467. * When this succeeds, we just return. We have nothing
  1468. * left to unlink.
  1469. */
  1470. if (ret && ret != EEXIST) {
  1471. try_rename:
  1472. if (!rename(tmpfile, filename))
  1473. goto out;
  1474. ret = errno;
  1475. }
  1476. unlink_or_warn(tmpfile);
  1477. if (ret) {
  1478. if (ret != EEXIST) {
  1479. return error_errno(_("unable to write file %s"), filename);
  1480. }
  1481. /* FIXME!!! Collision check here ? */
  1482. }
  1483. out:
  1484. if (adjust_shared_perm(filename))
  1485. return error(_("unable to set permission to '%s'"), filename);
  1486. return 0;
  1487. }
  1488. static int write_buffer(int fd, const void *buf, size_t len)
  1489. {
  1490. if (write_in_full(fd, buf, len) < 0)
  1491. return error_errno(_("file write error"));
  1492. return 0;
  1493. }
  1494. int hash_object_file(const void *buf, unsigned long len, const char *type,
  1495. struct object_id *oid)
  1496. {
  1497. char hdr[MAX_HEADER_LEN];
  1498. int hdrlen = sizeof(hdr);
  1499. write_object_file_prepare(buf, len, type, oid, hdr, &hdrlen);
  1500. return 0;
  1501. }
  1502. /* Finalize a file on disk, and close it. */
  1503. static void close_loose_object(int fd)
  1504. {
  1505. if (fsync_object_files)
  1506. fsync_or_die(fd, "loose object file");
  1507. if (close(fd) != 0)
  1508. die_errno(_("error when closing loose object file"));
  1509. }
  1510. /* Size of directory component, including the ending '/' */
  1511. static inline int directory_size(const char *filename)
  1512. {
  1513. const char *s = strrchr(filename, '/');
  1514. if (!s)
  1515. return 0;
  1516. return s - filename + 1;
  1517. }
  1518. /*
  1519. * This creates a temporary file in the same directory as the final
  1520. * 'filename'
  1521. *
  1522. * We want to avoid cross-directory filename renames, because those
  1523. * can have problems on various filesystems (FAT, NFS, Coda).
  1524. */
  1525. static int create_tmpfile(struct strbuf *tmp, const char *filename)
  1526. {
  1527. int fd, dirlen = directory_size(filename);
  1528. strbuf_reset(tmp);
  1529. strbuf_add(tmp, filename, dirlen);
  1530. strbuf_addstr(tmp, "tmp_obj_XXXXXX");
  1531. fd = git_mkstemp_mode(tmp->buf, 0444);
  1532. if (fd < 0 && dirlen && errno == ENOENT) {
  1533. /*
  1534. * Make sure the directory exists; note that the contents
  1535. * of the buffer are undefined after mkstemp returns an
  1536. * error, so we have to rewrite the whole buffer from
  1537. * scratch.
  1538. */
  1539. strbuf_reset(tmp);
  1540. strbuf_add(tmp, filename, dirlen - 1);
  1541. if (mkdir(tmp->buf, 0777) && errno != EEXIST)
  1542. return -1;
  1543. if (adjust_shared_perm(tmp->buf))
  1544. return -1;
  1545. /* Try again */
  1546. strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
  1547. fd = git_mkstemp_mode(tmp->buf, 0444);
  1548. }
  1549. return fd;
  1550. }
  1551. static int write_loose_object(const struct object_id *oid, char *hdr,
  1552. int hdrlen, const void *buf, unsigned long len,
  1553. time_t mtime)
  1554. {
  1555. int fd, ret;
  1556. unsigned char compressed[4096];
  1557. git_zstream stream;
  1558. git_hash_ctx c;
  1559. struct object_id parano_oid;
  1560. static struct strbuf tmp_file = STRBUF_INIT;
  1561. static struct strbuf filename = STRBUF_INIT;
  1562. loose_object_path(the_repository, &filename, oid);
  1563. fd = create_tmpfile(&tmp_file, filename.buf);
  1564. if (fd < 0) {
  1565. if (errno == EACCES)
  1566. return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
  1567. else
  1568. return error_errno(_("unable to create temporary file"));
  1569. }
  1570. /* Set it up */
  1571. git_deflate_init(&stream, zlib_compression_level);
  1572. stream.next_out = compressed;
  1573. stream.avail_out = sizeof(compressed);
  1574. the_hash_algo->init_fn(&c);
  1575. /* First header.. */
  1576. stream.next_in = (unsigned char *)hdr;
  1577. stream.avail_in = hdrlen;
  1578. while (git_deflate(&stream, 0) == Z_OK)
  1579. ; /* nothing */
  1580. the_hash_algo->update_fn(&c, hdr, hdrlen);
  1581. /* Then the data itself.. */
  1582. stream.next_in = (void *)buf;
  1583. stream.avail_in = len;
  1584. do {
  1585. unsigned char *in0 = stream.next_in;
  1586. ret = git_deflate(&stream, Z_FINISH);
  1587. the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
  1588. if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
  1589. die(_("unable to write loose object file"));
  1590. stream.next_out = compressed;
  1591. stream.avail_out = sizeof(compressed);
  1592. } while (ret == Z_OK);
  1593. if (ret != Z_STREAM_END)
  1594. die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
  1595. ret);
  1596. ret = git_deflate_end_gently(&stream);
  1597. if (ret != Z_OK)
  1598. die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
  1599. ret);
  1600. the_hash_algo->final_fn(parano_oid.hash, &c);
  1601. if (!oideq(oid, &parano_oid))
  1602. die(_("confused by unstable object source data for %s"),
  1603. oid_to_hex(oid));
  1604. close_loose_object(fd);
  1605. if (mtime) {
  1606. struct utimbuf utb;
  1607. utb.actime = mtime;
  1608. utb.modtime = mtime;
  1609. if (utime(tmp_file.buf, &utb) < 0)
  1610. warning_errno(_("failed utime() on %s"), tmp_file.buf);
  1611. }
  1612. return finalize_object_file(tmp_file.buf, filename.buf);
  1613. }
  1614. static int freshen_loose_object(const struct object_id *oid)
  1615. {
  1616. return check_and_freshen(oid, 1);
  1617. }
  1618. static int freshen_packed_object(const struct object_id *oid)
  1619. {
  1620. struct pack_entry e;
  1621. if (!find_pack_entry(the_repository, oid, &e))
  1622. return 0;
  1623. if (e.p->freshened)
  1624. return 1;
  1625. if (!freshen_file(e.p->pack_name))
  1626. return 0;
  1627. e.p->freshened = 1;
  1628. return 1;
  1629. }
  1630. int write_object_file(const void *buf, unsigned long len, const char *type,
  1631. struct object_id *oid)
  1632. {
  1633. char hdr[MAX_HEADER_LEN];
  1634. int hdrlen = sizeof(hdr);
  1635. /* Normally if we have it in the pack then we do not bother writing
  1636. * it out into .git/objects/??/?{38} file.
  1637. */
  1638. write_object_file_prepare(buf, len, type, oid, hdr, &hdrlen);
  1639. if (freshen_packed_object(oid) || freshen_loose_object(oid))
  1640. return 0;
  1641. return write_loose_object(oid, hdr, hdrlen, buf, len, 0);
  1642. }
  1643. int hash_object_file_literally(const void *buf, unsigned long len,
  1644. const char *type, struct object_id *oid,
  1645. unsigned flags)
  1646. {
  1647. char *header;
  1648. int hdrlen, status = 0;
  1649. /* type string, SP, %lu of the length plus NUL must fit this */
  1650. hdrlen = strlen(type) + MAX_HEADER_LEN;
  1651. header = xmalloc(hdrlen);
  1652. write_object_file_prepare(buf, len, type, oid, header, &hdrlen);
  1653. if (!(flags & HASH_WRITE_OBJECT))
  1654. goto cleanup;
  1655. if (freshen_packed_object(oid) || freshen_loose_object(oid))
  1656. goto cleanup;
  1657. status = write_loose_object(oid, header, hdrlen, buf, len, 0);
  1658. cleanup:
  1659. free(header);
  1660. return status;
  1661. }
  1662. int force_object_loose(const struct object_id *oid, time_t mtime)
  1663. {
  1664. void *buf;
  1665. unsigned long len;
  1666. enum object_type type;
  1667. char hdr[MAX_HEADER_LEN];
  1668. int hdrlen;
  1669. int ret;
  1670. if (has_loose_object(oid))
  1671. return 0;
  1672. buf = read_object(the_repository, oid, &type, &len);
  1673. if (!buf)
  1674. return error(_("cannot read object for %s"), oid_to_hex(oid));
  1675. hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
  1676. ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime);
  1677. free(buf);
  1678. return ret;
  1679. }
  1680. int repo_has_object_file_with_flags(struct repository *r,
  1681. const struct object_id *oid, int flags)
  1682. {
  1683. if (!startup_info->have_repository)
  1684. return 0;
  1685. return oid_object_info_extended(r, oid, NULL,
  1686. flags | OBJECT_INFO_SKIP_CACHED) >= 0;
  1687. }
  1688. int repo_has_object_file(struct repository *r,
  1689. const struct object_id *oid)
  1690. {
  1691. return repo_has_object_file_with_flags(r, oid, 0);
  1692. }
  1693. static void check_tree(const void *buf, size_t size)
  1694. {
  1695. struct tree_desc desc;
  1696. struct name_entry entry;
  1697. init_tree_desc(&desc, buf, size);
  1698. while (tree_entry(&desc, &entry))
  1699. /* do nothing
  1700. * tree_entry() will die() on malformed entries */
  1701. ;
  1702. }
  1703. static void check_commit(const void *buf, size_t size)
  1704. {
  1705. struct commit c;
  1706. memset(&c, 0, sizeof(c));
  1707. if (parse_commit_buffer(the_repository, &c, buf, size, 0))
  1708. die(_("corrupt commit"));
  1709. }
  1710. static void check_tag(const void *buf, size_t size)
  1711. {
  1712. struct tag t;
  1713. memset(&t, 0, sizeof(t));
  1714. if (parse_tag_buffer(the_repository, &t, buf, size))
  1715. die(_("corrupt tag"));
  1716. }
  1717. static int index_mem(struct index_state *istate,
  1718. struct object_id *oid, void *buf, size_t size,
  1719. enum object_type type,
  1720. const char *path, unsigned flags)
  1721. {
  1722. int ret, re_allocated = 0;
  1723. int write_object = flags & HASH_WRITE_OBJECT;
  1724. if (!type)
  1725. type = OBJ_BLOB;
  1726. /*
  1727. * Convert blobs to git internal format
  1728. */
  1729. if ((type == OBJ_BLOB) && path) {
  1730. struct strbuf nbuf = STRBUF_INIT;
  1731. if (convert_to_git(istate, path, buf, size, &nbuf,
  1732. get_conv_flags(flags))) {
  1733. buf = strbuf_detach(&nbuf, &size);
  1734. re_allocated = 1;
  1735. }
  1736. }
  1737. if (flags & HASH_FORMAT_CHECK) {
  1738. if (type == OBJ_TREE)
  1739. check_tree(buf, size);
  1740. if (type == OBJ_COMMIT)
  1741. check_commit(buf, size);
  1742. if (type == OBJ_TAG)
  1743. check_tag(buf, size);
  1744. }
  1745. if (write_object)
  1746. ret = write_object_file(buf, size, type_name(type), oid);
  1747. else
  1748. ret = hash_object_file(buf, size, type_name(type), oid);
  1749. if (re_allocated)
  1750. free(buf);
  1751. return ret;
  1752. }
  1753. static int index_stream_convert_blob(struct index_state *istate,
  1754. struct object_id *oid,
  1755. int fd,
  1756. const char *path,
  1757. unsigned flags)
  1758. {
  1759. int ret;
  1760. const int write_object = flags & HASH_WRITE_OBJECT;
  1761. struct strbuf sbuf = STRBUF_INIT;
  1762. assert(path);
  1763. assert(would_convert_to_git_filter_fd(istate, path));
  1764. convert_to_git_filter_fd(istate, path, fd, &sbuf,
  1765. get_conv_flags(flags));
  1766. if (write_object)
  1767. ret = write_object_file(sbuf.buf, sbuf.len, type_name(OBJ_BLOB),
  1768. oid);
  1769. else
  1770. ret = hash_object_file(sbuf.buf, sbuf.len, type_name(OBJ_BLOB),
  1771. oid);
  1772. strbuf_release(&sbuf);
  1773. return ret;
  1774. }
  1775. static int index_pipe(struct index_state *istate, struct object_id *oid,
  1776. int fd, enum object_type type,
  1777. const char *path, unsigned flags)
  1778. {
  1779. struct strbuf sbuf = STRBUF_INIT;
  1780. int ret;
  1781. if (strbuf_read(&sbuf, fd, 4096) >= 0)
  1782. ret = index_mem(istate, oid, sbuf.buf, sbuf.len, type, path, flags);
  1783. else
  1784. ret = -1;
  1785. strbuf_release(&sbuf);
  1786. return ret;
  1787. }
  1788. #define SMALL_FILE_SIZE (32*1024)
  1789. static int index_core(struct index_state *istate,
  1790. struct object_id *oid, int fd, size_t size,
  1791. enum object_type type, const char *path,
  1792. unsigned flags)
  1793. {
  1794. int ret;
  1795. if (!size) {
  1796. ret = index_mem(istate, oid, "", size, type, path, flags);
  1797. } else if (size <= SMALL_FILE_SIZE) {
  1798. char *buf = xmalloc(size);
  1799. ssize_t read_result = read_in_full(fd, buf, size);
  1800. if (read_result < 0)
  1801. ret = error_errno(_("read error while indexing %s"),
  1802. path ? path : "<unknown>");
  1803. else if (read_result != size)
  1804. ret = error(_("short read while indexing %s"),
  1805. path ? path : "<unknown>");
  1806. else
  1807. ret = index_mem(istate, oid, buf, size, type, path, flags);
  1808. free(buf);
  1809. } else {
  1810. void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
  1811. ret = index_mem(istate, oid, buf, size, type, path, flags);
  1812. munmap(buf, size);
  1813. }
  1814. return ret;
  1815. }
  1816. /*
  1817. * This creates one packfile per large blob unless bulk-checkin
  1818. * machinery is "plugged".
  1819. *
  1820. * This also bypasses the usual "convert-to-git" dance, and that is on
  1821. * purpose. We could write a streaming version of the converting
  1822. * functions and insert that before feeding the data to fast-import
  1823. * (or equivalent in-core API described above). However, that is
  1824. * somewhat complicated, as we do not know the size of the filter
  1825. * result, which we need to know beforehand when writing a git object.
  1826. * Since the primary motivation for trying to stream from the working
  1827. * tree file and to avoid mmaping it in core is to deal with large
  1828. * binary blobs, they generally do not want to get any conversion, and
  1829. * callers should avoid this code path when filters are requested.
  1830. */
  1831. static int index_stream(struct object_id *oid, int fd, size_t size,
  1832. enum object_type type, const char *path,
  1833. unsigned flags)
  1834. {
  1835. return index_bulk_checkin(oid, fd, size, type, path, flags);
  1836. }
  1837. int index_fd(struct index_state *istate, struct object_id *oid,
  1838. int fd, struct stat *st,
  1839. enum object_type type, const char *path, unsigned flags)
  1840. {
  1841. int ret;
  1842. /*
  1843. * Call xsize_t() only when needed to avoid potentially unnecessary
  1844. * die() for large files.
  1845. */
  1846. if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(istate, path))
  1847. ret = index_stream_convert_blob(istate, oid, fd, path, flags);
  1848. else if (!S_ISREG(st->st_mode))
  1849. ret = index_pipe(istate, oid, fd, type, path, flags);
  1850. else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
  1851. (path && would_convert_to_git(istate, path)))
  1852. ret = index_core(istate, oid, fd, xsize_t(st->st_size),
  1853. type, path, flags);
  1854. else
  1855. ret = index_stream(oid, fd, xsize_t(st->st_size), type, path,
  1856. flags);
  1857. close(fd);
  1858. return ret;
  1859. }
  1860. int index_path(struct index_state *istate, struct object_id *oid,
  1861. const char *path, struct stat *st, unsigned flags)
  1862. {
  1863. int fd;
  1864. struct strbuf sb = STRBUF_INIT;
  1865. int rc = 0;
  1866. switch (st->st_mode & S_IFMT) {
  1867. case S_IFREG:
  1868. fd = open(path, O_RDONLY);
  1869. if (fd < 0)
  1870. return error_errno("open(\"%s\")", path);
  1871. if (index_fd(istate, oid, fd, st, OBJ_BLOB, path, flags) < 0)
  1872. return error(_("%s: failed to insert into database"),
  1873. path);
  1874. break;
  1875. case S_IFLNK:
  1876. if (strbuf_readlink(&sb, path, st->st_size))
  1877. return error_errno("readlink(\"%s\")", path);
  1878. if (!(flags & HASH_WRITE_OBJECT))
  1879. hash_object_file(sb.buf, sb.len, blob_type, oid);
  1880. else if (write_object_file(sb.buf, sb.len, blob_type, oid))
  1881. rc = error(_("%s: failed to insert into database"), path);
  1882. strbuf_release(&sb);
  1883. break;
  1884. case S_IFDIR:
  1885. return resolve_gitlink_ref(path, "HEAD", oid);
  1886. default:
  1887. return error(_("%s: unsupported file type"), path);
  1888. }
  1889. return rc;
  1890. }
  1891. int read_pack_header(int fd, struct pack_header *header)
  1892. {
  1893. if (read_in_full(fd, header, sizeof(*header)) != sizeof(*header))
  1894. /* "eof before pack header was fully read" */
  1895. return PH_ERROR_EOF;
  1896. if (header->hdr_signature != htonl(PACK_SIGNATURE))
  1897. /* "protocol error (pack signature mismatch detected)" */
  1898. return PH_ERROR_PACK_SIGNATURE;
  1899. if (!pack_version_ok(header->hdr_version))
  1900. /* "protocol error (pack version unsupported)" */
  1901. return PH_ERROR_PROTOCOL;
  1902. return 0;
  1903. }
  1904. void assert_oid_type(const struct object_id *oid, enum object_type expect)
  1905. {
  1906. enum object_type type = oid_object_info(the_repository, oid, NULL);
  1907. if (type < 0)
  1908. die(_("%s is not a valid object"), oid_to_hex(oid));
  1909. if (type != expect)
  1910. die(_("%s is not a valid '%s' object"), oid_to_hex(oid),
  1911. type_name(expect));
  1912. }
  1913. int for_each_file_in_obj_subdir(unsigned int subdir_nr,
  1914. struct strbuf *path,
  1915. each_loose_object_fn obj_cb,
  1916. each_loose_cruft_fn cruft_cb,
  1917. each_loose_subdir_fn subdir_cb,
  1918. void *data)
  1919. {
  1920. size_t origlen, baselen;
  1921. DIR *dir;
  1922. struct dirent *de;
  1923. int r = 0;
  1924. struct object_id oid;
  1925. if (subdir_nr > 0xff)
  1926. BUG("invalid loose object subdirectory: %x", subdir_nr);
  1927. origlen = path->len;
  1928. strbuf_complete(path, '/');
  1929. strbuf_addf(path, "%02x", subdir_nr);
  1930. dir = opendir(path->buf);
  1931. if (!dir) {
  1932. if (errno != ENOENT)
  1933. r = error_errno(_("unable to open %s"), path->buf);
  1934. strbuf_setlen(path, origlen);
  1935. return r;
  1936. }
  1937. oid.hash[0] = subdir_nr;
  1938. strbuf_addch(path, '/');
  1939. baselen = path->len;
  1940. while ((de = readdir(dir))) {
  1941. size_t namelen;
  1942. if (is_dot_or_dotdot(de->d_name))
  1943. continue;
  1944. namelen = strlen(de->d_name);
  1945. strbuf_setlen(path, baselen);
  1946. strbuf_add(path, de->d_name, namelen);
  1947. if (namelen == the_hash_algo->hexsz - 2 &&
  1948. !hex_to_bytes(oid.hash + 1, de->d_name,
  1949. the_hash_algo->rawsz - 1)) {
  1950. if (obj_cb) {
  1951. r = obj_cb(&oid, path->buf, data);
  1952. if (r)
  1953. break;
  1954. }
  1955. continue;
  1956. }
  1957. if (cruft_cb) {
  1958. r = cruft_cb(de->d_name, path->buf, data);
  1959. if (r)
  1960. break;
  1961. }
  1962. }
  1963. closedir(dir);
  1964. strbuf_setlen(path, baselen - 1);
  1965. if (!r && subdir_cb)
  1966. r = subdir_cb(subdir_nr, path->buf, data);
  1967. strbuf_setlen(path, origlen);
  1968. return r;
  1969. }
  1970. int for_each_loose_file_in_objdir_buf(struct strbuf *path,
  1971. each_loose_object_fn obj_cb,
  1972. each_loose_cruft_fn cruft_cb,
  1973. each_loose_subdir_fn subdir_cb,
  1974. void *data)
  1975. {
  1976. int r = 0;
  1977. int i;
  1978. for (i = 0; i < 256; i++) {
  1979. r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
  1980. subdir_cb, data);
  1981. if (r)
  1982. break;
  1983. }
  1984. return r;
  1985. }
  1986. int for_each_loose_file_in_objdir(const char *path,
  1987. each_loose_object_fn obj_cb,
  1988. each_loose_cruft_fn cruft_cb,
  1989. each_loose_subdir_fn subdir_cb,
  1990. void *data)
  1991. {
  1992. struct strbuf buf = STRBUF_INIT;
  1993. int r;
  1994. strbuf_addstr(&buf, path);
  1995. r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb,
  1996. subdir_cb, data);
  1997. strbuf_release(&buf);
  1998. return r;
  1999. }
  2000. int for_each_loose_object(each_loose_object_fn cb, void *data,
  2001. enum for_each_object_flags flags)
  2002. {
  2003. struct object_directory *odb;
  2004. prepare_alt_odb(the_repository);
  2005. for (odb = the_repository->objects->odb; odb; odb = odb->next) {
  2006. int r = for_each_loose_file_in_objdir(odb->path, cb, NULL,
  2007. NULL, data);
  2008. if (r)
  2009. return r;
  2010. if (flags & FOR_EACH_OBJECT_LOCAL_ONLY)
  2011. break;
  2012. }
  2013. return 0;
  2014. }
  2015. static int append_loose_object(const struct object_id *oid, const char *path,
  2016. void *data)
  2017. {
  2018. oid_array_append(data, oid);
  2019. return 0;
  2020. }
  2021. struct oid_array *odb_loose_cache(struct object_directory *odb,
  2022. const struct object_id *oid)
  2023. {
  2024. int subdir_nr = oid->hash[0];
  2025. struct strbuf buf = STRBUF_INIT;
  2026. if (subdir_nr < 0 ||
  2027. subdir_nr >= ARRAY_SIZE(odb->loose_objects_subdir_seen))
  2028. BUG("subdir_nr out of range");
  2029. if (odb->loose_objects_subdir_seen[subdir_nr])
  2030. return &odb->loose_objects_cache[subdir_nr];
  2031. strbuf_addstr(&buf, odb->path);
  2032. for_each_file_in_obj_subdir(subdir_nr, &buf,
  2033. append_loose_object,
  2034. NULL, NULL,
  2035. &odb->loose_objects_cache[subdir_nr]);
  2036. odb->loose_objects_subdir_seen[subdir_nr] = 1;
  2037. strbuf_release(&buf);
  2038. return &odb->loose_objects_cache[subdir_nr];
  2039. }
  2040. void odb_clear_loose_cache(struct object_directory *odb)
  2041. {
  2042. int i;
  2043. for (i = 0; i < ARRAY_SIZE(odb->loose_objects_cache); i++)
  2044. oid_array_clear(&odb->loose_objects_cache[i]);
  2045. memset(&odb->loose_objects_subdir_seen, 0,
  2046. sizeof(odb->loose_objects_subdir_seen));
  2047. }
  2048. static int check_stream_oid(git_zstream *stream,
  2049. const char *hdr,
  2050. unsigned long size,
  2051. const char *path,
  2052. const struct object_id *expected_oid)
  2053. {
  2054. git_hash_ctx c;
  2055. struct object_id real_oid;
  2056. unsigned char buf[4096];
  2057. unsigned long total_read;
  2058. int status = Z_OK;
  2059. the_hash_algo->init_fn(&c);
  2060. the_hash_algo->update_fn(&c, hdr, stream->total_out);
  2061. /*
  2062. * We already read some bytes into hdr, but the ones up to the NUL
  2063. * do not count against the object's content size.
  2064. */
  2065. total_read = stream->total_out - strlen(hdr) - 1;
  2066. /*
  2067. * This size comparison must be "<=" to read the final zlib packets;
  2068. * see the comment in unpack_loose_rest for details.
  2069. */
  2070. while (total_read <= size &&
  2071. (status == Z_OK ||
  2072. (status == Z_BUF_ERROR && !stream->avail_out))) {
  2073. stream->next_out = buf;
  2074. stream->avail_out = sizeof(buf);
  2075. if (size - total_read < stream->avail_out)
  2076. stream->avail_out = size - total_read;
  2077. status = git_inflate(stream, Z_FINISH);
  2078. the_hash_algo->update_fn(&c, buf, stream->next_out - buf);
  2079. total_read += stream->next_out - buf;
  2080. }
  2081. git_inflate_end(stream);
  2082. if (status != Z_STREAM_END) {
  2083. error(_("corrupt loose object '%s'"), oid_to_hex(expected_oid));
  2084. return -1;
  2085. }
  2086. if (stream->avail_in) {
  2087. error(_("garbage at end of loose object '%s'"),
  2088. oid_to_hex(expected_oid));
  2089. return -1;
  2090. }
  2091. the_hash_algo->final_fn(real_oid.hash, &c);
  2092. if (!oideq(expected_oid, &real_oid)) {
  2093. error(_("hash mismatch for %s (expected %s)"), path,
  2094. oid_to_hex(expected_oid));
  2095. return -1;
  2096. }
  2097. return 0;
  2098. }
  2099. int read_loose_object(const char *path,
  2100. const struct object_id *expected_oid,
  2101. enum object_type *type,
  2102. unsigned long *size,
  2103. void **contents)
  2104. {
  2105. int ret = -1;
  2106. void *map = NULL;
  2107. unsigned long mapsize;
  2108. git_zstream stream;
  2109. char hdr[MAX_HEADER_LEN];
  2110. *contents = NULL;
  2111. map = map_loose_object_1(the_repository, path, NULL, &mapsize);
  2112. if (!map) {
  2113. error_errno(_("unable to mmap %s"), path);
  2114. goto out;
  2115. }
  2116. if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) {
  2117. error(_("unable to unpack header of %s"), path);
  2118. goto out;
  2119. }
  2120. *type = parse_loose_header(hdr, size);
  2121. if (*type < 0) {
  2122. error(_("unable to parse header of %s"), path);
  2123. git_inflate_end(&stream);
  2124. goto out;
  2125. }
  2126. if (*type == OBJ_BLOB && *size > big_file_threshold) {
  2127. if (check_stream_oid(&stream, hdr, *size, path, expected_oid) < 0)
  2128. goto out;
  2129. } else {
  2130. *contents = unpack_loose_rest(&stream, hdr, *size, expected_oid);
  2131. if (!*contents) {
  2132. error(_("unable to unpack contents of %s"), path);
  2133. git_inflate_end(&stream);
  2134. goto out;
  2135. }
  2136. if (check_object_signature(expected_oid, *contents,
  2137. *size, type_name(*type))) {
  2138. error(_("hash mismatch for %s (expected %s)"), path,
  2139. oid_to_hex(expected_oid));
  2140. free(*contents);
  2141. goto out;
  2142. }
  2143. }
  2144. ret = 0; /* everything checks out */
  2145. out:
  2146. if (map)
  2147. munmap(map, mapsize);
  2148. return ret;
  2149. }