THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Git Source Code Mirror - This is a publish-only repository and all pull requests are ignored. Please follow Documentation/SubmittingPatches procedure for any of your improvements.
git
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

631 lines
15KB

  1. #include "cache.h"
  2. #include "repository.h"
  3. #include "commit.h"
  4. #include "walker.h"
  5. #include "http.h"
  6. #include "list.h"
  7. #include "transport.h"
  8. #include "packfile.h"
  9. #include "object-store.h"
  10. struct alt_base {
  11. char *base;
  12. int got_indices;
  13. struct packed_git *packs;
  14. struct alt_base *next;
  15. };
  16. enum object_request_state {
  17. WAITING,
  18. ABORTED,
  19. ACTIVE,
  20. COMPLETE
  21. };
  22. struct object_request {
  23. struct walker *walker;
  24. struct object_id oid;
  25. struct alt_base *repo;
  26. enum object_request_state state;
  27. struct http_object_request *req;
  28. struct list_head node;
  29. };
  30. struct alternates_request {
  31. struct walker *walker;
  32. const char *base;
  33. struct strbuf *url;
  34. struct strbuf *buffer;
  35. struct active_request_slot *slot;
  36. int http_specific;
  37. };
  38. struct walker_data {
  39. const char *url;
  40. int got_alternates;
  41. struct alt_base *alt;
  42. };
  43. static LIST_HEAD(object_queue_head);
  44. static void fetch_alternates(struct walker *walker, const char *base);
  45. static void process_object_response(void *callback_data);
  46. static void start_object_request(struct walker *walker,
  47. struct object_request *obj_req)
  48. {
  49. struct active_request_slot *slot;
  50. struct http_object_request *req;
  51. req = new_http_object_request(obj_req->repo->base, &obj_req->oid);
  52. if (req == NULL) {
  53. obj_req->state = ABORTED;
  54. return;
  55. }
  56. obj_req->req = req;
  57. slot = req->slot;
  58. slot->callback_func = process_object_response;
  59. slot->callback_data = obj_req;
  60. /* Try to get the request started, abort the request on error */
  61. obj_req->state = ACTIVE;
  62. if (!start_active_slot(slot)) {
  63. obj_req->state = ABORTED;
  64. release_http_object_request(req);
  65. return;
  66. }
  67. }
  68. static void finish_object_request(struct object_request *obj_req)
  69. {
  70. if (finish_http_object_request(obj_req->req))
  71. return;
  72. if (obj_req->req->rename == 0)
  73. walker_say(obj_req->walker, "got %s\n", oid_to_hex(&obj_req->oid));
  74. }
  75. static void process_object_response(void *callback_data)
  76. {
  77. struct object_request *obj_req =
  78. (struct object_request *)callback_data;
  79. struct walker *walker = obj_req->walker;
  80. struct walker_data *data = walker->data;
  81. struct alt_base *alt = data->alt;
  82. process_http_object_request(obj_req->req);
  83. obj_req->state = COMPLETE;
  84. normalize_curl_result(&obj_req->req->curl_result,
  85. obj_req->req->http_code,
  86. obj_req->req->errorstr,
  87. sizeof(obj_req->req->errorstr));
  88. /* Use alternates if necessary */
  89. if (missing_target(obj_req->req)) {
  90. fetch_alternates(walker, alt->base);
  91. if (obj_req->repo->next != NULL) {
  92. obj_req->repo =
  93. obj_req->repo->next;
  94. release_http_object_request(obj_req->req);
  95. start_object_request(walker, obj_req);
  96. return;
  97. }
  98. }
  99. finish_object_request(obj_req);
  100. }
  101. static void release_object_request(struct object_request *obj_req)
  102. {
  103. if (obj_req->req !=NULL && obj_req->req->localfile != -1)
  104. error("fd leakage in release: %d", obj_req->req->localfile);
  105. list_del(&obj_req->node);
  106. free(obj_req);
  107. }
  108. #ifdef USE_CURL_MULTI
  109. static int fill_active_slot(struct walker *walker)
  110. {
  111. struct object_request *obj_req;
  112. struct list_head *pos, *tmp, *head = &object_queue_head;
  113. list_for_each_safe(pos, tmp, head) {
  114. obj_req = list_entry(pos, struct object_request, node);
  115. if (obj_req->state == WAITING) {
  116. if (has_object_file(&obj_req->oid))
  117. obj_req->state = COMPLETE;
  118. else {
  119. start_object_request(walker, obj_req);
  120. return 1;
  121. }
  122. }
  123. }
  124. return 0;
  125. }
  126. #endif
  127. static void prefetch(struct walker *walker, unsigned char *sha1)
  128. {
  129. struct object_request *newreq;
  130. struct walker_data *data = walker->data;
  131. newreq = xmalloc(sizeof(*newreq));
  132. newreq->walker = walker;
  133. hashcpy(newreq->oid.hash, sha1);
  134. newreq->repo = data->alt;
  135. newreq->state = WAITING;
  136. newreq->req = NULL;
  137. http_is_verbose = walker->get_verbosely;
  138. list_add_tail(&newreq->node, &object_queue_head);
  139. #ifdef USE_CURL_MULTI
  140. fill_active_slots();
  141. step_active_slots();
  142. #endif
  143. }
  144. static int is_alternate_allowed(const char *url)
  145. {
  146. const char *protocols[] = {
  147. "http", "https", "ftp", "ftps"
  148. };
  149. int i;
  150. if (http_follow_config != HTTP_FOLLOW_ALWAYS) {
  151. warning("alternate disabled by http.followRedirects: %s", url);
  152. return 0;
  153. }
  154. for (i = 0; i < ARRAY_SIZE(protocols); i++) {
  155. const char *end;
  156. if (skip_prefix(url, protocols[i], &end) &&
  157. starts_with(end, "://"))
  158. break;
  159. }
  160. if (i >= ARRAY_SIZE(protocols)) {
  161. warning("ignoring alternate with unknown protocol: %s", url);
  162. return 0;
  163. }
  164. if (!is_transport_allowed(protocols[i], 0)) {
  165. warning("ignoring alternate with restricted protocol: %s", url);
  166. return 0;
  167. }
  168. return 1;
  169. }
  170. static void process_alternates_response(void *callback_data)
  171. {
  172. struct alternates_request *alt_req =
  173. (struct alternates_request *)callback_data;
  174. struct walker *walker = alt_req->walker;
  175. struct walker_data *cdata = walker->data;
  176. struct active_request_slot *slot = alt_req->slot;
  177. struct alt_base *tail = cdata->alt;
  178. const char *base = alt_req->base;
  179. const char null_byte = '\0';
  180. char *data;
  181. int i = 0;
  182. normalize_curl_result(&slot->curl_result, slot->http_code,
  183. curl_errorstr, sizeof(curl_errorstr));
  184. if (alt_req->http_specific) {
  185. if (slot->curl_result != CURLE_OK ||
  186. !alt_req->buffer->len) {
  187. /* Try reusing the slot to get non-http alternates */
  188. alt_req->http_specific = 0;
  189. strbuf_reset(alt_req->url);
  190. strbuf_addf(alt_req->url, "%s/objects/info/alternates",
  191. base);
  192. curl_easy_setopt(slot->curl, CURLOPT_URL,
  193. alt_req->url->buf);
  194. active_requests++;
  195. slot->in_use = 1;
  196. if (slot->finished != NULL)
  197. (*slot->finished) = 0;
  198. if (!start_active_slot(slot)) {
  199. cdata->got_alternates = -1;
  200. slot->in_use = 0;
  201. if (slot->finished != NULL)
  202. (*slot->finished) = 1;
  203. }
  204. return;
  205. }
  206. } else if (slot->curl_result != CURLE_OK) {
  207. if (!missing_target(slot)) {
  208. cdata->got_alternates = -1;
  209. return;
  210. }
  211. }
  212. fwrite_buffer((char *)&null_byte, 1, 1, alt_req->buffer);
  213. alt_req->buffer->len--;
  214. data = alt_req->buffer->buf;
  215. while (i < alt_req->buffer->len) {
  216. int posn = i;
  217. while (posn < alt_req->buffer->len && data[posn] != '\n')
  218. posn++;
  219. if (data[posn] == '\n') {
  220. int okay = 0;
  221. int serverlen = 0;
  222. struct alt_base *newalt;
  223. if (data[i] == '/') {
  224. /*
  225. * This counts
  226. * http://git.host/pub/scm/linux.git/
  227. * -----------here^
  228. * so memcpy(dst, base, serverlen) will
  229. * copy up to "...git.host".
  230. */
  231. const char *colon_ss = strstr(base,"://");
  232. if (colon_ss) {
  233. serverlen = (strchr(colon_ss + 3, '/')
  234. - base);
  235. okay = 1;
  236. }
  237. } else if (!memcmp(data + i, "../", 3)) {
  238. /*
  239. * Relative URL; chop the corresponding
  240. * number of subpath from base (and ../
  241. * from data), and concatenate the result.
  242. *
  243. * The code first drops ../ from data, and
  244. * then drops one ../ from data and one path
  245. * from base. IOW, one extra ../ is dropped
  246. * from data than path is dropped from base.
  247. *
  248. * This is not wrong. The alternate in
  249. * http://git.host/pub/scm/linux.git/
  250. * to borrow from
  251. * http://git.host/pub/scm/linus.git/
  252. * is ../../linus.git/objects/. You need
  253. * two ../../ to borrow from your direct
  254. * neighbour.
  255. */
  256. i += 3;
  257. serverlen = strlen(base);
  258. while (i + 2 < posn &&
  259. !memcmp(data + i, "../", 3)) {
  260. do {
  261. serverlen--;
  262. } while (serverlen &&
  263. base[serverlen - 1] != '/');
  264. i += 3;
  265. }
  266. /* If the server got removed, give up. */
  267. okay = strchr(base, ':') - base + 3 <
  268. serverlen;
  269. } else if (alt_req->http_specific) {
  270. char *colon = strchr(data + i, ':');
  271. char *slash = strchr(data + i, '/');
  272. if (colon && slash && colon < data + posn &&
  273. slash < data + posn && colon < slash) {
  274. okay = 1;
  275. }
  276. }
  277. if (okay) {
  278. struct strbuf target = STRBUF_INIT;
  279. strbuf_add(&target, base, serverlen);
  280. strbuf_add(&target, data + i, posn - i);
  281. if (!strbuf_strip_suffix(&target, "objects")) {
  282. warning("ignoring alternate that does"
  283. " not end in 'objects': %s",
  284. target.buf);
  285. strbuf_release(&target);
  286. } else if (is_alternate_allowed(target.buf)) {
  287. warning("adding alternate object store: %s",
  288. target.buf);
  289. newalt = xmalloc(sizeof(*newalt));
  290. newalt->next = NULL;
  291. newalt->base = strbuf_detach(&target, NULL);
  292. newalt->got_indices = 0;
  293. newalt->packs = NULL;
  294. while (tail->next != NULL)
  295. tail = tail->next;
  296. tail->next = newalt;
  297. } else {
  298. strbuf_release(&target);
  299. }
  300. }
  301. }
  302. i = posn + 1;
  303. }
  304. cdata->got_alternates = 1;
  305. }
  306. static void fetch_alternates(struct walker *walker, const char *base)
  307. {
  308. struct strbuf buffer = STRBUF_INIT;
  309. struct strbuf url = STRBUF_INIT;
  310. struct active_request_slot *slot;
  311. struct alternates_request alt_req;
  312. struct walker_data *cdata = walker->data;
  313. /*
  314. * If another request has already started fetching alternates,
  315. * wait for them to arrive and return to processing this request's
  316. * curl message
  317. */
  318. #ifdef USE_CURL_MULTI
  319. while (cdata->got_alternates == 0) {
  320. step_active_slots();
  321. }
  322. #endif
  323. /* Nothing to do if they've already been fetched */
  324. if (cdata->got_alternates == 1)
  325. return;
  326. /* Start the fetch */
  327. cdata->got_alternates = 0;
  328. if (walker->get_verbosely)
  329. fprintf(stderr, "Getting alternates list for %s\n", base);
  330. strbuf_addf(&url, "%s/objects/info/http-alternates", base);
  331. /*
  332. * Use a callback to process the result, since another request
  333. * may fail and need to have alternates loaded before continuing
  334. */
  335. slot = get_active_slot();
  336. slot->callback_func = process_alternates_response;
  337. alt_req.walker = walker;
  338. slot->callback_data = &alt_req;
  339. curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
  340. curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
  341. curl_easy_setopt(slot->curl, CURLOPT_URL, url.buf);
  342. alt_req.base = base;
  343. alt_req.url = &url;
  344. alt_req.buffer = &buffer;
  345. alt_req.http_specific = 1;
  346. alt_req.slot = slot;
  347. if (start_active_slot(slot))
  348. run_active_slot(slot);
  349. else
  350. cdata->got_alternates = -1;
  351. strbuf_release(&buffer);
  352. strbuf_release(&url);
  353. }
  354. static int fetch_indices(struct walker *walker, struct alt_base *repo)
  355. {
  356. int ret;
  357. if (repo->got_indices)
  358. return 0;
  359. if (walker->get_verbosely)
  360. fprintf(stderr, "Getting pack list for %s\n", repo->base);
  361. switch (http_get_info_packs(repo->base, &repo->packs)) {
  362. case HTTP_OK:
  363. case HTTP_MISSING_TARGET:
  364. repo->got_indices = 1;
  365. ret = 0;
  366. break;
  367. default:
  368. repo->got_indices = 0;
  369. ret = -1;
  370. }
  371. return ret;
  372. }
  373. static int http_fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
  374. {
  375. struct packed_git *target;
  376. int ret;
  377. struct slot_results results;
  378. struct http_pack_request *preq;
  379. if (fetch_indices(walker, repo))
  380. return -1;
  381. target = find_sha1_pack(sha1, repo->packs);
  382. if (!target)
  383. return -1;
  384. if (walker->get_verbosely) {
  385. fprintf(stderr, "Getting pack %s\n",
  386. hash_to_hex(target->hash));
  387. fprintf(stderr, " which contains %s\n",
  388. hash_to_hex(sha1));
  389. }
  390. preq = new_http_pack_request(target, repo->base);
  391. if (preq == NULL)
  392. goto abort;
  393. preq->lst = &repo->packs;
  394. preq->slot->results = &results;
  395. if (start_active_slot(preq->slot)) {
  396. run_active_slot(preq->slot);
  397. if (results.curl_result != CURLE_OK) {
  398. error("Unable to get pack file %s\n%s", preq->url,
  399. curl_errorstr);
  400. goto abort;
  401. }
  402. } else {
  403. error("Unable to start request");
  404. goto abort;
  405. }
  406. ret = finish_http_pack_request(preq);
  407. release_http_pack_request(preq);
  408. if (ret)
  409. return ret;
  410. return 0;
  411. abort:
  412. return -1;
  413. }
  414. static void abort_object_request(struct object_request *obj_req)
  415. {
  416. release_object_request(obj_req);
  417. }
  418. static int fetch_object(struct walker *walker, unsigned char *hash)
  419. {
  420. char *hex = hash_to_hex(hash);
  421. int ret = 0;
  422. struct object_request *obj_req = NULL;
  423. struct http_object_request *req;
  424. struct list_head *pos, *head = &object_queue_head;
  425. list_for_each(pos, head) {
  426. obj_req = list_entry(pos, struct object_request, node);
  427. if (hasheq(obj_req->oid.hash, hash))
  428. break;
  429. }
  430. if (obj_req == NULL)
  431. return error("Couldn't find request for %s in the queue", hex);
  432. if (has_object_file(&obj_req->oid)) {
  433. if (obj_req->req != NULL)
  434. abort_http_object_request(obj_req->req);
  435. abort_object_request(obj_req);
  436. return 0;
  437. }
  438. #ifdef USE_CURL_MULTI
  439. while (obj_req->state == WAITING)
  440. step_active_slots();
  441. #else
  442. start_object_request(walker, obj_req);
  443. #endif
  444. /*
  445. * obj_req->req might change when fetching alternates in the callback
  446. * process_object_response; therefore, the "shortcut" variable, req,
  447. * is used only after we're done with slots.
  448. */
  449. while (obj_req->state == ACTIVE)
  450. run_active_slot(obj_req->req->slot);
  451. req = obj_req->req;
  452. if (req->localfile != -1) {
  453. close(req->localfile);
  454. req->localfile = -1;
  455. }
  456. normalize_curl_result(&req->curl_result, req->http_code,
  457. req->errorstr, sizeof(req->errorstr));
  458. if (obj_req->state == ABORTED) {
  459. ret = error("Request for %s aborted", hex);
  460. } else if (req->curl_result != CURLE_OK &&
  461. req->http_code != 416) {
  462. if (missing_target(req))
  463. ret = -1; /* Be silent, it is probably in a pack. */
  464. else
  465. ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
  466. req->errorstr, req->curl_result,
  467. req->http_code, hex);
  468. } else if (req->zret != Z_STREAM_END) {
  469. walker->corrupt_object_found++;
  470. ret = error("File %s (%s) corrupt", hex, req->url);
  471. } else if (!oideq(&obj_req->oid, &req->real_oid)) {
  472. ret = error("File %s has bad hash", hex);
  473. } else if (req->rename < 0) {
  474. struct strbuf buf = STRBUF_INIT;
  475. loose_object_path(the_repository, &buf, &req->oid);
  476. ret = error("unable to write sha1 filename %s", buf.buf);
  477. strbuf_release(&buf);
  478. }
  479. release_http_object_request(req);
  480. release_object_request(obj_req);
  481. return ret;
  482. }
  483. static int fetch(struct walker *walker, unsigned char *hash)
  484. {
  485. struct walker_data *data = walker->data;
  486. struct alt_base *altbase = data->alt;
  487. if (!fetch_object(walker, hash))
  488. return 0;
  489. while (altbase) {
  490. if (!http_fetch_pack(walker, altbase, hash))
  491. return 0;
  492. fetch_alternates(walker, data->alt->base);
  493. altbase = altbase->next;
  494. }
  495. return error("Unable to find %s under %s", hash_to_hex(hash),
  496. data->alt->base);
  497. }
  498. static int fetch_ref(struct walker *walker, struct ref *ref)
  499. {
  500. struct walker_data *data = walker->data;
  501. return http_fetch_ref(data->alt->base, ref);
  502. }
  503. static void cleanup(struct walker *walker)
  504. {
  505. struct walker_data *data = walker->data;
  506. struct alt_base *alt, *alt_next;
  507. if (data) {
  508. alt = data->alt;
  509. while (alt) {
  510. alt_next = alt->next;
  511. free(alt->base);
  512. free(alt);
  513. alt = alt_next;
  514. }
  515. free(data);
  516. walker->data = NULL;
  517. }
  518. }
  519. struct walker *get_http_walker(const char *url)
  520. {
  521. char *s;
  522. struct walker_data *data = xmalloc(sizeof(struct walker_data));
  523. struct walker *walker = xmalloc(sizeof(struct walker));
  524. data->alt = xmalloc(sizeof(*data->alt));
  525. data->alt->base = xstrdup(url);
  526. for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
  527. *s = 0;
  528. data->alt->got_indices = 0;
  529. data->alt->packs = NULL;
  530. data->alt->next = NULL;
  531. data->got_alternates = -1;
  532. walker->corrupt_object_found = 0;
  533. walker->fetch = fetch;
  534. walker->fetch_ref = fetch_ref;
  535. walker->prefetch = prefetch;
  536. walker->cleanup = cleanup;
  537. walker->data = data;
  538. #ifdef USE_CURL_MULTI
  539. add_fill_function(walker, (int (*)(void *)) fill_active_slot);
  540. #endif
  541. return walker;
  542. }