THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Git Source Code Mirror - This is a publish-only repository and all pull requests are ignored. Please follow Documentation/SubmittingPatches procedure for any of your improvements.
git
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

632 lines
15KB

  1. #include "cache.h"
  2. #include "repository.h"
  3. #include "commit.h"
  4. #include "walker.h"
  5. #include "http.h"
  6. #include "list.h"
  7. #include "transport.h"
  8. #include "packfile.h"
  9. #include "object-store.h"
  10. struct alt_base {
  11. char *base;
  12. int got_indices;
  13. struct packed_git *packs;
  14. struct alt_base *next;
  15. };
  16. enum object_request_state {
  17. WAITING,
  18. ABORTED,
  19. ACTIVE,
  20. COMPLETE
  21. };
  22. struct object_request {
  23. struct walker *walker;
  24. struct object_id oid;
  25. struct alt_base *repo;
  26. enum object_request_state state;
  27. struct http_object_request *req;
  28. struct list_head node;
  29. };
  30. struct alternates_request {
  31. struct walker *walker;
  32. const char *base;
  33. struct strbuf *url;
  34. struct strbuf *buffer;
  35. struct active_request_slot *slot;
  36. int http_specific;
  37. };
  38. struct walker_data {
  39. const char *url;
  40. int got_alternates;
  41. struct alt_base *alt;
  42. };
  43. static LIST_HEAD(object_queue_head);
  44. static void fetch_alternates(struct walker *walker, const char *base);
  45. static void process_object_response(void *callback_data);
  46. static void start_object_request(struct walker *walker,
  47. struct object_request *obj_req)
  48. {
  49. struct active_request_slot *slot;
  50. struct http_object_request *req;
  51. req = new_http_object_request(obj_req->repo->base, obj_req->oid.hash);
  52. if (req == NULL) {
  53. obj_req->state = ABORTED;
  54. return;
  55. }
  56. obj_req->req = req;
  57. slot = req->slot;
  58. slot->callback_func = process_object_response;
  59. slot->callback_data = obj_req;
  60. /* Try to get the request started, abort the request on error */
  61. obj_req->state = ACTIVE;
  62. if (!start_active_slot(slot)) {
  63. obj_req->state = ABORTED;
  64. release_http_object_request(req);
  65. return;
  66. }
  67. }
  68. static void finish_object_request(struct object_request *obj_req)
  69. {
  70. if (finish_http_object_request(obj_req->req))
  71. return;
  72. if (obj_req->req->rename == 0)
  73. walker_say(obj_req->walker, "got %s\n", oid_to_hex(&obj_req->oid));
  74. }
  75. static void process_object_response(void *callback_data)
  76. {
  77. struct object_request *obj_req =
  78. (struct object_request *)callback_data;
  79. struct walker *walker = obj_req->walker;
  80. struct walker_data *data = walker->data;
  81. struct alt_base *alt = data->alt;
  82. process_http_object_request(obj_req->req);
  83. obj_req->state = COMPLETE;
  84. /* Use alternates if necessary */
  85. if (missing_target(obj_req->req)) {
  86. fetch_alternates(walker, alt->base);
  87. if (obj_req->repo->next != NULL) {
  88. obj_req->repo =
  89. obj_req->repo->next;
  90. release_http_object_request(obj_req->req);
  91. start_object_request(walker, obj_req);
  92. return;
  93. }
  94. }
  95. finish_object_request(obj_req);
  96. }
  97. static void release_object_request(struct object_request *obj_req)
  98. {
  99. if (obj_req->req !=NULL && obj_req->req->localfile != -1)
  100. error("fd leakage in release: %d", obj_req->req->localfile);
  101. list_del(&obj_req->node);
  102. free(obj_req);
  103. }
  104. #ifdef USE_CURL_MULTI
  105. static int fill_active_slot(struct walker *walker)
  106. {
  107. struct object_request *obj_req;
  108. struct list_head *pos, *tmp, *head = &object_queue_head;
  109. list_for_each_safe(pos, tmp, head) {
  110. obj_req = list_entry(pos, struct object_request, node);
  111. if (obj_req->state == WAITING) {
  112. if (has_sha1_file(obj_req->oid.hash))
  113. obj_req->state = COMPLETE;
  114. else {
  115. start_object_request(walker, obj_req);
  116. return 1;
  117. }
  118. }
  119. }
  120. return 0;
  121. }
  122. #endif
  123. static void prefetch(struct walker *walker, unsigned char *sha1)
  124. {
  125. struct object_request *newreq;
  126. struct walker_data *data = walker->data;
  127. newreq = xmalloc(sizeof(*newreq));
  128. newreq->walker = walker;
  129. hashcpy(newreq->oid.hash, sha1);
  130. newreq->repo = data->alt;
  131. newreq->state = WAITING;
  132. newreq->req = NULL;
  133. http_is_verbose = walker->get_verbosely;
  134. list_add_tail(&newreq->node, &object_queue_head);
  135. #ifdef USE_CURL_MULTI
  136. fill_active_slots();
  137. step_active_slots();
  138. #endif
  139. }
  140. static int is_alternate_allowed(const char *url)
  141. {
  142. const char *protocols[] = {
  143. "http", "https", "ftp", "ftps"
  144. };
  145. int i;
  146. if (http_follow_config != HTTP_FOLLOW_ALWAYS) {
  147. warning("alternate disabled by http.followRedirects: %s", url);
  148. return 0;
  149. }
  150. for (i = 0; i < ARRAY_SIZE(protocols); i++) {
  151. const char *end;
  152. if (skip_prefix(url, protocols[i], &end) &&
  153. starts_with(end, "://"))
  154. break;
  155. }
  156. if (i >= ARRAY_SIZE(protocols)) {
  157. warning("ignoring alternate with unknown protocol: %s", url);
  158. return 0;
  159. }
  160. if (!is_transport_allowed(protocols[i], 0)) {
  161. warning("ignoring alternate with restricted protocol: %s", url);
  162. return 0;
  163. }
  164. return 1;
  165. }
  166. static void process_alternates_response(void *callback_data)
  167. {
  168. struct alternates_request *alt_req =
  169. (struct alternates_request *)callback_data;
  170. struct walker *walker = alt_req->walker;
  171. struct walker_data *cdata = walker->data;
  172. struct active_request_slot *slot = alt_req->slot;
  173. struct alt_base *tail = cdata->alt;
  174. const char *base = alt_req->base;
  175. const char null_byte = '\0';
  176. char *data;
  177. int i = 0;
  178. if (alt_req->http_specific) {
  179. if (slot->curl_result != CURLE_OK ||
  180. !alt_req->buffer->len) {
  181. /* Try reusing the slot to get non-http alternates */
  182. alt_req->http_specific = 0;
  183. strbuf_reset(alt_req->url);
  184. strbuf_addf(alt_req->url, "%s/objects/info/alternates",
  185. base);
  186. curl_easy_setopt(slot->curl, CURLOPT_URL,
  187. alt_req->url->buf);
  188. active_requests++;
  189. slot->in_use = 1;
  190. if (slot->finished != NULL)
  191. (*slot->finished) = 0;
  192. if (!start_active_slot(slot)) {
  193. cdata->got_alternates = -1;
  194. slot->in_use = 0;
  195. if (slot->finished != NULL)
  196. (*slot->finished) = 1;
  197. }
  198. return;
  199. }
  200. } else if (slot->curl_result != CURLE_OK) {
  201. if (!missing_target(slot)) {
  202. cdata->got_alternates = -1;
  203. return;
  204. }
  205. }
  206. fwrite_buffer((char *)&null_byte, 1, 1, alt_req->buffer);
  207. alt_req->buffer->len--;
  208. data = alt_req->buffer->buf;
  209. while (i < alt_req->buffer->len) {
  210. int posn = i;
  211. while (posn < alt_req->buffer->len && data[posn] != '\n')
  212. posn++;
  213. if (data[posn] == '\n') {
  214. int okay = 0;
  215. int serverlen = 0;
  216. struct alt_base *newalt;
  217. if (data[i] == '/') {
  218. /*
  219. * This counts
  220. * http://git.host/pub/scm/linux.git/
  221. * -----------here^
  222. * so memcpy(dst, base, serverlen) will
  223. * copy up to "...git.host".
  224. */
  225. const char *colon_ss = strstr(base,"://");
  226. if (colon_ss) {
  227. serverlen = (strchr(colon_ss + 3, '/')
  228. - base);
  229. okay = 1;
  230. }
  231. } else if (!memcmp(data + i, "../", 3)) {
  232. /*
  233. * Relative URL; chop the corresponding
  234. * number of subpath from base (and ../
  235. * from data), and concatenate the result.
  236. *
  237. * The code first drops ../ from data, and
  238. * then drops one ../ from data and one path
  239. * from base. IOW, one extra ../ is dropped
  240. * from data than path is dropped from base.
  241. *
  242. * This is not wrong. The alternate in
  243. * http://git.host/pub/scm/linux.git/
  244. * to borrow from
  245. * http://git.host/pub/scm/linus.git/
  246. * is ../../linus.git/objects/. You need
  247. * two ../../ to borrow from your direct
  248. * neighbour.
  249. */
  250. i += 3;
  251. serverlen = strlen(base);
  252. while (i + 2 < posn &&
  253. !memcmp(data + i, "../", 3)) {
  254. do {
  255. serverlen--;
  256. } while (serverlen &&
  257. base[serverlen - 1] != '/');
  258. i += 3;
  259. }
  260. /* If the server got removed, give up. */
  261. okay = strchr(base, ':') - base + 3 <
  262. serverlen;
  263. } else if (alt_req->http_specific) {
  264. char *colon = strchr(data + i, ':');
  265. char *slash = strchr(data + i, '/');
  266. if (colon && slash && colon < data + posn &&
  267. slash < data + posn && colon < slash) {
  268. okay = 1;
  269. }
  270. }
  271. if (okay) {
  272. struct strbuf target = STRBUF_INIT;
  273. strbuf_add(&target, base, serverlen);
  274. strbuf_add(&target, data + i, posn - i);
  275. if (!strbuf_strip_suffix(&target, "objects")) {
  276. warning("ignoring alternate that does"
  277. " not end in 'objects': %s",
  278. target.buf);
  279. strbuf_release(&target);
  280. } else if (is_alternate_allowed(target.buf)) {
  281. warning("adding alternate object store: %s",
  282. target.buf);
  283. newalt = xmalloc(sizeof(*newalt));
  284. newalt->next = NULL;
  285. newalt->base = strbuf_detach(&target, NULL);
  286. newalt->got_indices = 0;
  287. newalt->packs = NULL;
  288. while (tail->next != NULL)
  289. tail = tail->next;
  290. tail->next = newalt;
  291. } else {
  292. strbuf_release(&target);
  293. }
  294. }
  295. }
  296. i = posn + 1;
  297. }
  298. cdata->got_alternates = 1;
  299. }
  300. static void fetch_alternates(struct walker *walker, const char *base)
  301. {
  302. struct strbuf buffer = STRBUF_INIT;
  303. struct strbuf url = STRBUF_INIT;
  304. struct active_request_slot *slot;
  305. struct alternates_request alt_req;
  306. struct walker_data *cdata = walker->data;
  307. /*
  308. * If another request has already started fetching alternates,
  309. * wait for them to arrive and return to processing this request's
  310. * curl message
  311. */
  312. #ifdef USE_CURL_MULTI
  313. while (cdata->got_alternates == 0) {
  314. step_active_slots();
  315. }
  316. #endif
  317. /* Nothing to do if they've already been fetched */
  318. if (cdata->got_alternates == 1)
  319. return;
  320. /* Start the fetch */
  321. cdata->got_alternates = 0;
  322. if (walker->get_verbosely)
  323. fprintf(stderr, "Getting alternates list for %s\n", base);
  324. strbuf_addf(&url, "%s/objects/info/http-alternates", base);
  325. /*
  326. * Use a callback to process the result, since another request
  327. * may fail and need to have alternates loaded before continuing
  328. */
  329. slot = get_active_slot();
  330. slot->callback_func = process_alternates_response;
  331. alt_req.walker = walker;
  332. slot->callback_data = &alt_req;
  333. curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
  334. curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
  335. curl_easy_setopt(slot->curl, CURLOPT_URL, url.buf);
  336. alt_req.base = base;
  337. alt_req.url = &url;
  338. alt_req.buffer = &buffer;
  339. alt_req.http_specific = 1;
  340. alt_req.slot = slot;
  341. if (start_active_slot(slot))
  342. run_active_slot(slot);
  343. else
  344. cdata->got_alternates = -1;
  345. strbuf_release(&buffer);
  346. strbuf_release(&url);
  347. }
  348. static int fetch_indices(struct walker *walker, struct alt_base *repo)
  349. {
  350. int ret;
  351. if (repo->got_indices)
  352. return 0;
  353. if (walker->get_verbosely)
  354. fprintf(stderr, "Getting pack list for %s\n", repo->base);
  355. switch (http_get_info_packs(repo->base, &repo->packs)) {
  356. case HTTP_OK:
  357. case HTTP_MISSING_TARGET:
  358. repo->got_indices = 1;
  359. ret = 0;
  360. break;
  361. default:
  362. repo->got_indices = 0;
  363. ret = -1;
  364. }
  365. return ret;
  366. }
  367. static int http_fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
  368. {
  369. struct packed_git *target;
  370. int ret;
  371. struct slot_results results;
  372. struct http_pack_request *preq;
  373. if (fetch_indices(walker, repo))
  374. return -1;
  375. target = find_sha1_pack(sha1, repo->packs);
  376. if (!target)
  377. return -1;
  378. if (walker->get_verbosely) {
  379. fprintf(stderr, "Getting pack %s\n",
  380. sha1_to_hex(target->sha1));
  381. fprintf(stderr, " which contains %s\n",
  382. sha1_to_hex(sha1));
  383. }
  384. preq = new_http_pack_request(target, repo->base);
  385. if (preq == NULL)
  386. goto abort;
  387. preq->lst = &repo->packs;
  388. preq->slot->results = &results;
  389. if (start_active_slot(preq->slot)) {
  390. run_active_slot(preq->slot);
  391. if (results.curl_result != CURLE_OK) {
  392. error("Unable to get pack file %s\n%s", preq->url,
  393. curl_errorstr);
  394. goto abort;
  395. }
  396. } else {
  397. error("Unable to start request");
  398. goto abort;
  399. }
  400. ret = finish_http_pack_request(preq);
  401. release_http_pack_request(preq);
  402. if (ret)
  403. return ret;
  404. return 0;
  405. abort:
  406. return -1;
  407. }
  408. static void abort_object_request(struct object_request *obj_req)
  409. {
  410. release_object_request(obj_req);
  411. }
  412. static int fetch_object(struct walker *walker, unsigned char *sha1)
  413. {
  414. char *hex = sha1_to_hex(sha1);
  415. int ret = 0;
  416. struct object_request *obj_req = NULL;
  417. struct http_object_request *req;
  418. struct list_head *pos, *head = &object_queue_head;
  419. list_for_each(pos, head) {
  420. obj_req = list_entry(pos, struct object_request, node);
  421. if (hasheq(obj_req->oid.hash, sha1))
  422. break;
  423. }
  424. if (obj_req == NULL)
  425. return error("Couldn't find request for %s in the queue", hex);
  426. if (has_sha1_file(obj_req->oid.hash)) {
  427. if (obj_req->req != NULL)
  428. abort_http_object_request(obj_req->req);
  429. abort_object_request(obj_req);
  430. return 0;
  431. }
  432. #ifdef USE_CURL_MULTI
  433. while (obj_req->state == WAITING)
  434. step_active_slots();
  435. #else
  436. start_object_request(walker, obj_req);
  437. #endif
  438. /*
  439. * obj_req->req might change when fetching alternates in the callback
  440. * process_object_response; therefore, the "shortcut" variable, req,
  441. * is used only after we're done with slots.
  442. */
  443. while (obj_req->state == ACTIVE)
  444. run_active_slot(obj_req->req->slot);
  445. req = obj_req->req;
  446. if (req->localfile != -1) {
  447. close(req->localfile);
  448. req->localfile = -1;
  449. }
  450. /*
  451. * we turned off CURLOPT_FAILONERROR to avoid losing a
  452. * persistent connection and got CURLE_OK.
  453. */
  454. if (req->http_code >= 300 && req->curl_result == CURLE_OK &&
  455. (starts_with(req->url, "http://") ||
  456. starts_with(req->url, "https://"))) {
  457. req->curl_result = CURLE_HTTP_RETURNED_ERROR;
  458. xsnprintf(req->errorstr, sizeof(req->errorstr),
  459. "HTTP request failed");
  460. }
  461. if (obj_req->state == ABORTED) {
  462. ret = error("Request for %s aborted", hex);
  463. } else if (req->curl_result != CURLE_OK &&
  464. req->http_code != 416) {
  465. if (missing_target(req))
  466. ret = -1; /* Be silent, it is probably in a pack. */
  467. else
  468. ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
  469. req->errorstr, req->curl_result,
  470. req->http_code, hex);
  471. } else if (req->zret != Z_STREAM_END) {
  472. walker->corrupt_object_found++;
  473. ret = error("File %s (%s) corrupt", hex, req->url);
  474. } else if (!hasheq(obj_req->oid.hash, req->real_sha1)) {
  475. ret = error("File %s has bad hash", hex);
  476. } else if (req->rename < 0) {
  477. struct strbuf buf = STRBUF_INIT;
  478. sha1_file_name(the_repository, &buf, req->sha1);
  479. ret = error("unable to write sha1 filename %s", buf.buf);
  480. strbuf_release(&buf);
  481. }
  482. release_http_object_request(req);
  483. release_object_request(obj_req);
  484. return ret;
  485. }
  486. static int fetch(struct walker *walker, unsigned char *sha1)
  487. {
  488. struct walker_data *data = walker->data;
  489. struct alt_base *altbase = data->alt;
  490. if (!fetch_object(walker, sha1))
  491. return 0;
  492. while (altbase) {
  493. if (!http_fetch_pack(walker, altbase, sha1))
  494. return 0;
  495. fetch_alternates(walker, data->alt->base);
  496. altbase = altbase->next;
  497. }
  498. return error("Unable to find %s under %s", sha1_to_hex(sha1),
  499. data->alt->base);
  500. }
  501. static int fetch_ref(struct walker *walker, struct ref *ref)
  502. {
  503. struct walker_data *data = walker->data;
  504. return http_fetch_ref(data->alt->base, ref);
  505. }
  506. static void cleanup(struct walker *walker)
  507. {
  508. struct walker_data *data = walker->data;
  509. struct alt_base *alt, *alt_next;
  510. if (data) {
  511. alt = data->alt;
  512. while (alt) {
  513. alt_next = alt->next;
  514. free(alt->base);
  515. free(alt);
  516. alt = alt_next;
  517. }
  518. free(data);
  519. walker->data = NULL;
  520. }
  521. }
  522. struct walker *get_http_walker(const char *url)
  523. {
  524. char *s;
  525. struct walker_data *data = xmalloc(sizeof(struct walker_data));
  526. struct walker *walker = xmalloc(sizeof(struct walker));
  527. data->alt = xmalloc(sizeof(*data->alt));
  528. data->alt->base = xstrdup(url);
  529. for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
  530. *s = 0;
  531. data->alt->got_indices = 0;
  532. data->alt->packs = NULL;
  533. data->alt->next = NULL;
  534. data->got_alternates = -1;
  535. walker->corrupt_object_found = 0;
  536. walker->fetch = fetch;
  537. walker->fetch_ref = fetch_ref;
  538. walker->prefetch = prefetch;
  539. walker->cleanup = cleanup;
  540. walker->data = data;
  541. #ifdef USE_CURL_MULTI
  542. add_fill_function(walker, (int (*)(void *)) fill_active_slot);
  543. #endif
  544. return walker;
  545. }