THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Git Source Code Mirror - This is a publish-only repository and all pull requests are ignored. Please follow Documentation/SubmittingPatches procedure for any of your improvements.
git
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1867 lines
41KB

  1. #include "cache.h"
  2. #include "run-command.h"
  3. #include "exec-cmd.h"
  4. #include "sigchain.h"
  5. #include "argv-array.h"
  6. #include "thread-utils.h"
  7. #include "strbuf.h"
  8. #include "string-list.h"
  9. #include "quote.h"
  10. void child_process_init(struct child_process *child)
  11. {
  12. memset(child, 0, sizeof(*child));
  13. argv_array_init(&child->args);
  14. argv_array_init(&child->env_array);
  15. }
  16. void child_process_clear(struct child_process *child)
  17. {
  18. argv_array_clear(&child->args);
  19. argv_array_clear(&child->env_array);
  20. }
  21. struct child_to_clean {
  22. pid_t pid;
  23. struct child_process *process;
  24. struct child_to_clean *next;
  25. };
  26. static struct child_to_clean *children_to_clean;
  27. static int installed_child_cleanup_handler;
  28. static void cleanup_children(int sig, int in_signal)
  29. {
  30. struct child_to_clean *children_to_wait_for = NULL;
  31. while (children_to_clean) {
  32. struct child_to_clean *p = children_to_clean;
  33. children_to_clean = p->next;
  34. if (p->process && !in_signal) {
  35. struct child_process *process = p->process;
  36. if (process->clean_on_exit_handler) {
  37. trace_printf(
  38. "trace: run_command: running exit handler for pid %"
  39. PRIuMAX, (uintmax_t)p->pid
  40. );
  41. process->clean_on_exit_handler(process);
  42. }
  43. }
  44. kill(p->pid, sig);
  45. if (p->process && p->process->wait_after_clean) {
  46. p->next = children_to_wait_for;
  47. children_to_wait_for = p;
  48. } else {
  49. if (!in_signal)
  50. free(p);
  51. }
  52. }
  53. while (children_to_wait_for) {
  54. struct child_to_clean *p = children_to_wait_for;
  55. children_to_wait_for = p->next;
  56. while (waitpid(p->pid, NULL, 0) < 0 && errno == EINTR)
  57. ; /* spin waiting for process exit or error */
  58. if (!in_signal)
  59. free(p);
  60. }
  61. }
  62. static void cleanup_children_on_signal(int sig)
  63. {
  64. cleanup_children(sig, 1);
  65. sigchain_pop(sig);
  66. raise(sig);
  67. }
  68. static void cleanup_children_on_exit(void)
  69. {
  70. cleanup_children(SIGTERM, 0);
  71. }
  72. static void mark_child_for_cleanup(pid_t pid, struct child_process *process)
  73. {
  74. struct child_to_clean *p = xmalloc(sizeof(*p));
  75. p->pid = pid;
  76. p->process = process;
  77. p->next = children_to_clean;
  78. children_to_clean = p;
  79. if (!installed_child_cleanup_handler) {
  80. atexit(cleanup_children_on_exit);
  81. sigchain_push_common(cleanup_children_on_signal);
  82. installed_child_cleanup_handler = 1;
  83. }
  84. }
  85. static void clear_child_for_cleanup(pid_t pid)
  86. {
  87. struct child_to_clean **pp;
  88. for (pp = &children_to_clean; *pp; pp = &(*pp)->next) {
  89. struct child_to_clean *clean_me = *pp;
  90. if (clean_me->pid == pid) {
  91. *pp = clean_me->next;
  92. free(clean_me);
  93. return;
  94. }
  95. }
  96. }
  97. static inline void close_pair(int fd[2])
  98. {
  99. close(fd[0]);
  100. close(fd[1]);
  101. }
  102. int is_executable(const char *name)
  103. {
  104. struct stat st;
  105. if (stat(name, &st) || /* stat, not lstat */
  106. !S_ISREG(st.st_mode))
  107. return 0;
  108. #if defined(GIT_WINDOWS_NATIVE)
  109. /*
  110. * On Windows there is no executable bit. The file extension
  111. * indicates whether it can be run as an executable, and Git
  112. * has special-handling to detect scripts and launch them
  113. * through the indicated script interpreter. We test for the
  114. * file extension first because virus scanners may make
  115. * it quite expensive to open many files.
  116. */
  117. if (ends_with(name, ".exe"))
  118. return S_IXUSR;
  119. {
  120. /*
  121. * Now that we know it does not have an executable extension,
  122. * peek into the file instead.
  123. */
  124. char buf[3] = { 0 };
  125. int n;
  126. int fd = open(name, O_RDONLY);
  127. st.st_mode &= ~S_IXUSR;
  128. if (fd >= 0) {
  129. n = read(fd, buf, 2);
  130. if (n == 2)
  131. /* look for a she-bang */
  132. if (!strcmp(buf, "#!"))
  133. st.st_mode |= S_IXUSR;
  134. close(fd);
  135. }
  136. }
  137. #endif
  138. return st.st_mode & S_IXUSR;
  139. }
  140. /*
  141. * Search $PATH for a command. This emulates the path search that
  142. * execvp would perform, without actually executing the command so it
  143. * can be used before fork() to prepare to run a command using
  144. * execve() or after execvp() to diagnose why it failed.
  145. *
  146. * The caller should ensure that file contains no directory
  147. * separators.
  148. *
  149. * Returns the path to the command, as found in $PATH or NULL if the
  150. * command could not be found. The caller inherits ownership of the memory
  151. * used to store the resultant path.
  152. *
  153. * This should not be used on Windows, where the $PATH search rules
  154. * are more complicated (e.g., a search for "foo" should find
  155. * "foo.exe").
  156. */
  157. static char *locate_in_PATH(const char *file)
  158. {
  159. const char *p = getenv("PATH");
  160. struct strbuf buf = STRBUF_INIT;
  161. if (!p || !*p)
  162. return NULL;
  163. while (1) {
  164. const char *end = strchrnul(p, ':');
  165. strbuf_reset(&buf);
  166. /* POSIX specifies an empty entry as the current directory. */
  167. if (end != p) {
  168. strbuf_add(&buf, p, end - p);
  169. strbuf_addch(&buf, '/');
  170. }
  171. strbuf_addstr(&buf, file);
  172. if (is_executable(buf.buf))
  173. return strbuf_detach(&buf, NULL);
  174. if (!*end)
  175. break;
  176. p = end + 1;
  177. }
  178. strbuf_release(&buf);
  179. return NULL;
  180. }
  181. static int exists_in_PATH(const char *file)
  182. {
  183. char *r = locate_in_PATH(file);
  184. free(r);
  185. return r != NULL;
  186. }
  187. int sane_execvp(const char *file, char * const argv[])
  188. {
  189. #ifndef GIT_WINDOWS_NATIVE
  190. /*
  191. * execvp() doesn't return, so we all we can do is tell trace2
  192. * what we are about to do and let it leave a hint in the log
  193. * (unless of course the execvp() fails).
  194. *
  195. * we skip this for Windows because the compat layer already
  196. * has to emulate the execvp() call anyway.
  197. */
  198. int exec_id = trace2_exec(file, (const char **)argv);
  199. #endif
  200. if (!execvp(file, argv))
  201. return 0; /* cannot happen ;-) */
  202. #ifndef GIT_WINDOWS_NATIVE
  203. {
  204. int ec = errno;
  205. trace2_exec_result(exec_id, ec);
  206. errno = ec;
  207. }
  208. #endif
  209. /*
  210. * When a command can't be found because one of the directories
  211. * listed in $PATH is unsearchable, execvp reports EACCES, but
  212. * careful usability testing (read: analysis of occasional bug
  213. * reports) reveals that "No such file or directory" is more
  214. * intuitive.
  215. *
  216. * We avoid commands with "/", because execvp will not do $PATH
  217. * lookups in that case.
  218. *
  219. * The reassignment of EACCES to errno looks like a no-op below,
  220. * but we need to protect against exists_in_PATH overwriting errno.
  221. */
  222. if (errno == EACCES && !strchr(file, '/'))
  223. errno = exists_in_PATH(file) ? EACCES : ENOENT;
  224. else if (errno == ENOTDIR && !strchr(file, '/'))
  225. errno = ENOENT;
  226. return -1;
  227. }
  228. static const char **prepare_shell_cmd(struct argv_array *out, const char **argv)
  229. {
  230. if (!argv[0])
  231. BUG("shell command is empty");
  232. if (strcspn(argv[0], "|&;<>()$`\\\"' \t\n*?[#~=%") != strlen(argv[0])) {
  233. #ifndef GIT_WINDOWS_NATIVE
  234. argv_array_push(out, SHELL_PATH);
  235. #else
  236. argv_array_push(out, "sh");
  237. #endif
  238. argv_array_push(out, "-c");
  239. /*
  240. * If we have no extra arguments, we do not even need to
  241. * bother with the "$@" magic.
  242. */
  243. if (!argv[1])
  244. argv_array_push(out, argv[0]);
  245. else
  246. argv_array_pushf(out, "%s \"$@\"", argv[0]);
  247. }
  248. argv_array_pushv(out, argv);
  249. return out->argv;
  250. }
  251. #ifndef GIT_WINDOWS_NATIVE
  252. static int child_notifier = -1;
  253. enum child_errcode {
  254. CHILD_ERR_CHDIR,
  255. CHILD_ERR_DUP2,
  256. CHILD_ERR_CLOSE,
  257. CHILD_ERR_SIGPROCMASK,
  258. CHILD_ERR_ENOENT,
  259. CHILD_ERR_SILENT,
  260. CHILD_ERR_ERRNO
  261. };
  262. struct child_err {
  263. enum child_errcode err;
  264. int syserr; /* errno */
  265. };
  266. static void child_die(enum child_errcode err)
  267. {
  268. struct child_err buf;
  269. buf.err = err;
  270. buf.syserr = errno;
  271. /* write(2) on buf smaller than PIPE_BUF (min 512) is atomic: */
  272. xwrite(child_notifier, &buf, sizeof(buf));
  273. _exit(1);
  274. }
  275. static void child_dup2(int fd, int to)
  276. {
  277. if (dup2(fd, to) < 0)
  278. child_die(CHILD_ERR_DUP2);
  279. }
  280. static void child_close(int fd)
  281. {
  282. if (close(fd))
  283. child_die(CHILD_ERR_CLOSE);
  284. }
  285. static void child_close_pair(int fd[2])
  286. {
  287. child_close(fd[0]);
  288. child_close(fd[1]);
  289. }
  290. /*
  291. * parent will make it look like the child spewed a fatal error and died
  292. * this is needed to prevent changes to t0061.
  293. */
  294. static void fake_fatal(const char *err, va_list params)
  295. {
  296. vreportf("fatal: ", err, params);
  297. }
  298. static void child_error_fn(const char *err, va_list params)
  299. {
  300. const char msg[] = "error() should not be called in child\n";
  301. xwrite(2, msg, sizeof(msg) - 1);
  302. }
  303. static void child_warn_fn(const char *err, va_list params)
  304. {
  305. const char msg[] = "warn() should not be called in child\n";
  306. xwrite(2, msg, sizeof(msg) - 1);
  307. }
  308. static void NORETURN child_die_fn(const char *err, va_list params)
  309. {
  310. const char msg[] = "die() should not be called in child\n";
  311. xwrite(2, msg, sizeof(msg) - 1);
  312. _exit(2);
  313. }
  314. /* this runs in the parent process */
  315. static void child_err_spew(struct child_process *cmd, struct child_err *cerr)
  316. {
  317. static void (*old_errfn)(const char *err, va_list params);
  318. old_errfn = get_error_routine();
  319. set_error_routine(fake_fatal);
  320. errno = cerr->syserr;
  321. switch (cerr->err) {
  322. case CHILD_ERR_CHDIR:
  323. error_errno("exec '%s': cd to '%s' failed",
  324. cmd->argv[0], cmd->dir);
  325. break;
  326. case CHILD_ERR_DUP2:
  327. error_errno("dup2() in child failed");
  328. break;
  329. case CHILD_ERR_CLOSE:
  330. error_errno("close() in child failed");
  331. break;
  332. case CHILD_ERR_SIGPROCMASK:
  333. error_errno("sigprocmask failed restoring signals");
  334. break;
  335. case CHILD_ERR_ENOENT:
  336. error_errno("cannot run %s", cmd->argv[0]);
  337. break;
  338. case CHILD_ERR_SILENT:
  339. break;
  340. case CHILD_ERR_ERRNO:
  341. error_errno("cannot exec '%s'", cmd->argv[0]);
  342. break;
  343. }
  344. set_error_routine(old_errfn);
  345. }
  346. static int prepare_cmd(struct argv_array *out, const struct child_process *cmd)
  347. {
  348. if (!cmd->argv[0])
  349. BUG("command is empty");
  350. /*
  351. * Add SHELL_PATH so in the event exec fails with ENOEXEC we can
  352. * attempt to interpret the command with 'sh'.
  353. */
  354. argv_array_push(out, SHELL_PATH);
  355. if (cmd->git_cmd) {
  356. argv_array_push(out, "git");
  357. argv_array_pushv(out, cmd->argv);
  358. } else if (cmd->use_shell) {
  359. prepare_shell_cmd(out, cmd->argv);
  360. } else {
  361. argv_array_pushv(out, cmd->argv);
  362. }
  363. /*
  364. * If there are no '/' characters in the command then perform a path
  365. * lookup and use the resolved path as the command to exec. If there
  366. * are '/' characters, we have exec attempt to invoke the command
  367. * directly.
  368. */
  369. if (!strchr(out->argv[1], '/')) {
  370. char *program = locate_in_PATH(out->argv[1]);
  371. if (program) {
  372. free((char *)out->argv[1]);
  373. out->argv[1] = program;
  374. } else {
  375. argv_array_clear(out);
  376. errno = ENOENT;
  377. return -1;
  378. }
  379. }
  380. return 0;
  381. }
  382. static char **prep_childenv(const char *const *deltaenv)
  383. {
  384. extern char **environ;
  385. char **childenv;
  386. struct string_list env = STRING_LIST_INIT_DUP;
  387. struct strbuf key = STRBUF_INIT;
  388. const char *const *p;
  389. int i;
  390. /* Construct a sorted string list consisting of the current environ */
  391. for (p = (const char *const *) environ; p && *p; p++) {
  392. const char *equals = strchr(*p, '=');
  393. if (equals) {
  394. strbuf_reset(&key);
  395. strbuf_add(&key, *p, equals - *p);
  396. string_list_append(&env, key.buf)->util = (void *) *p;
  397. } else {
  398. string_list_append(&env, *p)->util = (void *) *p;
  399. }
  400. }
  401. string_list_sort(&env);
  402. /* Merge in 'deltaenv' with the current environ */
  403. for (p = deltaenv; p && *p; p++) {
  404. const char *equals = strchr(*p, '=');
  405. if (equals) {
  406. /* ('key=value'), insert or replace entry */
  407. strbuf_reset(&key);
  408. strbuf_add(&key, *p, equals - *p);
  409. string_list_insert(&env, key.buf)->util = (void *) *p;
  410. } else {
  411. /* otherwise ('key') remove existing entry */
  412. string_list_remove(&env, *p, 0);
  413. }
  414. }
  415. /* Create an array of 'char *' to be used as the childenv */
  416. ALLOC_ARRAY(childenv, env.nr + 1);
  417. for (i = 0; i < env.nr; i++)
  418. childenv[i] = env.items[i].util;
  419. childenv[env.nr] = NULL;
  420. string_list_clear(&env, 0);
  421. strbuf_release(&key);
  422. return childenv;
  423. }
  424. struct atfork_state {
  425. #ifndef NO_PTHREADS
  426. int cs;
  427. #endif
  428. sigset_t old;
  429. };
  430. #define CHECK_BUG(err, msg) \
  431. do { \
  432. int e = (err); \
  433. if (e) \
  434. BUG("%s: %s", msg, strerror(e)); \
  435. } while(0)
  436. static void atfork_prepare(struct atfork_state *as)
  437. {
  438. sigset_t all;
  439. if (sigfillset(&all))
  440. die_errno("sigfillset");
  441. #ifdef NO_PTHREADS
  442. if (sigprocmask(SIG_SETMASK, &all, &as->old))
  443. die_errno("sigprocmask");
  444. #else
  445. CHECK_BUG(pthread_sigmask(SIG_SETMASK, &all, &as->old),
  446. "blocking all signals");
  447. CHECK_BUG(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &as->cs),
  448. "disabling cancellation");
  449. #endif
  450. }
  451. static void atfork_parent(struct atfork_state *as)
  452. {
  453. #ifdef NO_PTHREADS
  454. if (sigprocmask(SIG_SETMASK, &as->old, NULL))
  455. die_errno("sigprocmask");
  456. #else
  457. CHECK_BUG(pthread_setcancelstate(as->cs, NULL),
  458. "re-enabling cancellation");
  459. CHECK_BUG(pthread_sigmask(SIG_SETMASK, &as->old, NULL),
  460. "restoring signal mask");
  461. #endif
  462. }
  463. #endif /* GIT_WINDOWS_NATIVE */
  464. static inline void set_cloexec(int fd)
  465. {
  466. int flags = fcntl(fd, F_GETFD);
  467. if (flags >= 0)
  468. fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
  469. }
  470. static int wait_or_whine(pid_t pid, const char *argv0, int in_signal)
  471. {
  472. int status, code = -1;
  473. pid_t waiting;
  474. int failed_errno = 0;
  475. while ((waiting = waitpid(pid, &status, 0)) < 0 && errno == EINTR)
  476. ; /* nothing */
  477. if (in_signal)
  478. return 0;
  479. if (waiting < 0) {
  480. failed_errno = errno;
  481. error_errno("waitpid for %s failed", argv0);
  482. } else if (waiting != pid) {
  483. error("waitpid is confused (%s)", argv0);
  484. } else if (WIFSIGNALED(status)) {
  485. code = WTERMSIG(status);
  486. if (code != SIGINT && code != SIGQUIT && code != SIGPIPE)
  487. error("%s died of signal %d", argv0, code);
  488. /*
  489. * This return value is chosen so that code & 0xff
  490. * mimics the exit code that a POSIX shell would report for
  491. * a program that died from this signal.
  492. */
  493. code += 128;
  494. } else if (WIFEXITED(status)) {
  495. code = WEXITSTATUS(status);
  496. } else {
  497. error("waitpid is confused (%s)", argv0);
  498. }
  499. clear_child_for_cleanup(pid);
  500. errno = failed_errno;
  501. return code;
  502. }
  503. static void trace_add_env(struct strbuf *dst, const char *const *deltaenv)
  504. {
  505. struct string_list envs = STRING_LIST_INIT_DUP;
  506. const char *const *e;
  507. int i;
  508. int printed_unset = 0;
  509. /* Last one wins, see run-command.c:prep_childenv() for context */
  510. for (e = deltaenv; e && *e; e++) {
  511. struct strbuf key = STRBUF_INIT;
  512. char *equals = strchr(*e, '=');
  513. if (equals) {
  514. strbuf_add(&key, *e, equals - *e);
  515. string_list_insert(&envs, key.buf)->util = equals + 1;
  516. } else {
  517. string_list_insert(&envs, *e)->util = NULL;
  518. }
  519. strbuf_release(&key);
  520. }
  521. /* "unset X Y...;" */
  522. for (i = 0; i < envs.nr; i++) {
  523. const char *var = envs.items[i].string;
  524. const char *val = envs.items[i].util;
  525. if (val || !getenv(var))
  526. continue;
  527. if (!printed_unset) {
  528. strbuf_addstr(dst, " unset");
  529. printed_unset = 1;
  530. }
  531. strbuf_addf(dst, " %s", var);
  532. }
  533. if (printed_unset)
  534. strbuf_addch(dst, ';');
  535. /* ... followed by "A=B C=D ..." */
  536. for (i = 0; i < envs.nr; i++) {
  537. const char *var = envs.items[i].string;
  538. const char *val = envs.items[i].util;
  539. const char *oldval;
  540. if (!val)
  541. continue;
  542. oldval = getenv(var);
  543. if (oldval && !strcmp(val, oldval))
  544. continue;
  545. strbuf_addf(dst, " %s=", var);
  546. sq_quote_buf_pretty(dst, val);
  547. }
  548. string_list_clear(&envs, 0);
  549. }
  550. static void trace_run_command(const struct child_process *cp)
  551. {
  552. struct strbuf buf = STRBUF_INIT;
  553. if (!trace_want(&trace_default_key))
  554. return;
  555. strbuf_addstr(&buf, "trace: run_command:");
  556. if (cp->dir) {
  557. strbuf_addstr(&buf, " cd ");
  558. sq_quote_buf_pretty(&buf, cp->dir);
  559. strbuf_addch(&buf, ';');
  560. }
  561. /*
  562. * The caller is responsible for initializing cp->env from
  563. * cp->env_array if needed. We only check one place.
  564. */
  565. if (cp->env)
  566. trace_add_env(&buf, cp->env);
  567. if (cp->git_cmd)
  568. strbuf_addstr(&buf, " git");
  569. sq_quote_argv_pretty(&buf, cp->argv);
  570. trace_printf("%s", buf.buf);
  571. strbuf_release(&buf);
  572. }
  573. int start_command(struct child_process *cmd)
  574. {
  575. int need_in, need_out, need_err;
  576. int fdin[2], fdout[2], fderr[2];
  577. int failed_errno;
  578. char *str;
  579. if (!cmd->argv)
  580. cmd->argv = cmd->args.argv;
  581. if (!cmd->env)
  582. cmd->env = cmd->env_array.argv;
  583. /*
  584. * In case of errors we must keep the promise to close FDs
  585. * that have been passed in via ->in and ->out.
  586. */
  587. need_in = !cmd->no_stdin && cmd->in < 0;
  588. if (need_in) {
  589. if (pipe(fdin) < 0) {
  590. failed_errno = errno;
  591. if (cmd->out > 0)
  592. close(cmd->out);
  593. str = "standard input";
  594. goto fail_pipe;
  595. }
  596. cmd->in = fdin[1];
  597. }
  598. need_out = !cmd->no_stdout
  599. && !cmd->stdout_to_stderr
  600. && cmd->out < 0;
  601. if (need_out) {
  602. if (pipe(fdout) < 0) {
  603. failed_errno = errno;
  604. if (need_in)
  605. close_pair(fdin);
  606. else if (cmd->in)
  607. close(cmd->in);
  608. str = "standard output";
  609. goto fail_pipe;
  610. }
  611. cmd->out = fdout[0];
  612. }
  613. need_err = !cmd->no_stderr && cmd->err < 0;
  614. if (need_err) {
  615. if (pipe(fderr) < 0) {
  616. failed_errno = errno;
  617. if (need_in)
  618. close_pair(fdin);
  619. else if (cmd->in)
  620. close(cmd->in);
  621. if (need_out)
  622. close_pair(fdout);
  623. else if (cmd->out)
  624. close(cmd->out);
  625. str = "standard error";
  626. fail_pipe:
  627. error("cannot create %s pipe for %s: %s",
  628. str, cmd->argv[0], strerror(failed_errno));
  629. child_process_clear(cmd);
  630. errno = failed_errno;
  631. return -1;
  632. }
  633. cmd->err = fderr[0];
  634. }
  635. trace2_child_start(cmd);
  636. trace_run_command(cmd);
  637. fflush(NULL);
  638. #ifndef GIT_WINDOWS_NATIVE
  639. {
  640. int notify_pipe[2];
  641. int null_fd = -1;
  642. char **childenv;
  643. struct argv_array argv = ARGV_ARRAY_INIT;
  644. struct child_err cerr;
  645. struct atfork_state as;
  646. if (prepare_cmd(&argv, cmd) < 0) {
  647. failed_errno = errno;
  648. cmd->pid = -1;
  649. if (!cmd->silent_exec_failure)
  650. error_errno("cannot run %s", cmd->argv[0]);
  651. goto end_of_spawn;
  652. }
  653. if (pipe(notify_pipe))
  654. notify_pipe[0] = notify_pipe[1] = -1;
  655. if (cmd->no_stdin || cmd->no_stdout || cmd->no_stderr) {
  656. null_fd = open("/dev/null", O_RDWR | O_CLOEXEC);
  657. if (null_fd < 0)
  658. die_errno(_("open /dev/null failed"));
  659. set_cloexec(null_fd);
  660. }
  661. childenv = prep_childenv(cmd->env);
  662. atfork_prepare(&as);
  663. /*
  664. * NOTE: In order to prevent deadlocking when using threads special
  665. * care should be taken with the function calls made in between the
  666. * fork() and exec() calls. No calls should be made to functions which
  667. * require acquiring a lock (e.g. malloc) as the lock could have been
  668. * held by another thread at the time of forking, causing the lock to
  669. * never be released in the child process. This means only
  670. * Async-Signal-Safe functions are permitted in the child.
  671. */
  672. cmd->pid = fork();
  673. failed_errno = errno;
  674. if (!cmd->pid) {
  675. int sig;
  676. /*
  677. * Ensure the default die/error/warn routines do not get
  678. * called, they can take stdio locks and malloc.
  679. */
  680. set_die_routine(child_die_fn);
  681. set_error_routine(child_error_fn);
  682. set_warn_routine(child_warn_fn);
  683. close(notify_pipe[0]);
  684. set_cloexec(notify_pipe[1]);
  685. child_notifier = notify_pipe[1];
  686. if (cmd->no_stdin)
  687. child_dup2(null_fd, 0);
  688. else if (need_in) {
  689. child_dup2(fdin[0], 0);
  690. child_close_pair(fdin);
  691. } else if (cmd->in) {
  692. child_dup2(cmd->in, 0);
  693. child_close(cmd->in);
  694. }
  695. if (cmd->no_stderr)
  696. child_dup2(null_fd, 2);
  697. else if (need_err) {
  698. child_dup2(fderr[1], 2);
  699. child_close_pair(fderr);
  700. } else if (cmd->err > 1) {
  701. child_dup2(cmd->err, 2);
  702. child_close(cmd->err);
  703. }
  704. if (cmd->no_stdout)
  705. child_dup2(null_fd, 1);
  706. else if (cmd->stdout_to_stderr)
  707. child_dup2(2, 1);
  708. else if (need_out) {
  709. child_dup2(fdout[1], 1);
  710. child_close_pair(fdout);
  711. } else if (cmd->out > 1) {
  712. child_dup2(cmd->out, 1);
  713. child_close(cmd->out);
  714. }
  715. if (cmd->dir && chdir(cmd->dir))
  716. child_die(CHILD_ERR_CHDIR);
  717. /*
  718. * restore default signal handlers here, in case
  719. * we catch a signal right before execve below
  720. */
  721. for (sig = 1; sig < NSIG; sig++) {
  722. /* ignored signals get reset to SIG_DFL on execve */
  723. if (signal(sig, SIG_DFL) == SIG_IGN)
  724. signal(sig, SIG_IGN);
  725. }
  726. if (sigprocmask(SIG_SETMASK, &as.old, NULL) != 0)
  727. child_die(CHILD_ERR_SIGPROCMASK);
  728. /*
  729. * Attempt to exec using the command and arguments starting at
  730. * argv.argv[1]. argv.argv[0] contains SHELL_PATH which will
  731. * be used in the event exec failed with ENOEXEC at which point
  732. * we will try to interpret the command using 'sh'.
  733. */
  734. execve(argv.argv[1], (char *const *) argv.argv + 1,
  735. (char *const *) childenv);
  736. if (errno == ENOEXEC)
  737. execve(argv.argv[0], (char *const *) argv.argv,
  738. (char *const *) childenv);
  739. if (errno == ENOENT) {
  740. if (cmd->silent_exec_failure)
  741. child_die(CHILD_ERR_SILENT);
  742. child_die(CHILD_ERR_ENOENT);
  743. } else {
  744. child_die(CHILD_ERR_ERRNO);
  745. }
  746. }
  747. atfork_parent(&as);
  748. if (cmd->pid < 0)
  749. error_errno("cannot fork() for %s", cmd->argv[0]);
  750. else if (cmd->clean_on_exit)
  751. mark_child_for_cleanup(cmd->pid, cmd);
  752. /*
  753. * Wait for child's exec. If the exec succeeds (or if fork()
  754. * failed), EOF is seen immediately by the parent. Otherwise, the
  755. * child process sends a child_err struct.
  756. * Note that use of this infrastructure is completely advisory,
  757. * therefore, we keep error checks minimal.
  758. */
  759. close(notify_pipe[1]);
  760. if (xread(notify_pipe[0], &cerr, sizeof(cerr)) == sizeof(cerr)) {
  761. /*
  762. * At this point we know that fork() succeeded, but exec()
  763. * failed. Errors have been reported to our stderr.
  764. */
  765. wait_or_whine(cmd->pid, cmd->argv[0], 0);
  766. child_err_spew(cmd, &cerr);
  767. failed_errno = errno;
  768. cmd->pid = -1;
  769. }
  770. close(notify_pipe[0]);
  771. if (null_fd >= 0)
  772. close(null_fd);
  773. argv_array_clear(&argv);
  774. free(childenv);
  775. }
  776. end_of_spawn:
  777. #else
  778. {
  779. int fhin = 0, fhout = 1, fherr = 2;
  780. const char **sargv = cmd->argv;
  781. struct argv_array nargv = ARGV_ARRAY_INIT;
  782. if (cmd->no_stdin)
  783. fhin = open("/dev/null", O_RDWR);
  784. else if (need_in)
  785. fhin = dup(fdin[0]);
  786. else if (cmd->in)
  787. fhin = dup(cmd->in);
  788. if (cmd->no_stderr)
  789. fherr = open("/dev/null", O_RDWR);
  790. else if (need_err)
  791. fherr = dup(fderr[1]);
  792. else if (cmd->err > 2)
  793. fherr = dup(cmd->err);
  794. if (cmd->no_stdout)
  795. fhout = open("/dev/null", O_RDWR);
  796. else if (cmd->stdout_to_stderr)
  797. fhout = dup(fherr);
  798. else if (need_out)
  799. fhout = dup(fdout[1]);
  800. else if (cmd->out > 1)
  801. fhout = dup(cmd->out);
  802. if (cmd->git_cmd)
  803. cmd->argv = prepare_git_cmd(&nargv, cmd->argv);
  804. else if (cmd->use_shell)
  805. cmd->argv = prepare_shell_cmd(&nargv, cmd->argv);
  806. cmd->pid = mingw_spawnvpe(cmd->argv[0], cmd->argv, (char**) cmd->env,
  807. cmd->dir, fhin, fhout, fherr);
  808. failed_errno = errno;
  809. if (cmd->pid < 0 && (!cmd->silent_exec_failure || errno != ENOENT))
  810. error_errno("cannot spawn %s", cmd->argv[0]);
  811. if (cmd->clean_on_exit && cmd->pid >= 0)
  812. mark_child_for_cleanup(cmd->pid, cmd);
  813. argv_array_clear(&nargv);
  814. cmd->argv = sargv;
  815. if (fhin != 0)
  816. close(fhin);
  817. if (fhout != 1)
  818. close(fhout);
  819. if (fherr != 2)
  820. close(fherr);
  821. }
  822. #endif
  823. if (cmd->pid < 0) {
  824. trace2_child_exit(cmd, -1);
  825. if (need_in)
  826. close_pair(fdin);
  827. else if (cmd->in)
  828. close(cmd->in);
  829. if (need_out)
  830. close_pair(fdout);
  831. else if (cmd->out)
  832. close(cmd->out);
  833. if (need_err)
  834. close_pair(fderr);
  835. else if (cmd->err)
  836. close(cmd->err);
  837. child_process_clear(cmd);
  838. errno = failed_errno;
  839. return -1;
  840. }
  841. if (need_in)
  842. close(fdin[0]);
  843. else if (cmd->in)
  844. close(cmd->in);
  845. if (need_out)
  846. close(fdout[1]);
  847. else if (cmd->out)
  848. close(cmd->out);
  849. if (need_err)
  850. close(fderr[1]);
  851. else if (cmd->err)
  852. close(cmd->err);
  853. return 0;
  854. }
  855. int finish_command(struct child_process *cmd)
  856. {
  857. int ret = wait_or_whine(cmd->pid, cmd->argv[0], 0);
  858. trace2_child_exit(cmd, ret);
  859. child_process_clear(cmd);
  860. return ret;
  861. }
  862. int finish_command_in_signal(struct child_process *cmd)
  863. {
  864. int ret = wait_or_whine(cmd->pid, cmd->argv[0], 1);
  865. trace2_child_exit(cmd, ret);
  866. return ret;
  867. }
  868. int run_command(struct child_process *cmd)
  869. {
  870. int code;
  871. if (cmd->out < 0 || cmd->err < 0)
  872. BUG("run_command with a pipe can cause deadlock");
  873. code = start_command(cmd);
  874. if (code)
  875. return code;
  876. return finish_command(cmd);
  877. }
  878. int run_command_v_opt(const char **argv, int opt)
  879. {
  880. return run_command_v_opt_cd_env(argv, opt, NULL, NULL);
  881. }
  882. int run_command_v_opt_tr2(const char **argv, int opt, const char *tr2_class)
  883. {
  884. return run_command_v_opt_cd_env_tr2(argv, opt, NULL, NULL, tr2_class);
  885. }
  886. int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const char *const *env)
  887. {
  888. return run_command_v_opt_cd_env_tr2(argv, opt, dir, env, NULL);
  889. }
  890. int run_command_v_opt_cd_env_tr2(const char **argv, int opt, const char *dir,
  891. const char *const *env, const char *tr2_class)
  892. {
  893. struct child_process cmd = CHILD_PROCESS_INIT;
  894. cmd.argv = argv;
  895. cmd.no_stdin = opt & RUN_COMMAND_NO_STDIN ? 1 : 0;
  896. cmd.git_cmd = opt & RUN_GIT_CMD ? 1 : 0;
  897. cmd.stdout_to_stderr = opt & RUN_COMMAND_STDOUT_TO_STDERR ? 1 : 0;
  898. cmd.silent_exec_failure = opt & RUN_SILENT_EXEC_FAILURE ? 1 : 0;
  899. cmd.use_shell = opt & RUN_USING_SHELL ? 1 : 0;
  900. cmd.clean_on_exit = opt & RUN_CLEAN_ON_EXIT ? 1 : 0;
  901. cmd.dir = dir;
  902. cmd.env = env;
  903. cmd.trace2_child_class = tr2_class;
  904. return run_command(&cmd);
  905. }
  906. #ifndef NO_PTHREADS
  907. static pthread_t main_thread;
  908. static int main_thread_set;
  909. static pthread_key_t async_key;
  910. static pthread_key_t async_die_counter;
  911. static void *run_thread(void *data)
  912. {
  913. struct async *async = data;
  914. intptr_t ret;
  915. if (async->isolate_sigpipe) {
  916. sigset_t mask;
  917. sigemptyset(&mask);
  918. sigaddset(&mask, SIGPIPE);
  919. if (pthread_sigmask(SIG_BLOCK, &mask, NULL) < 0) {
  920. ret = error("unable to block SIGPIPE in async thread");
  921. return (void *)ret;
  922. }
  923. }
  924. pthread_setspecific(async_key, async);
  925. ret = async->proc(async->proc_in, async->proc_out, async->data);
  926. return (void *)ret;
  927. }
  928. static NORETURN void die_async(const char *err, va_list params)
  929. {
  930. vreportf("fatal: ", err, params);
  931. if (in_async()) {
  932. struct async *async = pthread_getspecific(async_key);
  933. if (async->proc_in >= 0)
  934. close(async->proc_in);
  935. if (async->proc_out >= 0)
  936. close(async->proc_out);
  937. pthread_exit((void *)128);
  938. }
  939. exit(128);
  940. }
  941. static int async_die_is_recursing(void)
  942. {
  943. void *ret = pthread_getspecific(async_die_counter);
  944. pthread_setspecific(async_die_counter, (void *)1);
  945. return ret != NULL;
  946. }
  947. int in_async(void)
  948. {
  949. if (!main_thread_set)
  950. return 0; /* no asyncs started yet */
  951. return !pthread_equal(main_thread, pthread_self());
  952. }
  953. static void NORETURN async_exit(int code)
  954. {
  955. pthread_exit((void *)(intptr_t)code);
  956. }
  957. #else
  958. static struct {
  959. void (**handlers)(void);
  960. size_t nr;
  961. size_t alloc;
  962. } git_atexit_hdlrs;
  963. static int git_atexit_installed;
  964. static void git_atexit_dispatch(void)
  965. {
  966. size_t i;
  967. for (i=git_atexit_hdlrs.nr ; i ; i--)
  968. git_atexit_hdlrs.handlers[i-1]();
  969. }
  970. static void git_atexit_clear(void)
  971. {
  972. free(git_atexit_hdlrs.handlers);
  973. memset(&git_atexit_hdlrs, 0, sizeof(git_atexit_hdlrs));
  974. git_atexit_installed = 0;
  975. }
  976. #undef atexit
  977. int git_atexit(void (*handler)(void))
  978. {
  979. ALLOC_GROW(git_atexit_hdlrs.handlers, git_atexit_hdlrs.nr + 1, git_atexit_hdlrs.alloc);
  980. git_atexit_hdlrs.handlers[git_atexit_hdlrs.nr++] = handler;
  981. if (!git_atexit_installed) {
  982. if (atexit(&git_atexit_dispatch))
  983. return -1;
  984. git_atexit_installed = 1;
  985. }
  986. return 0;
  987. }
  988. #define atexit git_atexit
  989. static int process_is_async;
  990. int in_async(void)
  991. {
  992. return process_is_async;
  993. }
  994. static void NORETURN async_exit(int code)
  995. {
  996. exit(code);
  997. }
  998. #endif
  999. void check_pipe(int err)
  1000. {
  1001. if (err == EPIPE) {
  1002. if (in_async())
  1003. async_exit(141);
  1004. signal(SIGPIPE, SIG_DFL);
  1005. raise(SIGPIPE);
  1006. /* Should never happen, but just in case... */
  1007. exit(141);
  1008. }
  1009. }
  1010. int start_async(struct async *async)
  1011. {
  1012. int need_in, need_out;
  1013. int fdin[2], fdout[2];
  1014. int proc_in, proc_out;
  1015. need_in = async->in < 0;
  1016. if (need_in) {
  1017. if (pipe(fdin) < 0) {
  1018. if (async->out > 0)
  1019. close(async->out);
  1020. return error_errno("cannot create pipe");
  1021. }
  1022. async->in = fdin[1];
  1023. }
  1024. need_out = async->out < 0;
  1025. if (need_out) {
  1026. if (pipe(fdout) < 0) {
  1027. if (need_in)
  1028. close_pair(fdin);
  1029. else if (async->in)
  1030. close(async->in);
  1031. return error_errno("cannot create pipe");
  1032. }
  1033. async->out = fdout[0];
  1034. }
  1035. if (need_in)
  1036. proc_in = fdin[0];
  1037. else if (async->in)
  1038. proc_in = async->in;
  1039. else
  1040. proc_in = -1;
  1041. if (need_out)
  1042. proc_out = fdout[1];
  1043. else if (async->out)
  1044. proc_out = async->out;
  1045. else
  1046. proc_out = -1;
  1047. #ifdef NO_PTHREADS
  1048. /* Flush stdio before fork() to avoid cloning buffers */
  1049. fflush(NULL);
  1050. async->pid = fork();
  1051. if (async->pid < 0) {
  1052. error_errno("fork (async) failed");
  1053. goto error;
  1054. }
  1055. if (!async->pid) {
  1056. if (need_in)
  1057. close(fdin[1]);
  1058. if (need_out)
  1059. close(fdout[0]);
  1060. git_atexit_clear();
  1061. process_is_async = 1;
  1062. exit(!!async->proc(proc_in, proc_out, async->data));
  1063. }
  1064. mark_child_for_cleanup(async->pid, NULL);
  1065. if (need_in)
  1066. close(fdin[0]);
  1067. else if (async->in)
  1068. close(async->in);
  1069. if (need_out)
  1070. close(fdout[1]);
  1071. else if (async->out)
  1072. close(async->out);
  1073. #else
  1074. if (!main_thread_set) {
  1075. /*
  1076. * We assume that the first time that start_async is called
  1077. * it is from the main thread.
  1078. */
  1079. main_thread_set = 1;
  1080. main_thread = pthread_self();
  1081. pthread_key_create(&async_key, NULL);
  1082. pthread_key_create(&async_die_counter, NULL);
  1083. set_die_routine(die_async);
  1084. set_die_is_recursing_routine(async_die_is_recursing);
  1085. }
  1086. if (proc_in >= 0)
  1087. set_cloexec(proc_in);
  1088. if (proc_out >= 0)
  1089. set_cloexec(proc_out);
  1090. async->proc_in = proc_in;
  1091. async->proc_out = proc_out;
  1092. {
  1093. int err = pthread_create(&async->tid, NULL, run_thread, async);
  1094. if (err) {
  1095. error(_("cannot create async thread: %s"), strerror(err));
  1096. goto error;
  1097. }
  1098. }
  1099. #endif
  1100. return 0;
  1101. error:
  1102. if (need_in)
  1103. close_pair(fdin);
  1104. else if (async->in)
  1105. close(async->in);
  1106. if (need_out)
  1107. close_pair(fdout);
  1108. else if (async->out)
  1109. close(async->out);
  1110. return -1;
  1111. }
  1112. int finish_async(struct async *async)
  1113. {
  1114. #ifdef NO_PTHREADS
  1115. return wait_or_whine(async->pid, "child process", 0);
  1116. #else
  1117. void *ret = (void *)(intptr_t)(-1);
  1118. if (pthread_join(async->tid, &ret))
  1119. error("pthread_join failed");
  1120. return (int)(intptr_t)ret;
  1121. #endif
  1122. }
  1123. int async_with_fork(void)
  1124. {
  1125. #ifdef NO_PTHREADS
  1126. return 1;
  1127. #else
  1128. return 0;
  1129. #endif
  1130. }
  1131. const char *find_hook(const char *name)
  1132. {
  1133. static struct strbuf path = STRBUF_INIT;
  1134. strbuf_reset(&path);
  1135. strbuf_git_path(&path, "hooks/%s", name);
  1136. if (access(path.buf, X_OK) < 0) {
  1137. int err = errno;
  1138. #ifdef STRIP_EXTENSION
  1139. strbuf_addstr(&path, STRIP_EXTENSION);
  1140. if (access(path.buf, X_OK) >= 0)
  1141. return path.buf;
  1142. if (errno == EACCES)
  1143. err = errno;
  1144. #endif
  1145. if (err == EACCES && advice_ignored_hook) {
  1146. static struct string_list advise_given = STRING_LIST_INIT_DUP;
  1147. if (!string_list_lookup(&advise_given, name)) {
  1148. string_list_insert(&advise_given, name);
  1149. advise(_("The '%s' hook was ignored because "
  1150. "it's not set as executable.\n"
  1151. "You can disable this warning with "
  1152. "`git config advice.ignoredHook false`."),
  1153. path.buf);
  1154. }
  1155. }
  1156. return NULL;
  1157. }
  1158. return path.buf;
  1159. }
  1160. int run_hook_ve(const char *const *env, const char *name, va_list args)
  1161. {
  1162. struct child_process hook = CHILD_PROCESS_INIT;
  1163. const char *p;
  1164. p = find_hook(name);
  1165. if (!p)
  1166. return 0;
  1167. argv_array_push(&hook.args, p);
  1168. while ((p = va_arg(args, const char *)))
  1169. argv_array_push(&hook.args, p);
  1170. hook.env = env;
  1171. hook.no_stdin = 1;
  1172. hook.stdout_to_stderr = 1;
  1173. hook.trace2_hook_name = name;
  1174. return run_command(&hook);
  1175. }
  1176. int run_hook_le(const char *const *env, const char *name, ...)
  1177. {
  1178. va_list args;
  1179. int ret;
  1180. va_start(args, name);
  1181. ret = run_hook_ve(env, name, args);
  1182. va_end(args);
  1183. return ret;
  1184. }
  1185. struct io_pump {
  1186. /* initialized by caller */
  1187. int fd;
  1188. int type; /* POLLOUT or POLLIN */
  1189. union {
  1190. struct {
  1191. const char *buf;
  1192. size_t len;
  1193. } out;
  1194. struct {
  1195. struct strbuf *buf;
  1196. size_t hint;
  1197. } in;
  1198. } u;
  1199. /* returned by pump_io */
  1200. int error; /* 0 for success, otherwise errno */
  1201. /* internal use */
  1202. struct pollfd *pfd;
  1203. };
  1204. static int pump_io_round(struct io_pump *slots, int nr, struct pollfd *pfd)
  1205. {
  1206. int pollsize = 0;
  1207. int i;
  1208. for (i = 0; i < nr; i++) {
  1209. struct io_pump *io = &slots[i];
  1210. if (io->fd < 0)
  1211. continue;
  1212. pfd[pollsize].fd = io->fd;
  1213. pfd[pollsize].events = io->type;
  1214. io->pfd = &pfd[pollsize++];
  1215. }
  1216. if (!pollsize)
  1217. return 0;
  1218. if (poll(pfd, pollsize, -1) < 0) {
  1219. if (errno == EINTR)
  1220. return 1;
  1221. die_errno("poll failed");
  1222. }
  1223. for (i = 0; i < nr; i++) {
  1224. struct io_pump *io = &slots[i];
  1225. if (io->fd < 0)
  1226. continue;
  1227. if (!(io->pfd->revents & (POLLOUT|POLLIN|POLLHUP|POLLERR|POLLNVAL)))
  1228. continue;
  1229. if (io->type == POLLOUT) {
  1230. ssize_t len = xwrite(io->fd,
  1231. io->u.out.buf, io->u.out.len);
  1232. if (len < 0) {
  1233. io->error = errno;
  1234. close(io->fd);
  1235. io->fd = -1;
  1236. } else {
  1237. io->u.out.buf += len;
  1238. io->u.out.len -= len;
  1239. if (!io->u.out.len) {
  1240. close(io->fd);
  1241. io->fd = -1;
  1242. }
  1243. }
  1244. }
  1245. if (io->type == POLLIN) {
  1246. ssize_t len = strbuf_read_once(io->u.in.buf,
  1247. io->fd, io->u.in.hint);
  1248. if (len < 0)
  1249. io->error = errno;
  1250. if (len <= 0) {
  1251. close(io->fd);
  1252. io->fd = -1;
  1253. }
  1254. }
  1255. }
  1256. return 1;
  1257. }
  1258. static int pump_io(struct io_pump *slots, int nr)
  1259. {
  1260. struct pollfd *pfd;
  1261. int i;
  1262. for (i = 0; i < nr; i++)
  1263. slots[i].error = 0;
  1264. ALLOC_ARRAY(pfd, nr);
  1265. while (pump_io_round(slots, nr, pfd))
  1266. ; /* nothing */
  1267. free(pfd);
  1268. /* There may be multiple errno values, so just pick the first. */
  1269. for (i = 0; i < nr; i++) {
  1270. if (slots[i].error) {
  1271. errno = slots[i].error;
  1272. return -1;
  1273. }
  1274. }
  1275. return 0;
  1276. }
  1277. int pipe_command(struct child_process *cmd,
  1278. const char *in, size_t in_len,
  1279. struct strbuf *out, size_t out_hint,
  1280. struct strbuf *err, size_t err_hint)
  1281. {
  1282. struct io_pump io[3];
  1283. int nr = 0;
  1284. if (in)
  1285. cmd->in = -1;
  1286. if (out)
  1287. cmd->out = -1;
  1288. if (err)
  1289. cmd->err = -1;
  1290. if (start_command(cmd) < 0)
  1291. return -1;
  1292. if (in) {
  1293. io[nr].fd = cmd->in;
  1294. io[nr].type = POLLOUT;
  1295. io[nr].u.out.buf = in;
  1296. io[nr].u.out.len = in_len;
  1297. nr++;
  1298. }
  1299. if (out) {
  1300. io[nr].fd = cmd->out;
  1301. io[nr].type = POLLIN;
  1302. io[nr].u.in.buf = out;
  1303. io[nr].u.in.hint = out_hint;
  1304. nr++;
  1305. }
  1306. if (err) {
  1307. io[nr].fd = cmd->err;
  1308. io[nr].type = POLLIN;
  1309. io[nr].u.in.buf = err;
  1310. io[nr].u.in.hint = err_hint;
  1311. nr++;
  1312. }
  1313. if (pump_io(io, nr) < 0) {
  1314. finish_command(cmd); /* throw away exit code */
  1315. return -1;
  1316. }
  1317. return finish_command(cmd);
  1318. }
  1319. enum child_state {
  1320. GIT_CP_FREE,
  1321. GIT_CP_WORKING,
  1322. GIT_CP_WAIT_CLEANUP,
  1323. };
  1324. struct parallel_processes {
  1325. void *data;
  1326. int max_processes;
  1327. int nr_processes;
  1328. get_next_task_fn get_next_task;
  1329. start_failure_fn start_failure;
  1330. task_finished_fn task_finished;
  1331. struct {
  1332. enum child_state state;
  1333. struct child_process process;
  1334. struct strbuf err;
  1335. void *data;
  1336. } *children;
  1337. /*
  1338. * The struct pollfd is logically part of *children,
  1339. * but the system call expects it as its own array.
  1340. */
  1341. struct pollfd *pfd;
  1342. unsigned shutdown : 1;
  1343. int output_owner;
  1344. struct strbuf buffered_output; /* of finished children */
  1345. };
  1346. static int default_start_failure(struct strbuf *out,
  1347. void *pp_cb,
  1348. void *pp_task_cb)
  1349. {
  1350. return 0;
  1351. }
  1352. static int default_task_finished(int result,
  1353. struct strbuf *out,
  1354. void *pp_cb,
  1355. void *pp_task_cb)
  1356. {
  1357. return 0;
  1358. }
  1359. static void kill_children(struct parallel_processes *pp, int signo)
  1360. {
  1361. int i, n = pp->max_processes;
  1362. for (i = 0; i < n; i++)
  1363. if (pp->children[i].state == GIT_CP_WORKING)
  1364. kill(pp->children[i].process.pid, signo);
  1365. }
  1366. static struct parallel_processes *pp_for_signal;
  1367. static void handle_children_on_signal(int signo)
  1368. {
  1369. kill_children(pp_for_signal, signo);
  1370. sigchain_pop(signo);
  1371. raise(signo);
  1372. }
  1373. static void pp_init(struct parallel_processes *pp,
  1374. int n,
  1375. get_next_task_fn get_next_task,
  1376. start_failure_fn start_failure,
  1377. task_finished_fn task_finished,
  1378. void *data)
  1379. {
  1380. int i;
  1381. if (n < 1)
  1382. n = online_cpus();
  1383. pp->max_processes = n;
  1384. trace_printf("run_processes_parallel: preparing to run up to %d tasks", n);
  1385. pp->data = data;
  1386. if (!get_next_task)
  1387. BUG("you need to specify a get_next_task function");
  1388. pp->get_next_task = get_next_task;
  1389. pp->start_failure = start_failure ? start_failure : default_start_failure;
  1390. pp->task_finished = task_finished ? task_finished : default_task_finished;
  1391. pp->nr_processes = 0;
  1392. pp->output_owner = 0;
  1393. pp->shutdown = 0;
  1394. pp->children = xcalloc(n, sizeof(*pp->children));
  1395. pp->pfd = xcalloc(n, sizeof(*pp->pfd));
  1396. strbuf_init(&pp->buffered_output, 0);
  1397. for (i = 0; i < n; i++) {
  1398. strbuf_init(&pp->children[i].err, 0);
  1399. child_process_init(&pp->children[i].process);
  1400. pp->pfd[i].events = POLLIN | POLLHUP;
  1401. pp->pfd[i].fd = -1;
  1402. }
  1403. pp_for_signal = pp;
  1404. sigchain_push_common(handle_children_on_signal);
  1405. }
  1406. static void pp_cleanup(struct parallel_processes *pp)
  1407. {
  1408. int i;
  1409. trace_printf("run_processes_parallel: done");
  1410. for (i = 0; i < pp->max_processes; i++) {
  1411. strbuf_release(&pp->children[i].err);
  1412. child_process_clear(&pp->children[i].process);
  1413. }
  1414. free(pp->children);
  1415. free(pp->pfd);
  1416. /*
  1417. * When get_next_task added messages to the buffer in its last
  1418. * iteration, the buffered output is non empty.
  1419. */
  1420. strbuf_write(&pp->buffered_output, stderr);
  1421. strbuf_release(&pp->buffered_output);
  1422. sigchain_pop_common();
  1423. }
  1424. /* returns
  1425. * 0 if a new task was started.
  1426. * 1 if no new jobs was started (get_next_task ran out of work, non critical
  1427. * problem with starting a new command)
  1428. * <0 no new job was started, user wishes to shutdown early. Use negative code
  1429. * to signal the children.
  1430. */
  1431. static int pp_start_one(struct parallel_processes *pp)
  1432. {
  1433. int i, code;
  1434. for (i = 0; i < pp->max_processes; i++)
  1435. if (pp->children[i].state == GIT_CP_FREE)
  1436. break;
  1437. if (i == pp->max_processes)
  1438. BUG("bookkeeping is hard");
  1439. code = pp->get_next_task(&pp->children[i].process,
  1440. &pp->children[i].err,
  1441. pp->data,
  1442. &pp->children[i].data);
  1443. if (!code) {
  1444. strbuf_addbuf(&pp->buffered_output, &pp->children[i].err);
  1445. strbuf_reset(&pp->children[i].err);
  1446. return 1;
  1447. }
  1448. pp->children[i].process.err = -1;
  1449. pp->children[i].process.stdout_to_stderr = 1;
  1450. pp->children[i].process.no_stdin = 1;
  1451. if (start_command(&pp->children[i].process)) {
  1452. code = pp->start_failure(&pp->children[i].err,
  1453. pp->data,
  1454. pp->children[i].data);
  1455. strbuf_addbuf(&pp->buffered_output, &pp->children[i].err);
  1456. strbuf_reset(&pp->children[i].err);
  1457. if (code)
  1458. pp->shutdown = 1;
  1459. return code;
  1460. }
  1461. pp->nr_processes++;
  1462. pp->children[i].state = GIT_CP_WORKING;
  1463. pp->pfd[i].fd = pp->children[i].process.err;
  1464. return 0;
  1465. }
  1466. static void pp_buffer_stderr(struct parallel_processes *pp, int output_timeout)
  1467. {
  1468. int i;
  1469. while ((i = poll(pp->pfd, pp->max_processes, output_timeout)) < 0) {
  1470. if (errno == EINTR)
  1471. continue;
  1472. pp_cleanup(pp);
  1473. die_errno("poll");
  1474. }
  1475. /* Buffer output from all pipes. */
  1476. for (i = 0; i < pp->max_processes; i++) {
  1477. if (pp->children[i].state == GIT_CP_WORKING &&
  1478. pp->pfd[i].revents & (POLLIN | POLLHUP)) {
  1479. int n = strbuf_read_once(&pp->children[i].err,
  1480. pp->children[i].process.err, 0);
  1481. if (n == 0) {
  1482. close(pp->children[i].process.err);
  1483. pp->children[i].state = GIT_CP_WAIT_CLEANUP;
  1484. } else if (n < 0)
  1485. if (errno != EAGAIN)
  1486. die_errno("read");
  1487. }
  1488. }
  1489. }
  1490. static void pp_output(struct parallel_processes *pp)
  1491. {
  1492. int i = pp->output_owner;
  1493. if (pp->children[i].state == GIT_CP_WORKING &&
  1494. pp->children[i].err.len) {
  1495. strbuf_write(&pp->children[i].err, stderr);
  1496. strbuf_reset(&pp->children[i].err);
  1497. }
  1498. }
  1499. static int pp_collect_finished(struct parallel_processes *pp)
  1500. {
  1501. int i, code;
  1502. int n = pp->max_processes;
  1503. int result = 0;
  1504. while (pp->nr_processes > 0) {
  1505. for (i = 0; i < pp->max_processes; i++)
  1506. if (pp->children[i].state == GIT_CP_WAIT_CLEANUP)
  1507. break;
  1508. if (i == pp->max_processes)
  1509. break;
  1510. code = finish_command(&pp->children[i].process);
  1511. code = pp->task_finished(code,
  1512. &pp->children[i].err, pp->data,
  1513. pp->children[i].data);
  1514. if (code)
  1515. result = code;
  1516. if (code < 0)
  1517. break;
  1518. pp->nr_processes--;
  1519. pp->children[i].state = GIT_CP_FREE;
  1520. pp->pfd[i].fd = -1;
  1521. child_process_init(&pp->children[i].process);
  1522. if (i != pp->output_owner) {
  1523. strbuf_addbuf(&pp->buffered_output, &pp->children[i].err);
  1524. strbuf_reset(&pp->children[i].err);
  1525. } else {
  1526. strbuf_write(&pp->children[i].err, stderr);
  1527. strbuf_reset(&pp->children[i].err);
  1528. /* Output all other finished child processes */
  1529. strbuf_write(&pp->buffered_output, stderr);
  1530. strbuf_reset(&pp->buffered_output);
  1531. /*
  1532. * Pick next process to output live.
  1533. * NEEDSWORK:
  1534. * For now we pick it randomly by doing a round
  1535. * robin. Later we may want to pick the one with
  1536. * the most output or the longest or shortest
  1537. * running process time.
  1538. */
  1539. for (i = 0; i < n; i++)
  1540. if (pp->children[(pp->output_owner + i) % n].state == GIT_CP_WORKING)
  1541. break;
  1542. pp->output_owner = (pp->output_owner + i) % n;
  1543. }
  1544. }
  1545. return result;
  1546. }
  1547. int run_processes_parallel(int n,
  1548. get_next_task_fn get_next_task,
  1549. start_failure_fn start_failure,
  1550. task_finished_fn task_finished,
  1551. void *pp_cb)
  1552. {
  1553. int i, code;
  1554. int output_timeout = 100;
  1555. int spawn_cap = 4;
  1556. struct parallel_processes pp;
  1557. pp_init(&pp, n, get_next_task, start_failure, task_finished, pp_cb);
  1558. while (1) {
  1559. for (i = 0;
  1560. i < spawn_cap && !pp.shutdown &&
  1561. pp.nr_processes < pp.max_processes;
  1562. i++) {
  1563. code = pp_start_one(&pp);
  1564. if (!code)
  1565. continue;
  1566. if (code < 0) {
  1567. pp.shutdown = 1;
  1568. kill_children(&pp, -code);
  1569. }
  1570. break;
  1571. }
  1572. if (!pp.nr_processes)
  1573. break;
  1574. pp_buffer_stderr(&pp, output_timeout);
  1575. pp_output(&pp);
  1576. code = pp_collect_finished(&pp);
  1577. if (code) {
  1578. pp.shutdown = 1;
  1579. if (code < 0)
  1580. kill_children(&pp, -code);
  1581. }
  1582. }
  1583. pp_cleanup(&pp);
  1584. return 0;
  1585. }
  1586. int run_processes_parallel_tr2(int n, get_next_task_fn get_next_task,
  1587. start_failure_fn start_failure,
  1588. task_finished_fn task_finished, void *pp_cb,
  1589. const char *tr2_category, const char *tr2_label)
  1590. {
  1591. int result;
  1592. trace2_region_enter_printf(tr2_category, tr2_label, NULL, "max:%d",
  1593. ((n < 1) ? online_cpus() : n));
  1594. result = run_processes_parallel(n, get_next_task, start_failure,
  1595. task_finished, pp_cb);
  1596. trace2_region_leave(tr2_category, tr2_label, NULL);
  1597. return result;
  1598. }