THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Git Source Code Mirror - This is a publish-only repository and all pull requests are ignored. Please follow Documentation/SubmittingPatches procedure for any of your improvements.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

convert.h 4.1KB

convert: stream from fd to required clean filter to reduce used address space The data is streamed to the filter process anyway. Better avoid mapping the file if possible. This is especially useful if a clean filter reduces the size, for example if it computes a sha1 for binary data, like git media. The file size that the previous implementation could handle was limited by the available address space; large files for example could not be handled with (32-bit) msysgit. The new implementation can filter files of any size as long as the filter output is small enough. The new code path is only taken if the filter is required. The filter consumes data directly from the fd. If it fails, the original data is not immediately available. The condition can easily be handled as a fatal error, which is expected for a required filter anyway. If the filter was not required, the condition would need to be handled in a different way, like seeking to 0 and reading the data. But this would require more restructuring of the code and is probably not worth it. The obvious approach of falling back to reading all data would not help achieving the main purpose of this patch, which is to handle large files with limited address space. If reading all data is an option, we can simply take the old code path right away and mmap the entire file. The environment variable GIT_MMAP_LIMIT, which has been introduced in a previous commit is used to test that the expected code path is taken. A related test that exercises required filters is modified to verify that the data actually has been modified on its way from the file system to the object store. Signed-off-by: Steffen Prohaska <prohaska@zib.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. /*
  2. * Copyright (c) 2011, Google Inc.
  3. */
  4. #ifndef CONVERT_H
  5. #define CONVERT_H
  6. #include "string-list.h"
  7. struct index_state;
  8. struct object_id;
  9. struct strbuf;
  10. #define CONV_EOL_RNDTRP_DIE (1<<0) /* Die if CRLF to LF to CRLF is different */
  11. #define CONV_EOL_RNDTRP_WARN (1<<1) /* Warn if CRLF to LF to CRLF is different */
  12. #define CONV_EOL_RENORMALIZE (1<<2) /* Convert CRLF to LF */
  13. #define CONV_EOL_KEEP_CRLF (1<<3) /* Keep CRLF line endings as is */
  14. #define CONV_WRITE_OBJECT (1<<4) /* Content is written to the index */
  15. extern int global_conv_flags_eol;
  16. enum auto_crlf {
  17. AUTO_CRLF_FALSE = 0,
  18. AUTO_CRLF_TRUE = 1,
  19. AUTO_CRLF_INPUT = -1
  20. };
  21. extern enum auto_crlf auto_crlf;
  22. enum eol {
  23. EOL_UNSET,
  24. EOL_CRLF,
  25. EOL_LF,
  26. #ifdef NATIVE_CRLF
  27. EOL_NATIVE = EOL_CRLF
  28. #else
  29. EOL_NATIVE = EOL_LF
  30. #endif
  31. };
  32. enum ce_delay_state {
  33. CE_NO_DELAY = 0,
  34. CE_CAN_DELAY = 1,
  35. CE_RETRY = 2
  36. };
  37. struct delayed_checkout {
  38. /*
  39. * State of the currently processed cache entry. If the state is
  40. * CE_CAN_DELAY, then the filter can delay the current cache entry.
  41. * If the state is CE_RETRY, then this signals the filter that the
  42. * cache entry was requested before.
  43. */
  44. enum ce_delay_state state;
  45. /* List of filter drivers that signaled delayed blobs. */
  46. struct string_list filters;
  47. /* List of delayed blobs identified by their path. */
  48. struct string_list paths;
  49. };
  50. extern enum eol core_eol;
  51. extern char *check_roundtrip_encoding;
  52. const char *get_cached_convert_stats_ascii(const struct index_state *istate,
  53. const char *path);
  54. const char *get_wt_convert_stats_ascii(const char *path);
  55. const char *get_convert_attr_ascii(const struct index_state *istate,
  56. const char *path);
  57. /* returns 1 if *dst was used */
  58. int convert_to_git(const struct index_state *istate,
  59. const char *path, const char *src, size_t len,
  60. struct strbuf *dst, int conv_flags);
  61. int convert_to_working_tree(const struct index_state *istate,
  62. const char *path, const char *src,
  63. size_t len, struct strbuf *dst);
  64. int async_convert_to_working_tree(const struct index_state *istate,
  65. const char *path, const char *src,
  66. size_t len, struct strbuf *dst,
  67. void *dco);
  68. int async_query_available_blobs(const char *cmd,
  69. struct string_list *available_paths);
  70. int renormalize_buffer(const struct index_state *istate,
  71. const char *path, const char *src, size_t len,
  72. struct strbuf *dst);
  73. static inline int would_convert_to_git(const struct index_state *istate,
  74. const char *path)
  75. {
  76. return convert_to_git(istate, path, NULL, 0, NULL, 0);
  77. }
  78. /* Precondition: would_convert_to_git_filter_fd(path) == true */
  79. void convert_to_git_filter_fd(const struct index_state *istate,
  80. const char *path, int fd,
  81. struct strbuf *dst,
  82. int conv_flags);
  83. int would_convert_to_git_filter_fd(const struct index_state *istate,
  84. const char *path);
  85. /*
  86. * Reset the internal list of attributes used by convert_to_git and
  87. * convert_to_working_tree.
  88. */
  89. void reset_parsed_attributes(void);
  90. /*****************************************************************
  91. *
  92. * Streaming conversion support
  93. *
  94. *****************************************************************/
  95. struct stream_filter; /* opaque */
  96. struct stream_filter *get_stream_filter(const struct index_state *istate,
  97. const char *path,
  98. const struct object_id *);
  99. void free_stream_filter(struct stream_filter *);
  100. int is_null_stream_filter(struct stream_filter *);
  101. /*
  102. * Use as much input up to *isize_p and fill output up to *osize_p;
  103. * update isize_p and osize_p to indicate how much buffer space was
  104. * consumed and filled. Return 0 on success, non-zero on error.
  105. *
  106. * Some filters may need to buffer the input and look-ahead inside it
  107. * to decide what to output, and they may consume more than zero bytes
  108. * of input and still not produce any output. After feeding all the
  109. * input, pass NULL as input and keep calling this function, to let
  110. * such filters know there is no more input coming and it is time for
  111. * them to produce the remaining output based on the buffered input.
  112. */
  113. int stream_filter(struct stream_filter *,
  114. const char *input, size_t *isize_p,
  115. char *output, size_t *osize_p);
  116. #endif /* CONVERT_H */