packv4: nth_packed_object_sha1() packv4 support
[git/packv4.git] / convert.c
blob21908b10398492c0b07f705ed3b1ce7a06ac6b44
1 #include "cache.h"
2 #include "attr.h"
3 #include "run-command.h"
5 /*
6 * convert.c - convert a file when checking it out and checking it in.
8 * This should use the pathname to decide on whether it wants to do some
9 * more interesting conversions (automatic gzip/unzip, general format
10 * conversions etc etc), but by default it just does automatic CRLF<->LF
11 * translation when the "auto_crlf" option is set.
14 #define CRLF_GUESS (-1)
15 #define CRLF_BINARY 0
16 #define CRLF_TEXT 1
17 #define CRLF_INPUT 2
19 struct text_stat {
20 /* CR, LF and CRLF counts */
21 unsigned cr, lf, crlf;
23 /* These are just approximations! */
24 unsigned printable, nonprintable;
27 static void gather_stats(const char *buf, unsigned long size, struct text_stat *stats)
29 unsigned long i;
31 memset(stats, 0, sizeof(*stats));
33 for (i = 0; i < size; i++) {
34 unsigned char c = buf[i];
35 if (c == '\r') {
36 stats->cr++;
37 if (i+1 < size && buf[i+1] == '\n')
38 stats->crlf++;
39 continue;
41 if (c == '\n') {
42 stats->lf++;
43 continue;
45 if (c == 127)
46 /* DEL */
47 stats->nonprintable++;
48 else if (c < 32) {
49 switch (c) {
50 /* BS, HT, ESC and FF */
51 case '\b': case '\t': case '\033': case '\014':
52 stats->printable++;
53 break;
54 default:
55 stats->nonprintable++;
58 else
59 stats->printable++;
64 * The same heuristics as diff.c::mmfile_is_binary()
66 static int is_binary(unsigned long size, struct text_stat *stats)
69 if ((stats->printable >> 7) < stats->nonprintable)
70 return 1;
72 * Other heuristics? Average line length might be relevant,
73 * as might LF vs CR vs CRLF counts..
75 * NOTE! It might be normal to have a low ratio of CRLF to LF
76 * (somebody starts with a LF-only file and edits it with an editor
77 * that adds CRLF only to lines that are added..). But do we
78 * want to support CR-only? Probably not.
80 return 0;
83 static char *crlf_to_git(const char *path, const char *src, unsigned long *sizep, int action)
85 char *buffer, *dst;
86 unsigned long size, nsize;
87 struct text_stat stats;
89 if ((action == CRLF_BINARY) || !auto_crlf)
90 return NULL;
92 size = *sizep;
93 if (!size)
94 return NULL;
96 gather_stats(src, size, &stats);
98 /* No CR? Nothing to convert, regardless. */
99 if (!stats.cr)
100 return NULL;
102 if (action == CRLF_GUESS) {
104 * We're currently not going to even try to convert stuff
105 * that has bare CR characters. Does anybody do that crazy
106 * stuff?
108 if (stats.cr != stats.crlf)
109 return NULL;
112 * And add some heuristics for binary vs text, of course...
114 if (is_binary(size, &stats))
115 return NULL;
119 * Ok, allocate a new buffer, fill it in, and return it
120 * to let the caller know that we switched buffers.
122 nsize = size - stats.crlf;
123 buffer = xmalloc(nsize);
124 *sizep = nsize;
126 dst = buffer;
127 if (action == CRLF_GUESS) {
129 * If we guessed, we already know we rejected a file with
130 * lone CR, and we can strip a CR without looking at what
131 * follow it.
133 do {
134 unsigned char c = *src++;
135 if (c != '\r')
136 *dst++ = c;
137 } while (--size);
138 } else {
139 do {
140 unsigned char c = *src++;
141 if (! (c == '\r' && (1 < size && *src == '\n')))
142 *dst++ = c;
143 } while (--size);
146 return buffer;
149 static char *crlf_to_worktree(const char *path, const char *src, unsigned long *sizep, int action)
151 char *buffer, *dst;
152 unsigned long size, nsize;
153 struct text_stat stats;
154 unsigned char last;
156 if ((action == CRLF_BINARY) || (action == CRLF_INPUT) ||
157 auto_crlf <= 0)
158 return NULL;
160 size = *sizep;
161 if (!size)
162 return NULL;
164 gather_stats(src, size, &stats);
166 /* No LF? Nothing to convert, regardless. */
167 if (!stats.lf)
168 return NULL;
170 /* Was it already in CRLF format? */
171 if (stats.lf == stats.crlf)
172 return NULL;
174 if (action == CRLF_GUESS) {
175 /* If we have any bare CR characters, we're not going to touch it */
176 if (stats.cr != stats.crlf)
177 return NULL;
179 if (is_binary(size, &stats))
180 return NULL;
184 * Ok, allocate a new buffer, fill it in, and return it
185 * to let the caller know that we switched buffers.
187 nsize = size + stats.lf - stats.crlf;
188 buffer = xmalloc(nsize);
189 *sizep = nsize;
190 last = 0;
192 dst = buffer;
193 do {
194 unsigned char c = *src++;
195 if (c == '\n' && last != '\r')
196 *dst++ = '\r';
197 *dst++ = c;
198 last = c;
199 } while (--size);
201 return buffer;
204 static int filter_buffer(const char *path, const char *src,
205 unsigned long size, const char *cmd)
208 * Spawn cmd and feed the buffer contents through its stdin.
210 struct child_process child_process;
211 int pipe_feed[2];
212 int write_err, status;
214 memset(&child_process, 0, sizeof(child_process));
216 if (pipe(pipe_feed) < 0) {
217 error("cannot create pipe to run external filter %s", cmd);
218 return 1;
221 child_process.pid = fork();
222 if (child_process.pid < 0) {
223 error("cannot fork to run external filter %s", cmd);
224 close(pipe_feed[0]);
225 close(pipe_feed[1]);
226 return 1;
228 if (!child_process.pid) {
229 dup2(pipe_feed[0], 0);
230 close(pipe_feed[0]);
231 close(pipe_feed[1]);
232 execlp("sh", "sh", "-c", cmd, NULL);
233 return 1;
235 close(pipe_feed[0]);
237 write_err = (write_in_full(pipe_feed[1], src, size) < 0);
238 if (close(pipe_feed[1]))
239 write_err = 1;
240 if (write_err)
241 error("cannot feed the input to external filter %s", cmd);
243 status = finish_command(&child_process);
244 if (status)
245 error("external filter %s failed %d", cmd, -status);
246 return (write_err || status);
249 static char *apply_filter(const char *path, const char *src,
250 unsigned long *sizep, const char *cmd)
253 * Create a pipeline to have the command filter the buffer's
254 * contents.
256 * (child --> cmd) --> us
258 const int SLOP = 4096;
259 int pipe_feed[2];
260 int status;
261 char *dst;
262 unsigned long dstsize, dstalloc;
263 struct child_process child_process;
265 if (!cmd)
266 return NULL;
268 memset(&child_process, 0, sizeof(child_process));
270 if (pipe(pipe_feed) < 0) {
271 error("cannot create pipe to run external filter %s", cmd);
272 return NULL;
275 fflush(NULL);
276 child_process.pid = fork();
277 if (child_process.pid < 0) {
278 error("cannot fork to run external filter %s", cmd);
279 close(pipe_feed[0]);
280 close(pipe_feed[1]);
281 return NULL;
283 if (!child_process.pid) {
284 dup2(pipe_feed[1], 1);
285 close(pipe_feed[0]);
286 close(pipe_feed[1]);
287 exit(filter_buffer(path, src, *sizep, cmd));
289 close(pipe_feed[1]);
291 dstalloc = *sizep;
292 dst = xmalloc(dstalloc);
293 dstsize = 0;
295 while (1) {
296 ssize_t numread = xread(pipe_feed[0], dst + dstsize,
297 dstalloc - dstsize);
299 if (numread <= 0) {
300 if (!numread)
301 break;
302 error("read from external filter %s failed", cmd);
303 free(dst);
304 dst = NULL;
305 break;
307 dstsize += numread;
308 if (dstalloc <= dstsize + SLOP) {
309 dstalloc = dstsize + SLOP;
310 dst = xrealloc(dst, dstalloc);
313 if (close(pipe_feed[0])) {
314 error("read from external filter %s failed", cmd);
315 free(dst);
316 dst = NULL;
319 status = finish_command(&child_process);
320 if (status) {
321 error("external filter %s failed %d", cmd, -status);
322 free(dst);
323 dst = NULL;
326 if (dst)
327 *sizep = dstsize;
328 return dst;
331 static struct convert_driver {
332 const char *name;
333 struct convert_driver *next;
334 char *smudge;
335 char *clean;
336 } *user_convert, **user_convert_tail;
338 static int read_convert_config(const char *var, const char *value)
340 const char *ep, *name;
341 int namelen;
342 struct convert_driver *drv;
345 * External conversion drivers are configured using
346 * "filter.<name>.variable".
348 if (prefixcmp(var, "filter.") || (ep = strrchr(var, '.')) == var + 6)
349 return 0;
350 name = var + 7;
351 namelen = ep - name;
352 for (drv = user_convert; drv; drv = drv->next)
353 if (!strncmp(drv->name, name, namelen) && !drv->name[namelen])
354 break;
355 if (!drv) {
356 char *namebuf;
357 drv = xcalloc(1, sizeof(struct convert_driver));
358 namebuf = xmalloc(namelen + 1);
359 memcpy(namebuf, name, namelen);
360 namebuf[namelen] = 0;
361 drv->name = namebuf;
362 drv->next = NULL;
363 *user_convert_tail = drv;
364 user_convert_tail = &(drv->next);
367 ep++;
370 * filter.<name>.smudge and filter.<name>.clean specifies
371 * the command line:
373 * command-line
375 * The command-line will not be interpolated in any way.
378 if (!strcmp("smudge", ep)) {
379 if (!value)
380 return error("%s: lacks value", var);
381 drv->smudge = strdup(value);
382 return 0;
385 if (!strcmp("clean", ep)) {
386 if (!value)
387 return error("%s: lacks value", var);
388 drv->clean = strdup(value);
389 return 0;
391 return 0;
394 static void setup_convert_check(struct git_attr_check *check)
396 static struct git_attr *attr_crlf;
397 static struct git_attr *attr_ident;
398 static struct git_attr *attr_filter;
400 if (!attr_crlf) {
401 attr_crlf = git_attr("crlf", 4);
402 attr_ident = git_attr("ident", 5);
403 attr_filter = git_attr("filter", 6);
404 user_convert_tail = &user_convert;
405 git_config(read_convert_config);
407 check[0].attr = attr_crlf;
408 check[1].attr = attr_ident;
409 check[2].attr = attr_filter;
412 static int count_ident(const char *cp, unsigned long size)
415 * "$Id: 0000000000000000000000000000000000000000 $" <=> "$Id$"
417 int cnt = 0;
418 char ch;
420 while (size) {
421 ch = *cp++;
422 size--;
423 if (ch != '$')
424 continue;
425 if (size < 3)
426 break;
427 if (memcmp("Id", cp, 2))
428 continue;
429 ch = cp[2];
430 cp += 3;
431 size -= 3;
432 if (ch == '$')
433 cnt++; /* $Id$ */
434 if (ch != ':')
435 continue;
438 * "$Id: ... "; scan up to the closing dollar sign and discard.
440 while (size) {
441 ch = *cp++;
442 size--;
443 if (ch == '$') {
444 cnt++;
445 break;
449 return cnt;
452 static char *ident_to_git(const char *path, const char *src, unsigned long *sizep, int ident)
454 int cnt;
455 unsigned long size;
456 char *dst, *buf;
458 if (!ident)
459 return NULL;
460 size = *sizep;
461 cnt = count_ident(src, size);
462 if (!cnt)
463 return NULL;
464 buf = xmalloc(size);
466 for (dst = buf; size; size--) {
467 char ch = *src++;
468 *dst++ = ch;
469 if ((ch == '$') && (3 <= size) &&
470 !memcmp("Id:", src, 3)) {
471 unsigned long rem = size - 3;
472 const char *cp = src + 3;
473 do {
474 ch = *cp++;
475 if (ch == '$')
476 break;
477 rem--;
478 } while (rem);
479 if (!rem)
480 continue;
481 memcpy(dst, "Id$", 3);
482 dst += 3;
483 size -= (cp - src);
484 src = cp;
488 *sizep = dst - buf;
489 return buf;
492 static char *ident_to_worktree(const char *path, const char *src, unsigned long *sizep, int ident)
494 int cnt;
495 unsigned long size;
496 char *dst, *buf;
497 unsigned char sha1[20];
499 if (!ident)
500 return NULL;
502 size = *sizep;
503 cnt = count_ident(src, size);
504 if (!cnt)
505 return NULL;
507 hash_sha1_file(src, size, "blob", sha1);
508 buf = xmalloc(size + cnt * 43);
510 for (dst = buf; size; size--) {
511 const char *cp;
512 /* Fetch next source character, move the pointer on */
513 char ch = *src++;
514 /* Copy the current character to the destination */
515 *dst++ = ch;
516 /* If the current character is "$" or there are less than three
517 * remaining bytes or the two bytes following this one are not
518 * "Id", then simply read the next character */
519 if ((ch != '$') || (size < 3) || memcmp("Id", src, 2))
520 continue;
522 * Here when
523 * - There are more than 2 bytes remaining
524 * - The current three bytes are "$Id"
525 * with
526 * - ch == "$"
527 * - src[0] == "I"
531 * It's possible that an expanded Id has crept its way into the
532 * repository, we cope with that by stripping the expansion out
534 if (src[2] == ':') {
535 /* Expanded keywords have "$Id:" at the front */
537 /* discard up to but not including the closing $ */
538 unsigned long rem = size - 3;
539 /* Point at first byte after the ":" */
540 cp = src + 3;
542 * Throw away characters until either
543 * - we reach a "$"
544 * - we run out of bytes (rem == 0)
546 do {
547 ch = *cp;
548 if (ch == '$')
549 break;
550 cp++;
551 rem--;
552 } while (rem);
553 /* If the above finished because it ran out of characters, then
554 * this is an incomplete keyword, so don't run the expansion */
555 if (!rem)
556 continue;
557 } else if (src[2] == '$')
558 cp = src + 2;
559 else
560 /* Anything other than "$Id:XXX$" or $Id$ and we skip the
561 * expansion */
562 continue;
564 /* cp is now pointing at the last $ of the keyword */
566 memcpy(dst, "Id: ", 4);
567 dst += 4;
568 memcpy(dst, sha1_to_hex(sha1), 40);
569 dst += 40;
570 *dst++ = ' ';
572 /* Adjust for the characters we've discarded */
573 size -= (cp - src);
574 src = cp;
576 /* Copy the final "$" */
577 *dst++ = *src++;
578 size--;
581 *sizep = dst - buf;
582 return buf;
585 static int git_path_check_crlf(const char *path, struct git_attr_check *check)
587 const char *value = check->value;
589 if (ATTR_TRUE(value))
590 return CRLF_TEXT;
591 else if (ATTR_FALSE(value))
592 return CRLF_BINARY;
593 else if (ATTR_UNSET(value))
595 else if (!strcmp(value, "input"))
596 return CRLF_INPUT;
597 return CRLF_GUESS;
600 static struct convert_driver *git_path_check_convert(const char *path,
601 struct git_attr_check *check)
603 const char *value = check->value;
604 struct convert_driver *drv;
606 if (ATTR_TRUE(value) || ATTR_FALSE(value) || ATTR_UNSET(value))
607 return NULL;
608 for (drv = user_convert; drv; drv = drv->next)
609 if (!strcmp(value, drv->name))
610 return drv;
611 return NULL;
614 static int git_path_check_ident(const char *path, struct git_attr_check *check)
616 const char *value = check->value;
618 return !!ATTR_TRUE(value);
621 char *convert_to_git(const char *path, const char *src, unsigned long *sizep)
623 struct git_attr_check check[3];
624 int crlf = CRLF_GUESS;
625 int ident = 0;
626 char *filter = NULL;
627 char *buf, *buf2;
629 setup_convert_check(check);
630 if (!git_checkattr(path, ARRAY_SIZE(check), check)) {
631 struct convert_driver *drv;
632 crlf = git_path_check_crlf(path, check + 0);
633 ident = git_path_check_ident(path, check + 1);
634 drv = git_path_check_convert(path, check + 2);
635 if (drv && drv->clean)
636 filter = drv->clean;
639 buf = apply_filter(path, src, sizep, filter);
641 buf2 = crlf_to_git(path, buf ? buf : src, sizep, crlf);
642 if (buf2) {
643 free(buf);
644 buf = buf2;
647 buf2 = ident_to_git(path, buf ? buf : src, sizep, ident);
648 if (buf2) {
649 free(buf);
650 buf = buf2;
653 return buf;
656 char *convert_to_working_tree(const char *path, const char *src, unsigned long *sizep)
658 struct git_attr_check check[3];
659 int crlf = CRLF_GUESS;
660 int ident = 0;
661 char *filter = NULL;
662 char *buf, *buf2;
664 setup_convert_check(check);
665 if (!git_checkattr(path, ARRAY_SIZE(check), check)) {
666 struct convert_driver *drv;
667 crlf = git_path_check_crlf(path, check + 0);
668 ident = git_path_check_ident(path, check + 1);
669 drv = git_path_check_convert(path, check + 2);
670 if (drv && drv->smudge)
671 filter = drv->smudge;
674 buf = ident_to_worktree(path, src, sizep, ident);
676 buf2 = crlf_to_worktree(path, buf ? buf : src, sizep, crlf);
677 if (buf2) {
678 free(buf);
679 buf = buf2;
682 buf2 = apply_filter(path, buf ? buf : src, sizep, filter);
683 if (buf2) {
684 free(buf);
685 buf = buf2;
688 return buf;
691 void *convert_sha1_file(const char *path, const unsigned char *sha1,
692 unsigned int mode, enum object_type *type,
693 unsigned long *size)
695 void *buffer = read_sha1_file(sha1, type, size);
696 if (S_ISREG(mode) && buffer) {
697 void *converted = convert_to_working_tree(path, buffer, size);
698 if (converted) {
699 free(buffer);
700 buffer = converted;
703 return buffer;