t4200: use cut instead of sed
[git/peff.git] / builtin-fetch-pack.c
blob7b280242244094774c909f67d6a8ed7902389c5a
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "remote.h"
11 #include "run-command.h"
13 static int transfer_unpack_limit = -1;
14 static int fetch_unpack_limit = -1;
15 static int unpack_limit = 100;
16 static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
20 static const char fetch_pack_usage[] =
21 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
23 #define COMPLETE (1U << 0)
24 #define COMMON (1U << 1)
25 #define COMMON_REF (1U << 2)
26 #define SEEN (1U << 3)
27 #define POPPED (1U << 4)
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
33 #define MAX_IN_VAIN 256
35 static struct commit_list *rev_list;
36 static int non_common_revs, multi_ack, use_sideband;
38 static void rev_list_push(struct commit *commit, int mark)
40 if (!(commit->object.flags & mark)) {
41 commit->object.flags |= mark;
43 if (!(commit->object.parsed))
44 if (parse_commit(commit))
45 return;
47 insert_by_date(commit, &rev_list);
49 if (!(commit->object.flags & COMMON))
50 non_common_revs++;
54 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
56 struct object *o = deref_tag(parse_object(sha1), path, 0);
58 if (o && o->type == OBJ_COMMIT)
59 rev_list_push((struct commit *)o, SEEN);
61 return 0;
65 This function marks a rev and its ancestors as common.
66 In some cases, it is desirable to mark only the ancestors (for example
67 when only the server does not yet know that they are common).
70 static void mark_common(struct commit *commit,
71 int ancestors_only, int dont_parse)
73 if (commit != NULL && !(commit->object.flags & COMMON)) {
74 struct object *o = (struct object *)commit;
76 if (!ancestors_only)
77 o->flags |= COMMON;
79 if (!(o->flags & SEEN))
80 rev_list_push(commit, SEEN);
81 else {
82 struct commit_list *parents;
84 if (!ancestors_only && !(o->flags & POPPED))
85 non_common_revs--;
86 if (!o->parsed && !dont_parse)
87 if (parse_commit(commit))
88 return;
90 for (parents = commit->parents;
91 parents;
92 parents = parents->next)
93 mark_common(parents->item, 0, dont_parse);
99 Get the next rev to send, ignoring the common.
102 static const unsigned char* get_rev(void)
104 struct commit *commit = NULL;
106 while (commit == NULL) {
107 unsigned int mark;
108 struct commit_list *parents = NULL;
110 if (rev_list == NULL || non_common_revs == 0)
111 return NULL;
113 commit = rev_list->item;
114 if (!(commit->object.parsed))
115 if (!parse_commit(commit))
116 parents = commit->parents;
118 commit->object.flags |= POPPED;
119 if (!(commit->object.flags & COMMON))
120 non_common_revs--;
122 if (commit->object.flags & COMMON) {
123 /* do not send "have", and ignore ancestors */
124 commit = NULL;
125 mark = COMMON | SEEN;
126 } else if (commit->object.flags & COMMON_REF)
127 /* send "have", and ignore ancestors */
128 mark = COMMON | SEEN;
129 else
130 /* send "have", also for its ancestors */
131 mark = SEEN;
133 while (parents) {
134 if (!(parents->item->object.flags & SEEN))
135 rev_list_push(parents->item, mark);
136 if (mark & COMMON)
137 mark_common(parents->item, 1, 0);
138 parents = parents->next;
141 rev_list = rev_list->next;
144 return commit->object.sha1;
147 static int find_common(int fd[2], unsigned char *result_sha1,
148 struct ref *refs)
150 int fetching;
151 int count = 0, flushes = 0, retval;
152 const unsigned char *sha1;
153 unsigned in_vain = 0;
154 int got_continue = 0;
156 for_each_ref(rev_list_insert_ref, NULL);
158 fetching = 0;
159 for ( ; refs ; refs = refs->next) {
160 unsigned char *remote = refs->old_sha1;
161 struct object *o;
164 * If that object is complete (i.e. it is an ancestor of a
165 * local ref), we tell them we have it but do not have to
166 * tell them about its ancestors, which they already know
167 * about.
169 * We use lookup_object here because we are only
170 * interested in the case we *know* the object is
171 * reachable and we have already scanned it.
173 if (((o = lookup_object(remote)) != NULL) &&
174 (o->flags & COMPLETE)) {
175 continue;
178 if (!fetching)
179 packet_write(fd[1], "want %s%s%s%s%s%s%s%s\n",
180 sha1_to_hex(remote),
181 (multi_ack ? " multi_ack" : ""),
182 (use_sideband == 2 ? " side-band-64k" : ""),
183 (use_sideband == 1 ? " side-band" : ""),
184 (args.use_thin_pack ? " thin-pack" : ""),
185 (args.no_progress ? " no-progress" : ""),
186 (args.include_tag ? " include-tag" : ""),
187 " ofs-delta");
188 else
189 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
190 fetching++;
192 if (is_repository_shallow())
193 write_shallow_commits(fd[1], 1);
194 if (args.depth > 0)
195 packet_write(fd[1], "deepen %d", args.depth);
196 packet_flush(fd[1]);
197 if (!fetching)
198 return 1;
200 if (args.depth > 0) {
201 char line[1024];
202 unsigned char sha1[20];
203 int len;
205 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
206 if (!prefixcmp(line, "shallow ")) {
207 if (get_sha1_hex(line + 8, sha1))
208 die("invalid shallow line: %s", line);
209 register_shallow(sha1);
210 continue;
212 if (!prefixcmp(line, "unshallow ")) {
213 if (get_sha1_hex(line + 10, sha1))
214 die("invalid unshallow line: %s", line);
215 if (!lookup_object(sha1))
216 die("object not found: %s", line);
217 /* make sure that it is parsed as shallow */
218 if (!parse_object(sha1))
219 die("error in object: %s", line);
220 if (unregister_shallow(sha1))
221 die("no shallow found: %s", line);
222 continue;
224 die("expected shallow/unshallow, got %s", line);
228 flushes = 0;
229 retval = -1;
230 while ((sha1 = get_rev())) {
231 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
232 if (args.verbose)
233 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
234 in_vain++;
235 if (!(31 & ++count)) {
236 int ack;
238 packet_flush(fd[1]);
239 flushes++;
242 * We keep one window "ahead" of the other side, and
243 * will wait for an ACK only on the next one
245 if (count == 32)
246 continue;
248 do {
249 ack = get_ack(fd[0], result_sha1);
250 if (args.verbose && ack)
251 fprintf(stderr, "got ack %d %s\n", ack,
252 sha1_to_hex(result_sha1));
253 if (ack == 1) {
254 flushes = 0;
255 multi_ack = 0;
256 retval = 0;
257 goto done;
258 } else if (ack == 2) {
259 struct commit *commit =
260 lookup_commit(result_sha1);
261 mark_common(commit, 0, 1);
262 retval = 0;
263 in_vain = 0;
264 got_continue = 1;
266 } while (ack);
267 flushes--;
268 if (got_continue && MAX_IN_VAIN < in_vain) {
269 if (args.verbose)
270 fprintf(stderr, "giving up\n");
271 break; /* give up */
275 done:
276 packet_write(fd[1], "done\n");
277 if (args.verbose)
278 fprintf(stderr, "done\n");
279 if (retval != 0) {
280 multi_ack = 0;
281 flushes++;
283 while (flushes || multi_ack) {
284 int ack = get_ack(fd[0], result_sha1);
285 if (ack) {
286 if (args.verbose)
287 fprintf(stderr, "got ack (%d) %s\n", ack,
288 sha1_to_hex(result_sha1));
289 if (ack == 1)
290 return 0;
291 multi_ack = 1;
292 continue;
294 flushes--;
296 return retval;
299 static struct commit_list *complete;
301 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
303 struct object *o = parse_object(sha1);
305 while (o && o->type == OBJ_TAG) {
306 struct tag *t = (struct tag *) o;
307 if (!t->tagged)
308 break; /* broken repository */
309 o->flags |= COMPLETE;
310 o = parse_object(t->tagged->sha1);
312 if (o && o->type == OBJ_COMMIT) {
313 struct commit *commit = (struct commit *)o;
314 commit->object.flags |= COMPLETE;
315 insert_by_date(commit, &complete);
317 return 0;
320 static void mark_recent_complete_commits(unsigned long cutoff)
322 while (complete && cutoff <= complete->item->date) {
323 if (args.verbose)
324 fprintf(stderr, "Marking %s as complete\n",
325 sha1_to_hex(complete->item->object.sha1));
326 pop_most_recent_commit(&complete, COMPLETE);
330 static void filter_refs(struct ref **refs, int nr_match, char **match)
332 struct ref **return_refs;
333 struct ref *newlist = NULL;
334 struct ref **newtail = &newlist;
335 struct ref *ref, *next;
336 struct ref *fastarray[32];
338 if (nr_match && !args.fetch_all) {
339 if (ARRAY_SIZE(fastarray) < nr_match)
340 return_refs = xcalloc(nr_match, sizeof(struct ref *));
341 else {
342 return_refs = fastarray;
343 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
346 else
347 return_refs = NULL;
349 for (ref = *refs; ref; ref = next) {
350 next = ref->next;
351 if (!memcmp(ref->name, "refs/", 5) &&
352 check_ref_format(ref->name + 5))
353 ; /* trash */
354 else if (args.fetch_all &&
355 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
356 *newtail = ref;
357 ref->next = NULL;
358 newtail = &ref->next;
359 continue;
361 else {
362 int order = path_match(ref->name, nr_match, match);
363 if (order) {
364 return_refs[order-1] = ref;
365 continue; /* we will link it later */
368 free(ref);
371 if (!args.fetch_all) {
372 int i;
373 for (i = 0; i < nr_match; i++) {
374 ref = return_refs[i];
375 if (ref) {
376 *newtail = ref;
377 ref->next = NULL;
378 newtail = &ref->next;
381 if (return_refs != fastarray)
382 free(return_refs);
384 *refs = newlist;
387 static int everything_local(struct ref **refs, int nr_match, char **match)
389 struct ref *ref;
390 int retval;
391 unsigned long cutoff = 0;
393 save_commit_buffer = 0;
395 for (ref = *refs; ref; ref = ref->next) {
396 struct object *o;
398 o = parse_object(ref->old_sha1);
399 if (!o)
400 continue;
402 /* We already have it -- which may mean that we were
403 * in sync with the other side at some time after
404 * that (it is OK if we guess wrong here).
406 if (o->type == OBJ_COMMIT) {
407 struct commit *commit = (struct commit *)o;
408 if (!cutoff || cutoff < commit->date)
409 cutoff = commit->date;
413 if (!args.depth) {
414 for_each_ref(mark_complete, NULL);
415 if (cutoff)
416 mark_recent_complete_commits(cutoff);
420 * Mark all complete remote refs as common refs.
421 * Don't mark them common yet; the server has to be told so first.
423 for (ref = *refs; ref; ref = ref->next) {
424 struct object *o = deref_tag(lookup_object(ref->old_sha1),
425 NULL, 0);
427 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
428 continue;
430 if (!(o->flags & SEEN)) {
431 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
433 mark_common((struct commit *)o, 1, 1);
437 filter_refs(refs, nr_match, match);
439 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
440 const unsigned char *remote = ref->old_sha1;
441 unsigned char local[20];
442 struct object *o;
444 o = lookup_object(remote);
445 if (!o || !(o->flags & COMPLETE)) {
446 retval = 0;
447 if (!args.verbose)
448 continue;
449 fprintf(stderr,
450 "want %s (%s)\n", sha1_to_hex(remote),
451 ref->name);
452 continue;
455 hashcpy(ref->new_sha1, local);
456 if (!args.verbose)
457 continue;
458 fprintf(stderr,
459 "already have %s (%s)\n", sha1_to_hex(remote),
460 ref->name);
462 return retval;
465 static int sideband_demux(int fd, void *data)
467 int *xd = data;
469 return recv_sideband("fetch-pack", xd[0], fd, 2);
472 static int get_pack(int xd[2], char **pack_lockfile)
474 struct async demux;
475 const char *argv[20];
476 char keep_arg[256];
477 char hdr_arg[256];
478 const char **av;
479 int do_keep = args.keep_pack;
480 struct child_process cmd;
482 memset(&demux, 0, sizeof(demux));
483 if (use_sideband) {
484 /* xd[] is talking with upload-pack; subprocess reads from
485 * xd[0], spits out band#2 to stderr, and feeds us band#1
486 * through demux->out.
488 demux.proc = sideband_demux;
489 demux.data = xd;
490 if (start_async(&demux))
491 die("fetch-pack: unable to fork off sideband"
492 " demultiplexer");
494 else
495 demux.out = xd[0];
497 memset(&cmd, 0, sizeof(cmd));
498 cmd.argv = argv;
499 av = argv;
500 *hdr_arg = 0;
501 if (!args.keep_pack && unpack_limit) {
502 struct pack_header header;
504 if (read_pack_header(demux.out, &header))
505 die("protocol error: bad pack header");
506 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
507 ntohl(header.hdr_version), ntohl(header.hdr_entries));
508 if (ntohl(header.hdr_entries) < unpack_limit)
509 do_keep = 0;
510 else
511 do_keep = 1;
514 if (do_keep) {
515 if (pack_lockfile)
516 cmd.out = -1;
517 *av++ = "index-pack";
518 *av++ = "--stdin";
519 if (!args.quiet && !args.no_progress)
520 *av++ = "-v";
521 if (args.use_thin_pack)
522 *av++ = "--fix-thin";
523 if (args.lock_pack || unpack_limit) {
524 int s = sprintf(keep_arg,
525 "--keep=fetch-pack %d on ", getpid());
526 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
527 strcpy(keep_arg + s, "localhost");
528 *av++ = keep_arg;
531 else {
532 *av++ = "unpack-objects";
533 if (args.quiet)
534 *av++ = "-q";
536 if (*hdr_arg)
537 *av++ = hdr_arg;
538 *av++ = NULL;
540 cmd.in = demux.out;
541 cmd.git_cmd = 1;
542 if (start_command(&cmd))
543 die("fetch-pack: unable to fork off %s", argv[0]);
544 if (do_keep && pack_lockfile) {
545 *pack_lockfile = index_pack_lockfile(cmd.out);
546 close(cmd.out);
549 if (finish_command(&cmd))
550 die("%s failed", argv[0]);
551 if (use_sideband && finish_async(&demux))
552 die("error in sideband demultiplexer");
553 return 0;
556 static struct ref *do_fetch_pack(int fd[2],
557 const struct ref *orig_ref,
558 int nr_match,
559 char **match,
560 char **pack_lockfile)
562 struct ref *ref = copy_ref_list(orig_ref);
563 unsigned char sha1[20];
565 if (is_repository_shallow() && !server_supports("shallow"))
566 die("Server does not support shallow clients");
567 if (server_supports("multi_ack")) {
568 if (args.verbose)
569 fprintf(stderr, "Server supports multi_ack\n");
570 multi_ack = 1;
572 if (server_supports("side-band-64k")) {
573 if (args.verbose)
574 fprintf(stderr, "Server supports side-band-64k\n");
575 use_sideband = 2;
577 else if (server_supports("side-band")) {
578 if (args.verbose)
579 fprintf(stderr, "Server supports side-band\n");
580 use_sideband = 1;
582 if (everything_local(&ref, nr_match, match)) {
583 packet_flush(fd[1]);
584 goto all_done;
586 if (find_common(fd, sha1, ref) < 0)
587 if (!args.keep_pack)
588 /* When cloning, it is not unusual to have
589 * no common commit.
591 fprintf(stderr, "warning: no common commits\n");
593 if (get_pack(fd, pack_lockfile))
594 die("git-fetch-pack: fetch failed.");
596 all_done:
597 return ref;
600 static int remove_duplicates(int nr_heads, char **heads)
602 int src, dst;
604 for (src = dst = 0; src < nr_heads; src++) {
605 /* If heads[src] is different from any of
606 * heads[0..dst], push it in.
608 int i;
609 for (i = 0; i < dst; i++) {
610 if (!strcmp(heads[i], heads[src]))
611 break;
613 if (i < dst)
614 continue;
615 if (src != dst)
616 heads[dst] = heads[src];
617 dst++;
619 return dst;
622 static int fetch_pack_config(const char *var, const char *value)
624 if (strcmp(var, "fetch.unpacklimit") == 0) {
625 fetch_unpack_limit = git_config_int(var, value);
626 return 0;
629 if (strcmp(var, "transfer.unpacklimit") == 0) {
630 transfer_unpack_limit = git_config_int(var, value);
631 return 0;
634 return git_default_config(var, value);
637 static struct lock_file lock;
639 static void fetch_pack_setup(void)
641 static int did_setup;
642 if (did_setup)
643 return;
644 git_config(fetch_pack_config);
645 if (0 <= transfer_unpack_limit)
646 unpack_limit = transfer_unpack_limit;
647 else if (0 <= fetch_unpack_limit)
648 unpack_limit = fetch_unpack_limit;
649 did_setup = 1;
652 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
654 int i, ret, nr_heads;
655 struct ref *ref = NULL;
656 char *dest = NULL, **heads;
657 int fd[2];
658 struct child_process *conn;
660 nr_heads = 0;
661 heads = NULL;
662 for (i = 1; i < argc; i++) {
663 const char *arg = argv[i];
665 if (*arg == '-') {
666 if (!prefixcmp(arg, "--upload-pack=")) {
667 args.uploadpack = arg + 14;
668 continue;
670 if (!prefixcmp(arg, "--exec=")) {
671 args.uploadpack = arg + 7;
672 continue;
674 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
675 args.quiet = 1;
676 continue;
678 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
679 args.lock_pack = args.keep_pack;
680 args.keep_pack = 1;
681 continue;
683 if (!strcmp("--thin", arg)) {
684 args.use_thin_pack = 1;
685 continue;
687 if (!strcmp("--include-tag", arg)) {
688 args.include_tag = 1;
689 continue;
691 if (!strcmp("--all", arg)) {
692 args.fetch_all = 1;
693 continue;
695 if (!strcmp("-v", arg)) {
696 args.verbose = 1;
697 continue;
699 if (!prefixcmp(arg, "--depth=")) {
700 args.depth = strtol(arg + 8, NULL, 0);
701 continue;
703 if (!strcmp("--no-progress", arg)) {
704 args.no_progress = 1;
705 continue;
707 usage(fetch_pack_usage);
709 dest = (char *)arg;
710 heads = (char **)(argv + i + 1);
711 nr_heads = argc - i - 1;
712 break;
714 if (!dest)
715 usage(fetch_pack_usage);
717 conn = git_connect(fd, (char *)dest, args.uploadpack,
718 args.verbose ? CONNECT_VERBOSE : 0);
719 if (conn) {
720 get_remote_heads(fd[0], &ref, 0, NULL, 0);
722 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
723 close(fd[0]);
724 close(fd[1]);
725 if (finish_connect(conn))
726 ref = NULL;
727 } else {
728 ref = NULL;
730 ret = !ref;
732 if (!ret && nr_heads) {
733 /* If the heads to pull were given, we should have
734 * consumed all of them by matching the remote.
735 * Otherwise, 'git-fetch remote no-such-ref' would
736 * silently succeed without issuing an error.
738 for (i = 0; i < nr_heads; i++)
739 if (heads[i] && heads[i][0]) {
740 error("no such remote ref %s", heads[i]);
741 ret = 1;
744 while (ref) {
745 printf("%s %s\n",
746 sha1_to_hex(ref->old_sha1), ref->name);
747 ref = ref->next;
750 return ret;
753 struct ref *fetch_pack(struct fetch_pack_args *my_args,
754 int fd[], struct child_process *conn,
755 const struct ref *ref,
756 const char *dest,
757 int nr_heads,
758 char **heads,
759 char **pack_lockfile)
761 struct stat st;
762 struct ref *ref_cpy;
764 fetch_pack_setup();
765 memcpy(&args, my_args, sizeof(args));
766 if (args.depth > 0) {
767 if (stat(git_path("shallow"), &st))
768 st.st_mtime = 0;
771 if (heads && nr_heads)
772 nr_heads = remove_duplicates(nr_heads, heads);
773 if (!ref) {
774 packet_flush(fd[1]);
775 die("no matching remote head");
777 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
779 if (args.depth > 0) {
780 struct cache_time mtime;
781 char *shallow = git_path("shallow");
782 int fd;
784 mtime.sec = st.st_mtime;
785 #ifdef USE_NSEC
786 mtime.usec = st.st_mtim.usec;
787 #endif
788 if (stat(shallow, &st)) {
789 if (mtime.sec)
790 die("shallow file was removed during fetch");
791 } else if (st.st_mtime != mtime.sec
792 #ifdef USE_NSEC
793 || st.st_mtim.usec != mtime.usec
794 #endif
796 die("shallow file was changed during fetch");
798 fd = hold_lock_file_for_update(&lock, shallow, 1);
799 if (!write_shallow_commits(fd, 0)) {
800 unlink(shallow);
801 rollback_lock_file(&lock);
802 } else {
803 commit_lock_file(&lock);
807 return ref_cpy;