cg-fetch

   1 #!/usr/bin/env bash
   2 #
   3 # Fetch changes from a remote branch to the local repository
   4 # Copyright (c) Petr Baudis, 2005.
   5 #
   6 # Takes the branch name as an argument. If no branch name was specified,
   7 # the default remote branch for the current branch (as selected by
   8 # `cg-switch -o`, that is the 'branch."currenthead".merge' option) is used.
   9 # If no such option exists, 'origin' is used.
  10 #
  11 # This will fetch the latest changes from a remote repository to the
  12 # corresponding branch in your local repository. Note that this operation
  13 # does not involve merging those changes to your own branch - that is being
  14 # done by the `cg-merge` command. `cg-update` exists to conveniently bundle
  15 # the act of fetching and merging to your working branch together.
  16 #
  17 # Before the first fetch, you have to tell Cogito about the remote branch.
  18 # This should be done by the `cg-branch-add` command. See its documentation
  19 # for the list of supported fetching protocols and other details. Note that
  20 # one exception to this is the 'origin' branch, which was set to the location
  21 # of the source repository if you created yours using the `cg-clone` command.
  22 #
  23 # Note that in the GIT newspeak, the operation being performed by cg-update
  24 # is now called 'pull', even though in the past and in many other version
  25 # control systems systems, 'pull' is the name for the operation performed by
  26 # `cg-fetch`. Please do not let this confuse you. (Cogito won't call this
  27 # 'update' operation 'pull', since about everyone but GIT and BK users uses
  28 # it in the 'fetch' meaning.)
  29 #
  30 # OPTIONS
  31 # -------
  32 # -f:: Force the complete fetch even if the heads are the same.
  33 #       Force the complete fetch even if the heads are the same.
  34 #
  35 # -v:: Enable verbosity
  36 #       Display more verbose output - most notably list all the files
  37 #       touched by the fetched changes. Use twice to get even more verbosity,
  38 #       that is raw progress information instead of the progress bar.
  39 #
  40 # ENVIRONMENT
  41 # -----------
  42 # RSYNC::
  43 #       The command to invoke when we want to call the rsync tool (only used
  44 #       when fetching over the rsync protocol). Defaults to 'rsync'.
  45 #
  46 # RSYNC_FLAGS::
  47 #       Additional flags to be passed to the rsync tool when fetching over
  48 #       the rsync protocol.
  49
  50 # Testsuite: Largely covered (t91xx testsuite family, incomplete coverage)
  51
  52
  53 USAGE="cg-fetch [-f] [-v] [BRANCH_NAME]"
  54 _git_wc_unneeded=1
  55
  56 . "${COGITO_LIB}"cg-Xlib || exit 1
  57
  58
  59 fetch_progress()
  60 {
  61         [ $verbose -ge 2 ] && exec cat
  62         if [ -t 1 ]; then
  63                 exec "${COGITO_LIB}"cg-Xfetchprogress "$_git_objects"
  64         else
  65                 exec cat >/dev/null
  66         fi
  67 }
  68
  69 show_changes_summary()
  70 {
  71         local orig_head="$1"
  72         local new_head="$2"
  73         if [ ! "$orig_head" ]; then
  74                 echo "New branch: $new_head"
  75
  76         elif [ "$orig_head" != "$new_head" ]; then
  77                 echo "Tree change: $orig_head..$new_head"
  78                 [ $verbose -ge 1 ] && git-diff-tree --abbrev -r "$(cg-object-id -t "$orig_head")" "$(cg-object-id -t "$new_head")"
  79         else
  80                 echo "Up to date."
  81         fi
  82 }
  83
  84 initial_done()
  85 {
  86         rm -f ___
  87         git-update-ref "refs/heads/master" "$(get_ref "refs/heads/origin")" ||
  88                 die "initial checkout failed"
  89         if [ -s "$_git/info/cg-fetch-initial-wcless" ]; then
  90                 rm "$_git/info/cg-fetch-initial-wcless"
  91         else
  92                 git-read-tree HEAD &&
  93                 git-checkout-index -a &&
  94                 git-update-index --refresh ||
  95                 die "initial checkout failed"
  96         fi
  97         rm "$_git/info/cg-fetch-initial"
  98 }
  99
 100
 101 get_rsync()
 102 {
 103         [ "$1" = "-b" ] && shift
 104
 105         redir=
 106         if [ "$1" = "-i" ]; then # ignore-errors
 107                 redir="2>/dev/null"
 108                 shift
 109         fi
 110
 111         filter="cat"
 112         if [ "$1" = "-s" ]; then # subsequent
 113                 # We already saw the MOTD, thank you very much.
 114                 filter="grep -v ^MOTD:"
 115                 shift
 116         fi
 117
 118         appenduri=
 119         if [ "$1" = "-d" ]; then # directory
 120                 appenduri="/." # CowboyNeal
 121                 shift
 122         fi
 123
 124         echo "${RSYNC:-rsync}" $RSYNC_FLAGS -v --partial -Lr \
 125                 "$1$appenduri" "$2$appenduri" $redir
 126         eval '"${RSYNC:-rsync}"' $RSYNC_FLAGS -v --partial -Lr \
 127                 '"$1$appenduri"' '"$2$appenduri"' $redir | $filter
 128         return ${PIPESTATUS[0]}
 129 }
 130
 131 fetch_rsync_verify()
 132 {
 133         if [ $verbose -ge 2 ]; then
 134                 # We must not pipe to prevent buffered I/O
 135                 get_rsync -s -d "$2/objects" "$_git_objects"
 136         else
 137                 get_rsync -s -d "$2/objects" "$_git_objects" | fetch_progress
 138         fi
 139
 140         ret=${PIPESTATUS[0]}
 141         if [ "$3" ] && [ "$ret" -eq "0" ]; then
 142                 if [ "$orig_head" ]; then
 143                         git-rev-list --objects $new_head ^$orig_head |
 144                                 while read obj type; do
 145                                         git-cat-file -t $obj >/dev/null || exit $?
 146                                 done ||
 147                         die "rsync fetch incomplete, some objects missing"
 148                 fi
 149                 git-update-ref "refs/$3" "$1"
 150         fi
 151         return $ret
 152 }
 153
 154
 155 fetch_rsync()
 156 {
 157         if [ x"$1" = x"--stdin" ]; then
 158                 while read c w; do
 159                         git-update-ref "refs/$w" "$c"
 160                 done
 161         else
 162                 fetch_rsync_verify "$1" "$2" "$3"
 163         fi
 164 }
 165
 166 get_http()
 167 {
 168         [ "$1" = "-b" ] && shift
 169         [ "$1" = "-i" ] && shift
 170         [ "$1" = "-s" ] && shift
 171         [ "$1" = "-d" ] && die "INTERNAL ERROR: HTTP recursive not implemented"
 172
 173         src="$1"
 174         dest="$2"
 175
 176         curl_extra_args=
 177         [ "$GIT_SSL_NO_VERIFY" ] && curl_extra_args="-k"
 178         curl -nsfL $curl_extra_args -o "$dest" "$src"
 179 }
 180
 181 fetch_http()
 182 {
 183         whead=
 184         [ "$3" ] && whead="-w $3"
 185         (git-http-fetch -a -v $whead $recovery "$1" "$2/" 2>&1 /dev/null) | fetch_progress
 186         return ${PIPESTATUS[0]}
 187 }
 188
 189
 190 get_local()
 191 {
 192         #cp_flags_l="-v"
 193         cp_flags_l=
 194         if [ "$1" = "-b" ]; then
 195                 # Dereference symlinks
 196                 cp_flags_l="$cp_flags_l -L"
 197                 shift
 198         else
 199                 cp_flags_l="$cp_flags_l -pRP"
 200         fi
 201
 202         [ "$1" = "-i" ] && shift
 203         [ "$1" = "-s" ] && shift
 204         [ "$1" = "-d" ] && die "INTERNAL ERROR: local-fetch recursive not implemented"
 205
 206         src="$1"
 207         dest="$2"
 208
 209         cp $cp_flags_l "$src" "$dest"
 210 }
 211
 212 fetch_local()
 213 {
 214         whead=
 215         [ "$3" ] && whead="-w $3"
 216         (git-local-fetch -a -l -v $whead $recovery "$1" "$2" 2>&1 /dev/null) | fetch_progress
 217         return ${PIPESTATUS[0]}
 218 }
 219
 220
 221 fetch_tags()
 222 {
 223         echo -n "Fetching tags... "
 224
 225         # FIXME: Warn about conflicting tag names?
 226
 227         if [ "$get" = "get_rsync" ]; then
 228                 echo
 229                 warn "WHAT I'M DOING NOW IS RACY AND BROKEN IF YOU USE PACKED REFS!"
 230                 warn "Please switch from rsync to something else."
 231                 [ -d "$_git/refs/tags" ] || mkdir -p "$_git/refs/tags"
 232                 if ! $get -i -s -d "$uri/refs/tags" "$_git/refs/tags"; then
 233                         echo "unable to get tags list (non-fatal)" >&2
 234                         return $?
 235                 fi
 236         fi
 237
 238         last_tag=""
 239         (
 240         git-ls-remote --tags "$uri" |
 241                 # SHA1 refs/tags/v0.99.8^{} --> SHA1 tags/v0.99.8
 242                 # where SHA1 is the object v0.99.8 tag points at.
 243                 sed -n -e 's:\([^       ]\)     refs/\(tags/.*\)^{}$:\1 \2:p' \
 244                        -e 's:\([^       ]\)     refs/\(tags/.*\)$:\1 \2:p' | \
 245                 (while read sha1 tagname; do
 246                         # Do we have the tag itself?
 247                         exists_ref "refs/$tagname" && continue
 248                         # Do we have the object pointed at by the tag?
 249                         git-cat-file -t "$sha1" >/dev/null 2>&1 || continue
 250
 251                         # In common case we will get both "normal" and ^{} entries.
 252                         # Filter out the dupes.
 253                         [ "$last_tag" = "$tagname" ] && continue
 254                         last_tag="$tagname"
 255
 256                         # if so, fetch the tag -- which should be
 257                         # a cheap operation -- to complete the chain.
 258                         echo -n "${tagname#tags/} " >&3
 259                         echo -e "$tagname"\\t"$tagname"
 260                 done; echo >&3 ) |
 261                 sort | uniq | $fetch --stdin "$uri"
 262         ) 3>&1
 263         if [ "${PIPESTATUS[0]}" -ne 0 -o "$?" -ne 0 ]; then
 264                 echo "unable to fetch tags (non-fatal)" >&2
 265         fi
 266         return 0
 267 }
 268
 269
 270 recovery=
 271 verbose=0
 272 while optparse; do
 273         if optparse -f; then
 274                 # When forcing, let the fetch tools make more extensive
 275                 # walk over the dependency tree with --recover.
 276                 recovery=--recover
 277         elif optparse -v; then
 278                 verbose=$((verbose+1))
 279         else
 280                 optfail
 281         fi
 282 done
 283
 284 name="${ARGS[0]}"
 285
 286 [ "$name" ] || name="$(choose_origin branches "where to fetch from?")" || exit 1
 287 uri=$(cat "$_git/branches/$name" 2>/dev/null) || die "unknown branch: $name"
 288
 289 rembranch=
 290 if echo "$uri" | grep -q '#'; then
 291         rembranch=$(echo "$uri" | cut -d '#' -f 2)
 292         uri=$(echo "$uri" | cut -d '#' -f 1)
 293 fi
 294
 295 if [ "$_git_no_wc" ]; then
 296         [ -s "$_git/info/cg-fetch-initial" ] && [ ! -s "$_git/info/cg-fetch-initial-wcless" ] &&
 297                 die "you must run the initial cg-fetch from the working copy root directory"
 298 fi
 299
 300 # Some other process with the same pid might appear, that's why
 301 # we won't die but rather let the user check quickly.
 302 dirtyfile="$_git/info/cg-fetch-$(echo "$name" | sed -e 's/\//-.-/g')-dirty"
 303 if [ -s "$dirtyfile" ]; then
 304         kill -0 $(cat "$dirtyfile") 2>/dev/null && \
 305                 warn "aren't you fetching $name twice at once? (waiting 10s)" && \
 306                 sleep 10
 307         if [ -s "$_git/info/cg-fetch-initial" ]; then
 308                 echo "Recovering from a previously interrupted initial clone..."
 309         else
 310                 echo "Recovering from a previously interrupted fetch..."
 311         fi
 312         recovery=--recover
 313 fi
 314 mkdir -p "$_git/info"
 315 echo $$ > "$dirtyfile"
 316
 317
 318 orig_head="$(get_ref "refs/heads/$name")" || : may stay empty
 319
 320
 321 packed_transport=
 322
 323 if echo "$uri" | grep -q "^\(https\?\|ftp\)://"; then
 324         get=get_http
 325         fetch=fetch_http
 326 elif echo "$uri" | grep -q "^git+ssh://"; then
 327         packed_transport=ssh
 328 elif echo "$uri" | grep -q "^git://"; then
 329         packed_transport=git
 330 elif echo "$uri" | grep -q "^rsync://"; then
 331         echo "WARNING: The rsync access method is DEPRECATED and will be REMOVED in the future!" >&2
 332         get=get_rsync
 333         fetch=fetch_rsync
 334 elif echo "$uri" | grep -q ":"; then
 335         echo "WARNING: I guessed the host:path syntax was used and fell back to the git+ssh protocol." >&2
 336         echo "WARNING: The host:path syntax is evil because it is implicit. Please just use a URI." >&2
 337         packed_transport=ssh
 338 else
 339         [ -d "$uri/.git" ] && uri="$uri/.git"
 340         [ -d "$uri" ] || die "repository not found"
 341         get=get_local
 342         fetch=fetch_local
 343
 344         # Perhaps the object database is shared
 345         symlinked=
 346         is_same_repo "$_git_objects" "$uri/objects" && symlinked=1
 347
 348         # See if we can hardlink and add "-l" to cp flags.
 349         can_hardlink=
 350         sample_file="$(find "$uri" -type f -print | head -n 1)"
 351         rm -f "$_git/.,,lntest"
 352         if cp -fl "$sample_file" "$_git/.,,lntest" 2>/dev/null; then
 353                 can_hardlink=l
 354                 echo "Using hard links"
 355         else
 356                 echo "Hard links don't work - using copy"
 357         fi
 358         rm -f "$_git/.,,lntest"
 359 fi
 360
 361
 362 if [ "$packed_transport" ]; then
 363         # This is a really special case.
 364         [ "$rembranch" ] || rembranch="HEAD"
 365
 366         cloneorfetch= #fetch
 367         [ -s "$_git/info/cg-fetch-initial" ] && cloneorfetch=-k #clone
 368
 369         rm -f "$_git/info/cg-fetch-earlydie"
 370
 371         fetch_pack_recorder () {
 372                 while read sha1 remote_name; do
 373                         [ "$sha1" = "failed" ] && die "$2"
 374                         ref="$1"; [ "$ref" ] || ref="$remote_name"
 375                         git-update-ref "$ref" "$sha1"
 376                 done
 377         }
 378         echo "Fetching pack (head and objects)..."
 379         ( git-fetch-pack $cloneorfetch "$uri" "$rembranch" ||
 380           echo "failed" "$rembranch" ) |
 381                 fetch_pack_recorder "refs/heads/$name" "fetching pack failed" ||
 382                 exit
 383
 384         record_tags_to_fetch () {
 385                 ( cut -f 1 | tr '\n' '\0' |
 386                         xargs -0 git-fetch-pack $cloneorfetch "$uri" ||
 387                   echo "failed" "$rembranch" ) |
 388
 389                 fetch_pack_recorder "" "unable to retrieve tags (non-fatal)"
 390         }
 391         fetch=record_tags_to_fetch
 392         fetch_tags
 393
 394         rm "$dirtyfile"
 395         show_changes_summary "$orig_head" "$(cg-object-id "$name")"
 396         [ -s "$_git/info/cg-fetch-initial" ] && initial_done
 397         exit 0
 398 fi
 399
 400
 401 ### Behold, the fetch itself
 402
 403 ## Grab the head
 404 echo "Fetching head..."
 405
 406 tmpname="$(mktemp -t githead.XXXXXX)"
 407 cleanup () {
 408         rm "$tmpname"
 409 }
 410 cleanup_trap "cleanup"
 411
 412 if [ "$rembranch" ]; then
 413         $get -i "$uri/refs/heads/$rembranch" "$tmpname" ||
 414                 die "unable to get the head pointer of branch $rembranch"
 415 else
 416         $get -b "$uri/HEAD" "$tmpname" ||
 417                 die "unable to get the HEAD branch"
 418 fi
 419
 420 new_head="$(cat "$tmpname")"
 421 if [ "${new_head#ref:}" != "$new_head" ]; then
 422         new_head="$(echo "$new_head" | sed 's/^ref: *//')"
 423         $get -i "$uri/$new_head" "$tmpname" ||
 424                 die "unable to get the head pointer of branch $new_head (referenced by HEAD)"
 425         new_head="$(cat "$tmpname")"
 426 fi
 427
 428 rm -f "$_git/info/cg-fetch-earlydie"
 429
 430 echo "Fetching objects..."
 431 ## Fetch the objects
 432 if ! [ "$symlinked" ]; then
 433         if [ "$recovery" -o "$orig_head" != "$new_head" ]; then
 434                 [ -d "$_git_objects" ] || mkdir -p "$_git_objects"
 435                 $fetch "$(cat "$tmpname")" "$uri" "heads/$name" || die "objects fetch failed"
 436         fi
 437 else
 438         git-update-ref "refs/heads/$name" "$(cat "$tmpname")"
 439 fi
 440
 441 rm "$tmpname"
 442 cleanup_trap ""
 443
 444 ## Fetch the tags
 445 ret=0
 446 if ! fetch_tags; then
 447         ret=$?
 448 fi
 449
 450 rm "$dirtyfile"
 451 show_changes_summary "$orig_head" "$new_head"
 452 [ -s "$_git/info/cg-fetch-initial" ] && initial_done
 453 exit $ret