git-ftp.py

   1 #!/usr/bin/env python
   2
   3 """
   4 git-ftp: painless, quick and easy working copy syncing over FTP
   5
   6 Copyright (c) 2008-2012
   7 Edward Z. Yang <ezyang@mit.edu>, Mauro Lizaur <mauro@cacavoladora.org> and
   8 Niklas Fiekas <niklas.fiekas@googlemail.com>
   9
  10 Permission is hereby granted, free of charge, to any person
  11 obtaining a copy of this software and associated documentation
  12 files (the "Software"), to deal in the Software without
  13 restriction, including without limitation the rights to use,
  14 copy, modify, merge, publish, distribute, sublicense, and/or sell
  15 copies of the Software, and to permit persons to whom the
  16 Software is furnished to do so, subject to the following
  17 conditions:
  18
  19 The above copyright notice and this permission notice shall be
  20 included in all copies or substantial portions of the Software.
  21
  22 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  23 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
  24 OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  25 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
  26 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  27 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  28 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  29 OTHER DEALINGS IN THE SOFTWARE.
  30 """
  31
  32 import ftplib
  33 import re
  34 import sys
  35 import os.path
  36 import posixpath  # use this for ftp manipulation
  37 import getpass
  38 import optparse
  39 import logging
  40 import textwrap
  41 import fnmatch
  42 from io import BytesIO
  43 try:
  44     import configparser as ConfigParser
  45 except ImportError:
  46     import ConfigParser
  47
  48
  49 # Note about Tree.path/Blob.path: *real* Git trees and blobs don't
  50 # actually provide path information, but the git-python bindings, as a
  51 # convenience keep track of this if you access the blob from an index.
  52 # This ends up considerably simplifying our code, but do be careful!
  53
  54 from distutils.version import LooseVersion
  55 from git import __version__ as git_version
  56
  57 if LooseVersion(git_version) < '0.3.0':
  58     print('git-ftp requires git-python 0.3.0 or newer; %s provided.' % git_version)
  59     exit(1)
  60
  61 from git import Blob, Repo, Git, Submodule
  62
  63
  64 class BranchNotFound(Exception):
  65     pass
  66
  67
  68 class FtpDataOldVersion(Exception):
  69     pass
  70
  71
  72 class FtpSslNotSupported(Exception):
  73     pass
  74
  75
  76 class SectionNotFound(Exception):
  77     pass
  78
  79
  80 def split_pattern(path):  # TODO: Improve skeevy code
  81     path = fnmatch.translate(path).split('\\/')
  82     for i, p in enumerate(path[:-1]):
  83         if p:
  84             path[i] = p + '\\Z(?ms)'
  85     return path
  86
  87 # ezyang: This code is pretty skeevy; there is probably a better,
  88 # more obviously correct way of spelling it. Refactor me...
  89 def is_ignored(path, regex):
  90     regex = split_pattern(os.path.normcase(regex))
  91     path = os.path.normcase(path).split('/')
  92
  93     regex_pos = path_pos = 0
  94     if regex[0] == '':  # leading slash - root dir must match
  95         if path[0] != '' or not re.match(regex[1], path[1]):
  96             return False
  97         regex_pos = path_pos = 2
  98
  99     if not regex_pos:  # find beginning of regex
 100         for i, p in enumerate(path):
 101             if re.match(regex[0], p):
 102                 regex_pos = 1
 103                 path_pos = i + 1
 104                 break
 105         else:
 106             return False
 107
 108     if len(path[path_pos:]) < len(regex[regex_pos:]):
 109         return False
 110
 111     n = len(regex)
 112     for r in regex[regex_pos:]:  # match the rest
 113         if regex_pos + 1 == n:  # last item; if empty match anything
 114             if re.match(r, ''):
 115                 return True
 116
 117         if not re.match(r, path[path_pos]):
 118             return False
 119         path_pos += 1
 120         regex_pos += 1
 121
 122     return True
 123
 124
 125 def main():
 126     Git.git_binary = 'git'  # Windows doesn't like env
 127
 128     repo, options, args = parse_args()
 129
 130     if repo.is_dirty() and not options.commit:
 131         logging.warning("Working copy is dirty; uncommitted changes will NOT be uploaded")
 132
 133     base = options.ftp.remotepath
 134     logging.info("Base directory is %s", base)
 135     try:
 136         branch = next(h for h in repo.heads if h.name == options.branch)
 137     except StopIteration:
 138         raise BranchNotFound
 139     commit = branch.commit
 140     if options.commit:
 141         commit = repo.commit(options.commit)
 142     tree = commit.tree
 143     if options.ftp.ssl:
 144         if hasattr(ftplib, 'FTP_TLS'):  # SSL new in 2.7+
 145             ftp = ftplib.FTP_TLS(options.ftp.hostname, options.ftp.username, options.ftp.password)
 146             ftp.prot_p()
 147             logging.info("Using SSL")
 148         else:
 149             raise FtpSslNotSupported("Python is too old for FTP SSL. Try using Python 2.7 or later.")
 150     else:
 151         ftp = ftplib.FTP(options.ftp.hostname, options.ftp.username, options.ftp.password)
 152     ftp.cwd(base)
 153
 154     # Check revision
 155     hash = options.revision
 156     if not options.force and not hash:
 157         hashFile = BytesIO()
 158         try:
 159             ftp.retrbinary('RETR git-rev.txt', hashFile.write)
 160             hash = hashFile.getvalue().strip()
 161         except ftplib.error_perm:
 162             pass
 163
 164     # Load ftpignore rules, if any
 165     patterns = []
 166
 167     gitftpignore = os.path.join(repo.working_dir, options.ftp.gitftpignore)
 168     if os.path.isfile(gitftpignore):
 169         with open(gitftpignore, 'r') as ftpignore:
 170             patterns = parse_ftpignore(ftpignore)
 171         patterns.append('/' + options.ftp.gitftpignore)
 172
 173     if not hash:
 174         # Diffing against an empty tree will cause a full upload.
 175         oldtree = get_empty_tree(repo)
 176     else:
 177         oldtree = repo.commit(hash).tree
 178
 179     if oldtree.hexsha == tree.hexsha:
 180         logging.info('Nothing to do!')
 181     else:
 182         upload_diff(repo, oldtree, tree, ftp, [base], patterns)
 183
 184     ftp.storbinary('STOR git-rev.txt', BytesIO(commit.hexsha.encode('utf-8')))
 185     ftp.quit()
 186
 187
 188 def parse_ftpignore(rawPatterns):
 189     patterns = []
 190     for pat in rawPatterns:
 191         pat = pat.rstrip()
 192         if not pat or pat.startswith('#'):
 193             continue
 194         patterns.append(pat)
 195     return patterns
 196
 197
 198 def parse_args():
 199     usage = 'usage: %prog [OPTIONS] [DIRECTORY]'
 200     desc = """\
 201            This script uploads files in a Git repository to a
 202            website via FTP, but is smart and only uploads file
 203            that have changed.
 204            """
 205     parser = optparse.OptionParser(usage, description=textwrap.dedent(desc))
 206     parser.add_option('-f', '--force', dest="force", action="store_true", default=False,
 207             help="force the reupload of all files")
 208     parser.add_option('-q', '--quiet', dest="quiet", action="store_true", default=False,
 209             help="quiet output")
 210     parser.add_option('-r', '--revision', dest="revision", default=None,
 211             help="use this revision instead of the server stored one")
 212     parser.add_option('-b', '--branch', dest="branch", default=None,
 213             help="use this branch instead of the active one")
 214     parser.add_option('-c', '--commit', dest="commit", default=None,
 215             help="use this commit instead of HEAD")
 216     parser.add_option('-s', '--section', dest="section", default=None,
 217             help="use this section from ftpdata instead of branch name")
 218     options, args = parser.parse_args()
 219     configure_logging(options)
 220     if len(args) > 1:
 221         parser.error("too many arguments")
 222     if args:
 223         cwd = args[0]
 224     else:
 225         cwd = "."
 226     repo = Repo(cwd)
 227
 228     if not options.branch:
 229         options.branch = repo.active_branch.name
 230
 231     if not options.section:
 232         options.section = options.branch
 233
 234     get_ftp_creds(repo, options)
 235     return repo, options, args
 236
 237
 238 def configure_logging(options):
 239     logger = logging.getLogger()
 240     if not options.quiet:
 241         logger.setLevel(logging.INFO)
 242     ch = logging.StreamHandler(sys.stderr)
 243     formatter = logging.Formatter("%(levelname)s: %(message)s")
 244     ch.setFormatter(formatter)
 245     logger.addHandler(ch)
 246
 247
 248 def format_mode(mode):
 249     return "%o" % (mode & 0o777)
 250
 251
 252 class FtpData():
 253     password = None
 254     username = None
 255     hostname = None
 256     remotepath = None
 257     ssl = None
 258     gitftpignore = None
 259
 260
 261 def get_ftp_creds(repo, options):
 262     """
 263     Retrieves the data to connect to the FTP from .git/ftpdata
 264     or interactively.
 265
 266     ftpdata format example:
 267
 268         [branch]
 269         username=me
 270         password=s00perP4zzw0rd
 271         hostname=ftp.hostname.com
 272         remotepath=/htdocs
 273         ssl=yes
 274         gitftpignore=.gitftpignore
 275
 276     Please note that it isn't necessary to have this file,
 277     you'll be asked for the data every time you upload something.
 278     """
 279
 280     ftpdata = os.path.join(repo.git_dir, "ftpdata")
 281     options.ftp = FtpData()
 282     cfg = ConfigParser.ConfigParser()
 283     if os.path.isfile(ftpdata):
 284         logging.info("Using .git/ftpdata")
 285         cfg.read(ftpdata)
 286
 287         if (not cfg.has_section(options.section)):
 288             if cfg.has_section('ftp'):
 289                 raise FtpDataOldVersion("Please rename the [ftp] section to [branch]. " +
 290                                         "Take a look at the README for more information")
 291             else:
 292                 raise SectionNotFound("Your .git/ftpdata file does not contain a section " +
 293                                      "named '%s'" % options.section)
 294
 295         # just in case you do not want to store your ftp password.
 296         try:
 297             options.ftp.password = cfg.get(options.section, 'password')
 298         except ConfigParser.NoOptionError:
 299             options.ftp.password = getpass.getpass('FTP Password: ')
 300
 301         options.ftp.username = cfg.get(options.section, 'username')
 302         options.ftp.hostname = cfg.get(options.section, 'hostname')
 303         options.ftp.remotepath = cfg.get(options.section, 'remotepath')
 304         try:
 305             options.ftp.ssl = boolish(cfg.get(options.section, 'ssl'))
 306         except ConfigParser.NoOptionError:
 307             options.ftp.ssl = False
 308
 309         try:
 310             options.ftp.gitftpignore = cfg.get(options.section, 'gitftpignore')
 311         except ConfigParser.NoOptionError:
 312             options.ftp.gitftpignore = '.gitftpignore'
 313     else:
 314         print("Please configure settings for branch '%s'" % options.section)
 315         options.ftp.username = raw_input('FTP Username: ')
 316         options.ftp.password = getpass.getpass('FTP Password: ')
 317         options.ftp.hostname = raw_input('FTP Hostname: ')
 318         options.ftp.remotepath = raw_input('Remote Path: ')
 319         if hasattr(ftplib, 'FTP_TLS'):
 320             options.ftp.ssl = ask_ok('Use SSL? ')
 321         else:
 322             logging.warning("SSL not supported, defaulting to no")
 323
 324         # set default branch
 325         if ask_ok("Should I write ftp details to .git/ftpdata? "):
 326             cfg.add_section(options.section)
 327             cfg.set(options.section, 'username', options.ftp.username)
 328             cfg.set(options.section, 'password', options.ftp.password)
 329             cfg.set(options.section, 'hostname', options.ftp.hostname)
 330             cfg.set(options.section, 'remotepath', options.ftp.remotepath)
 331             cfg.set(options.section, 'ssl', options.ftp.ssl)
 332             f = open(ftpdata, 'w')
 333             cfg.write(f)
 334
 335
 336 def get_empty_tree(repo):
 337     return repo.tree(repo.git.hash_object('-w', '-t', 'tree', os.devnull))
 338
 339
 340 def upload_diff(repo, oldtree, tree, ftp, base, ignored):
 341     """
 342     Upload  and/or delete items according to a Git diff between two trees.
 343
 344     upload_diff requires, that the ftp working directory is set to the base
 345     of the current repository before it is called.
 346
 347     Keyword arguments:
 348     repo    -- The git.Repo to upload objects from
 349     oldtree -- The old tree to diff against. An empty tree will cause a full
 350                upload of the new tree.
 351     tree    -- The new tree. An empty tree will cause a full removal of all
 352                objects of the old tree.
 353     ftp     -- The active ftplib.FTP object to upload contents to
 354     base    -- The list of base directory and submodule paths to upload contents
 355                to in ftp.
 356                For example, base = ['www', 'www']. base must exist and must not
 357                have a trailing slash.
 358     ignored -- The list of patterns explicitly ignored by gitftpignore.
 359
 360     """
 361     # -z is used so we don't have to deal with quotes in path matching
 362     diff = repo.git.diff("--name-status", "--no-renames", "-z", oldtree.hexsha, tree.hexsha)
 363     diff = iter(diff.split("\0"))
 364     for line in diff:
 365         if not line:
 366             continue
 367         status, file = line, next(diff)
 368         assert status in ['A', 'D', 'M']
 369
 370         filepath = posixpath.join(*(['/'] + base[1:] + [file]))
 371         if is_ignored_path(filepath, ignored):
 372             logging.info('Skipped ' + filepath)
 373             continue
 374
 375         if status == "D":
 376             try:
 377                 ftp.delete(file)
 378                 logging.info('Deleted ' + file)
 379             except ftplib.error_perm:
 380                 logging.warning('Failed to delete ' + file)
 381
 382             # Now let's see if we need to remove some subdirectories
 383             def generate_parent_dirs(x):
 384                 # invariant: x is a filename
 385                 while '/' in x:
 386                     x = posixpath.dirname(x)
 387                     yield x
 388             for dir in generate_parent_dirs(file):
 389                 try:
 390                     # unfortunately, dir in tree doesn't work for subdirs
 391                     tree[dir]
 392                 except KeyError:
 393                     try:
 394                         ftp.rmd(dir)
 395                         logging.debug('Cleaned away ' + dir)
 396                     except ftplib.error_perm:
 397                         logging.info('Did not clean away ' + dir)
 398                         break
 399         else:
 400             node = tree[file]
 401
 402             if status == "A":
 403                 # try building up the parent directory
 404                 subtree = tree
 405                 if isinstance(node, Blob):
 406                     directories = file.split("/")[:-1]
 407                 else:
 408                     # for submodules also add the directory itself
 409                     assert isinstance(node, Submodule)
 410                     directories = file.split("/")
 411                 for c in directories:
 412                     subtree = subtree / c
 413                     try:
 414                         ftp.mkd(subtree.path)
 415                     except ftplib.error_perm:
 416                         pass
 417
 418             if isinstance(node, Blob):
 419                 upload_blob(node, ftp)
 420             else:
 421                 module = node.module()
 422                 module_tree = module.commit(node.hexsha).tree
 423                 if status == "A":
 424                     module_oldtree = get_empty_tree(module)
 425                 else:
 426                     oldnode = oldtree[file]
 427                     assert isinstance(oldnode, Submodule)  # TODO: What if not?
 428                     module_oldtree = module.commit(oldnode.hexsha).tree
 429                 module_base = base + [node.path]
 430                 logging.info('Entering submodule %s', node.path)
 431                 ftp.cwd(posixpath.join(*module_base))
 432                 upload_diff(module, module_oldtree, module_tree, ftp, module_base, ignored)
 433                 logging.info('Leaving submodule %s', node.path)
 434                 ftp.cwd(posixpath.join(*base))
 435
 436
 437 def is_ignored_path(path, patterns, quiet=False):
 438     """Returns true if a filepath is ignored by gitftpignore."""
 439     if is_special_file(path):
 440         return True
 441     for pat in patterns:
 442         if is_ignored(path, pat):
 443             return True
 444     return False
 445
 446
 447 def is_special_file(name):
 448     """Returns true if a file is some special Git metadata and not content."""
 449     return posixpath.basename(name) in ['.gitignore', '.gitattributes', '.gitmodules']
 450
 451
 452 def upload_blob(blob, ftp, quiet=False):
 453     """
 454     Uploads a blob.  Pre-condition on ftp is that our current working
 455     directory is the root directory of the repository being uploaded
 456     (that means DON'T use ftp.cwd; we'll use full paths appropriately).
 457     """
 458     if not quiet:
 459         logging.info('Uploading ' + blob.path)
 460     try:
 461         ftp.delete(blob.path)
 462     except ftplib.error_perm:
 463         pass
 464     ftp.storbinary('STOR ' + blob.path, blob.data_stream)
 465     try:
 466         ftp.voidcmd('SITE CHMOD ' + format_mode(blob.mode) + ' ' + blob.path)
 467     except ftplib.error_perm:
 468         # Ignore Windows chmod errors
 469         logging.warning('Failed to chmod ' + blob.path)
 470         pass
 471
 472
 473 def boolish(s):
 474     if s in ('1', 'true', 'y', 'ye', 'yes', 'on'):
 475         return True
 476     if s in ('0', 'false', 'n', 'no', 'off'):
 477         return False
 478     return None
 479
 480
 481 def ask_ok(prompt, retries=4, complaint='Yes or no, please!'):
 482     while True:
 483         ok = raw_input(prompt).lower()
 484         r = boolish(ok)
 485         if r is not None:
 486             return r
 487         retries = retries - 1
 488         if retries < 0:
 489             raise IOError('Wrong user input.')
 490         print(complaint)
 491
 492 if __name__ == "__main__":
 493     main()