Logo Search packages:      
Sourcecode: aap version File versions


# Part of the A-A-P recipe executive: copy and move files (remotely)

# Copyright (C) 2002-2003 Stichting NLnet Labs
# Permission to copy and use this file is specified in the file COPYING.
# If this file is missing you can find it here: http://www.a-a-p.org/COPYING

# These are functions to copy and move files.  Not only on the local system,
# also to remote systems, using http://, ftp://, etc.

import os
import os.path
import string
import re

from Dictlist import str2dictlist
from Error import *
from Process import recipe_error, option_error
from Util import *
from Work import getrpstack
from Message import *

def copy_move(line_nr, recdict, arg, copy):
    """Implementation of ":copy -x from to" and ":move -x from to".
       When "copy" is non-zero copying is done, otherwise moving.
       "arg" is the whole argument.
       "line_nr" is used for error messages."""

    # Skip when not actually building.
    cmdname = (copy and ':copy') or ':move'
    if skip_commands():
        msg_skip(line_nr, recdict, cmdname + ' ' + arg)
    rpstack = getrpstack(recdict, line_nr)

    # Get the arguments and check the options.
    # TODO: check if all options are handled properly
    from Commands import get_args
    opt = {"f": "force", "force": "force",
           "i": "interactive", "interactive": "interactive",
           "e": "exist", "exist": "exist", "exists": "exist",
           "m": "mkdir", "mkdir": "mkdir",
           "c": "continue", "continue": "continue",
           "q": "quiet", "quiet": "quiet"}
    if copy:
                "u": "unlink", "unlink": "unlink",
                "p": "preserve", "preserve": "preserve",
                "r": "recursive", "recursive": "recursive",
                "k": "keepdir", "keepdir": "keepdir"})

    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg, opt)
    if attrdict:
        option_error(rpstack, attrdict, cmdname)
    if len(argdictlist) < 2:
        recipe_error(rpstack, _("%s command requires at least two arguments")
                                                                     % cmdname)

    remote_copy_move(rpstack, recdict, copy, argdictlist[:-1],
                                    argdictlist[-1], optiondict, 1, errmsg = 1)

# Schemes handled by scheme_copy().  When changing this also need to change
# code and comments below!
scheme_copy_names = ["rcp", "scp", "rsync"]

def scheme_copy(rpstack, recdict, scheme, fromfile, tmach, destpath, fcount):
    Use rcp, scp or rsync, specified with "scheme" to copy "fromfile" (with
    "fcount" file names in quotes) to tmach:destpath.
    Return non-zero for success.
    # Install rcp/scp/rsync when needed.
    from DoInstall import assert_pkg
    assert_pkg(rpstack, recdict, scheme)

    from Remote import get_progname_rsync, get_progname_rcp, get_progname_scp
    if scheme == "rcp":
        cmd = "%s %s '%s:%s'" % (get_progname_rcp(recdict),
                                                     fromfile, tmach, destpath)
    elif scheme == "scp":
        cmd = "%s %s '%s:%s'" % (get_progname_scp(recdict),
                                                     fromfile, tmach, destpath)
        # A bug in rsync: it can't handle "file(with)parenthesis" in the
        # destination, but escaping them in the source is not allowed!
        d = re.sub("([()])", r"\\\1", destpath)
        cmd = "%s %s '%s:%s'" % (get_progname_rsync(recdict),
                                                            fromfile, tmach, d)

    if os.name == "posix":
        # On Unix we can use "tee".
        ok, text = redir_system_int(recdict, cmd)
        # Scp may ask for a password.  When we can't use "tee" we need to
        # see the message.  Can't catch the text output then...
        ok = logged_system(recdict, "{interactive} " + cmd) == 0
        text = ''

    if text:
        msg_log(recdict, text)
        # Check for an error connecting, scp doesn't exit with an error value
        # then.  Usual messages are:
        #    "Secure connection to vim.sf.net refused."
        #    "lost connection"
        #    "Read-only file system"
        i = string.find(text, "connection to")
        if i > 0:
            e = string.find(text, "\n", i)
            if e > 0:
                i = string.find(text, "refused", i, e)
                i = -1
        if i > 0 or string.find(text, "lost connection") >= 0:
            ok = 0
        elif string.find(text, "Permission denied") >= 0:
            # Happens when a file exists and permissions don't allow
            # overwriting: scp: vim/new/con_cvs.php: Permission denied
            # TODO: check the file name and only mark this one as failed.
            ok = 0
        elif string.find(text, "Read-only file system") >= 0:
            # Happens when the file system was mounted read-only.
            # TODO: check the file name and only mark this one as failed.
            ok = 0
        elif string.find(text, "Not a directory") >= 0:
            # Happens when copying multiple files into a file.
            ok = 0
        elif string.find(text, "No such file or directory") >= 0:
            ok = 0
                     _('looks like the directory does not exist, creating it'))
            # TODO: This is just guessing; how do we know if the destination
            # is a directory or a file?
            if fcount > 1:
                dirname = destpath
                dirname = os.path.dirname(destpath)
            mkdircmd = ('ssh %s mkdir -p %s' % (tmach, dirname))
            if logged_system(recdict, mkdircmd) == 0:
                # Try copying again.
                ok = (logged_system(recdict, cmd) == 0)

    return ok

# Cached ftp connections.
ftp_conn = {}
did_set_exitfunc = 0
old_exitfunc = None

def ftpCloseAll():
    Close all cached ftp connections.  Invoked on exit.
    import ftplib

    if old_exitfunc:
        old_exitfunc()      # Invoke previous exit function.

    global ftp_conn
    for ftp in ftp_conn.values():
        except ftplib.all_errors, e:
            msg_log({}, _("could not quit an ftp connection: %s") % e)
    ftp_conn = {}

# Cached passwords for ftp connections.
ftp_pass = {}

def ftpConnect(mach):
    Connect to an ftp server "mach".
    If "mach" is in the form "user:passwd@ftp.com" use that.
    Otherwise consult the netrc file or ask the user.
    Returns a tuple.  The first item is an ftp object.
    If the connection was successful the second item is an empty string.
    If the connection failed the second item is an error message.

    # Use a cached connection if there is one.
    if mach in ftp_conn.keys():
        return ftp_conn[mach], ''

    passwd = ''
    acct = ''
    key = None
    msg = ''

    # For "user:passwd@ftp.com" split at the @
    i = string.find(mach, '@')
    if i > 0:
        user = mach[:i]
        machname = mach[i+1:]
        i = string.find(user, ':')
        if i > 0:
            passwd = user[i+1:]
            user = user[:i]

            key = user + "@" + machname
            passwd = ftp_pass.get(key)
            if not passwd:
                prompt = (_('Enter password for user %s at %s: ')
                                                        % (user, machname))
                    import getpass
                    passwd = getpass.getpass(prompt)
                    # TODO: should display stars for typed chars
                    passwd = raw_input(prompt)
        user = ''
        machname = mach

        import aapnetrc
        # obtain the login name and password from the netrc file
            n = aapnetrc.netrc()
            res = n.authenticators(machname)
            if res is None:
                user = ''
                user, acct, passwd = res
        except (IOError, aapnetrc.NetrcParseError), e:

    # Try to open the connection to the ftp server.
    import ftplib
        ftp = ftplib.FTP()

        i = string.find(machname, ':')
        if i > 0:
            # Port number specified.
            ftp.connect(machname[0:i], machname[i+1:])
            # Use default port number.

        if user != '':
            if passwd != '':
                if acct != '':
                    ftp.login(user, passwd, acct)
                    ftp.login(user, passwd)
    except ftplib.all_errors, e:
        msg = (_('Cannot open connection to "%s": %s') % (mach, str(e)))
        if key and passwd:
            # Remember the typed password, so that the user doesn't have to
            # type it again.
            ftp_pass[key] = passwd

        # Cache the connection for another time.
        ftp_conn[mach] = ftp

        # Setup the exit function to close the connection.
        global did_set_exitfunc
        global old_exitfunc
        if not did_set_exitfunc:
            did_set_exitfunc = 1
            old_exitfunc = getattr(sys, 'exitfunc', None)
            sys.exitfunc = ftpCloseAll

    return ftp, msg

def ftp_may_mkdir(recdict, error, ftp, destpath):
    """If "error" indicates we failed to upload through ftp because the
       directory doesn't exist try creating that directory.  This works
       Return non-zero if the directory was created."""
    if string.find(str(error), "No such file or dir") >= 0:
        adir = os.path.dirname(destpath)
        if len(adir) < len(destpath):
            import ftplib
            msg_info(recdict, _('Directory "%s" does not appear to exist, creating it...') % adir)
            except ftplib.all_errors, e:
                # Maybe the upper directory doesn't exist, try creating it.
                if ftp_may_mkdir(recdict, e, ftp, adir):
                    except ftplib.all_errors, e:
                        msg_info(recdict, _('Could not create directory "%s"') % adir)
                        return 0
                    return 0
            return 1
    return 0

def remote_copy_move(rpstack, recdict, copy, from_items, to_item,
                                            optiondict, argexpand, errmsg = 0):
    """Copy or move command.  Copy when "copy" is non-zero.
       "from_items" is a dictlist of the files to copy or move from.
       "to_item" is the dictionary of the destination.
       "optiondict" is a dictionary with options:
            recursive: recursive copy.
            exist: skip if destination exists
            interactive: ask about overwriting.
            preserve: keep mode bits and timestamp
            quiet: don't report copied/moved files
            continue: ignore wildcards without matches
            mkdir: create directory when needed
       "argexpand" is non-zero when wildcards are to be expanded (called from
       ":copy" or ":move").
       When "errmsg" is non-zero, call recipe_error() for an error.
       Returns a list of filenames that failed."""
    from Remote import url_split3
    import glob
    failed = []

    # -l: change symlinks to real files
    #if optiondict.get("unlink"):
    #    keepsymlinks = 0
    #    keepsymlinks = 1

    # Expand and check the "from" arguments.
    fromlist = []
    for a in from_items:
        fname = a["name"]
        fscheme, fmach, fpath = url_split3(fname)
        if fscheme == '':
            # It's a local file, may expand ~user and wildcards.
            f = os.path.expanduser(fname)
            if argexpand:
                fl = glob.glob(f)
                if len(fl) == 0:
                    if optiondict.get("continue") and has_wildcard(f):
                    recipe_error(rpstack, _('No match for "%s"') % fname)
                fl = [ f ]
            # For copy without {r} sources can't be a directory.
            if copy and not optiondict.get("recursive"):
                for l in fl:
                    if os.path.isdir(l):
                             _('Copying a directory requires {r} flag: %s') % l)

            # It's a URL, append without expanding.
            if not copy:
                recipe_error(rpstack, _('Cannot move from a URL yet: "%s"')
                                                                       % fname)
            if argexpand:
                # Do change [*] to *, we might support expanding later.
                f = expand_unescape(fname)
                f = fname

    # Expand and check the "to" argument.
    tname = to_item["name"]
    tscheme, tmach, tpath = url_split3(tname)
    if tscheme == '':
        # For a local file expand "~/" and "~user/".  Don't use "tname", it may
        # start with "file:".
        tpath = os.path.expanduser(tpath)
    if argexpand:
        if tscheme == '':
            # For a local destination file expand wildcards.  If expanding
            # results in no matches use the name without expansion.
            l = glob.glob(tpath)
            if len(l) > 1:
                recipe_error(rpstack, _('More than one match for "%s"') % tname)
            if len(l) == 1:
                tpath = l[0]
            # Can't expand wildcards in a URL yet, at least remove escaped
            # wildcards.
            tpath = expand_unescape(tpath)

    if tscheme == '' and optiondict.get("mkdir"):
        # Create local target directory when it doesn't exist.
        if len(fromlist) > 1:
            dirpath = tpath;
            dirpath = os.path.dirname(tpath)
        may_create_dir(recdict, rpstack, dirpath, optiondict.get("quiet"))

    # If there is more than one source, target must be a directory.
    if tscheme == '' and len(fromlist) > 1 and not os.path.isdir(tpath):
        recipe_error(rpstack, _('Destination must be a directory: "%s"')
                                                                       % tname)

    msg = ''
    if tscheme == "ftp":
        # Prepare for uploading through ftp.
        ftp, msg = ftpConnect(tmach)

    elif tscheme in scheme_copy_names:
        # TODO: Prepare for uploading through rcp/scp/rsync.

    elif tscheme != '':
        msg = _('Can only upload to rcp://, scp://, rsync:// and ftp://')

    # If copy/move isn't possible either give an error message or return the
    # whole list of sources.
    if msg:
        if argexpand:
            msg_error(recdict, msg)
            return map(lambda x : x["name"], from_items)
        recipe_error(rpstack, msg)

    # files to be copied with scp or rsync later.
    scheme_files = {}
    scheme_filelist = {}
    for scheme in scheme_copy_names:
        scheme_files[scheme] = ''
        scheme_filelist[scheme] = []

    # Loop over all "from" files.
    for fname in fromlist:
        fscheme, fmach, fpath = url_split3(fname)

        # If the destination is a directory, append the source file name to the
        # destination directory.
        # "dest" includes the scheme (may be "file:" when "tscheme" is empty).
        if ((tscheme != '' and len(fromlist) > 1)
                                  or (tscheme == '' and os.path.isdir(tpath))):
            if optiondict.get("keepdir"):
                dest = os.path.join(tname, fpath)
                destpath = os.path.join(tpath, fpath)
                if tscheme == '' and os.path.dirname(fpath):
                    # Create the directory if it doesn't exist.
                    dirpath = os.path.dirname(destpath)
                    may_create_dir(recdict, rpstack, dirpath,
                dest = os.path.join(tname, os.path.basename(fname))
                destpath = os.path.join(tpath, os.path.basename(fname))
            dest = tname
            destpath = tpath

        # If destination is a local file and exists:
        # - When {exist} option used, silently ignore.
        # - When {interactive} option used, ask for overwriting.  Use a special
        #   string to allow translating the response characters.
        if tscheme == '' and os.path.exists(destpath):
            if optiondict.get("exist"):
                msg = _('file already exists: "%s"') % dest
                msg_note(recdict, msg)
            if optiondict.get("interactive"):
                reply = raw_input(_('"%s" exists, overwrite? (y/n) ') % dest)
                if (len(reply) == 0 or not reply[0]
                              in _("yY   up to four chars that mean yes")[:4]):
                    if copy:
                        msg_note(recdict, _("file not copied"))
                        msg_note(recdict, _("file not moved"))

        if fscheme == '' and tscheme == '':
            # local file copy or move

            if not copy:
                    os.rename(fpath, destpath)
                    done = 1
                    done = 0        # renaming failed, try copying

            if copy or not done:
                    import shutil

                    # TODO: handle symlinks and "keepsymlinks".
                    if os.path.isdir(fpath):
                        # Cannot use shutil.copytree here, it fails when the
                        # destination already exists.
                        if not os.path.exists(destpath):
                        # Call ourselves recursively
                        from Commands import dir_contents
                        flist = map(lambda x: {"name": x}, dir_contents(fpath))

                        if flist:
                            # Remove the "keepdir" attribute, the directory
                            # name is already in "destpath" now.
                            if optiondict.has_key("keepdir"):
                                opt = optiondict.copy()
                                del opt["keepdir"]
                                opt = optiondict

                            f = remote_copy_move(rpstack, recdict, copy, flist,
                                         {"name": destpath}, opt, 0, errmsg)
                            if f:
                    elif not copy or optiondict.get("preserve"):
                        shutil.copy2(fpath, destpath)
                        shutil.copy(fpath, destpath)
                except (IOError, OSError), e:
                    msg = (_('Cannot copy "%s" to "%s": %s')
                                                       % (fname, dest, str(e)))
                    if not argexpand and not errmsg:
                        msg_note(recdict, msg)
                        recipe_error(rpstack, msg)
                if not copy:
                    if os.path.isdir(fpath):

            if not optiondict.get("quiet"):
                f = shorten_name(fpath)
                if copy:
                    msg_info(recdict, _('Copied "%s" to "%s"') % (f, dest))
                    msg_info(recdict, _('Moved "%s" to "%s"') % (f, dest))

            if fscheme != '':
                # download to local file
                from Remote import url_download

                    if tscheme != '':
                        tmpfile, rtime = url_download(recdict, fname, '')
                        tmpfile, rtime = url_download(recdict, fname, destpath)
                except IOError, e:
                    msg = (_('Cannot download "%s" to "%s": %s')
                                                       % (fname, dest, str(e)))
                    if not argexpand and not errmsg:
                        msg_note(recdict, msg)
                        recipe_error(rpstack, msg)

            if tscheme != '':
                if fscheme != '':
                    # use temporary file
                    fromfile = tmpfile
                    fromfile = fpath

                postponed = 0
                if tscheme == 'ftp':
                    # No system command, give a message about what we're doing,
                    # it may take a while.
                    if fscheme != '':
                        msg_info(recdict, _('Uploading to "%s"' % dest))
                        if not copy:
                            msg_info(recdict, _('Moving "%s" to "%s"'
                                                              % (fname, dest)))
                            msg_info(recdict, _('Uploading "%s" to "%s"'
                                                              % (fname, dest)))

                    # FTP must use '/' path separator characters,
                    # os.path.join() may have used something else.
                    if os.sep != '/':
                        destpath = string.replace(destpath, os.sep, '/')

                    def store_file(ftp, fromfile, destpath):
                        import ftplib       # avoid warning message
                        error = None
                            f = open(fromfile, "rb")
                        except IOError, e:
                            return e
                            ftp.storbinary("STOR " + destpath, f, 8192)
                        except ftplib.all_errors, e:
                            error = e
                        return error

                    error = store_file(ftp, fromfile, destpath)

                    # If it looks like the directory doesn't exist, try
                    # creating it.
                    if error and ftp_may_mkdir(recdict, error, ftp, destpath):
                        msg_info(recdict, _('Directory created, attempt uploading again...'))
                        error = store_file(ftp, fromfile, destpath)

                    if error:
                        msg = (_('Cannot upload "%s" to "%s": %s')
                                                   % (fname, dest, str(error)))
                        if not argexpand and not errmsg:
                            msg_note(recdict, msg)
                            recipe_error(rpstack, msg)
                        if fscheme != '':
                            os.remove(tmpfile)      # delete temporary file

                elif tscheme in scheme_copy_names:
                    if fscheme == '' and len(fromlist) > 1:
                        # do all local files at once below
                        scheme_files[tscheme] = (scheme_files[tscheme]
                                                       + "'" + fromfile + "' ")
                        postponed = 1
                    elif not scheme_copy(rpstack, recdict, tscheme,
                                     "'" + fromfile + "'", tmach, destpath, 1):
                        msg = _('Cannot upload "%s" to "%s"') % (fname, dest)
                        if not argexpand and not errmsg:
                            msg_note(recdict, msg)
                            recipe_error(rpstack, msg)
                        if fscheme != '':
                            os.remove(tmpfile)      # delete temporary file

                if fscheme != '':
                    os.remove(tmpfile)      # delete temporary file
                    msg_info(recdict, _('Uploaded to "%s"' % dest))
                    if not copy:
                        msg_info(recdict, _('Moved "%s" to "%s"'
                                                % (shorten_name(fname), dest)))
                    elif not postponed:
                        msg_info(recdict, _('Uploaded "%s" to "%s"'
                                                % (shorten_name(fname), dest)))

    # End of loop over all "from" files.

    if fscheme != '':
        from Remote import url_cleanup

    # Copy collected files with rcp, scp and rsync.
    for scheme in scheme_copy_names:
        files = scheme_files[scheme]
        if files:
            filelist = scheme_filelist[scheme]

            # If there is only one file to copy, add its basename to the
            # destination path to avoid creating a file by the name of the
            # directory of that file.
            if len(filelist) == 1:
                tpath = os.path.join(tpath,

            if scheme_copy(rpstack, recdict, scheme, files, tmach, tpath,
                dl = shorten_dictlist(str2dictlist(rpstack, files))
                flist = map(lambda x: x["name"], dl)
                msg_info(recdict, _('Uploaded "%s" to "%s"' % (flist, tname)))
                msg = _('Cannot upload "%s" to "%s"') % (files, tname)
                if not argexpand and not errmsg:
                    msg_note(recdict, msg)
                    recipe_error(rpstack, msg)

    return failed

def may_create_dir(recdict, rpstack, dir, quiet):
    Create directory "dir" if it does not exist.
    if not os.path.exists(dir):
            if not quiet:
                msg_info(recdict, _('Created directory "%s"') % dir)
        except StandardError, e:
            recipe_error(rpstack, _('Destination directory "%s" cannot be created: %s') % (dir, str(e)))

# vim: set sw=4 et sts=4 tw=79 fo+=l:

Generated by  Doxygen 1.6.0   Back to index