#!/bin/zsh # http://Yost.com/computers/yostupload # See notice of copyright and license at end. # To do: # * BUG: download from a subdir gets the whole tree # * Provide a way for the root file to specify # files and folders that are not to be uploaded # in other words, an easy front end to rsync --exclude # * specify permissions for dirs and files # instead of always complaining if not world-accessible # * BUG: yostupload -v -r works but not -r -v # * The .YostUploadRoot file can cause the dir you're in to be excluded #------------------------------------------------------------------------------- # Constants # If $0 is a symlink, this is the invoked name, not the symlink referent name declare -r commandName="${0##*/}" declare -r rootFileDefault=.YostUploadRoot declare -r rsyncStdArgsUp=-rRhHLtp declare -r rsyncStdArgsDown=-rRhHtp #------------------------------------------------------------------------------- echo2() { echo 1>&2 $@ } summary() { echo 1>&2 " Usage: $commandName [ -d [ -o dir ] ] [ -v ] [ -q ] [ -n ] [ -r rootFile ] \\ [ --rsync-args args ] [ path ... ] $commandName -l [ -r rootFile ] [ path ] $commandName -R [ -r rootFile ] [ path ] $commandName -u [ -r rootFile ] [ path ] $commandName -i user host remote-dir [ rsync-args ]" } optionError() { echo 1>&2 " $commandName: $1 Try --help " exit 2 } usage() { if [[ $1 != '' ]] ; then echo 1>&2 "\n$1" ; fi summary echo 1>&2 " path defaults to the current directory. Uploads files unless -d to download instead. Or: use -l to ssh to the server. The remote shell's current directory will correspond to path. Or: use -R to output rootFile path. Or: use -u to output the URL for path. Or: use -i to quickly set up the current directory for use with $commandName. The -i option usage creates a $rootFileDefault file in the current directory so that from then on you can use $commandName there. Append --no-progress and/or --no-partial if you don't like the defaults. All path arguments (files and/or folders) must be within a folder hierarchy that serves as a local staging area for a folder on the server. yostupload uploads (or downloads) everything between their corresponding locations in the local folder hierarchy and the server folder hierarchy. The root of the local folder hierarchy is identified by the fact that it contains a parameters file called \"$rootFileDefault\"; $commandName executes this file and expects it to output key-value pairs, as in this example output: protocol rsync command /usr/local/bin/rsync host upload.yourhost.com url https://somedomain.com/ port 12345 user neddie dir public_html rsync-args-up --bwlimit=48 --partial finishUp echo Upload succeeded \$(date) Because \"$rootFileDefault\" is executed, it can run another script containing favorite defaults, as this example does. Values from multiple lines with the same key are catenated. Notes on the settings: * protocol can be rsync, ftp, or sftp, or omitted (default is rsync). * command is optional and defaults to the protocol name. * host is a domain name or IP address with no port appended; or, if protocol is rsync, host may be omitted for uploading to a folder on the same host. * url is optional; defaults to http://\$host; used only by the -u option. * port is optional. * dir is the path on the server; default is your home directory. * rsync-args, rsync-args-up, and rsync-args-down are optional; if present, these arguments are passed to rsync for up/download after the standard upload args $rsyncStdArgsUp or download args $rsyncStdArgsDown. Don't use --rsh here. * finishUp is a command to be run via ssh on host after a successful rsync upload. For example, after uploading crontab, this installs that crontab file on host and reports back the result: finishUp crontab crontab ; crontab -l Use rsync if possible; ftp and sftp copy with world-readable permissions, and they aren't smart (like rsync is) about transferring only what needs to be transferred. The rsync command sets permissions on the server to match those in the local hierarchy, so when the destination is a public web site, local dirs should be searchable by all and local files should be readable by all; $commandName warns if some permissions are too restrictive for web access. The -v option shows the rsync, ftp, or sftp transfer command actually used and puts the transfer command itself into verbose mode. The -q option refrains from warning about files with restricted permissions. The -n option sets -v and refrains from actually running the command. The -r rootFile option specifies a different root filename instead of \"$rootFileDefault\". The -o dir option argument is used as the destination directory for a download. http://Yost.com/computers/yostupload " exit 2 } #------------------------------------------------------------------------------- # Gobals server= rootDir= file= sshPortOption= trouble= sourceDirs=() sourceFiles=() # rootFile options protocol= host= port= user= remoteDir= downloadDir=. rsyncArgs= rsyncArgsUp= rsyncArgsDown= finishUp= #--------------------- argsRsync=() argRoot=() argVerbose=() argQuiet=() argDryRun=() argDownload=() argDownloadDir=() argLogin=() argInit=() argDump=() argNoWarn=() # undocumented if ! zparseopts -D -K - -help=argHelp \ -rsync-args:=argsRsync \ r:=argRoot \ R=argReportRootPath \ q=argQuiet \ v=argVerbose \ n=argDryRun \ d=argDownload \ o:=argDownloadDir \ l=argLogin \ u=argURL \ w=argNoWarn \ -dump=argDump \ i=argInit ; then optionError "" fi # Max one of these: -d -R -i -l -u # Max one path arg for -R and -u # Bug: must fix the code below to enforce these. bunch=($argDownload $argReportRootPath $argInit $argLogin $argURL) if [[ $#bunch > 1 ]] ; then optionError "No more than one of these: $bunch" fi if [[ $#argsRsync > 1 ]] ; then local bunch=($argReportRootPath $argInit $argLogin $argURL) if [[ $#bunch != 0 ]] ; then optionError "--rsync-args is incompatible with: $bunch" fi fi case $1 in -*) optionError "Unknown option: $1" ;; esac if [[ $#argHelp != 0 ]] ; then usage fi if [[ $#argDryRun != 0 ]] ; then argVerbose=(-v) fi argsRsync=${argsRsync:#--rsync-args} if [[ $#argRoot != 0 ]] ; then rootFile=${argRoot[2]} else rootFile=$rootFileDefault fi # To Do: # If rootFile is a directory, $commandName assumes that each file directly under # that directory should be executed in turn, to do multiple up/downloads. # # if [[ -d $rootFile ]] ; then # rootFiles=$(find $rootFile -type f -maxdepth 1) # fi if [[ $#argDownloadDir != 0 ]] ; then if [[ $#argDownload = 0 ]] ; then optionError "-o can be used only with -d" fi downloadDir=${argDownloadDir[2]} fi #------------------------------------------------------------------------------- echo-quoted() { local noNewlineArg case "$1" in -n) noNewlineArg=-n ; shift ;; *) noNewlineArg= ;; esac local first=YES local output='' while [[ $# != 0 ]] do case "$first" in YES) first=NO ;; *) output=$output' ' ;; esac case "$1" in "") output=$output"''" ;; # warning: what you see on the following line is really *' \t '*) *['" []()&$*|;<>?`'"'"]*) case "$1" in *'"'*"'"* | *"'"*'"'*) # both kinds of quote - output in '' output=$output"'" output=$output`echo "$1" | sed "s/'/\\\\\\'/g"` output=$output"'" ;; *"'"*) # single quote only - output in "" output=$output'"' output=$output`echo "$1" | sed 's/\([\`$"]\)/\\\\\1/g'` output=$output'"' ;; *) # double quote or no quotes - output in '' output=$output"'$1'" esac ;; *) output=$output$1 esac shift done echo $noNewlineArg "$output" } echo-command() { echo -n 1>&2 "[ " echo-quoted -n 1>&2 $@ echo 1>&2 " ]" } echodo() { if [[ $#argVerbose != 0 ]] ; then eval "echo-command $@" fi eval "$@" } checkOK() { local rootTmp="$1" local name="$2" local value="$3" if [[ ! -z "$value" \ && "${value##* }" != "$value" \ ]] then echo 1>&2 "$commandName: $rootTmp/$rootFile $name is bad: \"$value\"" trouble=true fi } # sets $server setServer() { local node=$1 local params if [[ ! -z "$server" \ && "$server" != $node \ ]] \ then echo echo 1>&2 "$commandName: can't upload files from two different roots: $server and $node" trouble=true return 1 fi server=$node if ! params="$($node/$rootFile)" ; then echo echo 1>&2 "$commandName: Trouble executing: $node/$rootFile" trouble=true return 1 fi # The extra level of echo collapses multiple lines into one. protocol=$(echo $(echo "$params" | sed -n 's,^[ ]*protocol[ ][ ]*,,p' )) host=$(echo $(echo "$params" | sed -n 's,^[ ]*host[ ][ ]*,,p' )) url=$(echo $(echo "$params" | sed -n 's,^[ ]*url[ ][ ]*,,p' )) port=$(echo $(echo "$params" | sed -n 's,^[ ]*port[ ][ ]*,,p' )) user=$(echo $(echo "$params" | sed -n 's,^[ ]*user[ ][ ]*,,p' )) remoteDir=$(echo $(echo "$params" | sed -n 's,^[ ]*dir[ ][ ]*,,p' )) rsyncArgs=$(echo $(echo "$params" | sed -n 's,^[ ]*rsync-args[ ][ ]*,,p' )) rsyncArgsUp=$(echo $(echo "$params" | sed -n 's,^[ ]*rsync-args-up[ ][ ]*,,p')) rsyncArgsDown=$(echo $(echo "$params" | sed -n 's,^[ ]*rsync-args-down[ ][ ]*,,p')) finishUp=$(echo $(echo "$params" | sed -n 's,^[ ]*finishUp[ ][ ]*,,p' )) if [[ $host == localhost ]] ; then host= fi if [[ -n $host ]] ; then if [[ -z "$user" ]] ; then echo 1>&2 "$commandName: user must be specified in $node/$rootFile" trouble=true else checkOK $node user "$user" fi else if [[ -n "$user" ]] ; then echo 1>&2 "$commandName: host is localhost, so user setting is ignored." user= fi port= fi if [[ -z $host ]] ; then if [[ $#argURL != 0 ]] ; then echo 1>&2 "$commandName: -u option makes no sense with no host setting." exit 2 fi if [[ $#argLogin != 0 ]] ; then echo 1>&2 "$commandName: -l option makes no sense with no host setting" exit 2 fi fi if [[ -z $url ]] ; then url="http://$host/" fi if [[ -n $port ]] ; then sshPortOption=" -p $port" fi case "$protocol" in rsync | "") protocol=rsync if [[ -n $host ]] ; then command="rsync --rsh 'ssh -l $user$sshPortOption'" else command="rsync" fi ;; ftp|sftp) if [[ -z "$host" ]] ; then echo 1>&2 "$commandName: host must be specified in $node/$rootFile" trouble=true return 1 else checkOK $node host "$host" fi local portOption if [[ -n $port ]] ; then portOption=":$port" fi command="$protocol $user@${host}$portOption" ;; *) echo 1>&2 "$commandName: protocol in $node/$rootFile must rsync, ftp, or sftp." trouble=true esac if [[ -z "$command" ]] ; then command="$(echo "$params" | sed -n 's,^[ ]*command[ ]*,,p'))" fi rootDir=$node/ if [[ $#argDump != 0 ]] ; then if [[ -n $protocol ]] ; then echo "protocol $protocol" ; fi if [[ -n $host ]] ; then echo "host $host" ; fi if [[ -n $url ]] ; then echo "url $url" ; fi if [[ -n $port ]] ; then echo "port $port" ; fi if [[ -n $user ]] ; then echo "user $user" ; fi if [[ -n $remoteDir ]] ; then echo "remoteDir $remoteDir" ; fi if [[ -n $rsyncArgs ]] ; then echo "rsyncArgs $rsyncArgs" ; fi if [[ -n $rsyncArgsUp ]] ; then echo "rsyncArgsUp $rsyncArgsUp" ; fi if [[ -n $rsyncArgsDown ]] ; then echo "rsyncArgsDown $rsycnArgsDown" ; fi if [[ -n $finishUp ]] ; then echo "finishUp $finishUp" ; fi if [[ -n $command ]] ; then echo "command $command" ; fi if [[ -n $sshPortOption ]] ; then echo "sshPortOption $sshPortOption" ; fi fi return 0 } mkdirp() { local node=$1 while [[ $node != . ]] do echo mkdir '"'"$node"'"' node=$node:h done } # Find rootDir for the given path. # Sets $rootDir to the dir containing $rootFile. # Sets $file to the part of the path that starts at $rootDir. findRoot() { local arg="$1" local node="$arg" file= if [[ $#argDownload != 0 ]] ; then # Download # Deal with possibility that whole path doesn't exist locally. while [[ "$node" != / ]] do if [[ -e "$node" ]] ; then break fi if [[ -z "$file" ]] ; then file=$node:t else file=${node:t}/$file fi node=$node:h done fi if [[ ! -d "$node" ]] ; then file=${node##*/} node=${node%/*} if [[ "$node" == "$file" ]] ; then node=. fi fi #echo \===1 node=$node file=$file if [[ "$node" == *..* ]] ; then node="$(removeDotDots "$(pwd -L)/$node")" fi #echo \===2 node=$node file=$file if [[ "$node" == "${node#/}" ]] ; then if [[ "$node" == . ]] ; then node="$(pwd -L)" else node="$(pwd -L)/$node" fi fi #echo \===3 node=$node file=$file host= user= remoteDir= rootDir= local foundRoot= while [[ $node != / ]] do if [[ -x $node/$rootFile ]] ; then if ! setServer "$node" ; then # $trouble is set return 1 fi foundRoot=true if [[ -z "$file" ]] ; then file= fi break fi if [[ -z "$file" ]] ; then file=$node:t else file=${node:t}/$file fi node=$node:h done if [[ ! -z "$trouble" ]] ; then return 1 fi if [[ -z "$foundRoot" ]] ; then echo echo 1>&2 "$commandName: Can't find $rootFile file for $arg" return 1 fi return 0 } removeDotDots() { local result=$1 local previous= while [[ $result != $previous ]] { previous=$result result=$( echo $result \ | sed 's,[^/][^/]*//*\.\.//*,,g' ) } echo $result } # Add an argument to sourceDirs or sourceFiles as appropriate # after finding its root directory. examine() { local arg="$1" if [[ $#argDownload == 0 ]] ; then # Upload if [[ $#argURL = 0 && ! -e $arg ]] ; then echo echo 1>&2 "$commandName: Does not exist: $arg" trouble=true return 1 fi fi if ! findRoot "$arg" ; then trouble=true return fi #echo file=$file if [[ -d "$rootDir$file" ]] ; then sourceDirs+="${file:-.}" else sourceFiles+="$file" fi } finish() { if [[ ! -z $finishUp ]] then if [[ $#argVerbose != 0 && ! -z $finishUp ]] ; then echo "[ Execute on $host: $finishUp ]" fi sshCommand "$finishUp" fi } sshCommand() { local baseDir if [[ -n $sourceDirs && -n $sourceDirs[1] ]] then baseDir=$sourceDirs[1] else baseDir=$sourceFiles[1]:h fi # Unfortunately, there is no way to make the following code work on all remote shells. # The code as I provide it works when zsh is the remote shell. If you're using bash # as the remote shell, move the -l before the $0. if [[ $#argVerbose != 0 ]] then echo 1>&2 "[ ssh -t -l $user $host$sshPortOption "'"'"cd ${remoteDir:+${(qq)remoteDir}/}${(qq)baseDir} && $@"'"'" ]" fi if [[ $#argDryRun != 0 ]] then return 0 fi if ssh -t -l $user $host$sshPortOption "cd ${remoteDir:+${(qq)remoteDir}/}${(qq)baseDir} && $@" ; then return 0 else echo 1>&2 "$commandName: trouble executing ssh" return 2 fi } #------------------------------------------------------------------------------- # Do -i if [[ $#argInit != 0 ]] ; then # Initialize this dir if [[ -e $rootFileDefault ]] ; then echo 1>&2 "$commandName: $rootFileDefault already exists" exit 2 fi if [[ $#@ < 3 ]] ; then optionError "-i requires more arguments" fi user=$1 ; shift host=$1 ; shift dir=$1 ; shift echo > $rootFileDefault \ "#!/bin/sh echo user $user echo host $host echo dir $dir echo rsync-args --partial --progress echo rsync-args $@" chmod a+x $rootFileDefault exit 0 fi # Examine the args to find $rootFile, etc. if [[ $# < 1 ]] ; then # No arguments; do current directory. if [[ $#argVerbose != 0 ]] ; then echo -n $commandName: examining current directory' ' fi examine . else # With arguments if [[ $#argVerbose != 0 ]] ; then echo -n $commandName: examining arguments' ' fi for arg in "$@" do if [[ $#argVerbose != 0 ]] ; then echo -n . fi examine "$arg" done fi if [[ ! -z "$trouble" ]] ; then exit 2 fi if [[ $#argVerbose != 0 ]] ; then echo fi #echo rootDir=$rootDir #echo sourceDirs="${sourceDirs[@]}" #echo sourceFiles="${sourceFiles[@]}" if [[ $#argLogin != 0 ]] ; then # ssh to the remote dir exec sshCommand 'exec $0 -l' exit 2 fi if [[ $#argReportRootPath != 0 ]] ; then echo ${rootDir}$rootFile exit 0 fi if [[ $#argURL != 0 ]] ; then for x in $sourceFiles[@] if [[ -n $url ]] ; then echo ${url%%/}/$x else echo $x fi exit 0 fi set -eu if [[ $#argVerbose != 0 ]] ; then echo "[ cd ${(qq)rootDir} ]" fi cd $rootDir checkDirPermissions() { if [[ "${(u)#}" != 0 ]] ; then local dirs="$(find -L "${(u)@}" -type d ! -perm -111)" complainAboutFilePermissions "$dirs" folders fi } checkFilePermissions() { if [[ "${(u)#}" != 0 ]] ; then local files="$(find -L "${(u)@}" -type f ! -perm -444)" complainAboutFilePermissions "$files" files fi } complainAboutFilePermissions() { if [[ $#argNoWarn == 0 ]] ; then if [[ -n $1 ]] ; then echo 1>&2 -n "$commandName: Warning: permissions are too restrictive on some $2" if [[ $#argVerbose != 0 ]] ; then echo : echo "$1" | sed 's,^, ,' 1>&2 else echo fi fi fi } case $protocol in rsync) local hostPrefix if [[ -n $host ]] ; then hostPrefix=$host: fi ( if [[ $#argDownload == 0 ]] ; then # Upload if [[ $#argQuiet == 0 ]] ; then checkDirPermissions "${(u)sourceDirs[@]}" checkFilePermissions "${(u)sourceDirs[@]}" checkFilePermissions "${(u)sourceFiles[@]}" fi echodo \ $command \ $rsyncStdArgsUp \ ${rsyncArgs[@]} \ ${rsyncArgsUp[@]} \ ${argsRsync[@]} \ ${argVerbose:+--progress} \ $argDryRun \ ${(qq)sourceDirs[@]} \ ${(qq)sourceFiles[@]} \ ${hostPrefix}${(qq)remoteDir} \ && finish else # Download local downloadArgs if true ; then # Support for --files-from is not yet universal local items items=($sourceDirs[@] $sourceFiles[@]) for ((i = 1 ; i <= $#items ; ++i)) do if [[ "${items[i]##* }" != "$items[i]" ]] ; then items[i]="${items[i]// /\ }" fi done if [[ -n $host ]] ; then downloadArgs="--rsync-path='cd ${(qq)remoteDir};rsync' ${hostPrefix}'$items[@]'" else # not clear this works right downloadArgs="${(qq)remoteDir}/'$items[@]'" fi else # This is another way in more recent versions of rsync. local items= for x in "$sourceDirs[@]" "$sourceFiles[@]" do items="$items$x " done echo "$items" > $rootDir/.rsyncFileList chmod 600 $rootDir/.rsyncFileList echodo rsync -p \ $argVerbose \ $rootDir/.rsyncFileList \ $host${(qq)remoteDir} downloadArgs="--files-from=:${remoteDir:+${(qq)remoteDir}/}.rsyncFileList $items" fi echodo \ $command \ $rsyncStdArgsDown \ ${rsyncArgs[@]} \ ${rsyncArgsDown[@]} \ ${argsRsync[@]} \ ${argVerbose:+--progress} \ $argDryRun \ $downloadArgs \ $downloadDir fi ) ;; ftp|sftp) if [[ $#argDownload != 0 ]] ; then echo 1>&2 "$commandName: Only upload supported for ftp in this release." exit 2 fi if [[ $#argVerbose != 0 ]] ; then echo $protocol $user@$host fi if [[ $#argDownload != 0 ]] ; then echo 1>&2 "$commandName: Only upload supported for ftp in this release." exit 2 fi ( echo "binary on" if [[ $#argVerbose != 0 ]] ; then echo "verbose on" else echo "verbose off" fi if [[ ! -z "${(qq)remoteDir}" ]] ; then echo "cd ${(qq)remoteDir}" fi ( local file for file in "${(ou)sourceDirs[@]}" do mkdirp "$file" find "$file" -type d | sed 's,.*,mkdir "&",' done ) | sort -u ( local file for file in "${(ou)sourceDirs[@]}" do find "$file" -type f | sed 's,.*,put "&" "&",' done ) | sort -u ) \ | tee /tmp/$commandName-$(date '+%Y-%m-%dT%H:%M:%S') \ | \ if [[ $#argVerbose != 0 ]] ; then tee /dev/tty else cat fi \ | \ if [[ $#argDryRun == 0 ]] ; then eval ${command[@]} fi ;; esac # Copyright 2005-2008 Dave Yost # All rights reserved. # This version is # yostupload 4.1 2011-07-21 # which at time of this publication can be found at: # http://Yost.com/computers/yostupload # Redistribution and use in the form of source code or derivative data built # from the source code, with or without modification, are permitted provided # that the following conditions are met: # 1. THE USER AGREES THAT THERE IS NO WARRANTY. # 2. If and only if appropriate, the above phrase "This version is" must be # followed by the phrase "a modified form of" or "extracted from" or # "extracted and modified from". # 3. Redistributions of source code must retain this notice intact. # 4. Redistributions in the form of derivative data built from the source # code must reproduce this notice intact in the documentation and/or other # materials provided with the distribution, and each file in the derivative # data must reproduce any Yost.com URI included in the original distribution. # 5. Neither the name of Dave Yost nor the names of its contributors may be # used to endorse or promote products derived from this software without # specific prior written permission. # 6. Written permission by the author is required for redistribution as part # of a commercial product. # This notice comprises all text from "Copyright" above through here.