#!/bin/bash # # Author: J.G. Messchendorp, messchendorp@kvi.nl # # USAGE: movefiles [arg1] [arg2] [arg3] [arg4] # # with # # [arg1] ($1): source directory # [arg2] ($2): destination # [arg3] ($3): in case you want to copy files, set to "1" # [arg4] ($4): in case you want to delete the source files after copying, set to "1" # # Examples: # # 1) movefiles /scratch/mydata root://kvip81.kvi.nl:1094///daq/mydata 1 1 # 2) movefiles johan@opteron:~/stuff /scratch/stuff 1 0 # 3) movefiles /scratch/0 whatever 0 1 # # 1) moves files from local filesystem or nfs system "/scratch/mydata" to remote storage, # located at "kvip81.kvi.nl" and on "/daq/mydata" using xrdcp program (xrootd protocol) # 2) copies files from remote system, "opteron:~/stuff" using "scp" and "johan" as username # to local or nfs filesystem "/scratch/stuff" # 3) removes the directory or file "/scratch/0" # if [ ! "$#" = "4" ]; then echo echo " Not enough or too many arguments supplied." echo echo " Usage: movefiles [source] [destination] [copy files (0/1)] [delete source (0/1)]" echo exit 1 fi # # If the location of ssh, scp, or xrdcp programs are not garantueed to be set up properly # on your system, please enrich your PATH variable accordingly here.... # export PATH=$PATH:$HOME/bin # # Below some important local variables and settings # RETVAL=0 # Default return value (0=OK, anything else is ERROR) NROFTRIES=100 # Maximum number of attempts for data transfer BACKUP="no" # Set to "yes" if you want old data to be kept KEEPLOG="no" # Keep the logfile in case of error or remove it (yes/no) SCPPROG="`which scp`" # path and filename of scp program XRDPROG="`which xrdcp`" # path and filename of xrdcp program COMPROT="`which ssh`" # Use ssh for communication LOGFILE="/tmp/mv_$$.log" # The name and location of the temporarily logfile # # Return if source and destination are the same # if [ ${1%/} = ${2%/} ]; then exit $RETVAL fi # # Analyze the destination # arg=$2 dir=${arg#*:} host=${arg%%:*} argsrc=$1 dirsrc=${argsrc#*:} hostsrc=${argsrc%%:*} # # Check whether ssh exists # if [ -z "$COMPROT" ]; then echo " Could not find \"ssh\", please make sure your PATH is set properly!" RETVAL=1 exit $RETVAL fi prot="$COMPROT -n $host" # Use ssh for communication if [ $host = $dir ]; then host="localhost" prot="" # In case of localhost fi # # Check whether xrdcp is going to be used # if [ $hostsrc = "root" ] || [ $host = "root" ]; then if [ -z "$XRDPROG" ]; then echo " Could not find \"xrdcp\", please make sure your PATH is set properly!" RETVAL=1 exit $RETVAL fi CPPROT="$XRDPROG -R -s" else if [ -z "$SCPPROG" ]; then echo " Could not find \"scp\", please make sure your PATH is set properly!" RETVAL=1 exit $RETVAL fi CPPROT="$SCPPROG -B -r" fi # # Try to create logfile, otherwise use /dev/null instead (no log) # touch $LOGFILE RETVAL=$? if [ ! $RETVAL==0 ]; then echo " Error creating logfile $LOGFILE, no log produced!" LOGFILE="/dev/null" RETVAL=0 fi # # Check whether output directory exists # If so, remove or rename it, depending on BACKUP # Note that this only works in case SCPPROG is used as CPPROT # if [ ! $host = "root" ]; then isthere="yes" $prot ls $dir 2>&1 | grep -i "No such file or directory" >> $LOGFILE 2>&1 && isthere="no" if [ $dir = "." ] || [ $dir = "./" ] || [ $dir = `pwd` ]; then isthere="no" fi if [ $isthere = "yes" ]; then if [ $BACKUP = "yes" ]; then date=`date +%d_%m_%y` newdir="${dir%/*}/oldstuff_$date" isthere="yes" $prot ls $newdir 2>&1 | grep -i "No such file or directory" >> $LOGFILE 2>&1 && isthere="no" if [ $isthere = "no" ]; then $prot mkdir $newdir >> $LOGFILE 2>&1 else checkdir="$newdir/${dir##*/}" $prot rm -rf $checkdir >> $LOGFILE 2>&1 fi $prot mv $dir $newdir/ >> $LOGFILE 2>&1 else $prot rm -rf $dir >> $LOGFILE 2>&1 fi fi fi # # Copy files if $3 set # if [ "$3" = "1" ]; then # # Check whether destination subdirectory exists # If it doesnt exist, try to create it. # Note that this only works in case SCPPROG is used as CPPROT # if [ ! $host = "root" ]; then subdir=${dir%/*} isthere="yes" $prot ls $subdir 2>&1 | grep -i "No such file or directory" >> $LOGFILE 2>&1 && isthere="no" if [ $isthere = "no" ]; then echo " Destination directory $subdir does not exist, I will try to create it..." $prot mkdir -p $subdir >> $LOGFILE 2>&1 fi fi # # Copy files.... # FAILURE=1 while [ $FAILURE != 0 ] do if [ $NROFTRIES == 0 ] ; then FAILURE=0 RETVAL=1 else if [ $NROFTRIES == 1 ] ; then $CPPROT $1 $2 >> $LOGFILE 2>&1 else $CPPROT $1 $2 > /dev/null 2>&1 fi FAILURE=$? let NROFTRIES=NROFTRIES-1 fi done fi # # In case of failure, make an error mark # if [ $RETVAL != 0 ] ; then echo " The \"movefiles\" process was not successful:" cat $LOGFILE if [ $KEEPLOG = "no" ]; then rm -f $LOGFILE > /dev/null 2>&1 fi exit $RETVAL fi # # Remove files in $4 set # if [ "$4" = "1" ]; then prot="$COMPROT -n $hostsrc" if [ $hostsrc = $dirsrc ]; then hostsrc="localhost" prot="" fi $prot rm -rf $dirsrc >> $LOGFILE 2>&1 RETVAL=$? fi # # Clean the log and exit successfully # rm -f $LOGFILE > /dev/null 2>&1 exit $RETVAL