Ci-dessous, les différences entre deux révisions de la page.
| Les deux révisions précédentesRévision précédente | |||
| 04-linux:40-rsync [2016/11/16 12:09] – Roge | 04-linux:40-rsync [2017/10/06 23:38] (Version actuelle) – modification externe 127.0.0.1 | ||
|---|---|---|---|
| Ligne 1: | Ligne 1: | ||
| + | ====== rsync & ssh ====== | ||
| + | |||
| + | Pour réaliser le backup d'un dossier vers un dossier d'une autre machine il faut : | ||
| + | - Disposer d'un accès à la machine distante via SSH ([[04-linux: | ||
| + | |||
| + | - [[04-linux: | ||
| + | - Utiliser les scripts de backup. | ||
| + | |||
| + | ===== Le script bash de backup principal (générique) ===== | ||
| + | Ce script n'est rien d' | ||
| + | |||
| + | <file bash sshDirSynchro.sh> | ||
| + | #!/bin/bash | ||
| + | # sshDirSynchro.sh | ||
| + | # | ||
| + | # | ||
| + | # It is just an Rsync wrapper. | ||
| + | # | ||
| + | # V0.1 - 2014-08-17 | ||
| + | # V0.2 - 2014-08-18 | ||
| + | # Suppression du rapport de syncho intégré | ||
| + | # Suppression de messages | ||
| + | # V0.3 - 2014-08-18 | ||
| + | # l' | ||
| + | # l' | ||
| + | # l' | ||
| + | # lors d'une copie depuis Windows (on fixe les droits chmod=u+rwx). | ||
| + | |||
| + | TIME_START_1970=$(date +%s) | ||
| + | |||
| + | # | ||
| + | # Define usage | ||
| + | # | ||
| + | |||
| + | usage() { | ||
| + | cat << | ||
| + | |||
| + | sshDirSynchro.sh: | ||
| + | | ||
| + | Usage: sshDirSynchro.sh --backup|--restore -ld < | ||
| + | ] | ||
| + | |||
| + | Mandatory parameters: | ||
| + | --backup | ||
| + | copy from local to remote | ||
| + | --restore | ||
| + | copy from remote to local | ||
| + | -ld, --local-dir | ||
| + | Local directory from where the data are copied | ||
| + | -rh, --remote-host | ||
| + | Remote host to where the data are copied | ||
| + | -ru, --remote-user | ||
| + | Authorized user for the remote host | ||
| + | -rd, --remote-dir | ||
| + | Remote directory to where the data are copied | ||
| + | |||
| + | Optionnal parameters: | ||
| + | -h, --help | ||
| + | Display usage information. | ||
| + | -p, --ssh-port | ||
| + | Default port is 22. | ||
| + | -bwl, --bandwidth-limit | ||
| + | Embeded " | ||
| + | -ksd, --kill-sleep-delay | ||
| + | Permit to kill the script after a user defined delay. | ||
| + | Exemple: -ksd " | ||
| + | See man sleep for delay format. | ||
| + | -lf, --log-file | ||
| + | If no log file is set, the report is writen on stdout. | ||
| + | If -lf is set a log file is created for the report. | ||
| + | --delete | ||
| + | rsync delete extraneous files from dest dirs | ||
| + | --chmod | ||
| + | ensure chmod=u+rwx on remote host | ||
| + | --inconv | ||
| + | convert chrset from Windows to Linux | ||
| + | --check | ||
| + | compare local and remote dir content | ||
| + | |||
| + | *** Caution: | ||
| + | This script use several ssh commands in batch mode. To avoid password asking ssh keys must be installed. | ||
| + | See man ssh-keygen | ||
| + | | ||
| + | _usage_statement_ | ||
| + | } | ||
| + | |||
| + | # | ||
| + | # Analyse les arguments de la ligne de commande. | ||
| + | # | ||
| + | SSH_PORT=22 | ||
| + | # echo -e " | ||
| + | while [[ $# > 0 ]] | ||
| + | do | ||
| + | # echo -e "$# - OPTION: $1" | ||
| + | case $1 in | ||
| + | --backup) | ||
| + | if [ " | ||
| + | then | ||
| + | echo -e " | ||
| + | exit 1 | ||
| + | fi | ||
| + | SYNC_OPTION=" | ||
| + | shift | ||
| + | ;; | ||
| + | --restore) | ||
| + | if [ " | ||
| + | then | ||
| + | echo -e " | ||
| + | exit 1 | ||
| + | fi | ||
| + | SYNC_OPTION=" | ||
| + | shift | ||
| + | ;; | ||
| + | -ld|--local-dir) | ||
| + | LOCAL_WORKING_DIR=$2; | ||
| + | shift 2 | ||
| + | ;; | ||
| + | -rh|--remote-host) | ||
| + | REMOTE_HOST=$2 | ||
| + | shift 2 | ||
| + | ;; | ||
| + | -ru|--remote-user) | ||
| + | REMOTE_USER=$2 | ||
| + | shift 2 | ||
| + | ;; | ||
| + | -rd|--remote-dir) | ||
| + | REMOTE_WORKING_DIR=$2 | ||
| + | shift 2 | ||
| + | ;; | ||
| + | -p|--ssh-port) | ||
| + | SSH_PORT=$2 | ||
| + | shift 2 | ||
| + | ;; | ||
| + | -bwl|--bandwidth-limit) | ||
| + | BAND_WIDTH_LIMIT=$2 | ||
| + | shift 2 | ||
| + | ;; | ||
| + | -ksd|--kill-sleep-delay) | ||
| + | KILL_SLEEP_DELAY=$2 | ||
| + | shift 2 | ||
| + | ;; | ||
| + | -lf|--log-file) | ||
| + | LOGFILE=" | ||
| + | shift | ||
| + | ;; | ||
| + | --delete) | ||
| + | DELETE_OPTION=" | ||
| + | shift | ||
| + | ;; | ||
| + | --chmod) | ||
| + | CHMOD_OPTION=" | ||
| + | shift | ||
| + | ;; | ||
| + | --iconv) | ||
| + | ICONV_OPTION=" | ||
| + | shift | ||
| + | ;; | ||
| + | --check) | ||
| + | CHECK=$1 | ||
| + | shift | ||
| + | ;; | ||
| + | -h|--help) | ||
| + | usage | ||
| + | exit 0 | ||
| + | ;; | ||
| + | *) # Unknown option | ||
| + | echo -e "Error : Unknown option $1\n" | ||
| + | usage | ||
| + | exit -1 | ||
| + | ;; | ||
| + | esac | ||
| + | done | ||
| + | |||
| + | [ " | ||
| + | [ " | ||
| + | [ ${REMOTE_HOST} ] || echo -e " | ||
| + | [ ${REMOTE_USER} ] || echo -e " | ||
| + | [ ${REMOTE_WORKING_DIR} ] || echo -e " | ||
| + | |||
| + | if [ ! " | ||
| + | then | ||
| + | usage | ||
| + | exit -1 | ||
| + | fi | ||
| + | |||
| + | # | ||
| + | # Create a custom logger | ||
| + | # | ||
| + | |||
| + | if [ ${LOGFILE} ] | ||
| + | then | ||
| + | | ||
| + | #echo -e " | ||
| + | echo -e " | ||
| + | } | ||
| + | else | ||
| + | | ||
| + | #echo -e " | ||
| + | echo -e " | ||
| + | } | ||
| + | fi | ||
| + | |||
| + | # | ||
| + | # Test local diretory is ok | ||
| + | # | ||
| + | |||
| + | # if [ ! -d ${LOCAL_WORKING_DIR} ] | ||
| + | # then | ||
| + | # logprint | ||
| + | # exit -1 | ||
| + | # fi | ||
| + | |||
| + | # | ||
| + | # Test remote host is ok | ||
| + | # | ||
| + | |||
| + | if [ $LOGFILE ] | ||
| + | then | ||
| + | echo -e " | ||
| + | fi | ||
| + | |||
| + | # if ( ! ssh $REMOTE_USER@$REMOTE_HOST -p $SSH_PORT " | ||
| + | # then | ||
| + | # logprint | ||
| + | # exit -1 | ||
| + | # fi | ||
| + | |||
| + | # if ( ! ssh $REMOTE_USER@$REMOTE_HOST -p $SSH_PORT "cd $REMOTE_WORKING_DIR" | ||
| + | # then | ||
| + | # logprint | ||
| + | # exit -1 | ||
| + | # fi | ||
| + | |||
| + | # | ||
| + | # Lancement du kill de ce processus en tâche de fond | ||
| + | # pour limiter la durée d' | ||
| + | # | ||
| + | # Attention: | ||
| + | # la commande rsync s' | ||
| + | # l' | ||
| + | # il faut donc faire un "pkill -P $$" pour tuer l' | ||
| + | |||
| + | if [ ${KILL_SLEEP_DELAY} ] | ||
| + | then | ||
| + | | ||
| + | |||
| + | sleep $KILL_SLEEP_DELAY \ | ||
| + | && logprint | ||
| + | && logprint | ||
| + | && logprint | ||
| + | && pkill -P $$ & | ||
| + | # variante d' | ||
| + | # logprint | ||
| + | |||
| + | # On stocke le PID de process lancé en backgound pour pouvoir le tuer | ||
| + | | ||
| + | fi | ||
| + | |||
| + | # | ||
| + | # Recupération sur erreur | ||
| + | # | ||
| + | function traitement_erreur { | ||
| + | kill $KILL_ME_PID > /dev/null 2>&1 | ||
| + | | ||
| + | | ||
| + | | ||
| + | | ||
| + | | ||
| + | # remote_report | ||
| + | exit -1 | ||
| + | } | ||
| + | trap traitement_erreur SIGHUP SIGINT SIGTERM SIGKILL | ||
| + | |||
| + | # | ||
| + | # Set rsync options | ||
| + | # | ||
| + | # options rsync: | ||
| + | # -a, --archive | ||
| + | # -v, --verbose | ||
| + | # -S, --sparse | ||
| + | # -P équivalent à --partial --progress | ||
| + | # | ||
| + | # | ||
| + | # | ||
| + | # | ||
| + | # --no-p | ||
| + | # --no-g | ||
| + | # --chmod=ugo=rwX | ||
| + | # --delete-excluded | ||
| + | # --exclude=PATTERN | ||
| + | # -s, --protect-args | ||
| + | # --iconv=ISO-8859-1, | ||
| + | |||
| + | # RSYNC_OPTIONS=" | ||
| + | # le -e \"ssh -p $SSH_PORT \" pose problème dans RSYNC_OPTIONS | ||
| + | # le --chmod=u=rwX semble nécessaire pour le Raspi | ||
| + | |||
| + | RSYNC_OPTIONS=" | ||
| + | [ ${CHMOD_OPTION} ] && RSYNC_OPTIONS=" | ||
| + | [ ${DELETE_OPTION} ] && RSYNC_OPTIONS=" | ||
| + | [ " | ||
| + | [ ${BAND_WIDTH_LIMIT} ] && RSYNC_OPTIONS=" | ||
| + | # | ||
| + | |||
| + | #logprint " | ||
| + | logprint " | ||
| + | if [[ ${SYNC_OPTION} = " | ||
| + | then | ||
| + | logprint " | ||
| + | logprint " | ||
| + | fi | ||
| + | if [[ ${SYNC_OPTION} = " | ||
| + | then | ||
| + | logprint " | ||
| + | logprint " | ||
| + | fi | ||
| + | logprint " | ||
| + | logprint " | ||
| + | |||
| + | |||
| + | # | ||
| + | # Build final command | ||
| + | # | ||
| + | |||
| + | RSYNC_COMMAND=" | ||
| + | [ " | ||
| + | [ " | ||
| + | [ ${LOGFILE} ] && RSYNC_COMMAND=" | ||
| + | |||
| + | logprint " | ||
| + | |||
| + | # !!!! eval introduces potential vulnerabilities and unexpected behavior situations so must be used !!! | ||
| + | |||
| + | eval ${RSYNC_COMMAND} | ||
| + | | ||
| + | | ||
| + | | ||
| + | | ||
| + | exit -2 | ||
| + | } | ||
| + | |||
| + | [ " | ||
| + | |||
| + | # | ||
| + | # On tue le process de kill (qu'il existe ou pas) | ||
| + | # afin qu'il n' | ||
| + | # | ||
| + | if [ ${KILL_SLEEP_DELAY} ] | ||
| + | then | ||
| + | # logprint | ||
| + | kill $KILL_ME_PID > /dev/null 2>&1 | ||
| + | fi | ||
| + | |||
| + | # | ||
| + | # fin des opération | ||
| + | # | ||
| + | TIME_END_1970=$(date +%s) | ||
| + | logprint | ||
| + | logprint " | ||
| + | |||
| + | </ | ||
| + | |||
| + | ===== Le script contrôle du backup (générique) ===== | ||
| + | |||
| + | |||
| + | Ce script compare des repertoires qui euvent être locaux et/ou distants. [[00-linux: | ||
| + | |||
| + | <file bash compareDirs.sh> | ||
| + | #!/bin/bash | ||
| + | # | ||
| + | # | ||
| + | # | ||
| + | # V0.1 - 2014-11-23 | ||
| + | # Missing: SSH port management | ||
| + | |||
| + | # | ||
| + | # Define usage | ||
| + | # | ||
| + | |||
| + | usage() { | ||
| + | cat << | ||
| + | |||
| + | compareDirs.sh compares the top level content of 2 local or remote directories using du. | ||
| + | | ||
| + | Usage: compareDirs.sh [[user@]host1: | ||
| + | | ||
| + | Note: SSH port management is missing. | ||
| + | |||
| + | _usage_statement_ | ||
| + | } | ||
| + | |||
| + | # | ||
| + | # Analyse les arguments de la ligne de commande. | ||
| + | # | ||
| + | |||
| + | echo -e " | ||
| + | |||
| + | [[ $# != 2 ]] && { | ||
| + | usage | ||
| + | exit -1 | ||
| + | } | ||
| + | |||
| + | function getDirFromParam { | ||
| + | | ||
| + | if [ $idx == 0 ] | ||
| + | then | ||
| + | echo $1 | ||
| + | else | ||
| + | echo ${1:$idx} | ||
| + | fi | ||
| + | } | ||
| + | |||
| + | function getHostFromParam { | ||
| + | | ||
| + | if [ $idx != 0 ] | ||
| + | then | ||
| + | ((idx--)) | ||
| + | echo ${1:0:$idx} | ||
| + | fi | ||
| + | } | ||
| + | |||
| + | function checkHostAndDir { | ||
| + | | ||
| + | | ||
| + | if [ $host ] | ||
| + | then | ||
| + | # echo "$host - $dir --> REMOTE" | ||
| + | if ( ! ssh $host " | ||
| + | then | ||
| + | | ||
| + | | ||
| + | elif ( ! ssh $host "cd $dir" > /dev/null 2>&1 ) | ||
| + | then | ||
| + | | ||
| + | | ||
| + | fi | ||
| + | | ||
| + | # echo " | ||
| + | [ -d " | ||
| + | echo -e "Error $dir must be a directory!" | ||
| + | | ||
| + | } | ||
| + | fi | ||
| + | | ||
| + | } | ||
| + | |||
| + | dir1=$(getDirFromParam $1) | ||
| + | host1=$(getHostFromParam $1) | ||
| + | dir2=$(getDirFromParam $2) | ||
| + | host2=$(getHostFromParam $2) | ||
| + | |||
| + | # echo -e "dir1 : $dir1 - host1 : $host1\ndir2 : $dir2 - host2 : $host2" | ||
| + | |||
| + | checkHostAndDir " | ||
| + | [ $? -eq 0 ] || exit -1 | ||
| + | checkHostAndDir " | ||
| + | [ $? -eq 0 ] || exit -1 | ||
| + | |||
| + | # | ||
| + | # VARS | ||
| + | # | ||
| + | |||
| + | tmp_report_1="/ | ||
| + | tmp_report_2="/ | ||
| + | |||
| + | # | ||
| + | # Recupération sur erreur | ||
| + | # | ||
| + | function traitement_erreur { | ||
| + | | ||
| + | | ||
| + | | ||
| + | | ||
| + | | ||
| + | rm -f $tmp_report_1 $tmp_report_2 | ||
| + | exit 1 | ||
| + | } | ||
| + | trap traitement_erreur SIGHUP SIGINT SIGTERM SIGKILL | ||
| + | |||
| + | # | ||
| + | # fonction de bilan de synchro | ||
| + | # déclarée ici pour pouvoir être utilisée par le traitement d' | ||
| + | # | ||
| + | |||
| + | function local_report { | ||
| + | | ||
| + | echo $1 | sed ': | ||
| + | } | ||
| + | function stats_folder() { | ||
| + | # Attention : le nom du dossier peut contenir des blancs | ||
| + | # il faut donc le traiter entre "" | ||
| + | local folder=" | ||
| + | local items=`find " | ||
| + | local taille=0 | ||
| + | echo -e " | ||
| + | # echo -e " | ||
| + | if [ $items -gt 0 ]; then | ||
| + | taille=$(du -bc " | ||
| + | | ||
| + | | ||
| + | echo -e " | ||
| + | else | ||
| + | echo vide | ||
| + | fi | ||
| + | } | ||
| + | local curdir=$(pwd) | ||
| + | local dir=" | ||
| + | echo -e "LOCAL report:" | ||
| + | echo -e " | ||
| + | cd " | ||
| + | for f in * ; do | ||
| + | # Attention : le nom du dossier peut contenir des blancs | ||
| + | # il faut donc le passer entre "" | ||
| + | #echo "---f: $f" | ||
| + | [ -d " | ||
| + | done | ||
| + | echo -e " | ||
| + | stats_folder " | ||
| + | # echo "--- pwd: $(pwd) " | ||
| + | cd " | ||
| + | echo | ||
| + | } | ||
| + | |||
| + | function remote_report { | ||
| + | local dir=" | ||
| + | local host=" | ||
| + | ssh $host " | ||
| + | function numFormat { | ||
| + | echo \$1 | sed ': | ||
| + | } | ||
| + | function stats_folder() { | ||
| + | # Attention : le nom du dossier peut contenir des blancs | ||
| + | # il faut donc le traiter entre \" | ||
| + | local folder=\" | ||
| + | local items=\`find \" | ||
| + | local taille=0 | ||
| + | | ||
| + | if test \$items -ne 0; then | ||
| + | taille=\$(du -bc \" | ||
| + | taille_f=\$(numFormat \$taille) | ||
| + | items_f=\$(numFormat \$items) | ||
| + | echo -e \" | ||
| + | | ||
| + | echo vide | ||
| + | fi | ||
| + | } | ||
| + | |||
| + | dir=$(readlink -f " | ||
| + | echo \" | ||
| + | echo $dir | ||
| + | cd $dir | ||
| + | for folder in * ; do | ||
| + | # Attention : le nom du dossier peut contenir des blancs | ||
| + | # il faut donc le passer entre \" | ||
| + | [ -d \" | ||
| + | done | ||
| + | echo -e \" | ||
| + | stats_folder \" | ||
| + | echo | ||
| + | " | ||
| + | } | ||
| + | echo | ||
| + | |||
| + | # set -x | ||
| + | |||
| + | if [ $host1 ] | ||
| + | then | ||
| + | | ||
| + | else | ||
| + | | ||
| + | fi | ||
| + | |||
| + | if [ $host2 ] | ||
| + | then | ||
| + | | ||
| + | else | ||
| + | | ||
| + | fi | ||
| + | |||
| + | sdiff -atw 140 $tmp_report_1 $tmp_report_2 | ||
| + | rm -f $tmp_report_1 $tmp_report_2 | ||
| + | </ | ||
| + | |||
| + | ===== Un exemple de script de backup ===== | ||
| + | |||
| + | < | ||
| + | #!/bin/bash | ||
| + | |||
| + | N73SM=" | ||
| + | RASPI=" | ||
| + | BUREAU=" | ||
| + | |||
| + | DIR_SCRIPTS_BUREAU="/ | ||
| + | DIR_SCRIPTS_N73SM="/ | ||
| + | DIR_SCRIPTS_RASPI="/ | ||
| + | |||
| + | case `hostname` in | ||
| + | $N73SM) | ||
| + | # Scripts : N73SM --> Raspi | ||
| + | echo -e " | ||
| + | ./ | ||
| + | -rd $DIR_SCRIPTS_RASPI \ | ||
| + | -ld $DIR_SCRIPTS_N73SM | ||
| + | ;; | ||
| + | $BUREAU) | ||
| + | # Scripts : Bureau --> N73SM | ||
| + | echo -e " | ||
| + | | ||
| + | -rd $DIR_SCRIPTS_N73SM \ | ||
| + | -ld $DIR_SCRIPTS_BUREAU | ||
| + | |||
| + | # execution sur N73SM pour alimenter le Raspi | ||
| + | ssh roge@n73sm "cd / | ||
| + | ;; | ||
| + | *) # Unknown option | ||
| + | echo " | ||
| + | exit -1 | ||
| + | ;; | ||
| + | esac | ||
| + | </ | ||
| + | |||
| + | ===== Un exemple de rapport de backup : ===== | ||
| + | Le rapport suivant est produit par le script ci-dessus : | ||
| + | < | ||
| + | $ ./ | ||
| + | Backup Scripts : Bureau --> N73SM | ||
| + | 09:18:47: ------------------------------------------------------ | ||
| + | 09:18:47: Synchronisation | ||
| + | 09: | ||
| + | 09: | ||
| + | 09: | ||
| + | 09: | ||
| + | 09:18:47: ------------------------------------------------------ | ||
| + | 09:18:47: | ||
| + | 09:18:47: >> rsync -avS --no-p --no-g --delete-before -e "ssh -p 22" "/ | ||
| + | building file list ... done | ||
| + | deleting sshDirectorySynchro.sh | ||
| + | ./ | ||
| + | archive/ | ||
| + | archive/ | ||
| + | |||
| + | sent 1429 bytes received 121 bytes 3100.00 bytes/sec | ||
| + | total size is 175567 | ||
| + | 09:18:47: ******** Fin du traitment OK! ********** | ||
| + | 09:18:47: | ||
| + | |||
| + | -------------- | ||
| + | LOCAL report : | ||
| + | -------------- | ||
| + | LOCAL_WORKING_DIR: | ||
| + | - archive: 21 fichiers, 133K | ||
| + | - exemples: 6 fichiers, 24K - exemples: 6 fichiers, 24K | ||
| + | - restore: 18 fichiers, 96K - restore: 18 fichiers, 96K | ||
| + | - test: 1 fichiers, 4,0K - test: 1 fichiers, 4,0K | ||
| + | -------------------------- | ||
| + | Total: | ||
| + | - .: 57 fichiers, 329K | ||
| + | </ | ||
| + | |||
| + | |||
| + | ===== Autres solutions ===== | ||
| + | |||
| + | Tester [[https:// | ||
| + | |||