diff --git a/backups-manager.sh b/backups-manager.sh new file mode 100644 index 0000000..d0e37b6 --- /dev/null +++ b/backups-manager.sh @@ -0,0 +1,789 @@ + #!/bin/bash + # Copyright (C) 2019-2021 Jesus Perez Lorenzo + # This file is free software; as a special exception the author gives + # unlimited permission to copy and/or distribute it, with or without + # modifications, as long as this notice is preserved. + # License MIT + + VERBOSE=${VERBOSE:--v} + ROOT=${ROOT:-/data} + CLSTR=${CLSTR:-/clstr} + # shellcheck disable=SC1091 + [ -r "$CLSTR/bin/env" ] && . "$CLSTR"/bin/env + # shellcheck disable=SC1091 + [ -r "$ROOT/bin/env" ] && . "$ROOT"/bin/env + CMD=${CMD:-/usr/local/bin/restic} + JQ=${JQ:-/usr/bin/jq} + + ARCHIVER=${ARCHIVER:-/usr/local/bin/s3cmd} + + CMD_OPS=${CMD_OPS:--q} + ARCHIVER_OPS=${ARCHIVER_OPS:--q} + TASKS_LIST=${TASKS_LIST:-$ROOT/tasks_list} + ROOT_BACKUP_PATH=${ROOT_BACKUP_PATH:-$ROOT/backup} + BACKUP_SERVER_PROTOCOL=${BACKUP_SERVER_PROTOCOL:-http} + BACKUP_SERVER_HOST=${BACKUP_SERVER_HOST:-10.0.0.45} + BACKUP_SERVER_PORT=${BACKUP_SERVER_PORT:-9009} + S3_BACKUP_SERVER_PROTOCOL=${S3_BACKUP_SERVER_PROTOCOL:-s3} + S3_BACKUP_SERVER_HOST=${S3_BACKUP_SERVER_HOST:-10.0.0.45} + S3_BACKUP_SERVER_PORT=${S3_BACKUP_SERVER_PORT:-} + BACKUP_SERVER_TARGET=${BACKUP_SERVER_TARGET:-server} + BACKUP_PREPARE_CMD=${BACKUP_PREPARE_CMD:-} + RESTORE_PREPARE_CMD=${RESTORE_PREPARE_CMD:-} + ARCHIVE_PREPARE_CMD=${ARCHIVE_PREPARE_CMD:-} + BACKUP_CLEAN_DAYS=${BACKUP_CLEAN_DAYS:-} + # Will exit in _check_remove_target() + CLEAN_BACKUPS=${CLEAN_BACKUPS:-} + HOST_ARGS=${HOST_ARGS:-} + KEEP_NUM_ITEMS=${KEEP_NUM_ITEMS:-1} + + PRFX_NAME=${PRFX_NAME:-$(hostname -s | cut -f1 -d"-")} + case "$BACKUP_SERVER_TARGET" in + "aws") + BACKUP_SERVER=${BACKUP_SERVER:-$S3_BACKUP_SERVER_PROTOCOL:$S3_BACKUP_SERVER_HOST} + BACKUP_SERVER_PUBLIC=${BACKUP_SERVER:-$S3_BACKUP_SERVER_PROTOCOL:$S3_BACKUP_SERVER_HOST} + ;; + "upcloud") + BACKUP_SERVER=${BACKUP_SERVER:-$S3_BACKUP_SERVER_PROTOCOL:$S3_BACKUP_SERVER_HOST} + BACKUP_SERVER_PUBLIC=${BACKUP_SERVER_PUBLIC:-$S3_BACKUP_SERVER_PROTOCOL:$S3_BACKUP_SERVER_HOST} + ;; + "server") + BACKUP_SERVER=${BACKUP_SERVER:-$BACKUP_SERVER_PROTOCOL://$BACKUP_SERVER_HOST:$BACKUP_SERVER_PORT} + BACKUP_SERVER_PUBLIC=${BACKUP_SERVER:-$BACKUP_SERVER_PROTOCOL://$BACKUP_SERVER_HOST:$BACKUP_SERVER_PORT} + ;; + *) + echo "$BACKUP_SERVER_TARGET not defined" + exit 1 + ;; + esac + BACKUP_PATH=$ROOT_BACKUP_PATH"/" + ENV_HOST_CRD=${ENV_HOST_CRD:-bcksrvr} + + WORK_PATH=${WORK_PATH:-$ROOT/clstr} + if [ ! -d "$WORK_PATH" ] ; then + ! mkdir -p "$WORK_PATH" && echo "Unable to create WORK_PATH: $WORK_PATH " && exit 1 + fi + + TMP_VAR="" + + _usage() { + echo -e " + backups-manager -v(verbose) -l tasks-list-path -m match-name + [ task option for CMD ($CMD) or ARCHIVER ($ARCHIVER) ] [user] [source-paths] [backup-path | id | last . ] [restore-target-path] + + tasks will use: + CMD ($CMD) with options: copy restore show mount clean + or ARCHIVER ($ARCHIVER) for: archive-copy archive-restore archive-show archive-delete archive-create + + with [task user source ] parameters assigned 'tasks-list' is not processed + parameters with value take precedence over the one in 'tasks-list' + parameters can be occupied their space with '-' to be ignores, + in 'restore' task using '.' or 'last' as 'backup-path' option, will get last saved copy (ej: backup restore - - . /tmp/e) + CMD ($CMD) uses a passworfile RESTIC_PASSWORD_FILE generated with user|password encrypted + + use '-h tasks-list' to get help on 'tasks-list' sintax + + backups-manager home page: + License MIT + " + } + _tasks_list_usage() { + echo -e " + Task list format (fields serpartor '|' ): + + 1 - title name + 2 - user (if start with '@' is also password) + 3 - task: copy restore show mount clean (by adding '::' will include options for command [-v,-q]) + 4 - source-paths: + \t \t \t if separate by ';' will do relative path each one + \t \t \t by '::' for several absolute paths in one time + \t \t \t '__prfx__' will be replaced with PRFX_NAME ($PRFX_NAME) + 5 - target-path: + \t \t \t starting with '_backsrvr@' will use a default BACKUP_SERVER ($BACKUP_SERVER) + \t \t \t with '_path@' will set default ROOT_BACKUP_PATH ($ROOT_BACKUP_PATH) + \t \t \t '__prfx__' will be replaced with PRFX_NAME ($PRFX_NAME) + " + } + _show_curr_values() { + echo -e " + Current values: + + CMD \t\t\t $CMD + CMD_OPS \t\t\t $CMD_OPS + ARCHIVER \t\t\t $ARCHIVER + ARCHIVER_OPS \t\t $ARCHIVER_OPS + TASKS_LIST \t\t\t $TASKS_LIST + BACKUP_SERVER \t\t $BACKUP_SERVER + BACKUP_SERVER_PUBLIC \t $BACKUP_SERVER_PUBLIC + ROOT_BACKUP_PATH \t\t $ROOT_BACKUP_PATH + PRFX_NAME \t\t\t $PRFX_NAME + VERBOSE \t\t\t $VERBOSE + " + } + if [ "$1" == "-h" ] ; then + [ "$2" == "tasks-list" ] && _tasks_list_usage && _show_curr_values && exit + _usage + _show_curr_values + exit + fi + [ "$1" == "-h" ] && echo "$USAGE" && _tasks_list_usage && exit + [ "$1" == "-v" ] && VERBOSE=$1 && shift + [ "$1" == "-l" ] && TASKS_LIST=$2 && shift 2 + [ "$1" == "-m" ] && MATCH_NAME=$2 && shift 2 + [ -n "$1" ] && TASK_CMD=$1 + [ -n "$2" ] && [ "$2" != "-" ] && USER_CMD=$2 + [ -n "$3" ] && [ "$3" != "-" ] && BACKUP_PATH_SOURCE=$3 + [ -n "$4" ] && [ "$4" != "-" ] && BACKUP_PATH_TARGET=$4 + [ -n "$5" ] && RESTORE_PATH_TARGET=$5 + + + CMD_INIT="$CMD init" + CMD_COPY="$CMD backup" + CMD_SHOW="$CMD snapshots" + CMD_SHOW_LAST="$CMD list snapshots" + CMD_MOUNT="$CMD mount" + CMD_CLEAN="$CMD prune" + CMD_RESTORE="$CMD restore" + + CMD_ARCHIVE="$ARCHIVER -c $ROOT/.c " + CMD_ARCHIVE_CREATE="$CMD_ARCHIVE mb" + CMD_ARCHIVE_COPY="$CMD_ARCHIVE put" + CMD_ARCHIVE_RESTORE="$CMD_ARCHIVE get" + CMD_ARCHIVE_DELETE="$CMD_ARCHIVE rm" + CMD_ARCHIVE_SHOW="$CMD_ARCHIVE ls" + CMD_ARCHIVE_OPS=${ARCHIVER_OPS:-} + CMD_ARCHIVE_OPS_HOST="--no-ssl" + + CRDS=${CRDS:-$ROOT/.crd} + ROOT_ETC=${ROOT_ETC:-$ROOT/etc} + ORG=$(pwd) + CMD_ERROR_LOG="/tmp/backup_err.$$" + + # shellcheck disable=SC1091 + [ -x "$ROOT/bin/prepare-tasks" ] && . "$ROOT"/bin/prepare-tasks + + if [ ! -d "$ROOT/tmp" ] ; then + ! mkdir -p "$ROOT"/tmp && echo "Unable to create $ROOT/tmp " && exit 1 + fi + export RESTIC_PASSWORD_FILE=$ROOT/tmp/.tmp_$$ + + [ ! -r "$TASKS_LIST" ] && echo "No tasks list found" && exit + + _last_month_day() { + #local m=$(( 10#$(date +%m) + 1 )) + local m=$(($(date +%-m) + 1 )) + local os="" + os=$(grep "^ID=" /etc/os-release | sed 's/ID=//g') + if [ "$os" == "alpine" ] ; then + date -d "$(date +%Y-$m-01) -1" +%d + else + date -d "$(date +%Y-$m-01) -1 day" +%d + fi + } + _remove_until_last() { + local target=$1 + local snap="" + snap=$($CMD -r "$target" snapshots --json | $JQ -r .[].short_id) + # shellcheck disable=SC2206 + local snps_list=($snap) + local lst=$((${#snps_list[@]}-1)) + [ "$lst" -lt 1 ] && return + unset snps_list[$lst] + $CMD -r "$target" unlock --remove-all + for it in "${snps_list[@]}" + do + $CMD -r "$target" forget "$it" --prune + done + } + _check_remove_target() { + [ -z "$BACKUP_CLEAN_DAYS" ] && return + local target=$1 + local day="" + day=$(date +%d) + for it in ${BACKUP_CLEAN_DAYS//,/ } + do + [ "$it" == "last" ] && it=$(_last_month_day) + [ "$it" == "$day" ] && _remove_until_last "$target" && break + [ "$it" == "$day" ] && $CMD "$HOST_ARGS" -r "$target" forget --keep-last "$KEEP_NUM_ITEMS" --prune && break + done + [ -n "$CLEAN_BACKUPS" ] && exit + } + _set_cmd_ops() { + if [ -n "$VERBOSE" ] || [ -n "$1" ]; then + local has_v="" + has_v=$(echo "$CMD_OPS" | grep "v") + local has_dash="" + has_dash=$(echo "$CMD_OPS" | grep "-") + [ -z "$has_dash" ] && CMD_OPS=$CMD_OPS"-" + [ -z "$has_v" ] && CMD_OPS=${CMD_OPS//q/}"v" + [[ ! "$1" =~ "v" ]] && CMD_OPS=$CMD_OPS""$1 + fi + } + _get_ops_base64() { + local os="" + os=$(uname -s) + local opsBase="" + case "$os" in + *arwin) opsBase="-D" ;; + *) opsBase="-d" + esac + echo $opsBase + } + _get_kys() { + [ -z "$1" ] && return + (base64 "$(_get_ops_base64)" < "$CRDS" | grep "^$1|" | sed "s/^$1|//g") 2>/dev/null + #(base64 _get_ops_base64 < "$CRDS" | grep "^$1|" | sed "s/^$1|//g") 2>/dev/null + } + _set_passwd_file() { + [ -z "$1" ] && echo "no user found " && return 1 + echo "" > "$RESTIC_PASSWORD_FILE" + local user_crd="" + if [[ "$1" =~ ^@.* ]] ; then + user_crd=${1//@/} + else + user_crd=$(_get_kys "$1") + fi + #[ -z "$user_crd" ] && echo "No credentials found for $1" && return 1 + [ -z "$user_crd" ] && return 1 + echo "$user_crd" | cut -f2 -d"|" > "$RESTIC_PASSWORD_FILE" + } + _get_ky() { + [ -z "$1" ] && echo "No server env found" && return + local srv_file=$1 + if [ ! -r "$ROOT_ETC/.$srv_file" ] ; then + srv_file=$ENV_HOST_CRD + [ ! -r "$ROOT_ETC/.$srv_file" ] && echo "No server env found" && return + fi + local has_export="" + has_export=$(grep -m1 "export" "$ROOT_ETC/.$srv_file") + # shellcheck disable=SC1091,SC1090 + [ -n "$has_export" ] && . "$ROOT_ETC/.$srv_file" && return + local n="" + n=$(cut -f2 -d"@" < "$ROOT_ETC"/."$srv_file") + local k="" + k=$(cut -f1 -d"@" < "$ROOT_ETC"/."$srv_file") + local opsBase="" + opsBase=$(_get_ops_base64) + for (( i=1; i<=n; i++)) + do + k=$(echo "$k" | base64 "$opsBase") + done + TMP_VAR=$k + } + _env_srvr() { + [ -z "$1" ] && echo "No server env found" && return + TMP_VAR="" + _get_ky "$1" + local k=$TMP_VAR + [ -z "$k" ] && return + TMP_VAR="" + local srv_id="" + srv_id=$(echo "$k" | cut -d"|" -f1) + local srv_ky="" + srv_ky=$(echo "$k" | cut -d"|" -f2) + export AWS_ACCESS_KEY_ID=$srv_id + export AWS_SECRET_ACCESS_KEY=$srv_ky + export ACCESS_KEY=$srv_id + export SECRET_KEY=$srv_ky + } + _env_s3() { + [ -z "$1" ] && echo "No server env found" && return + TMP_VAR="" + _get_ky "$1" + local k=$TMP_VAR + [ -z "$k" ] && return + TMP_VAR="" + local srv_id="" + srv_id=$(echo "$k" | cut -d"|" -f1) + local srv_ky="" + srv_ky=$(echo "$k" | cut -d"|" -f2) + export AWS_ACCESS_KEY_ID=$srv_id + export AWS_SECRET_ACCESS_KEY=$srv_ky + echo "export AWS_ACCESS_KEY_ID=$srv_id + export AWS_SECRET_ACCESS_KEY=$srv_ky + export host_base=$S3_BACKUP_SERVER_HOST:$S3_BACKUP_SERVER_PORT + export host_bucket=$S3_BACKUP_SERVER_HOST:$S3_BACKUP_SERVER_PORT" > "$ROOT"/.c + } + _get_target() { + [ -z "$1" ] && return 1 + local trgt=$1 + local is_remote=0 + local target="" + case "$trgt" in + _backsrvr@*) + [[ "$BACKUP_SERVER" =~ "://" ]] && target="s3:" + target=$target${trgt//_backsrvr@/$BACKUP_SERVER} + is_remote=1 + ;; + _path@*) + target=${trgt//_path@/$BACKUP_PATH} + ;; + *) target=$trgt + esac + [[ "$target" =~ "@" ]] && target=$(echo "$target" | cut -d"@" -f2) + target=${target//__prfx__/$PRFX_NAME} + echo "$target" + } + _do_archive_create() { + [ -z "$1" ] && echo "no target found " && return + local target=$1 + [ -n "$VERBOSE" ] && echo "create $target ..." + local bucket="" + bucket=${target//$BACKUP_SERVER} + bucket=${bucket//s3:/s3:/} + # shellcheck disable=SC2086 + $CMD_ARCHIVE_CREATE $CMD_ARCHIVE_OPS $CMD_ARCHIVE_OPS_HOST $bucket 2>/dev/null && [ -n "$VERBOSE" ] && echo "$target created " + } + _get_source() { + [ -z "$1" ] && echo "No source found " && return 1 + [ -z "$2" ] &&echo "No user found " && return 1 + local src=$1 + local usr=$2 + local is_dply=0 + local source=$src + if [[ "$src" =~ "@" ]] ; then + source=$(echo "$src" | cut -d"@" -f2) + is_dply=1 + fi + source=${source//__prfx__/$PRFX_NAME} + [ "$is_dply" != "1" ] && TMP_VAR=$source && return + [ -r "$ROOT_ETC/.kconfig" ] && export KUBECONFIG=$ROOT_ETC/.kconfig + local source_dir="" + source_dir=$(dirname "$source") + local source_name="" + source_name=$(basename "$source") + local target_file=$source_name".tar.gz" + local dply_hostname="" + dply_hostname=$(echo "$src" | cut -d"@" -f1) + #TMP_VAR=$WORK_PATH/$usr/$source_name + #return + [ -n "$VERBOSE" ] && echo "getting source from $dply_hostname ..." + local namespace="" + namespace=$(echo "$dply_hostname" | cut -d"." -f2) + local dply_name="" + dply_name=$(echo "$dply_hostname" | cut -d"." -f1) + [ -z "$namespace" ] || [ -z "$dply_name" ] && return + local dply_id="" + dply_id=$(kubectl get pods -n "$namespace" -o wide --no-headers=true | grep "$dply_name" |awk 'NR==1{print $1}') + [ -z "$dply_id" ] && return + [ -n "$VERBOSE" ] && echo -n "archiving $source_name from $source_dir in $namespace $dply_id ..." + local k_ops="" + [ "$dply_name" == "site" ] && k_ops="-c site" + ! kubectl exec -n "$namespace" "$dply_id" "$k_ops" -- tar czf /tmp/"$target_file" -C "$source_dir" "$source_name" && echo "" && return + [ -n "$VERBOSE" ] && echo "" + [ ! -d "$WORK_PATH/$usr" ] && mkdir -p "$WORK_PATH/$usr" + [ -n "$VERBOSE" ] && echo -n "copy $target_file from $namespace $dply_id ..." + # shellcheck disable=SC2086 + ! kubectl cp -n $namespace $k_ops $dply_id:tmp/$target_file $WORK_PATH/$usr/$target_file && echo "" && return + # shellcheck disable=SC2086 + kubectl exec -n $namespace $dply_id $k_ops -- rm -f /tmp/$target_file + [ -n "$VERBOSE" ] && echo "" + cd "$WORK_PATH/$usr" || return + [ -n "$VERBOSE" ] && echo -n "extracting $target_file to $WORK_PATH/$usr/$target_file ..." + ! tar xzf "$WORK_PATH/$usr/$target_file" && echo "" && return + [ -n "$VERBOSE" ] && echo "" + [ -d "$WORK_PATH/$usr/$source_name" ] && rm -f "$WORK_PATH/$usr/$target_file" + cd "$ORG" || return + TMP_VAR=$WORK_PATH/$usr/$source_name + } + _do_copy() { + [ -z "$1" ] && echo "no user found " && return + [ -z "$2" ] && echo "no source found " && return + [ -z "$3" ] && echo "no target found " && return + ! _set_passwd_file "$1" && echo "No credentials found for $1" && return 1 + local _src=$2 + TMP_VAR="" + _get_source "$_src" "$1" + local src=$TMP_VAR + TMP_VAR="" + [ -z "$src" ] && echo "No source found for $_src - $1" && return 1 + local target="" + target=$(_get_target "$3") + [ -n "$VERBOSE" ] && echo "check-in $target ..." + _check_remove_target "$target" + # shellcheck disable=SC2086 + if $CMD_INIT $CMD_OPS -r $target 2> "$CMD_ERROR_LOG" ; then + echo "$target initiated " 2> "$CMD_ERROR_LOG" + else + local err_info="" + err_info=$(grep "already initialized" "$CMD_ERROR_LOG") + if [ -z "$err_info" ] ; then + echo "Error $target initiated: $(cat "$CMD_ERROR_LOG")" + return 1 + else + echo "$target already initiated " + fi + fi + rm -f "$CMD_ERROR_LOG" + #$CMD_INIT $CMD_OPS -r $target 2>/dev/null && echo "$target initiated " + local is_in_work_path=0 + if [[ "$src" =~ $WORK_PATH ]] ; then + local work_dir="" + work_dir=$(dirname "$src") + cd "$work_dir" || return + src=$(basename "$src") + is_in_work_path=1 + fi + [ -n "$BACKUP_PREPARE_CMD" ] && bash "$BACKUP_PREPARE_CMD" + if [[ "$src" =~ ";" ]] ; then + for it in ${src//;/ } + do + local base_path=$it + local dir_path="" + dir_path=$(dirname "$it") + if [ -d "$dir_path" ] ; then + cd "$dir_path" || continue + base_path=$(basename "$it") + fi + [ -n "$VERBOSE" ] && echo "copying $target ..." + # shellcheck disable=SC2086 + $CMD_COPY $HOST_ARGS $CMD_OPS -r $target $base_path && [ -n "$VERBOSE" ] && echo "$base_path copied in $target " + if [ "$base_path" != "$it" ] ; then + cd "$ORG" || continue + fi + if [ "$is_in_work_path" == "1" ] ; then + cd "$WORK_PATH" || continue + fi + done + else + [ -n "$VERBOSE" ] && echo "copying $target ..." + local src_path=${src//::/ } + local dir_path="" + dir_path=$(dirname "$src_path") + local base_path="" + base_path=$(basename "$src_path") + cd "$dir_path" || return + # shellcheck disable=SC2086 + $CMD_COPY $HOST_ARGS $CMD_OPS -r $target $base_path && [ -n "$VERBOSE" ] && echo "$dir_path->$base_path copied in $target" + fi + if [ "$is_in_work_path" == "1" ] ; then + cd "$ORG" || return + # rm -rf $work_dir + fi + } + _do_archive_copy_target() { + [ -z "$1" ] && echo "no target found " && return + [ -z "$2" ] && echo "no dir_path found " && return + [ -z "$3" ] && echo "no base_path found " && return + local target=$1 + local dir_path=$2 + local base_path=$3 + [ -n "$VERBOSE" ] && echo "copying $dir_path/$base_path to $target/$base_path.tar.gz ..." + if tar czf "$base_path.tar.gz" -C "$dir_path" "$base_path" ; then + local bucket="" + bucket=$(echo "$target" | sed "s,$BACKUP_SERVER,,g" | sed "s,s3:,s3:/,g")".tar.gz" + # shellcheck disable=SC2086 + if $CMD_ARCHIVE_COPY $CMD_ARCHIVE_OPS $base_path.tar.gz $CMD_ARCHIVE_OPS_HOST $bucket ; then + [ -n "$VERBOSE" ] && echo "$base_path/$base_path.tar.gz copied in $target/$base_path.tar.gz " + fi + fi + } + _do_archive_copy() { + [ -z "$1" ] && echo "no user found " && return + [ -z "$2" ] && echo "no source found " && return + [ -z "$3" ] && echo "no target found " && return + local _src=$2 + TMP_VAR="" + _get_source "$_src" "$1" + local src=$TMP_VAR + TMP_VAR="" + local base_path="" + local dir_path="" + local target="" + local work_dir="" + [ -z "$src" ] && echo "No source found for $_src - $1" && return 1 + target=$(_get_target "$3") + local is_in_work_path=0 + if [[ "$src" =~ $WORK_PATH ]] ; then + work_dir=$(dirname "$src") + cd "$work_dir" || return + src=$(basename "$src") + is_in_work_path=1 + fi + [ -n "$ARCHIVE_PREPARE_CMD" ] && bash "$ARCHIVE_PREPARE_CMD" + if [[ "$src" =~ ";" ]] ; then + for it in ${src//;/ } + do + base_path=$it + dir_path=$(dirname "$it") + if [ -d "$dir_path" ] ; then + cd "$dir_path" || continue + base_path=$(basename "$it") + fi + _do_archive_create "$(dirname "$target")" + _do_archive_copy_target "$target" "$dir_path" "$base_path" + if [ "$base_path" != "$it" ] ; then + cd "$ORG" || continue + fi + if [ "$is_in_work_path" == "1" ] ; then + cd "$WORK_PATH" || continue + fi + done + else + base_path=$(basename "$src") + dir_path=$(dirname "$src") + _do_archive_create "$(dirname "$target")" + _do_archive_copy_target "$target" "$dir_path" "$base_path" + fi + if [ "$is_in_work_path" == "1" ] ; then + cd "$ORG" || return + fi + } + _do_restore() { + [ -z "$1" ] && echo "no user found " && return + [ -z "$2" ] && echo "no source found " && return + [ -z "$3" ] && echo "no id found " && return + [ -z "$4" ] && echo "no target found " && return + ! _set_passwd_file "$1" && echo "No credentials found for $1" && return 1 + local source="" + source=$(_get_target "$2") + local target=$3 + case "$target" in + last|.) + target="latest" + #`$CMD_SHOW_LAST -r $source | tail -1` + ;; + esac + # shellcheck disable=SC2086 + $CMD_RESTORE $HOST_ARGS $CMD_OPS -r $source $target --target $4 + } + _do_archive_restore_target() { + [ -z "$1" ] && echo "no target found " && return + [ -z "$2" ] && echo "no dir_path found " && return + [ -z "$3" ] && echo "no base_path found " && return + local target=$1 + local dir_path=$2 + local base_path=$3 + [ -n "$VERBOSE" ] && echo "restoring $dir_path/$base_path from $target/$dir_path ..." + local bucket="" + bucket=$(echo "$target/$dir_path" | sed "s,$BACKUP_SERVER,,g" | sed "s,s3:,s3:/,g")".tar.gz" + # shellcheck disable=SC2086 + if $CMD_ARCHIVE_RESTORE $CMD_ARCHIVE_OPS $CMD_ARCHIVE_OPS_HOST $bucket $base_path.tar.gz ; then + tar xzf "$base_path.tar.gz" && [ -n "$VERBOSE" ] && echo "$base_path.tar.gz restore from $target/$dir_path" + fi + } + _do_archive_restore() { + [ -z "$1" ] && echo "no user found " && return + [ -z "$2" ] && echo "no source found " && return + [ -z "$3" ] && echo "no target found " && return + local _src=$2 + local base_path="" + local dir_path="" + local target="" + local is_in_work_path=0 + TMP_VAR="" + _get_source "$_src" "$1" + TMP_VAR="" + local src=$TMP_VAR + [ -z "$src" ] && echo "No source found for $_src - $1" && return 1 + target=$(_get_target "$3") + if [[ "$src" =~ $WORK_PATH ]] ; then + local work_dir="" + work_dir=$(dirname "$src") + cd "$work_dir" || return + src=$(basename "$src") + is_in_work_path=1 + fi + [ -n "$RESTORE_PREPARE_CMD" ] && bash "$RESTORE_PREPARE_CMD" + if [[ "$src" =~ ";" ]] ; then + # $(echo "$src" | sed 's/;/ /') + for it in ${src//;/} + do + base_path=$it + dir_path=$(dirname "$it") + if [ -d "$dir_path" ] ; then + cd "$dir_path" || continue + base_path=$(basename "$it") + fi + _do_archive_copy_target "$target" "$dir_path" "$base_path" + if [ "$base_path" != "$it" ] ; then + cd "$ORG" || continue + fi + if [ "$is_in_work_path" == "1" ] ; then + cd "$WORK_PATH" || continue + fi + done + else + base_path=$(basename "$src") + dir_path=$(dirname "$src") + _do_archive_copy_target "$target" "$dir_path" "$base_path" + fi + [ "$is_in_work_path" == "1" ] && cd "$ORG" || return + } + _do_archive_delete() { + [ -z "$1" ] && echo "no target found " && return + [ -z "$2" ] && echo "no dir_path found " && return + [ -z "$3" ] && echo "no base_path found " && return + local target=$1 + local dir_path=$2 + local base_path=$3 + [ -n "$VERBOSE" ] && echo "restoring $dir_path/$base_path from $target ..." + tar czf "$base_path.tar.gz" -C "$dir_path $base_path" + local bucket="" + bucket=${target//$BACKUP_SERVER/} + bucket=${bucket//s3:/s3:/}".tar.gz" + if $CMD_ARCHIVE_DELETE "$CMD_ARCHIVE_OPS" $CMD_ARCHIVE_OPS_HOST "$bucket/$base_path.tar.gz" ; then + [ -n "$VERBOSE" ] && echo "$target deleted " + fi + } + _do_show() { + [ -z "$1" ] && echo "no user found " && return + [ -z "$2" ] && echo "no source found " && return + _set_passwd_file "$1" && echo "No credentials found for $1" && return 1 + local source="" + source=$(_get_target "$2") + # shellcheck disable=SC2086 + $CMD_SHOW $HOST_ARGS $CMD_OPS -r $source + } + _do_archive_show() { + [ -z "$1" ] && echo "no target found " && return + [ -z "$2" ] && echo "no dir_path found " && return + #[ -z "$3" ] && echo "no base_path found " && return + local target=$1 + local dir_path=$2 + local base_path=$3 + local bucket="" + [ -n "$VERBOSE" ] && echo "show $dir_path/$base_path from $target ..." + bucket=$(echo "$target" | sed "s,$BACKUP_SERVER,,g" | sed "s,s3:,s3:/,g") + # shellcheck disable=SC2086 + if $CMD_ARCHIVE_SHOW $CMD_ARCHIVE_OPS $CMD_ARCHIVE_OPS_HOST $bucket/$base_path; then + [ -n "$VERBOSE" ] && echo "$target listed " + fi + } + _do_clean() { + [ -z "$1" ] && echo "no user found " && return + [ -z "$2" ] && echo "no source found " && return + _set_passwd_file "$1" && echo "No credentials found for $1" && return 1 + local source="" + source=$(_get_target "$2") + # shellcheck disable=SC2086 + $CMD_CLEAN $HOST_ARGS $CMD_OPS -r $source + } + _do_mount() { + [ -z "$1" ] && echo "no user found " && return + [ -z "$2" ] && echo "no source found " && return + [ -z "$3" ] && echo "no target found " && return + _set_passwd_file "$1" && echo "No credentials found for $1" && return 1 + local target=$3 + if [[ "$target" =~ "@" ]] ; then + target=$(echo "$target" | cut -d"@" -f2 | sed 's,^/,,g') + target=$ROOT/mnt/$(basename "$target") + fi + [ ! -d "$target" ] && mkdir -p "$target" + [ ! -d "$target" ] && echo "Directory $target can not be used " && return 1 + local src="" + src=$(_get_target "$2") + # shellcheck disable=SC2086 + $CMD_MOUNT $HOST_ARGS $CMD_OPS -r $src $target + } + _do_task() { + [ -z "$1" ] && echo "no name found " && return + [ -z "$2" ] && echo "no user found " && return + [ -z "$3" ] && echo "no task found " && return + [ -z "$4" ] && echo "no source found " && return + [ -n "$VERBOSE" ] && echo "Making $1 ..." + local task=$3 + if [[ "$task" =~ ":" ]] ; then + task=$(echo "$3" | cut -f1 -d":") + ops=$(echo "$3" | cut -f2 -d":" | sed 's/,/ /g' | sed 's/;/ /g') + [ -n "$ops" ] && CMD_OPS=$CMD_OPS" "$ops + fi + case "$task" in + copy) + [ -z "$5" ] && echo "no target found " && return + _set_cmd_ops v + _do_copy "$2" "$4" "$5" + ;; + archive-copy) + [ -z "$5" ] && echo "no target found " && return + _do_archive_copy "$2" "$4" "$5" + ;; + restore) + local id_path=$5 + [ -n "$BACKUP_PATH_TARGET" ] && id_path=$BACKUP_PATH_TARGET + [ -z "$id_path" ] && echo "no id found " && return + local restore_path=$6 + [ -n "$RESTORE_PATH_TARGET" ] && restore_path=$RESTORE_PATH_TARGET + [ -z "$restore_path" ] && echo "no target found " && return + [ "$restore_path" == "." ] && restore_path=$5 # / + _set_cmd_ops v + _do_restore "$2" "$4" "$id_path" "$restore_path" + ;; + archive-restore) + [ -z "$5" ] && echo "no target found " && return + _do_archive_restore "$2" "$4" "$5" + ;; + show) + _do_show "$2" "$4" "$5" + ;; + archive-show) + _do_archive_show "$2" "$4" "$5" + ;; + archive-create) + _do_archive_create "$2" "$4" "$5" + ;; + mount) + [ -z "$5" ] && echo "no target found " && return + local restore_path=$5 + [ -n "$RESTORE_PATH_TARGET" ] && restore_path=$RESTORE_PATH_TARGET + _set_cmd_ops + _do_mount "$2" "$4" "$restore_path" + ;; + clear|clean) + _set_cmd_ops + _do_clean "$2" "$4" "$5" + ;; + *) echo "$task not configured !" + esac + [ -n "$VERBOSE" ] && echo "$1 done !" + } + _parse_line() { + [ -z "$1" ] && return + local has_comment="" + has_comment=$(echo "$1" | grep "^#") + [ -n "$has_comment" ] && return + local name="" + name=$(echo "$1" | cut -f1 -d"|") + local user="" + user=$(echo "$1" | cut -f2 -d"|" | sed 's/ //g') + local task="" + local source="" + local target="" + if [ -n "$TASK_CMD" ] ; then + task=$TASK_CMD + case "$TASK_CMD" in + copy|archive) + target=$(echo "$1" | cut -f5 -d"|" | sed 's/ //g') + source=$(echo "$1" | cut -f4 -d"|" | sed 's/ //g') + ;; + *) + target=$(echo "$1" | cut -f4 -d"|" | sed 's/ //g') + source=$(echo "$1" | cut -f5 -d"|" | sed 's/ //g') + ;; + esac + else + task=$(echo "$1" | cut -f3 -d"|" | sed 's/ //g') + source=$(echo "$1" | cut -f4 -d"|" | sed 's/ //g') + target=$(echo "$1" | cut -f5 -d"|" | sed 's/ //g') + fi + [ -n "$VERBOSE" ] && echo "$name ($user) -> $task from $source to $target" + _do_task "$name" "$user" "$task" "$source" "$target" + } + if [ "$BACKUP_SERVER_TARGET" == "server" ] ; then + _env_srvr "$BACKUP_SERVER_TARGET""_""$ENV_HOST_CRD" + else + _env_s3 "s3_$ENV_HOST_CRD" + fi + if [ -n "$TASK_CMD" ] && [ -n "$BACKUP_PATH_SOURCE" ] && [ -n "$USER_CMD" ]; then + _do_task "task" "$USER_CMD" "$TASK_CMD" "$BACKUP_PATH_SOURCE" "$BACKUP_PATH_TARGET" "$RESTORE_PATH_TARGET" + else + while IFS= read -r line + do + [ -n "$MATCH_NAME" ] && [[ ! "$line" =~ ^$MATCH_NAME.* ]] && continue + _parse_line "$line" + done < "$TASKS_LIST" + fi + + rm -f "$RESTIC_PASSWORD_FILE" "$ROOT"/tmp/.tmp* "$ROOT"/.c "$CMD_ERROR_LOG" + +