808 lines
27 KiB
Bash
808 lines
27 KiB
Bash
#!/bin/bash
|
|
# Copyright (C) 2019-2021 Jesus Perez Lorenzo <https://jesusperez.pro>
|
|
# This file is free software; as a special exception the author gives
|
|
# unlimited permission to copy and/or distribute it, with or without
|
|
# modifications, as long as this notice is preserved.
|
|
# License MIT
|
|
|
|
VERBOSE=${VERBOSE:--v}
|
|
ROOT=${ROOT:-/data}
|
|
CLSTR=${CLSTR:-/clstr}
|
|
# shellcheck disable=SC1091
|
|
[ -r "$CLSTR/bin/env" ] && . "$CLSTR"/bin/env
|
|
# shellcheck disable=SC1091
|
|
[ -r "$ROOT/bin/env" ] && . "$ROOT"/bin/env
|
|
CMD=${CMD:-/usr/local/bin/restic}
|
|
JQ=${JQ:-/usr/bin/jq}
|
|
|
|
ARCHIVER=${ARCHIVER:-/usr/local/bin/s3cmd}
|
|
|
|
CMD_OPS=${CMD_OPS:--q}
|
|
ARCHIVER_OPS=${ARCHIVER_OPS:--q}
|
|
TASKS_LIST=${TASKS_LIST:-$ROOT/tasks_list}
|
|
ROOT_BACKUP_PATH=${ROOT_BACKUP_PATH:-$ROOT/backup}
|
|
BACKUP_SERVER_PROTOCOL=${BACKUP_SERVER_PROTOCOL:-http}
|
|
BACKUP_SERVER_HOST=${BACKUP_SERVER_HOST:-10.0.0.45}
|
|
BACKUP_SERVER_PORT=${BACKUP_SERVER_PORT:-9009}
|
|
S3_BACKUP_SERVER_PROTOCOL=${S3_BACKUP_SERVER_PROTOCOL:-s3}
|
|
S3_BACKUP_SERVER_HOST=${S3_BACKUP_SERVER_HOST:-10.0.0.45}
|
|
S3_BACKUP_SERVER_PORT=${S3_BACKUP_SERVER_PORT:-}
|
|
BACKUP_SERVER_TARGET=${BACKUP_SERVER_TARGET:-server}
|
|
BACKUP_PREPARE_CMD=${BACKUP_PREPARE_CMD:-}
|
|
RESTORE_PREPARE_CMD=${RESTORE_PREPARE_CMD:-}
|
|
ARCHIVE_PREPARE_CMD=${ARCHIVE_PREPARE_CMD:-}
|
|
BACKUP_CLEAN_DAYS=${BACKUP_CLEAN_DAYS:-}
|
|
# Will exit in _check_remove_target()
|
|
CLEAN_BACKUPS=${CLEAN_BACKUPS:-}
|
|
HOST_ARGS=${HOST_ARGS:-}
|
|
KEEP_NUM_ITEMS=${KEEP_NUM_ITEMS:-1}
|
|
|
|
PRFX_NAME=${PRFX_NAME:-$(hostname -s | cut -f1 -d"-")}
|
|
case "$BACKUP_SERVER_TARGET" in
|
|
"aws")
|
|
BACKUP_SERVER=${BACKUP_SERVER:-$S3_BACKUP_SERVER_PROTOCOL:$S3_BACKUP_SERVER_HOST}
|
|
BACKUP_SERVER_PUBLIC=${BACKUP_SERVER:-$S3_BACKUP_SERVER_PROTOCOL:$S3_BACKUP_SERVER_HOST}
|
|
;;
|
|
"upcloud")
|
|
BACKUP_SERVER=${BACKUP_SERVER:-$S3_BACKUP_SERVER_PROTOCOL:$S3_BACKUP_SERVER_HOST}
|
|
BACKUP_SERVER_PUBLIC=${BACKUP_SERVER_PUBLIC:-$S3_BACKUP_SERVER_PROTOCOL:$S3_BACKUP_SERVER_HOST}
|
|
;;
|
|
"server")
|
|
BACKUP_SERVER=${BACKUP_SERVER:-$BACKUP_SERVER_PROTOCOL://$BACKUP_SERVER_HOST:$BACKUP_SERVER_PORT}
|
|
BACKUP_SERVER_PUBLIC=${BACKUP_SERVER:-$BACKUP_SERVER_PROTOCOL://$BACKUP_SERVER_HOST:$BACKUP_SERVER_PORT}
|
|
;;
|
|
*)
|
|
echo "$BACKUP_SERVER_TARGET not defined"
|
|
exit 1
|
|
;;
|
|
esac
|
|
BACKUP_PATH=$ROOT_BACKUP_PATH"/"
|
|
ENV_HOST_CRD=${ENV_HOST_CRD:-bcksrvr}
|
|
|
|
WORK_PATH=${WORK_PATH:-$ROOT/clstr}
|
|
if [ ! -d "$WORK_PATH" ] ; then
|
|
! mkdir -p "$WORK_PATH" && echo "Unable to create WORK_PATH: $WORK_PATH " && exit 1
|
|
fi
|
|
|
|
TMP_VAR=""
|
|
|
|
_usage() {
|
|
echo -e "
|
|
backups-manager -v(verbose) -l tasks-list-path -m match-name
|
|
[ task option for CMD ($CMD) or ARCHIVER ($ARCHIVER) ] [user] [source-paths] [backup-path | id | last . ] [restore-target-path]
|
|
|
|
tasks will use:
|
|
CMD ($CMD) with options: copy restore show mount clean
|
|
or ARCHIVER ($ARCHIVER) for: archive-copy archive-restore archive-show archive-delete archive-create
|
|
|
|
with [task user source ] parameters assigned 'tasks-list' is not processed
|
|
parameters with value take precedence over the one in 'tasks-list'
|
|
parameters can be occupied their space with '-' to be ignores,
|
|
in 'restore' task using '.' or 'last' as 'backup-path' option, will get last saved copy (ej: backup restore - - . /tmp/e)
|
|
CMD ($CMD) uses a passworfile RESTIC_PASSWORD_FILE generated with user|password encrypted
|
|
|
|
use '-h tasks-list' to get help on 'tasks-list' sintax
|
|
|
|
backups-manager home page: <https://repo.librecloud.online/backups-manager>
|
|
License MIT
|
|
"
|
|
}
|
|
_tasks_list_usage() {
|
|
echo -e "
|
|
Task list format (fields serpartor '|' ):
|
|
|
|
1 - title name
|
|
2 - user (if start with '@' is also password)
|
|
3 - task: copy restore show mount clean (by adding '::' will include options for command [-v,-q])
|
|
4 - source-paths:
|
|
\t \t \t if separate by ';' will do relative path each one
|
|
\t \t \t by '::' for several absolute paths in one time
|
|
\t \t \t '__prfx__' will be replaced with PRFX_NAME ($PRFX_NAME)
|
|
5 - target-path:
|
|
\t \t \t starting with '_backsrvr@' will use a default BACKUP_SERVER ($BACKUP_SERVER)
|
|
\t \t \t with '_path@' will set default ROOT_BACKUP_PATH ($ROOT_BACKUP_PATH)
|
|
\t \t \t '__prfx__' will be replaced with PRFX_NAME ($PRFX_NAME)
|
|
"
|
|
}
|
|
_show_curr_values() {
|
|
echo -e "
|
|
Current values:
|
|
|
|
CMD \t\t\t $CMD
|
|
CMD_OPS \t\t\t $CMD_OPS
|
|
ARCHIVER \t\t\t $ARCHIVER
|
|
ARCHIVER_OPS \t\t $ARCHIVER_OPS
|
|
TASKS_LIST \t\t\t $TASKS_LIST
|
|
BACKUP_SERVER \t\t $BACKUP_SERVER
|
|
BACKUP_SERVER_PUBLIC \t $BACKUP_SERVER_PUBLIC
|
|
ROOT_BACKUP_PATH \t\t $ROOT_BACKUP_PATH
|
|
PRFX_NAME \t\t\t $PRFX_NAME
|
|
VERBOSE \t\t\t $VERBOSE
|
|
"
|
|
}
|
|
if [ "$1" == "-h" ] ; then
|
|
[ "$2" == "tasks-list" ] && _tasks_list_usage && _show_curr_values && exit
|
|
_usage
|
|
_show_curr_values
|
|
exit
|
|
fi
|
|
[ "$1" == "-h" ] && echo "$USAGE" && _tasks_list_usage && exit
|
|
[ "$1" == "-v" ] && VERBOSE=$1 && shift
|
|
[ "$1" == "-l" ] && TASKS_LIST=$2 && shift 2
|
|
[ "$1" == "-m" ] && MATCH_NAME=$2 && shift 2
|
|
[ -n "$1" ] && TASK_CMD=$1
|
|
[ -n "$2" ] && [ "$2" != "-" ] && USER_CMD=$2
|
|
[ -n "$3" ] && [ "$3" != "-" ] && BACKUP_PATH_SOURCE=$3
|
|
[ -n "$4" ] && [ "$4" != "-" ] && BACKUP_PATH_TARGET=$4
|
|
[ -n "$5" ] && RESTORE_PATH_TARGET=$5
|
|
[ -n "$6" ] && TARGET_ITEM=$6
|
|
|
|
CMD_INIT="$CMD init"
|
|
CMD_COPY="$CMD backup"
|
|
CMD_SHOW="$CMD snapshots"
|
|
CMD_SHOW_LAST="$CMD list snapshots"
|
|
CMD_MOUNT="$CMD mount"
|
|
CMD_CLEAN="$CMD prune"
|
|
CMD_RESTORE="$CMD restore"
|
|
CMD_FORGET="$CMD forget"
|
|
|
|
CMD_ARCHIVE="$ARCHIVER -c $ROOT/.c "
|
|
CMD_ARCHIVE_CREATE="$CMD_ARCHIVE mb"
|
|
CMD_ARCHIVE_COPY="$CMD_ARCHIVE put"
|
|
CMD_ARCHIVE_RESTORE="$CMD_ARCHIVE get"
|
|
CMD_ARCHIVE_DELETE="$CMD_ARCHIVE rm"
|
|
CMD_ARCHIVE_SHOW="$CMD_ARCHIVE ls"
|
|
CMD_ARCHIVE_OPS=${ARCHIVER_OPS:-}
|
|
CMD_ARCHIVE_OPS_HOST="--no-ssl"
|
|
|
|
CRDS=${CRDS:-$ROOT/.crd}
|
|
ROOT_ETC=${ROOT_ETC:-$ROOT/etc}
|
|
ORG=$(pwd)
|
|
CMD_ERROR_LOG="/tmp/backup_err.$$"
|
|
|
|
# shellcheck disable=SC1091
|
|
[ -x "$ROOT/bin/prepare-tasks" ] && . "$ROOT"/bin/prepare-tasks
|
|
|
|
if [ ! -d "$ROOT/tmp" ] ; then
|
|
! mkdir -p "$ROOT"/tmp && echo "Unable to create $ROOT/tmp " && exit 1
|
|
fi
|
|
export RESTIC_PASSWORD_FILE=$ROOT/tmp/.tmp_$$
|
|
|
|
[ ! -r "$TASKS_LIST" ] && echo "No tasks list found" && exit
|
|
|
|
_last_month_day() {
|
|
#local m=$(( 10#$(date +%m) + 1 ))
|
|
local m=$(($(date +%-m) + 1 ))
|
|
local os=""
|
|
os=$(grep "^ID=" /etc/os-release | sed 's/ID=//g')
|
|
if [ "$os" == "alpine" ] ; then
|
|
date -d "$(date +%Y-$m-01) -1" +%d
|
|
else
|
|
date -d "$(date +%Y-$m-01) -1 day" +%d
|
|
fi
|
|
}
|
|
_remove_until_last() {
|
|
local target=$1
|
|
local snap=""
|
|
snap=$($CMD -r "$target" snapshots --json | $JQ -r .[].short_id)
|
|
# shellcheck disable=SC2206
|
|
local snps_list=($snap)
|
|
local lst=$((${#snps_list[@]}-1))
|
|
[ "$lst" -lt 1 ] && return
|
|
unset snps_list[$lst]
|
|
$CMD -r "$target" unlock --remove-all
|
|
for it in "${snps_list[@]}"
|
|
do
|
|
$CMD -r "$target" forget "$it" --prune
|
|
done
|
|
}
|
|
_check_remove_target() {
|
|
[ -z "$BACKUP_CLEAN_DAYS" ] && return
|
|
local target=$1
|
|
local day=""
|
|
day=$(date +%d)
|
|
for it in ${BACKUP_CLEAN_DAYS//,/ }
|
|
do
|
|
[ "$it" == "last" ] && it=$(_last_month_day)
|
|
[ "$it" == "$day" ] && _remove_until_last "$target" && break
|
|
[ "$it" == "$day" ] && $CMD "$HOST_ARGS" -r "$target" forget --keep-last "$KEEP_NUM_ITEMS" --prune && break
|
|
done
|
|
[ -n "$CLEAN_BACKUPS" ] && exit
|
|
}
|
|
_set_cmd_ops() {
|
|
if [ -n "$VERBOSE" ] || [ -n "$1" ]; then
|
|
local has_v=""
|
|
has_v=$(echo "$CMD_OPS" | grep "v")
|
|
local has_dash=""
|
|
has_dash=$(echo "$CMD_OPS" | grep "-")
|
|
[ -z "$has_dash" ] && CMD_OPS=$CMD_OPS"-"
|
|
[ -z "$has_v" ] && CMD_OPS=${CMD_OPS//q/}"v"
|
|
[[ ! "$1" =~ "v" ]] && CMD_OPS=$CMD_OPS""$1
|
|
fi
|
|
}
|
|
_get_ops_base64() {
|
|
local os=""
|
|
os=$(uname -s)
|
|
local opsBase=""
|
|
case "$os" in
|
|
*arwin) opsBase="-D" ;;
|
|
*) opsBase="-d"
|
|
esac
|
|
echo $opsBase
|
|
}
|
|
_get_kys() {
|
|
[ -z "$1" ] && return
|
|
(base64 "$(_get_ops_base64)" < "$CRDS" | grep "^$1|" | sed "s/^$1|//g") 2>/dev/null
|
|
#(base64 _get_ops_base64 < "$CRDS" | grep "^$1|" | sed "s/^$1|//g") 2>/dev/null
|
|
}
|
|
_set_passwd_file() {
|
|
[ -z "$1" ] && echo "no user found " && return 1
|
|
echo "" > "$RESTIC_PASSWORD_FILE"
|
|
local user_crd=""
|
|
if [[ "$1" =~ ^@.* ]] ; then
|
|
user_crd=${1//@/}
|
|
else
|
|
user_crd=$(_get_kys "$1")
|
|
fi
|
|
#[ -z "$user_crd" ] && echo "No credentials found for $1" && return 1
|
|
[ -z "$user_crd" ] && return 1
|
|
echo "$user_crd" | cut -f2 -d"|" > "$RESTIC_PASSWORD_FILE"
|
|
}
|
|
_get_ky() {
|
|
[ -z "$1" ] && echo "No server env found" && return
|
|
local srv_file=$1
|
|
if [ ! -r "$ROOT_ETC/.$srv_file" ] ; then
|
|
srv_file=$ENV_HOST_CRD
|
|
[ ! -r "$ROOT_ETC/.$srv_file" ] && echo "No server env found" && return
|
|
fi
|
|
local has_export=""
|
|
has_export=$(grep -m1 "export" "$ROOT_ETC/.$srv_file")
|
|
# shellcheck disable=SC1091,SC1090
|
|
[ -n "$has_export" ] && . "$ROOT_ETC/.$srv_file" && return
|
|
local n=""
|
|
n=$(cut -f2 -d"@" < "$ROOT_ETC"/."$srv_file")
|
|
local k=""
|
|
k=$(cut -f1 -d"@" < "$ROOT_ETC"/."$srv_file")
|
|
local opsBase=""
|
|
opsBase=$(_get_ops_base64)
|
|
for (( i=1; i<=n; i++))
|
|
do
|
|
k=$(echo "$k" | base64 "$opsBase")
|
|
done
|
|
TMP_VAR=$k
|
|
}
|
|
_env_srvr() {
|
|
[ -z "$1" ] && echo "No server env found" && return
|
|
TMP_VAR=""
|
|
_get_ky "$1"
|
|
local k=$TMP_VAR
|
|
[ -z "$k" ] && return
|
|
TMP_VAR=""
|
|
local srv_id=""
|
|
srv_id=$(echo "$k" | cut -d"|" -f1)
|
|
local srv_ky=""
|
|
srv_ky=$(echo "$k" | cut -d"|" -f2)
|
|
export AWS_ACCESS_KEY_ID=$srv_id
|
|
export AWS_SECRET_ACCESS_KEY=$srv_ky
|
|
export ACCESS_KEY=$srv_id
|
|
export SECRET_KEY=$srv_ky
|
|
}
|
|
_env_s3() {
|
|
[ -z "$1" ] && echo "No server env found" && return
|
|
TMP_VAR=""
|
|
_get_ky "$1"
|
|
local k=$TMP_VAR
|
|
[ -z "$k" ] && return
|
|
TMP_VAR=""
|
|
local srv_id=""
|
|
srv_id=$(echo "$k" | cut -d"|" -f1)
|
|
local srv_ky=""
|
|
srv_ky=$(echo "$k" | cut -d"|" -f2)
|
|
export AWS_ACCESS_KEY_ID=$srv_id
|
|
export AWS_SECRET_ACCESS_KEY=$srv_ky
|
|
echo "export AWS_ACCESS_KEY_ID=$srv_id
|
|
export AWS_SECRET_ACCESS_KEY=$srv_ky
|
|
export host_base=$S3_BACKUP_SERVER_HOST:$S3_BACKUP_SERVER_PORT
|
|
export host_bucket=$S3_BACKUP_SERVER_HOST:$S3_BACKUP_SERVER_PORT" > "$ROOT"/.c
|
|
}
|
|
_get_target() {
|
|
[ -z "$1" ] && return 1
|
|
local trgt=$1
|
|
local is_remote=0
|
|
local target=""
|
|
case "$trgt" in
|
|
_backsrvr@*)
|
|
[[ "$BACKUP_SERVER" =~ "://" ]] && target="s3:"
|
|
target=$target${trgt//_backsrvr@/$BACKUP_SERVER}
|
|
is_remote=1
|
|
;;
|
|
_path@*)
|
|
target=${trgt//_path@/$BACKUP_PATH}
|
|
;;
|
|
*) target=$trgt
|
|
esac
|
|
[[ "$target" =~ "@" ]] && target=$(echo "$target" | cut -d"@" -f2)
|
|
target=${target//__prfx__/$PRFX_NAME}
|
|
echo "$target"
|
|
}
|
|
_do_archive_create() {
|
|
[ -z "$1" ] && echo "no target found " && return
|
|
local target=$1
|
|
[ -n "$VERBOSE" ] && echo "create $target ..."
|
|
local bucket=""
|
|
bucket=${target//$BACKUP_SERVER}
|
|
bucket=${bucket//s3:/s3:/}
|
|
# shellcheck disable=SC2086
|
|
$CMD_ARCHIVE_CREATE $CMD_ARCHIVE_OPS $CMD_ARCHIVE_OPS_HOST $bucket 2>/dev/null && [ -n "$VERBOSE" ] && echo "$target created "
|
|
}
|
|
_get_source() {
|
|
[ -z "$1" ] && echo "No source found " && return 1
|
|
[ -z "$2" ] &&echo "No user found " && return 1
|
|
local src=$1
|
|
local usr=$2
|
|
local is_dply=0
|
|
local source=$src
|
|
if [[ "$src" =~ "@" ]] ; then
|
|
source=$(echo "$src" | cut -d"@" -f2)
|
|
is_dply=1
|
|
fi
|
|
source=${source//__prfx__/$PRFX_NAME}
|
|
[ "$is_dply" != "1" ] && TMP_VAR=$source && return
|
|
[ -r "$ROOT_ETC/.kconfig" ] && export KUBECONFIG=$ROOT_ETC/.kconfig
|
|
local source_dir=""
|
|
source_dir=$(dirname "$source")
|
|
local source_name=""
|
|
source_name=$(basename "$source")
|
|
local target_file=$source_name".tar.gz"
|
|
local dply_hostname=""
|
|
dply_hostname=$(echo "$src" | cut -d"@" -f1)
|
|
#TMP_VAR=$WORK_PATH/$usr/$source_name
|
|
#return
|
|
[ -n "$VERBOSE" ] && echo "getting source from $dply_hostname ..."
|
|
local namespace=""
|
|
namespace=$(echo "$dply_hostname" | cut -d"." -f2)
|
|
local dply_name=""
|
|
dply_name=$(echo "$dply_hostname" | cut -d"." -f1)
|
|
[ -z "$namespace" ] || [ -z "$dply_name" ] && return
|
|
local dply_id=""
|
|
dply_id=$(kubectl get pods -n "$namespace" -o wide --no-headers=true | grep "$dply_name" |awk 'NR==1{print $1}')
|
|
[ -z "$dply_id" ] && return
|
|
[ -n "$VERBOSE" ] && echo -n "archiving $source_name from $source_dir in $namespace $dply_id ..."
|
|
local k_ops=""
|
|
[ "$dply_name" == "site" ] && k_ops="-c site"
|
|
! kubectl exec -n "$namespace" "$dply_id" "$k_ops" -- tar czf /tmp/"$target_file" -C "$source_dir" "$source_name" && echo "" && return
|
|
[ -n "$VERBOSE" ] && echo ""
|
|
[ ! -d "$WORK_PATH/$usr" ] && mkdir -p "$WORK_PATH/$usr"
|
|
[ -n "$VERBOSE" ] && echo -n "copy $target_file from $namespace $dply_id ..."
|
|
# shellcheck disable=SC2086
|
|
! kubectl cp -n $namespace $k_ops $dply_id:tmp/$target_file $WORK_PATH/$usr/$target_file && echo "" && return
|
|
# shellcheck disable=SC2086
|
|
kubectl exec -n $namespace $dply_id $k_ops -- rm -f /tmp/$target_file
|
|
[ -n "$VERBOSE" ] && echo ""
|
|
cd "$WORK_PATH/$usr" || return
|
|
[ -n "$VERBOSE" ] && echo -n "extracting $target_file to $WORK_PATH/$usr/$target_file ..."
|
|
! tar xzf "$WORK_PATH/$usr/$target_file" && echo "" && return
|
|
[ -n "$VERBOSE" ] && echo ""
|
|
[ -d "$WORK_PATH/$usr/$source_name" ] && rm -f "$WORK_PATH/$usr/$target_file"
|
|
cd "$ORG" || return
|
|
TMP_VAR=$WORK_PATH/$usr/$source_name
|
|
}
|
|
_do_copy() {
|
|
[ -z "$1" ] && echo "no user found " && return
|
|
[ -z "$2" ] && echo "no source found " && return
|
|
[ -z "$3" ] && echo "no target found " && return
|
|
! _set_passwd_file "$1" && echo "No credentials found for $1" && return 1
|
|
local _src=$2
|
|
TMP_VAR=""
|
|
_get_source "$_src" "$1"
|
|
local src=$TMP_VAR
|
|
TMP_VAR=""
|
|
[ -z "$src" ] && echo "No source found for $_src - $1" && return 1
|
|
local target=""
|
|
target=$(_get_target "$3")
|
|
[ -n "$VERBOSE" ] && echo "check-in $target ..."
|
|
_check_remove_target "$target"
|
|
# shellcheck disable=SC2086
|
|
if $CMD_INIT $CMD_OPS -r $target 2> "$CMD_ERROR_LOG" ; then
|
|
echo "$target initiated " 2> "$CMD_ERROR_LOG"
|
|
else
|
|
local err_info=""
|
|
err_info=$(grep "already initialized" "$CMD_ERROR_LOG")
|
|
if [ -z "$err_info" ] ; then
|
|
echo "Error $target initiated: $(cat "$CMD_ERROR_LOG")"
|
|
return 1
|
|
else
|
|
echo "$target already initiated "
|
|
fi
|
|
fi
|
|
rm -f "$CMD_ERROR_LOG"
|
|
#$CMD_INIT $CMD_OPS -r $target 2>/dev/null && echo "$target initiated "
|
|
local is_in_work_path=0
|
|
if [[ "$src" =~ $WORK_PATH ]] ; then
|
|
local work_dir=""
|
|
work_dir=$(dirname "$src")
|
|
cd "$work_dir" || return
|
|
src=$(basename "$src")
|
|
is_in_work_path=1
|
|
fi
|
|
[ -n "$BACKUP_PREPARE_CMD" ] && bash "$BACKUP_PREPARE_CMD"
|
|
if [[ "$src" =~ ";" ]] ; then
|
|
for it in ${src//;/ }
|
|
do
|
|
local base_path=$it
|
|
local dir_path=""
|
|
dir_path=$(dirname "$it")
|
|
if [ -d "$dir_path" ] ; then
|
|
cd "$dir_path" || continue
|
|
base_path=$(basename "$it")
|
|
fi
|
|
[ -n "$VERBOSE" ] && echo "copying $target ..."
|
|
# shellcheck disable=SC2086
|
|
$CMD_COPY $HOST_ARGS $CMD_OPS -r $target $base_path && [ -n "$VERBOSE" ] && echo "$base_path copied in $target "
|
|
if [ "$base_path" != "$it" ] ; then
|
|
cd "$ORG" || continue
|
|
fi
|
|
if [ "$is_in_work_path" == "1" ] ; then
|
|
cd "$WORK_PATH" || continue
|
|
fi
|
|
done
|
|
else
|
|
[ -n "$VERBOSE" ] && echo "copying $target ..."
|
|
local src_path=${src//::/ }
|
|
local dir_path=""
|
|
dir_path=$(dirname "$src_path")
|
|
local base_path=""
|
|
base_path=$(basename "$src_path")
|
|
cd "$dir_path" || return
|
|
# shellcheck disable=SC2086
|
|
$CMD_COPY $HOST_ARGS $CMD_OPS -r $target $base_path && [ -n "$VERBOSE" ] && echo "$dir_path->$base_path copied in $target"
|
|
fi
|
|
if [ "$is_in_work_path" == "1" ] ; then
|
|
cd "$ORG" || return
|
|
# rm -rf $work_dir
|
|
fi
|
|
}
|
|
_do_archive_copy_target() {
|
|
[ -z "$1" ] && echo "no target found " && return
|
|
[ -z "$2" ] && echo "no dir_path found " && return
|
|
[ -z "$3" ] && echo "no base_path found " && return
|
|
local target=$1
|
|
local dir_path=$2
|
|
local base_path=$3
|
|
[ -n "$VERBOSE" ] && echo "copying $dir_path/$base_path to $target/$base_path.tar.gz ..."
|
|
if tar czf "$base_path.tar.gz" -C "$dir_path" "$base_path" ; then
|
|
local bucket=""
|
|
bucket=$(echo "$target" | sed "s,$BACKUP_SERVER,,g" | sed "s,s3:,s3:/,g")".tar.gz"
|
|
# shellcheck disable=SC2086
|
|
if $CMD_ARCHIVE_COPY $CMD_ARCHIVE_OPS $base_path.tar.gz $CMD_ARCHIVE_OPS_HOST $bucket ; then
|
|
[ -n "$VERBOSE" ] && echo "$base_path/$base_path.tar.gz copied in $target/$base_path.tar.gz "
|
|
fi
|
|
fi
|
|
}
|
|
_do_archive_copy() {
|
|
[ -z "$1" ] && echo "no user found " && return
|
|
[ -z "$2" ] && echo "no source found " && return
|
|
[ -z "$3" ] && echo "no target found " && return
|
|
local _src=$2
|
|
TMP_VAR=""
|
|
_get_source "$_src" "$1"
|
|
local src=$TMP_VAR
|
|
TMP_VAR=""
|
|
local base_path=""
|
|
local dir_path=""
|
|
local target=""
|
|
local work_dir=""
|
|
[ -z "$src" ] && echo "No source found for $_src - $1" && return 1
|
|
target=$(_get_target "$3")
|
|
local is_in_work_path=0
|
|
if [[ "$src" =~ $WORK_PATH ]] ; then
|
|
work_dir=$(dirname "$src")
|
|
cd "$work_dir" || return
|
|
src=$(basename "$src")
|
|
is_in_work_path=1
|
|
fi
|
|
[ -n "$ARCHIVE_PREPARE_CMD" ] && bash "$ARCHIVE_PREPARE_CMD"
|
|
if [[ "$src" =~ ";" ]] ; then
|
|
for it in ${src//;/ }
|
|
do
|
|
base_path=$it
|
|
dir_path=$(dirname "$it")
|
|
if [ -d "$dir_path" ] ; then
|
|
cd "$dir_path" || continue
|
|
base_path=$(basename "$it")
|
|
fi
|
|
_do_archive_create "$(dirname "$target")"
|
|
_do_archive_copy_target "$target" "$dir_path" "$base_path"
|
|
if [ "$base_path" != "$it" ] ; then
|
|
cd "$ORG" || continue
|
|
fi
|
|
if [ "$is_in_work_path" == "1" ] ; then
|
|
cd "$WORK_PATH" || continue
|
|
fi
|
|
done
|
|
else
|
|
base_path=$(basename "$src")
|
|
dir_path=$(dirname "$src")
|
|
_do_archive_create "$(dirname "$target")"
|
|
_do_archive_copy_target "$target" "$dir_path" "$base_path"
|
|
fi
|
|
if [ "$is_in_work_path" == "1" ] ; then
|
|
cd "$ORG" || return
|
|
fi
|
|
}
|
|
_do_restore() {
|
|
[ -z "$1" ] && echo "no user found " && return
|
|
[ -z "$2" ] && echo "no source found " && return
|
|
[ -z "$3" ] && echo "no id found " && return
|
|
[ -z "$4" ] && echo "no target found " && return
|
|
! _set_passwd_file "$1" && echo "No credentials found for $1" && return 1
|
|
local source=""
|
|
source=$(_get_target "$2")
|
|
local target=$3
|
|
case "$target" in
|
|
last|.)
|
|
target="latest"
|
|
#`$CMD_SHOW_LAST -r $source | tail -1`
|
|
;;
|
|
esac
|
|
# shellcheck disable=SC2086
|
|
$CMD_RESTORE $HOST_ARGS $CMD_OPS -r $source $target --target $4
|
|
}
|
|
_do_archive_restore_target() {
|
|
[ -z "$1" ] && echo "no target found " && return
|
|
[ -z "$2" ] && echo "no dir_path found " && return
|
|
[ -z "$3" ] && echo "no base_path found " && return
|
|
local target=$1
|
|
local dir_path=$2
|
|
local base_path=$3
|
|
[ -n "$VERBOSE" ] && echo "restoring $dir_path/$base_path from $target/$dir_path ..."
|
|
local bucket=""
|
|
bucket=$(echo "$target/$dir_path" | sed "s,$BACKUP_SERVER,,g" | sed "s,s3:,s3:/,g")".tar.gz"
|
|
# shellcheck disable=SC2086
|
|
if $CMD_ARCHIVE_RESTORE $CMD_ARCHIVE_OPS $CMD_ARCHIVE_OPS_HOST $bucket $base_path.tar.gz ; then
|
|
tar xzf "$base_path.tar.gz" && [ -n "$VERBOSE" ] && echo "$base_path.tar.gz restore from $target/$dir_path"
|
|
fi
|
|
}
|
|
_do_archive_restore() {
|
|
[ -z "$1" ] && echo "no user found " && return
|
|
[ -z "$2" ] && echo "no source found " && return
|
|
[ -z "$3" ] && echo "no target found " && return
|
|
local _src=$2
|
|
local base_path=""
|
|
local dir_path=""
|
|
local target=""
|
|
local is_in_work_path=0
|
|
TMP_VAR=""
|
|
_get_source "$_src" "$1"
|
|
TMP_VAR=""
|
|
local src=$TMP_VAR
|
|
[ -z "$src" ] && echo "No source found for $_src - $1" && return 1
|
|
target=$(_get_target "$3")
|
|
if [[ "$src" =~ $WORK_PATH ]] ; then
|
|
local work_dir=""
|
|
work_dir=$(dirname "$src")
|
|
cd "$work_dir" || return
|
|
src=$(basename "$src")
|
|
is_in_work_path=1
|
|
fi
|
|
[ -n "$RESTORE_PREPARE_CMD" ] && bash "$RESTORE_PREPARE_CMD"
|
|
if [[ "$src" =~ ";" ]] ; then
|
|
# $(echo "$src" | sed 's/;/ /')
|
|
for it in ${src//;/}
|
|
do
|
|
base_path=$it
|
|
dir_path=$(dirname "$it")
|
|
if [ -d "$dir_path" ] ; then
|
|
cd "$dir_path" || continue
|
|
base_path=$(basename "$it")
|
|
fi
|
|
_do_archive_copy_target "$target" "$dir_path" "$base_path"
|
|
if [ "$base_path" != "$it" ] ; then
|
|
cd "$ORG" || continue
|
|
fi
|
|
if [ "$is_in_work_path" == "1" ] ; then
|
|
cd "$WORK_PATH" || continue
|
|
fi
|
|
done
|
|
else
|
|
base_path=$(basename "$src")
|
|
dir_path=$(dirname "$src")
|
|
_do_archive_copy_target "$target" "$dir_path" "$base_path"
|
|
fi
|
|
[ "$is_in_work_path" == "1" ] && cd "$ORG" || return
|
|
}
|
|
_do_archive_delete() {
|
|
[ -z "$1" ] && echo "no target found " && return
|
|
[ -z "$2" ] && echo "no dir_path found " && return
|
|
[ -z "$3" ] && echo "no base_path found " && return
|
|
local target=$1
|
|
local dir_path=$2
|
|
local base_path=$3
|
|
[ -n "$VERBOSE" ] && echo "restoring $dir_path/$base_path from $target ..."
|
|
tar czf "$base_path.tar.gz" -C "$dir_path $base_path"
|
|
local bucket=""
|
|
bucket=${target//$BACKUP_SERVER/}
|
|
bucket=${bucket//s3:/s3:/}".tar.gz"
|
|
if $CMD_ARCHIVE_DELETE "$CMD_ARCHIVE_OPS" $CMD_ARCHIVE_OPS_HOST "$bucket/$base_path.tar.gz" ; then
|
|
[ -n "$VERBOSE" ] && echo "$target deleted "
|
|
fi
|
|
}
|
|
_do_show() {
|
|
[ -z "$1" ] && echo "no user found " && return
|
|
[ -z "$2" ] && echo "no source found " && return
|
|
! _set_passwd_file "$1" && echo "No credentials found for $1" && return 1
|
|
local source=""
|
|
source=$(_get_target "$2")
|
|
# shellcheck disable=SC2086
|
|
$CMD_SHOW $HOST_ARGS $CMD_OPS -r $source
|
|
}
|
|
_do_archive_show() {
|
|
[ -z "$1" ] && echo "no target found " && return
|
|
[ -z "$2" ] && echo "no dir_path found " && return
|
|
#[ -z "$3" ] && echo "no base_path found " && return
|
|
local target=$1
|
|
local dir_path=$2
|
|
local base_path=$3
|
|
local bucket=""
|
|
[ -n "$VERBOSE" ] && echo "show $dir_path/$base_path from $target ..."
|
|
bucket=$(echo "$target" | sed "s,$BACKUP_SERVER,,g" | sed "s,s3:,s3:/,g")
|
|
# shellcheck disable=SC2086
|
|
if $CMD_ARCHIVE_SHOW $CMD_ARCHIVE_OPS $CMD_ARCHIVE_OPS_HOST $bucket/$base_path; then
|
|
[ -n "$VERBOSE" ] && echo "$target listed "
|
|
fi
|
|
}
|
|
_do_clean() {
|
|
[ -z "$1" ] && echo "no user found " && return
|
|
[ -z "$2" ] && echo "no source found " && return
|
|
! _set_passwd_file "$1" && echo "No credentials found for $1" && return 1
|
|
local source=""
|
|
source=$(_get_target "$2")
|
|
# shellcheck disable=SC2086
|
|
$CMD_CLEAN $HOST_ARGS $CMD_OPS -r $source
|
|
}
|
|
_do_forget() {
|
|
[ -z "$1" ] && echo "no user found " && return
|
|
[ -z "$2" ] && echo "no source found " && return
|
|
[ -z "$3" ] && echo "no target found target_item " && return
|
|
! _set_passwd_file "$1" && echo "No credentials found for $1" && return 1
|
|
local target=$3
|
|
local source=""
|
|
source=$(_get_target "$2")
|
|
# shellcheck disable=SC2086
|
|
$CMD_FORGET $HOST_ARGS $CMD_OPS -r $source $target
|
|
}
|
|
_do_mount() {
|
|
[ -z "$1" ] && echo "no user found " && return
|
|
[ -z "$2" ] && echo "no source found " && return
|
|
[ -z "$3" ] && echo "no target found " && return
|
|
! _set_passwd_file "$1" && echo "No credentials found for $1" && return 1
|
|
local target=$3
|
|
if [[ "$target" =~ "@" ]] ; then
|
|
target=$(echo "$target" | cut -d"@" -f2 | sed 's,^/,,g')
|
|
target=$ROOT/mnt/$(basename "$target")
|
|
fi
|
|
[ ! -d "$target" ] && mkdir -p "$target"
|
|
[ ! -d "$target" ] && echo "Directory $target can not be used " && return 1
|
|
local src=""
|
|
src=$(_get_target "$2")
|
|
# shellcheck disable=SC2086
|
|
$CMD_MOUNT $HOST_ARGS $CMD_OPS -r $src $target
|
|
}
|
|
_do_task() {
|
|
[ -z "$1" ] && echo "no name found " && return
|
|
[ -z "$2" ] && echo "no user found " && return
|
|
[ -z "$3" ] && echo "no task found " && return
|
|
[ -z "$4" ] && echo "no source found " && return
|
|
[ -n "$VERBOSE" ] && echo "Making $1 ..."
|
|
local task=$3
|
|
if [[ "$task" =~ ":" ]] ; then
|
|
task=$(echo "$3" | cut -f1 -d":")
|
|
ops=$(echo "$3" | cut -f2 -d":" | sed 's/,/ /g' | sed 's/;/ /g')
|
|
[ -n "$ops" ] && CMD_OPS=$CMD_OPS" "$ops
|
|
fi
|
|
case "$task" in
|
|
copy)
|
|
[ -z "$5" ] && echo "no target found " && return
|
|
_set_cmd_ops v
|
|
_do_copy "$2" "$4" "$5"
|
|
;;
|
|
archive-copy)
|
|
[ -z "$5" ] && echo "no target found " && return
|
|
_do_archive_copy "$2" "$4" "$5"
|
|
;;
|
|
restore)
|
|
local id_path=$5
|
|
[ -n "$BACKUP_PATH_TARGET" ] && id_path=$BACKUP_PATH_TARGET
|
|
[ -z "$id_path" ] && echo "no id found " && return
|
|
local restore_path=$6
|
|
[ -n "$RESTORE_PATH_TARGET" ] && restore_path=$RESTORE_PATH_TARGET
|
|
[ -z "$restore_path" ] && echo "no target found " && return
|
|
[ "$restore_path" == "." ] && restore_path=$5 # /
|
|
_set_cmd_ops v
|
|
_do_restore "$2" "$4" "$id_path" "$restore_path"
|
|
;;
|
|
archive-restore)
|
|
[ -z "$5" ] && echo "no target found " && return
|
|
_do_archive_restore "$2" "$4" "$5"
|
|
;;
|
|
show)
|
|
_do_show "$2" "$4" "$5"
|
|
;;
|
|
archive-show)
|
|
_do_archive_show "$2" "$4" "$5"
|
|
;;
|
|
archive-create)
|
|
_do_archive_create "$2" "$4" "$5"
|
|
;;
|
|
mount)
|
|
[ -z "$5" ] && echo "no target found " && return
|
|
local restore_path=$5
|
|
[ -n "$RESTORE_PATH_TARGET" ] && restore_path=$RESTORE_PATH_TARGET
|
|
_set_cmd_ops
|
|
_do_mount "$2" "$4" "$restore_path"
|
|
;;
|
|
forget)
|
|
_set_cmd_ops
|
|
local target_item=""
|
|
[ -n "$TARGET_ITEM" ] && target_item=$TARGET_ITEM
|
|
_do_forget "$2" "$4" "$target_item"
|
|
;;
|
|
clear|clean)
|
|
_set_cmd_ops
|
|
_do_clean "$2" "$4" "$5"
|
|
;;
|
|
*) echo "$task not configured !"
|
|
esac
|
|
[ -n "$VERBOSE" ] && echo "$1 done !"
|
|
}
|
|
_parse_line() {
|
|
[ -z "$1" ] && return
|
|
local has_comment=""
|
|
has_comment=$(echo "$1" | grep "^#")
|
|
[ -n "$has_comment" ] && return
|
|
local name=""
|
|
name=$(echo "$1" | cut -f1 -d"|")
|
|
local user=""
|
|
user=$(echo "$1" | cut -f2 -d"|" | sed 's/ //g')
|
|
local task=""
|
|
local source=""
|
|
local target=""
|
|
if [ -n "$TASK_CMD" ] ; then
|
|
task=$TASK_CMD
|
|
case "$TASK_CMD" in
|
|
copy|archive)
|
|
target=$(echo "$1" | cut -f5 -d"|" | sed 's/ //g')
|
|
source=$(echo "$1" | cut -f4 -d"|" | sed 's/ //g')
|
|
;;
|
|
*)
|
|
target=$(echo "$1" | cut -f4 -d"|" | sed 's/ //g')
|
|
source=$(echo "$1" | cut -f5 -d"|" | sed 's/ //g')
|
|
;;
|
|
esac
|
|
else
|
|
task=$(echo "$1" | cut -f3 -d"|" | sed 's/ //g')
|
|
source=$(echo "$1" | cut -f4 -d"|" | sed 's/ //g')
|
|
target=$(echo "$1" | cut -f5 -d"|" | sed 's/ //g')
|
|
fi
|
|
[ -n "$VERBOSE" ] && echo "$name ($user) -> $task from $source to $target"
|
|
_do_task "$name" "$user" "$task" "$source" "$target"
|
|
}
|
|
if [ "$BACKUP_SERVER_TARGET" == "server" ] ; then
|
|
_env_srvr "$BACKUP_SERVER_TARGET""_""$ENV_HOST_CRD"
|
|
else
|
|
_env_s3 "s3_$ENV_HOST_CRD"
|
|
fi
|
|
if [ -n "$TASK_CMD" ] && [ -n "$BACKUP_PATH_SOURCE" ] && [ -n "$USER_CMD" ]; then
|
|
_do_task "task" "$USER_CMD" "$TASK_CMD" "$BACKUP_PATH_SOURCE" "$BACKUP_PATH_TARGET" "$RESTORE_PATH_TARGET"
|
|
else
|
|
while IFS= read -r line
|
|
do
|
|
[ -n "$MATCH_NAME" ] && [[ ! "$line" =~ ^$MATCH_NAME.* ]] && continue
|
|
_parse_line "$line"
|
|
done < "$TASKS_LIST"
|
|
fi
|
|
|
|
rm -f "$RESTIC_PASSWORD_FILE" "$ROOT"/tmp/.tmp* "$ROOT"/.c "$CMD_ERROR_LOG"
|
|
|
|
|