initial commit

dates/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

orchestration/backup-all.service

@@ -0,0 +1,5 @@
+[Unit]
+Description=Perform systemd-enabled backups (backuprunner@.service)
+
+[Service]
+ExecStart=/usr/bin/echo Performing all enabled backups

orchestration/backup-all.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Weekly remote-site backups
+After=network-online.target
+
+[Timer]
+OnCalendar=Sun, 04:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target

orchestration/backup-out-of-date@.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Out-of-date notification for %I backup
+After=local-fs.target
+Wants=local-fs.target
+
+[Service]
+Type=oneshot
+ExecStart=/opt/backuprunner/scripts/notify.sh %I
+WorkingDirectory=/opt/backuprunner

orchestration/backuprunner@.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Backup job for %I
+Requires=backupserver.service
+After=backupserver.service
+
+[Service]
+Type=oneshot
+User=backupuser
+ExecStart=/opt/backuprunner/scripts/backuprunner.sh %I
+ExecStartPost=/usr/bin/touch /opt/backuprunner/dates/%I
+AmbientCapabilities=CAP_DAC_READ_SEARCH
+
+[Install]
+RequiredBy=backup-all.service

orchestration/backupserver.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Ensure backup server is online
+StopWhenUnneeded=yes
+
+[Service]
+Type=oneshot
+User=backupuser
+ExecStart=/opt/backuprunner/scripts/backupserver-poweron.sh
+ExecStop=/opt/backuprunner/scripts/backupserver-shutdown.sh
+RemainAfterExit=yes

orchestration/is-backup-recent@.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Check last backup date for %I
+After=local-fs.target
+Wants=local-fs.target
+OnFailure=backup-out-of-date@.service
+
+[Service]
+Type=oneshot
+ExecStart=/opt/backuprunner/scripts/is-recent.sh %I
+WorkingDirectory=/opt/backuprunner

orchestration/is-backup-recent@.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Periodically check last backup date for %I
+
+[Timer]
+OnCalendar=00:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target

scripts/.gitignore

@@ -0,0 +1,2 @@
+backupserver-netrc
+ntfy-url.sh

scripts/backuprunner.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 TARGET"
+    exit -2
+fi
+
+err_report() {
+    echo "Error on line $1"
+    backup_post
+    exit -1
+}
+
+trap 'err_report $LINENO' ERR
+
+ssh() {
+    /usr/bin/env -S ssh -o PasswordAuthentication=no "$@"
+}
+
+backup_host="rex.do.netdeneb.com"
+backup_user="backupuser"
+timestamp="$(date +%Y-%m-%d_%H%M)"
+
+# MUST set up the following:
+# - backup_pre()
+# - backup_post()
+# - $local_path     (directly or in backup_pre())
+# - $backup_path    (directly or in backup_pre())
+# MUST NOT call backup_pre or backup_post
+# MAY set up $rsync_extra_args
+# MAY rely on $backup_host, $backup_user and $timestamp
+source "/opt/backuprunner/targets/$1.sh"
+
+
+backup_pre
+
+
+if ! ssh "$backup_user"@"$backup_host" "stat \"$backup_path\"" >/dev/null 2>&1; then
+    echo "Creating repository '$backup_path'"
+    borg init \
+        --encryption=none \
+        --make-parent-dirs \
+        "$backup_user"@"$backup_host":"$backup_path"
+fi
+
+pushd "$local_path"
+echo "Creating archive '$backup_path::$timestamp'"
+borg create \
+    --stats \
+    --compression=zstd,5 \
+    "${borg_extra_args[@]}" \
+    "$backup_user"@"$backup_host":"$backup_path"::"$timestamp" \
+    .
+popd
+
+
+#ssh "$backup_user"@"$backup_host" "mkdir -p \"$backup_path\""
+
+#echo "Synchronise backup folder with local state"
+#rsync --compress --archive --delete \
+#      "${rsync_extra_args[@]}" \
+#      "$local_path" \
+#      "$backup_user"@"$backup_host":"$backup_path"
+
+
+backup_post

scripts/backupserver-poweron.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e
+
+ping_host="rex.do.netdeneb.com"      
+irmc_host="rex-irmc.do.netdeneb.com"
+
+netrc_path="/opt/backuprunner/scripts/backupserver-netrc"
+
+timeout=600
+
+err_report() {
+    echo "Error on line $1"
+    exit 255
+}
+
+trap 'err_report $LINENO' ERR
+                                 
+if ! ping -c4 "$ping_host"; then          
+    curl "$irmc_host/9#restart" -X POST \
+    --digest --netrc-file "$netrc_path" \
+        -H "Content-Type: application/x-www-form-urlencoded" \
+        -d 'APPLY=0&a=1&P0=Confirm' \
+        -s -o /dev/null
+                
+    echo "Waiting for $ping_host..." 
+    while [[ "$timeout" -gt 0 ]]; do                  
+        ping -c1 -W10 "$ping_host" >/dev/null && break
+        echo "timeout in $timeout" 
+        timeout=$(( $timeout - 10 ))
+    done
+else
+    echo "$ping_host is already up"
+fi

scripts/backupserver-shutdown.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+backup_host="rex.do.netdeneb.com"
+backup_user="backupuser"
+
+err_report() {
+    echo "Error on line $1"
+    exit 255
+}
+
+trap 'err_report $LINENO' ERR
+
+ssh() {
+    /usr/bin/env -S ssh -o PasswordAuthentication=no "$@"
+}
+
+ssh "$backup_user"@"$backup_host" sudo poweroff

scripts/is-backup-recent.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+file="$1"
+dir="${2:-./dates}"
+
+threshold=$(( 8 * 24 * 3600 )) # eight days
+
+if [ -z "$file" ]; then
+	echo "filename not specified" >2
+	echo "usage: $0 <filename> [<dirname>]" >2
+	exit 255
+fi
+
+(( ($(stat -c%Y "$dir/$file") + threshold) > $(date +%s) )) || exit 1
+
+exit 0

scripts/notify.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+file="$1"
+dir="${2:-/opt/backuprunner/dates}"
+
+date="$(stat -c%y "$dir/$file" | cut -d'.' -f1) (UTC)"
+
+title="$file backup is out of date!"
+description="Last successful backup: $date"
+
+. /opt/backuprunner/scripts/ntfy-url.sh
+
+curl -H "Title: $title" \
+     -H "Priority: high" \
+     -d "$description" \
+     $NTFY_URL

targets/conduit.sh

@@ -0,0 +1,13 @@
+local_path="/srv/conduit"
+backup_path="conduit"
+borg_extra_args=(
+    '--exclude=*.service'
+    '--exclude=*.yml'
+    '--exclude=*.toml'
+    '--exclude=.local'
+    '--exclude=.config'
+    '--exclude=.cache'
+)
+
+backup_pre() { :; }
+backup_post() { :; }

targets/nextcloud.sh

@@ -0,0 +1,63 @@
+local_path="/mnt/barracuda/Nextcloud/"
+backup_path="nextcloud"
+borg_extra_args=('--exclude=appdata*/preview' '--exclude=updater*')
+
+backup_pre() {
+    echo "Enable maintenance mode"
+    _maintenance_mode on
+}
+
+backup_post() {
+    echo "Disable maintenance mode"
+    _maintenance_mode off
+}
+
+
+_maintenance_mode() {
+    ssh backupuser@trex-nextcloud.ibk.netdeneb.com \
+        NEXTCLOUD_PHP=/usr/bin/php NEXTCLOUD_PHP_CONFIG=/etc/webapps/nextcloud/php.ini \
+            sudo --preserve-env="NEXTCLOUD_PHP,NEXTCLOUD_PHP_CONFIG" \
+                occ maintenance:mode --${1:-on}
+}
+
+## old (reflink+rsync) method
+#local_path="/mnt/barracuda/Nextcloud/"
+#rsync_extra_args=('--exclude=appdata*/preview' '--exclude=updater*')
+#_backup_path_base="Nextcloud/"
+#
+#backup_pre() {
+#    echo "Enable maintenance mode"
+#    _maintenance_mode on
+#
+#    #_backup_subpath="${timestamp}_inc"
+#    #backup_path="$_backup_path_base/$_backup_subpath/mnt/nextcloud/"
+#
+#    #_copy_last "$_backup_subpath"
+#}
+#
+#_copy_last() {
+#    echo "Create new backup directory"
+#    ssh "$backup_user"@"$backup_host" <<EOF
+#        next="$1"
+#        if [ -z "\$next" ]; then
+#            echo "\\\$next not specified"
+#        fi
+#
+#        cd "\$HOME/$_backup_path_base"
+#
+#        last="\$(find . -maxdepth 1 -name '*_inc' -type d | sort | tail -n1)"
+#
+#        echo "\$last" "\$next"
+#        
+#        if [ "\$next" == "\$last" ]; then
+#            echo "\$next == last timestamp; aborting"
+#            exit 2
+#        fi
+#        if [ -e "\$next" ]; then
+#            echo "\$next already exists; using"
+#            exit 0
+#        fi
+#        
+#        cp -rp --reflink "\$last" "\$next"
+#EOF
+#}

targets/syncthing.sh

@@ -0,0 +1,5 @@
+local_path="/mnt/butter/syncthing"
+backup_path="syncthing"
+
+backup_pre() { :; }
+backup_post() { :; }

targets/vaultwarden.sh

@@ -0,0 +1,13 @@
+local_path="/srv/vaultwarden"
+backup_path="vaultwarden"
+borg_extra_args=(
+    '--exclude=*.service'
+    '--exclude=*.yml'
+    '--exclude=*.toml'
+    '--exclude=.local'
+    '--exclude=.config'
+    '--exclude=.cache'
+)
+
+backup_pre() { :; }
+backup_post() { :; }