From 20ba5378c25d4ba7356875cd7fef81c8d0e37134 Mon Sep 17 00:00:00 2001
From: deneb <deneb@screee.ee>
Date: Mon, 13 Jan 2025 21:55:16 +0000
Subject: [PATCH] initial commit
---
dates/.gitignore | 2 +
orchestration/backup-all.service | 5 ++
orchestration/backup-all.timer | 10 ++++
orchestration/backup-out-of-date@.service | 9 ++++
orchestration/backuprunner@.service | 14 +++++
orchestration/backupserver.service | 10 ++++
orchestration/is-backup-recent@.service | 10 ++++
orchestration/is-backup-recent@.timer | 9 ++++
scripts/.gitignore | 2 +
scripts/backuprunner.sh | 66 +++++++++++++++++++++++
scripts/backupserver-poweron.sh | 34 ++++++++++++
scripts/backupserver-shutdown.sh | 17 ++++++
scripts/is-backup-recent.sh | 15 ++++++
scripts/notify.sh | 15 ++++++
targets/conduit.sh | 13 +++++
targets/nextcloud.sh | 63 ++++++++++++++++++++++
targets/syncthing.sh | 5 ++
targets/vaultwarden.sh | 13 +++++
18 files changed, 312 insertions(+)
create mode 100644 dates/.gitignore
create mode 100644 orchestration/backup-all.service
create mode 100644 orchestration/backup-all.timer
create mode 100644 orchestration/backup-out-of-date@.service
create mode 100644 orchestration/backuprunner@.service
create mode 100644 orchestration/backupserver.service
create mode 100644 orchestration/is-backup-recent@.service
create mode 100644 orchestration/is-backup-recent@.timer
create mode 100644 scripts/.gitignore
create mode 100755 scripts/backuprunner.sh
create mode 100755 scripts/backupserver-poweron.sh
create mode 100755 scripts/backupserver-shutdown.sh
create mode 100755 scripts/is-backup-recent.sh
create mode 100755 scripts/notify.sh
create mode 100644 targets/conduit.sh
create mode 100644 targets/nextcloud.sh
create mode 100644 targets/syncthing.sh
create mode 100644 targets/vaultwarden.sh
diff --git a/dates/.gitignore b/dates/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/dates/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/orchestration/backup-all.service b/orchestration/backup-all.service
new file mode 100644
index 0000000..8f8c6dc
--- /dev/null
+++ b/orchestration/backup-all.service
@@ -0,0 +1,5 @@
+[Unit]
+Description=Perform systemd-enabled backups (backuprunner@.service)
+
+[Service]
+ExecStart=/usr/bin/echo Performing all enabled backups
diff --git a/orchestration/backup-all.timer b/orchestration/backup-all.timer
new file mode 100644
index 0000000..a911032
--- /dev/null
+++ b/orchestration/backup-all.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Weekly remote-site backups
+After=network-online.target
+
+[Timer]
+OnCalendar=Sun, 04:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/orchestration/backup-out-of-date@.service b/orchestration/backup-out-of-date@.service
new file mode 100644
index 0000000..ddb93f5
--- /dev/null
+++ b/orchestration/backup-out-of-date@.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Out-of-date notification for %I backup
+After=local-fs.target
+Wants=local-fs.target
+
+[Service]
+Type=oneshot
+ExecStart=/opt/backuprunner/scripts/notify.sh %I
+WorkingDirectory=/opt/backuprunner
diff --git a/orchestration/backuprunner@.service b/orchestration/backuprunner@.service
new file mode 100644
index 0000000..8add9d5
--- /dev/null
+++ b/orchestration/backuprunner@.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Backup job for %I
+Requires=backupserver.service
+After=backupserver.service
+
+[Service]
+Type=oneshot
+User=backupuser
+ExecStart=/opt/backuprunner/scripts/backuprunner.sh %I
+ExecStartPost=/usr/bin/touch /opt/backuprunner/dates/%I
+AmbientCapabilities=CAP_DAC_READ_SEARCH
+
+[Install]
+RequiredBy=backup-all.service
diff --git a/orchestration/backupserver.service b/orchestration/backupserver.service
new file mode 100644
index 0000000..738ccc2
--- /dev/null
+++ b/orchestration/backupserver.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Ensure backup server is online
+StopWhenUnneeded=yes
+
+[Service]
+Type=oneshot
+User=backupuser
+ExecStart=/opt/backuprunner/scripts/backupserver-poweron.sh
+ExecStop=/opt/backuprunner/scripts/backupserver-shutdown.sh
+RemainAfterExit=yes
diff --git a/orchestration/is-backup-recent@.service b/orchestration/is-backup-recent@.service
new file mode 100644
index 0000000..fa2c1e2
--- /dev/null
+++ b/orchestration/is-backup-recent@.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Check last backup date for %I
+After=local-fs.target
+Wants=local-fs.target
+OnFailure=backup-out-of-date@.service
+
+[Service]
+Type=oneshot
+ExecStart=/opt/backuprunner/scripts/is-recent.sh %I
+WorkingDirectory=/opt/backuprunner
diff --git a/orchestration/is-backup-recent@.timer b/orchestration/is-backup-recent@.timer
new file mode 100644
index 0000000..af15907
--- /dev/null
+++ b/orchestration/is-backup-recent@.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Periodically check last backup date for %I
+
+[Timer]
+OnCalendar=00:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/scripts/.gitignore b/scripts/.gitignore
new file mode 100644
index 0000000..b0e99e8
--- /dev/null
+++ b/scripts/.gitignore
@@ -0,0 +1,2 @@
+backupserver-netrc
+ntfy-url.sh
diff --git a/scripts/backuprunner.sh b/scripts/backuprunner.sh
new file mode 100755
index 0000000..99d75ed
--- /dev/null
+++ b/scripts/backuprunner.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+if [ -z "$1" ]; then
+ echo "Usage: $0 TARGET"
+ exit -2
+fi
+
+err_report() {
+ echo "Error on line $1"
+ backup_post
+ exit -1
+}
+
+trap 'err_report $LINENO' ERR
+
+ssh() {
+ /usr/bin/env -S ssh -o PasswordAuthentication=no "$@"
+}
+
+backup_host="rex.do.netdeneb.com"
+backup_user="backupuser"
+timestamp="$(date +%Y-%m-%d_%H%M)"
+
+# MUST set up the following:
+# - backup_pre()
+# - backup_post()
+# - $local_path (directly or in backup_pre())
+# - $backup_path (directly or in backup_pre())
+# MUST NOT call backup_pre or backup_post
+# MAY set up $rsync_extra_args
+# MAY rely on $backup_host, $backup_user and $timestamp
+source "/opt/backuprunner/targets/$1.sh"
+
+
+backup_pre
+
+
+if ! ssh "$backup_user"@"$backup_host" "stat \"$backup_path\"" >/dev/null 2>&1; then
+ echo "Creating repository '$backup_path'"
+ borg init \
+ --encryption=none \
+ --make-parent-dirs \
+ "$backup_user"@"$backup_host":"$backup_path"
+fi
+
+pushd "$local_path"
+echo "Creating archive '$backup_path::$timestamp'"
+borg create \
+ --stats \
+ --compression=zstd,5 \
+ "${borg_extra_args[@]}" \
+ "$backup_user"@"$backup_host":"$backup_path"::"$timestamp" \
+ .
+popd
+
+
+#ssh "$backup_user"@"$backup_host" "mkdir -p \"$backup_path\""
+
+#echo "Synchronise backup folder with local state"
+#rsync --compress --archive --delete \
+# "${rsync_extra_args[@]}" \
+# "$local_path" \
+# "$backup_user"@"$backup_host":"$backup_path"
+
+
+backup_post
diff --git a/scripts/backupserver-poweron.sh b/scripts/backupserver-poweron.sh
new file mode 100755
index 0000000..deefb8d
--- /dev/null
+++ b/scripts/backupserver-poweron.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e
+
+ping_host="rex.do.netdeneb.com"
+irmc_host="rex-irmc.do.netdeneb.com"
+
+netrc_path="/opt/backuprunner/scripts/backupserver-netrc"
+
+timeout=600
+
+err_report() {
+ echo "Error on line $1"
+ exit 255
+}
+
+trap 'err_report $LINENO' ERR
+
+if ! ping -c4 "$ping_host"; then
+ curl "$irmc_host/9#restart" -X POST \
+ --digest --netrc-file "$netrc_path" \
+ -H "Content-Type: application/x-www-form-urlencoded" \
+ -d 'APPLY=0&a=1&P0=Confirm' \
+ -s -o /dev/null
+
+ echo "Waiting for $ping_host..."
+ while [[ "$timeout" -gt 0 ]]; do
+ ping -c1 -W10 "$ping_host" >/dev/null && break
+ echo "timeout in $timeout"
+ timeout=$(( $timeout - 10 ))
+ done
+else
+ echo "$ping_host is already up"
+fi
diff --git a/scripts/backupserver-shutdown.sh b/scripts/backupserver-shutdown.sh
new file mode 100755
index 0000000..9524411
--- /dev/null
+++ b/scripts/backupserver-shutdown.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+backup_host="rex.do.netdeneb.com"
+backup_user="backupuser"
+
+err_report() {
+ echo "Error on line $1"
+ exit 255
+}
+
+trap 'err_report $LINENO' ERR
+
+ssh() {
+ /usr/bin/env -S ssh -o PasswordAuthentication=no "$@"
+}
+
+ssh "$backup_user"@"$backup_host" sudo poweroff
diff --git a/scripts/is-backup-recent.sh b/scripts/is-backup-recent.sh
new file mode 100755
index 0000000..3fbebac
--- /dev/null
+++ b/scripts/is-backup-recent.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+file="$1"
+dir="${2:-./dates}"
+
+threshold=$(( 8 * 24 * 3600 )) # eight days
+
+if [ -z "$file" ]; then
+ echo "filename not specified" >2
+ echo "usage: $0 <filename> [<dirname>]" >2
+ exit 255
+fi
+
+(( ($(stat -c%Y "$dir/$file") + threshold) > $(date +%s) )) || exit 1
+
+exit 0
diff --git a/scripts/notify.sh b/scripts/notify.sh
new file mode 100755
index 0000000..6354bc5
--- /dev/null
+++ b/scripts/notify.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+file="$1"
+dir="${2:-/opt/backuprunner/dates}"
+
+date="$(stat -c%y "$dir/$file" | cut -d'.' -f1) (UTC)"
+
+title="$file backup is out of date!"
+description="Last successful backup: $date"
+
+. /opt/backuprunner/scripts/ntfy-url.sh
+
+curl -H "Title: $title" \
+ -H "Priority: high" \
+ -d "$description" \
+ $NTFY_URL
diff --git a/targets/conduit.sh b/targets/conduit.sh
new file mode 100644
index 0000000..c3d42a6
--- /dev/null
+++ b/targets/conduit.sh
@@ -0,0 +1,13 @@
+local_path="/srv/conduit"
+backup_path="conduit"
+borg_extra_args=(
+ '--exclude=*.service'
+ '--exclude=*.yml'
+ '--exclude=*.toml'
+ '--exclude=.local'
+ '--exclude=.config'
+ '--exclude=.cache'
+)
+
+backup_pre() { :; }
+backup_post() { :; }
diff --git a/targets/nextcloud.sh b/targets/nextcloud.sh
new file mode 100644
index 0000000..e313a9e
--- /dev/null
+++ b/targets/nextcloud.sh
@@ -0,0 +1,63 @@
+local_path="/mnt/barracuda/Nextcloud/"
+backup_path="nextcloud"
+borg_extra_args=('--exclude=appdata*/preview' '--exclude=updater*')
+
+backup_pre() {
+ echo "Enable maintenance mode"
+ _maintenance_mode on
+}
+
+backup_post() {
+ echo "Disable maintenance mode"
+ _maintenance_mode off
+}
+
+
+_maintenance_mode() {
+ ssh backupuser@trex-nextcloud.ibk.netdeneb.com \
+ NEXTCLOUD_PHP=/usr/bin/php NEXTCLOUD_PHP_CONFIG=/etc/webapps/nextcloud/php.ini \
+ sudo --preserve-env="NEXTCLOUD_PHP,NEXTCLOUD_PHP_CONFIG" \
+ occ maintenance:mode --${1:-on}
+}
+
+## old (reflink+rsync) method
+#local_path="/mnt/barracuda/Nextcloud/"
+#rsync_extra_args=('--exclude=appdata*/preview' '--exclude=updater*')
+#_backup_path_base="Nextcloud/"
+#
+#backup_pre() {
+# echo "Enable maintenance mode"
+# _maintenance_mode on
+#
+# #_backup_subpath="${timestamp}_inc"
+# #backup_path="$_backup_path_base/$_backup_subpath/mnt/nextcloud/"
+#
+# #_copy_last "$_backup_subpath"
+#}
+#
+#_copy_last() {
+# echo "Create new backup directory"
+# ssh "$backup_user"@"$backup_host" <<EOF
+# next="$1"
+# if [ -z "\$next" ]; then
+# echo "\\\$next not specified"
+# fi
+#
+# cd "\$HOME/$_backup_path_base"
+#
+# last="\$(find . -maxdepth 1 -name '*_inc' -type d | sort | tail -n1)"
+#
+# echo "\$last" "\$next"
+#
+# if [ "\$next" == "\$last" ]; then
+# echo "\$next == last timestamp; aborting"
+# exit 2
+# fi
+# if [ -e "\$next" ]; then
+# echo "\$next already exists; using"
+# exit 0
+# fi
+#
+# cp -rp --reflink "\$last" "\$next"
+#EOF
+#}
diff --git a/targets/syncthing.sh b/targets/syncthing.sh
new file mode 100644
index 0000000..f85de4d
--- /dev/null
+++ b/targets/syncthing.sh
@@ -0,0 +1,5 @@
+local_path="/mnt/butter/syncthing"
+backup_path="syncthing"
+
+backup_pre() { :; }
+backup_post() { :; }
diff --git a/targets/vaultwarden.sh b/targets/vaultwarden.sh
new file mode 100644
index 0000000..da1615d
--- /dev/null
+++ b/targets/vaultwarden.sh
@@ -0,0 +1,13 @@
+local_path="/srv/vaultwarden"
+backup_path="vaultwarden"
+borg_extra_args=(
+ '--exclude=*.service'
+ '--exclude=*.yml'
+ '--exclude=*.toml'
+ '--exclude=.local'
+ '--exclude=.config'
+ '--exclude=.cache'
+)
+
+backup_pre() { :; }
+backup_post() { :; }