Repository navigation
135 lines (122 loc) · 6.59 KB
/
Copy pathbackup.yml
File metadata and controls
135 lines (122 loc) · 6.59 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
name: Scheduled Backup
on:
schedule:
- cron: '0 3 * * *' # Daily at 3 AM UTC
workflow_dispatch:
env:
PROJECT_DIR: /home/vasyl/projects/onlinelib/textstack
BACKUP_DIR: /home/vasyl/backups/textstack
# Opt JS actions into Node 24 ahead of June 2026 forced cutover.
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
# No checkout and no API calls: the job token gets nothing.
permissions: {}
jobs:
backup:
runs-on: self-hosted
# 13m typical.
timeout-minutes: 45
steps:
- name: Set backup date
run: |
echo "BACKUP_DATE=$(date +%F)" >> $GITHUB_ENV
# The V2 database (ADR-030) holds all live data since the cutover; the legacy one is
# read-only and archived, so it is no longer dumped. Files are prefixed by the V2 name.
v2=$(sed -n 's/^V2_POSTGRES_DB=//p' $PROJECT_DIR/.env | tr -d "\"' ")
echo "V2_DB=${v2:-textstack_v2}" >> $GITHUB_ENV
- name: Create backup directory
run: mkdir -p $BACKUP_DIR
# pg-dump-checked.sh: a missing database or a dump without pg_dump's trailer fails the job, and
# the file gets its real name only once complete.
- name: Backup V2 database
run: |
POSTGRES_USER=textstack_prod bash $PROJECT_DIR/infra/scripts/pg-dump-checked.sh \
$V2_DB $BACKUP_DIR/$V2_DB-db-$BACKUP_DATE.sql.gz
# data/storage + data/library (the V2 volume, #1110; a notice while it does not exist). A file
# that changed while read is accepted, an unreadable one fails. Contract + self-check: the script.
- name: Backup storage
run: bash $PROJECT_DIR/infra/scripts/backup-files-tar.sh $PROJECT_DIR/data $BACKUP_DIR/storage-$BACKUP_DATE.tar.gz
# The legacy db-* and pre-deploy-* files are no longer written; the 2 newest of each stay.
- name: Cleanup old backups (keep 2 newest of each type — history lives in R2)
run: |
cd $BACKUP_DIR
ls -1t db-*.sql.gz 2>/dev/null | tail -n +3 | xargs -r rm -v
ls -1t storage-*.tar.gz 2>/dev/null | tail -n +3 | xargs -r rm -v
ls -1t pre-deploy-*.sql.gz 2>/dev/null | tail -n +3 | xargs -r rm -v
ls -1t $V2_DB-db-*.sql.gz 2>/dev/null | tail -n +3 | xargs -r rm -v
ls -1t $V2_DB-pre-deploy-*.sql.gz 2>/dev/null | tail -n +3 | xargs -r rm -v
- name: Verify archive integrity
run: |
ls -lh $BACKUP_DIR/*.gz | tail -5
gunzip -t $BACKUP_DIR/$V2_DB-db-$BACKUP_DATE.sql.gz
- name: Verify backup restores into fresh postgres
run: |
cd $PROJECT_DIR
./infra/scripts/backup-verify.sh $BACKUP_DIR/$V2_DB-db-$BACKUP_DATE.sql.gz
# Off-site copy. Everything above lives on the same box as the data it protects; this is the
# only copy that survives losing the server. restic (in Docker, nothing installed on the host)
# to Cloudflare R2: encrypted client-side, deduplicated, so the storage folder costs its size
# once, not once per night. Keeps 7 nightly + 3 monthly snapshots.
# .env goes too — without it a restored box cannot start.
# RESTIC_PASSWORD must also live in the owner's password manager: without it the copy is noise.
# Skipped (with a warning) until the secrets exist, so the job stays green before setup.
- name: Off-site backup to R2 (restic)
env:
RESTIC_REPOSITORY: ${{ secrets.RESTIC_REPOSITORY }}
RESTIC_PASSWORD: ${{ secrets.RESTIC_PASSWORD }}
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
run: |
set -euo pipefail
if [ -z "$RESTIC_REPOSITORY" ]; then
echo "::warning::RESTIC_REPOSITORY secret not set — off-site backup skipped"
exit 0
fi
restic() {
docker run --rm -i \
-e RESTIC_REPOSITORY -e RESTIC_PASSWORD -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \
-v "$HOME/.cache/restic:/root/.cache/restic" \
-v "$PROJECT_DIR/data/storage:/backup/storage:ro" \
-v "$PROJECT_DIR/data/library:/backup/library:ro" \
-v "$PROJECT_DIR/.env:/backup/env:ro" \
restic/restic:0.17.3@sha256:8f5a62b422a2cb1277ea0dd6e826fe1acf649e5b9f02d60e5268d5fd1976255a "$@"
}
mkdir -p "$HOME/.cache/restic"
# Guard: stay inside R2's free 10 GB. Over 9 GB the job fails (email) instead of quietly
# costing money. Dedup means retention barely matters — size ≈ live data (~3-4 GB, 2026-10).
LIMIT=$((9 * 1024 * 1024 * 1024))
repo_size() { restic stats --mode raw-data --json | grep -o '"total_size":[0-9]*' | cut -d: -f2; }
restic cat config >/dev/null 2>&1 || restic init
size=$(repo_size)
if [ "$size" -gt "$LIMIT" ]; then
echo "::error::R2 backup repo is $((size / 1024 / 1024)) MB (> 9 GB) — not adding a snapshot"
exit 1
fi
restic backup --host textstack-prod --tag nightly /backup
# The dump goes in UNcompressed, through stdin. A .gz differs byte-for-byte every night,
# so restic stored all of it each time (2026-10-05). Plain SQL dedups by chunk: a night adds
# only what changed; restic compresses it itself. Before forget/check and the size guard, so
# it is pruned, integrity-checked and counted against 9 GB like the rest. The stdin path
# stays /db_v2.sql: restore-drill.yml reads it. The last legacy /db.sql snapshots stay:
# forget keeps the newest of each path, and none is added any more.
zcat "$BACKUP_DIR/$V2_DB-db-$BACKUP_DATE.sql.gz" \
| restic backup --host textstack-prod --tag nightly --stdin --stdin-filename db_v2.sql
restic forget --host textstack-prod --keep-daily 7 --keep-monthly 3 --prune
restic check
size=$(repo_size)
echo "R2 backup repo: $((size / 1024 / 1024)) MB"
if [ "$size" -gt "$LIMIT" ]; then
echo "::error::R2 backup repo is $((size / 1024 / 1024)) MB (> 9 GB)"
exit 1
fi
# Runs even when the backup failed — a full disk is the usual reason it does.
# Failing here fires the same failure notification as a broken backup.
- name: Disk space alarm (>= 85%)
if: always()
run: |
bad=0
for mnt in / "$(docker info -f '{{.DockerRootDir}}')"; do
pct=$(df --output=pcent "$mnt" | tail -1 | tr -dc '0-9')
echo "$mnt: ${pct}% used"
[ "$pct" -lt 85 ] || { echo "::error::$mnt is ${pct}% full"; bad=1; }
done
exit $bad