Add B2 automated backup with restic
- Add services.postgresqlBackup for daily DB dumps (2 AM) - New modules/backup-b2.nix: restic backup to B2 (3 AM daily) - Weekly integrity check (Sunday 4 AM) - Retention: 7 daily, 4 weekly, 6 monthly - B2 bucket: ops-jrz1-backup with scoped app key Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Dan
parent
ff34cee51e
commit
31d388d21c
|
|
@ -20,6 +20,7 @@
|
||||||
../modules/security/ssh-hardening.nix
|
../modules/security/ssh-hardening.nix
|
||||||
../modules/matrix-secrets
|
../modules/matrix-secrets
|
||||||
../modules/backup.nix
|
../modules/backup.nix
|
||||||
|
../modules/backup-b2.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
# System configuration
|
# System configuration
|
||||||
|
|
@ -102,6 +103,9 @@
|
||||||
# Local backup service (Phase 1: manual trigger)
|
# Local backup service (Phase 1: manual trigger)
|
||||||
services.backup.enable = true;
|
services.backup.enable = true;
|
||||||
|
|
||||||
|
# B2 offsite backup (daily automated via restic)
|
||||||
|
services.backup-b2.enable = true;
|
||||||
|
|
||||||
# Security hardening - DISABLED pending fixes
|
# Security hardening - DISABLED pending fixes
|
||||||
# security.fail2ban-enhanced.enable = true;
|
# security.fail2ban-enhanced.enable = true;
|
||||||
# security.ssh-hardening.enable = true;
|
# security.ssh-hardening.enable = true;
|
||||||
|
|
|
||||||
148
modules/backup-b2.nix
Normal file
148
modules/backup-b2.nix
Normal file
|
|
@ -0,0 +1,148 @@
|
||||||
|
# B2 backup module using restic
|
||||||
|
# Backs up PostgreSQL dumps and application state to Backblaze B2
|
||||||
|
{ config, pkgs, lib, ... }:
|
||||||
|
|
||||||
|
with lib;
|
||||||
|
|
||||||
|
let
|
||||||
|
cfg = config.services.backup-b2;
|
||||||
|
|
||||||
|
backupPaths = [
|
||||||
|
"/var/backup/postgresql" # DB dumps from services.postgresqlBackup
|
||||||
|
"/var/lib/matrix-continuwuity" # Matrix homeserver (RocksDB)
|
||||||
|
"/var/lib/forgejo" # Git hosting (repos, avatars)
|
||||||
|
"/var/lib/mautrix-slack" # Bridge state
|
||||||
|
"/var/lib/maubot" # Bot data
|
||||||
|
];
|
||||||
|
|
||||||
|
excludePatterns = [
|
||||||
|
"*.sock"
|
||||||
|
"*.pid"
|
||||||
|
"*.log"
|
||||||
|
"**/.git/objects/pack/*.tmp" # Git temp files
|
||||||
|
];
|
||||||
|
|
||||||
|
# Script to set up restic environment from sops secrets
|
||||||
|
resticEnvScript = ''
|
||||||
|
export RESTIC_PASSWORD_FILE="${config.sops.secrets."restic/password".path}"
|
||||||
|
export B2_ACCOUNT_ID="$(cat "${config.sops.secrets."restic/b2_account_id".path}")"
|
||||||
|
export B2_ACCOUNT_KEY="$(cat "${config.sops.secrets."restic/b2_account_key".path}")"
|
||||||
|
export RESTIC_REPOSITORY="$(cat "${config.sops.secrets."restic/b2_repo".path}")"
|
||||||
|
'';
|
||||||
|
in
|
||||||
|
{
|
||||||
|
options.services.backup-b2 = {
|
||||||
|
enable = mkEnableOption "B2 backup service using restic";
|
||||||
|
};
|
||||||
|
|
||||||
|
config = mkIf cfg.enable {
|
||||||
|
# Restic package available system-wide for manual operations
|
||||||
|
environment.systemPackages = [ pkgs.restic ];
|
||||||
|
|
||||||
|
# sops secrets for restic (defined here, values in secrets.yaml)
|
||||||
|
sops.secrets = {
|
||||||
|
"restic/password" = {
|
||||||
|
mode = "0400";
|
||||||
|
};
|
||||||
|
"restic/b2_account_id" = {
|
||||||
|
mode = "0400";
|
||||||
|
};
|
||||||
|
"restic/b2_account_key" = {
|
||||||
|
mode = "0400";
|
||||||
|
};
|
||||||
|
"restic/b2_repo" = {
|
||||||
|
mode = "0400";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
# Backup service
|
||||||
|
systemd.services.backup-b2 = {
|
||||||
|
description = "Restic backup to Backblaze B2";
|
||||||
|
after = [ "network-online.target" "postgresql.service" ];
|
||||||
|
wants = [ "network-online.target" ];
|
||||||
|
# Don't require postgres - backup should still run even if DB is down
|
||||||
|
# (will just skip the dump files if they don't exist)
|
||||||
|
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "oneshot";
|
||||||
|
User = "root";
|
||||||
|
# Low priority
|
||||||
|
IOSchedulingClass = "idle";
|
||||||
|
Nice = 19;
|
||||||
|
};
|
||||||
|
|
||||||
|
path = [ pkgs.restic ];
|
||||||
|
|
||||||
|
script = ''
|
||||||
|
set -euo pipefail
|
||||||
|
${resticEnvScript}
|
||||||
|
|
||||||
|
# Initialize repo if it doesn't exist (idempotent)
|
||||||
|
if ! restic snapshots &>/dev/null; then
|
||||||
|
echo "Initializing restic repository..."
|
||||||
|
restic init
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Starting backup..."
|
||||||
|
restic backup \
|
||||||
|
${concatMapStringsSep " " (p: lib.escapeShellArg p) backupPaths} \
|
||||||
|
${concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludePatterns} \
|
||||||
|
--tag ops-jrz1 \
|
||||||
|
--tag automated \
|
||||||
|
--verbose
|
||||||
|
|
||||||
|
echo "Pruning old snapshots..."
|
||||||
|
restic forget \
|
||||||
|
--keep-daily 7 \
|
||||||
|
--keep-weekly 4 \
|
||||||
|
--keep-monthly 6 \
|
||||||
|
--prune
|
||||||
|
|
||||||
|
echo "Backup complete."
|
||||||
|
restic snapshots --latest 3
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
# Daily backup timer - 3 AM (after postgresql backup at 2 AM)
|
||||||
|
systemd.timers.backup-b2 = {
|
||||||
|
wantedBy = [ "timers.target" ];
|
||||||
|
timerConfig = {
|
||||||
|
OnCalendar = "*-*-* 03:00:00";
|
||||||
|
Persistent = true; # Run if missed (e.g., server was down)
|
||||||
|
RandomizedDelaySec = "5m";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
# Weekly integrity check service
|
||||||
|
systemd.services.backup-b2-check = {
|
||||||
|
description = "Verify B2 backup integrity";
|
||||||
|
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "oneshot";
|
||||||
|
User = "root";
|
||||||
|
IOSchedulingClass = "idle";
|
||||||
|
Nice = 19;
|
||||||
|
};
|
||||||
|
|
||||||
|
path = [ pkgs.restic ];
|
||||||
|
|
||||||
|
script = ''
|
||||||
|
set -euo pipefail
|
||||||
|
${resticEnvScript}
|
||||||
|
|
||||||
|
echo "Checking backup integrity (5% data sample)..."
|
||||||
|
restic check --read-data-subset=5%
|
||||||
|
echo "Integrity check passed."
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
# Weekly integrity check timer - Sunday 4 AM
|
||||||
|
systemd.timers.backup-b2-check = {
|
||||||
|
wantedBy = [ "timers.target" ];
|
||||||
|
timerConfig = {
|
||||||
|
OnCalendar = "Sun *-*-* 04:00:00";
|
||||||
|
Persistent = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -116,6 +116,14 @@ in
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# PostgreSQL backup service (dumps for restic to pick up)
|
||||||
|
services.postgresqlBackup = {
|
||||||
|
enable = true;
|
||||||
|
databases = [ "forgejo" ] ++ optional cfg.slackBridge.enable "mautrix_slack";
|
||||||
|
location = "/var/backup/postgresql";
|
||||||
|
startAt = "*-*-* 02:00:00"; # 2 AM daily, before restic at 3 AM
|
||||||
|
};
|
||||||
|
|
||||||
# Matrix Continuwuity server
|
# Matrix Continuwuity server
|
||||||
systemd.services.matrix-continuwuity = mkIf cfg.matrix.enable {
|
systemd.services.matrix-continuwuity = mkIf cfg.matrix.enable {
|
||||||
description = "Continuwuity Matrix homeserver";
|
description = "Continuwuity Matrix homeserver";
|
||||||
|
|
|
||||||
|
|
@ -1,32 +1,37 @@
|
||||||
matrix-registration-token: ENC[AES256_GCM,data:j7i/qtPol4dFtIdcBBfiJPrmIcNv0oeGU0Et/rbYwiC7eqAfh4v0xcS9ymzMJZXt75ikLEy8gJBm0i3kzuY8Tw==,iv:t5vQ7NQ3Mq1xnplgkdWu/XJlN/YEedVp+hRCbazy7YM=,tag:soQm891iwqgxZuYNoNFVYw==,type:str]
|
matrix-registration-token: ENC[AES256_GCM,data:SiF+6nXlxpPddr83/CR0o1rs3jde6ewxKxMXpJ9t4vz6x0xaC4tFEIWh6u8xyrX/6ORqqPZr0mWsbWwNJM+MfA==,iv:Z6/irfWu8YCuSvtYmVGSkz8GhVUeuN5fP7qsVXlNUdE=,tag:fS8NfiT5ErhVMDOkKyac1w==,type:str]
|
||||||
acme-email: ENC[AES256_GCM,data:178eat1kqzemxmHJ4w==,iv:27x07i90//RA/Lvs/N8ITOU+abcrfpOoCZiOV932MAY=,tag:NStHMV22Bsq/nbyobbR54w==,type:str]
|
acme-email: ENC[AES256_GCM,data:A97cinBoMWHpCpAM9A==,iv:VrROWl9HfVKZT4aq1T23puCUkbeoCbDRJbCqpOzCKG8=,tag:eXQ3IiMjn2njYgue1NvQog==,type:str]
|
||||||
slack-app-token: ENC[AES256_GCM,data:s9TAQvQH4QpRyBQFAU3aVgjyLzLLIqqTCXVV8mHv2ITNyFNWd5lveyGFzmuDmU3qPW5/S0ZuYMkuSkZrREVPH57Kbv19dR9/fTe0keIbtLC9FmZn2yRZdjKvjgGMIKeWsA==,iv:mwxEVj6bsghkXZ0v6IH6JA2JZfCoknjyOa8FTdOP2OQ=,tag:9N5EoUXsMU41hICaWMtVaw==,type:str]
|
slack-app-token: ENC[AES256_GCM,data:eXNpliGcPwrnc3pqRlaLmbjjVTKmJOQlchKt7uDlEEH4mWnzsm87VrRdi3usrVAJT+il7zapd+VQK5fEOcJwP1b4hGp1PvEFdiN1oofDKenRGNyptzolRA3cyhdQsCZVZA==,iv:Qc9aYgm5g7T6kG4hG9TGdK6RufosGnVbSV0IhguhHuk=,tag:fY9V4AKTcUvHKcMp/OoG2w==,type:str]
|
||||||
maubot-admin-password: ENC[AES256_GCM,data:iJ3lZKaPWIOoVrkC0qx5tzxOdbks7c3J9WrR1c3KgpSrtzfiZtl36PPZFqs=,iv:P708rzoWfrcUWqDdU1Vw80xGQVYwwRYI1g/i6rDhOMU=,tag:x/Xjq7w/k7Qr4hcuuSEHYg==,type:str]
|
maubot-admin-password: ENC[AES256_GCM,data:C8s7rPrPI3V7NYksNVw8CW10QGR8iAnWo2yVO2i3Jv/3AU/dza3pwbu4bRQ=,iv:qVLpFC3BYQ48hem3I5msRt5s8nqf2WSGyeOIw1Ior70=,tag:nOXpD1pdb+GRBBPjhobqKg==,type:str]
|
||||||
maubot-secret-key: ENC[AES256_GCM,data:8GZjOJo/Txl7aQf/jlHgctcmwk47CFP75tZyLfnbnlcgsEjCbclxxKJEv0zYZ4z270pQ9ieMx6JNGD6z61iSpw==,iv:3H9DJYAZiNaW4DSbREajaLnUXufxo1h9BUm2gYFPW/Y=,tag:y+7hQDlszhopnTaoHZ48yg==,type:str]
|
maubot-secret-key: ENC[AES256_GCM,data:k8meKgyRYhYqR86GjpH0xCPJsrvKxy4LTHM4PJct0TmZnBatqpWoO6abqzPnsuyA8PHipAz4Yw7+VyXsUdifkQ==,iv:uUYuMja1X8U7FMYj1oOGVIZ/Opfoi/Zo/a7gQIS4FCQ=,tag:z8vVRohp5iREil3lW4ZoHg==,type:str]
|
||||||
slack-bot-token: ENC[AES256_GCM,data:JRV8Fw2I9YMXttXWqPTlm1/2chF4m8KOilzsPuIyX8V7BYYb4uXlgW53MgfVcScXrgp981q7jL0=,iv:xJGHpI1WkZmRt5n9ZJmiu8IbdazrQJAb/ztHw5v7mXA=,tag:ajjAtWsOkXLV1iNQOf/h1g==,type:str]
|
slack-bot-token: ENC[AES256_GCM,data:2d1GNPvNwJBN8S2fBzL6E8fh6D2hGU8aFPEaNYHCfM+AhrzGctnzk3pgTOTpUWkXHDp5bCaxFGw=,iv:7lHPLQyL+GzH1siujx517BPQ+BlQXbuDbHMpaNH+MrQ=,tag:Qt/KiiFBHnbU5lz9mUWhvg==,type:str]
|
||||||
forgejo-admin-password: ENC[AES256_GCM,data:+ckJhKS7ive6h/dxru7IZ2fjW7MRnMW2,iv:om0MfDwFRTqPgVETcelnmmKX4BtZfbO1feBseZ2kO0s=,tag:FirgrQwCM+fUab85liBWiw==,type:str]
|
forgejo-admin-password: ENC[AES256_GCM,data:ih8u+gNJpYcmtWD/0voNBc+dyDf7aQaC,iv:WIrD3IgwUJZIR8BrO5ok1uL5YXdG4I+HgEzwvjnLaiw=,tag:Xon/rtoXQF1SbDtfDZYaiQ==,type:str]
|
||||||
forgejo-api-token: ENC[AES256_GCM,data:a7IZXAnzg6CS+GHS/grqaw5InbMoTs9igDI66sMe4z+A3tH4xDCtWQ==,iv:u+cSLF5w4MxO4yWblHscfEi1KzJnbSqONL5LMYBpQE4=,tag:XaxX1t3gcxfN2Z/xKfN8SA==,type:str]
|
forgejo-api-token: ENC[AES256_GCM,data:eOYykB65PbfMnFeW3U0l5HmV3yBtQ7pPlYdUfZsupRIO/8UTrCqfLw==,iv:8vw18fr/e8kSD2U09BN3GWzmJ7GzdSPssZDExazY6Jc=,tag:rMuAD+iqSZCfOtEWY1fJgA==,type:str]
|
||||||
|
restic:
|
||||||
|
password: ENC[AES256_GCM,data:ydVeGcvZThSYtXpMsgVxkVvykQDlvI5niy/YTHJ14h0=,iv:PLDgROTsRMbLXgnBkkPosfoJ/SV6Ejx46o8FFYZAGPQ=,tag:vxARTz5XAsaHr+jrtVjt9w==,type:str]
|
||||||
|
b2_account_id: ENC[AES256_GCM,data:Arz8ZZ8ahVNjvlPlVEbRbLzWMmLS3AXdgQ==,iv:YcOcAFIs3KjWEpMVOM7mtBGmdrh4IG47/esZIyxeUTg=,tag:eOi7Z4CW9DgzDFjjQwGsdg==,type:str]
|
||||||
|
b2_account_key: ENC[AES256_GCM,data:9jpuaRaGtGxz+Bp9wlOr5apECRorkpsC7+4+wrxUAQ==,iv:WFwKWc1yR8sU7xKVZ40RkCCGVzChGMxLLc5L6D12UBY=,tag:SwLzS6MwA0O1G6XwHo1LtA==,type:str]
|
||||||
|
b2_repo: ENC[AES256_GCM,data:JcvCGIJunXFq99mfo7+GQVjR,iv:ym7mu7yNwNZ2MuViKaEe6WXLuS58k++nFA42ZJHsbis=,tag:0mBsoVKY7aTzv4SxB8Jcyg==,type:str]
|
||||||
sops:
|
sops:
|
||||||
age:
|
age:
|
||||||
- recipient: age1vuxcwvdvzl2u7w6kudqvnnf45czrnhwv9aevjq9hyjjpa409jvkqhkz32q
|
- recipient: age1vuxcwvdvzl2u7w6kudqvnnf45czrnhwv9aevjq9hyjjpa409jvkqhkz32q
|
||||||
enc: |
|
enc: |
|
||||||
-----BEGIN AGE ENCRYPTED FILE-----
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwMUhsUGl6dncxaGVScTVo
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuTm1DNzZjM1VmSVU2NlA0
|
||||||
a3crTjdGaXpvNm5Xb2lHN3UwNWFzckRUempJCmNzK1E4dW5BVlZPMk5JS3ZtS21K
|
RUFpZ2h0Snc0ZlVsc04zanBQZmcyaFh0aUc4CnNOQ0ZUS25xeTUwV1A0MzZFQzBT
|
||||||
Nm40bGZzV0M3Z3JjaFA1Qit6S001dVkKLS0tIGQwODVJT1ZTeTNsSHZNd0IvSnNu
|
NmhjVzVZR2gzb21iTFM2cDJRV0NDa3cKLS0tIG85RnRzYWQvNjNUMnN2bUJ6UUNn
|
||||||
SmhqQ1RtUHloL1RwTWhlN0NxMTFrc0kKiJRArc6hfwRNQqI9zWnJjvgpD+RrYT8S
|
Smp1ZkMwZ1RBb1NmWExYc280c20zYjAKwyChuUih0BTk7nYsit6aBkGDAzJV0xBa
|
||||||
huj/komeDL3+gUJgXdbxvXKczLjtUf6bOjSm/BgwFiLG/dr1meWV/Q==
|
gi2/bk5uLk7cW+JU46IrK9VPN6VhexDqN0k9ub3YMXNxfurn3wMNIg==
|
||||||
-----END AGE ENCRYPTED FILE-----
|
-----END AGE ENCRYPTED FILE-----
|
||||||
- recipient: age18ue40q4fw8uggdlfag7jf5nrawvfvsnv93nurschhuynus200yjsd775v3
|
- recipient: age18ue40q4fw8uggdlfag7jf5nrawvfvsnv93nurschhuynus200yjsd775v3
|
||||||
enc: |
|
enc: |
|
||||||
-----BEGIN AGE ENCRYPTED FILE-----
|
-----BEGIN AGE ENCRYPTED FILE-----
|
||||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBazZpazREcER4OGpRVEhl
|
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkWlVpbGFPY04wbjVhQndX
|
||||||
UFE4My9paDdZMW45ZUQzbC9IT0s5NXc1NldNCm5GaHIvM29lN1ZiL0FobVJvWjBP
|
cEpuVzliTkFocUhDMGhnNmNxek9pcXNLeEFRCmNWc1dHMFBpMWcxYkg2MEdYTjRs
|
||||||
MEVQVFpMSHZWTkxOQkgyYkxzVjVYN00KLS0tIHVHeE5MU2QwSVNzcldaWURzdDlo
|
YjVreWpHbng0dTBIQkM2QzBHM0EwMmMKLS0tIGFkYVprTytHSEc4TXR1NGc0V2VC
|
||||||
RU1TR24zOGxnd0syeVlvc3JHeGFhTTAKwg1TvZ6ixaDgBfz+3auoLVjdXnHuzvGv
|
TFF5cmRNSFE2ODBjOVBTemFNUzQrREEKyzPRDrmR68VKmjDLoJ89Yz+9A0tQPMB9
|
||||||
pN+pUklEsaymDCun3rEUGiI0xA08WML1HAE7AyfqKa32wSJUOpmcMA==
|
1+0/F+3OAbk66FTycap4E2mIqxLFWifW2h7tOfP2exxXFktCQcgmyA==
|
||||||
-----END AGE ENCRYPTED FILE-----
|
-----END AGE ENCRYPTED FILE-----
|
||||||
lastmodified: "2026-01-09T01:45:06Z"
|
lastmodified: "2026-01-10T21:48:44Z"
|
||||||
mac: ENC[AES256_GCM,data:gbYiINSWDI0Bhgrlv8A1ImUciQg17WT47RlN8VZfrbGfa6PKdnUQncrjuKBjMdVXhk2e6sIvLO2OGXJp2dznK8DEKJOeeHamKo6k5PnrAe81tLLI5wub6+q4vARwqV8SC5JJAMDT8+H+PQZ25ao98usJ4A9EJ4zD6EQ4Tbff4g8=,iv:CElQ1xzT7Z0VZW452hTR/DcSnfplW5GBfYOUalOP4nU=,tag:+nSgn2eEstS8aZD99jxuYw==,type:str]
|
mac: ENC[AES256_GCM,data:HGGz77ONHpz/OjwJU1+F+D+MJyHJP/UrCytjrYKTRK1pirNsJWyCwWDSKkpXvLt3vgJBlnWLgzbCk9Bp7NpYOO+QooRETdIqaZHSpEGoQjcJjY1o/8j4/THxwTb0Yh5mVZKQg39tEGIFOIcYc8HLPBLGQEbh6JGN2F/4r5PseWI=,iv:70QniATCsMmRfpdPbDspUle35Okxj1y4AhEJvY1CpQI=,tag:HnsKH/qqC9bUsk2aHSkTZQ==,type:str]
|
||||||
unencrypted_suffix: _unencrypted
|
unencrypted_suffix: _unencrypted
|
||||||
version: 3.11.0
|
version: 3.11.0
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue