diff --git a/hosts/ops-jrz1.nix b/hosts/ops-jrz1.nix index ad2118b..4c5f6b2 100644 --- a/hosts/ops-jrz1.nix +++ b/hosts/ops-jrz1.nix @@ -20,6 +20,7 @@ ../modules/security/ssh-hardening.nix ../modules/matrix-secrets ../modules/backup.nix + ../modules/backup-b2.nix ]; # System configuration @@ -102,6 +103,9 @@ # Local backup service (Phase 1: manual trigger) services.backup.enable = true; + # B2 offsite backup (daily automated via restic) + services.backup-b2.enable = true; + # Security hardening - DISABLED pending fixes # security.fail2ban-enhanced.enable = true; # security.ssh-hardening.enable = true; diff --git a/modules/backup-b2.nix b/modules/backup-b2.nix new file mode 100644 index 0000000..aacde64 --- /dev/null +++ b/modules/backup-b2.nix @@ -0,0 +1,148 @@ +# B2 backup module using restic +# Backs up PostgreSQL dumps and application state to Backblaze B2 +{ config, pkgs, lib, ... }: + +with lib; + +let + cfg = config.services.backup-b2; + + backupPaths = [ + "/var/backup/postgresql" # DB dumps from services.postgresqlBackup + "/var/lib/matrix-continuwuity" # Matrix homeserver (RocksDB) + "/var/lib/forgejo" # Git hosting (repos, avatars) + "/var/lib/mautrix-slack" # Bridge state + "/var/lib/maubot" # Bot data + ]; + + excludePatterns = [ + "*.sock" + "*.pid" + "*.log" + "**/.git/objects/pack/*.tmp" # Git temp files + ]; + + # Script to set up restic environment from sops secrets + resticEnvScript = '' + export RESTIC_PASSWORD_FILE="${config.sops.secrets."restic/password".path}" + export B2_ACCOUNT_ID="$(cat "${config.sops.secrets."restic/b2_account_id".path}")" + export B2_ACCOUNT_KEY="$(cat "${config.sops.secrets."restic/b2_account_key".path}")" + export RESTIC_REPOSITORY="$(cat "${config.sops.secrets."restic/b2_repo".path}")" + ''; +in +{ + options.services.backup-b2 = { + enable = mkEnableOption "B2 backup service using restic"; + }; + + config = mkIf cfg.enable { + # Restic package available system-wide for manual operations + environment.systemPackages = [ pkgs.restic ]; + + # sops secrets for restic (defined here, values in secrets.yaml) + sops.secrets = { + "restic/password" = { + mode = "0400"; + }; + "restic/b2_account_id" = { + mode = "0400"; + }; + "restic/b2_account_key" = { + mode = "0400"; + }; + "restic/b2_repo" = { + mode = "0400"; + }; + }; + + # Backup service + systemd.services.backup-b2 = { + description = "Restic backup to Backblaze B2"; + after = [ "network-online.target" "postgresql.service" ]; + wants = [ "network-online.target" ]; + # Don't require postgres - backup should still run even if DB is down + # (will just skip the dump files if they don't exist) + + serviceConfig = { + Type = "oneshot"; + User = "root"; + # Low priority + IOSchedulingClass = "idle"; + Nice = 19; + }; + + path = [ pkgs.restic ]; + + script = '' + set -euo pipefail + ${resticEnvScript} + + # Initialize repo if it doesn't exist (idempotent) + if ! restic snapshots &>/dev/null; then + echo "Initializing restic repository..." + restic init + fi + + echo "Starting backup..." + restic backup \ + ${concatMapStringsSep " " (p: lib.escapeShellArg p) backupPaths} \ + ${concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludePatterns} \ + --tag ops-jrz1 \ + --tag automated \ + --verbose + + echo "Pruning old snapshots..." + restic forget \ + --keep-daily 7 \ + --keep-weekly 4 \ + --keep-monthly 6 \ + --prune + + echo "Backup complete." + restic snapshots --latest 3 + ''; + }; + + # Daily backup timer - 3 AM (after postgresql backup at 2 AM) + systemd.timers.backup-b2 = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "*-*-* 03:00:00"; + Persistent = true; # Run if missed (e.g., server was down) + RandomizedDelaySec = "5m"; + }; + }; + + # Weekly integrity check service + systemd.services.backup-b2-check = { + description = "Verify B2 backup integrity"; + + serviceConfig = { + Type = "oneshot"; + User = "root"; + IOSchedulingClass = "idle"; + Nice = 19; + }; + + path = [ pkgs.restic ]; + + script = '' + set -euo pipefail + ${resticEnvScript} + + echo "Checking backup integrity (5% data sample)..." + restic check --read-data-subset=5% + echo "Integrity check passed." + ''; + }; + + # Weekly integrity check timer - Sunday 4 AM + systemd.timers.backup-b2-check = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "Sun *-*-* 04:00:00"; + Persistent = true; + }; + }; + }; +} diff --git a/modules/dev-services.nix b/modules/dev-services.nix index b4f4dd6..d7c9c9b 100644 --- a/modules/dev-services.nix +++ b/modules/dev-services.nix @@ -116,6 +116,14 @@ in }; }; + # PostgreSQL backup service (dumps for restic to pick up) + services.postgresqlBackup = { + enable = true; + databases = [ "forgejo" ] ++ optional cfg.slackBridge.enable "mautrix_slack"; + location = "/var/backup/postgresql"; + startAt = "*-*-* 02:00:00"; # 2 AM daily, before restic at 3 AM + }; + # Matrix Continuwuity server systemd.services.matrix-continuwuity = mkIf cfg.matrix.enable { description = "Continuwuity Matrix homeserver"; diff --git a/secrets/secrets.yaml b/secrets/secrets.yaml index ea484b6..2dee5d8 100644 --- a/secrets/secrets.yaml +++ b/secrets/secrets.yaml @@ -1,32 +1,37 @@ -matrix-registration-token: ENC[AES256_GCM,data:j7i/qtPol4dFtIdcBBfiJPrmIcNv0oeGU0Et/rbYwiC7eqAfh4v0xcS9ymzMJZXt75ikLEy8gJBm0i3kzuY8Tw==,iv:t5vQ7NQ3Mq1xnplgkdWu/XJlN/YEedVp+hRCbazy7YM=,tag:soQm891iwqgxZuYNoNFVYw==,type:str] -acme-email: ENC[AES256_GCM,data:178eat1kqzemxmHJ4w==,iv:27x07i90//RA/Lvs/N8ITOU+abcrfpOoCZiOV932MAY=,tag:NStHMV22Bsq/nbyobbR54w==,type:str] -slack-app-token: ENC[AES256_GCM,data:s9TAQvQH4QpRyBQFAU3aVgjyLzLLIqqTCXVV8mHv2ITNyFNWd5lveyGFzmuDmU3qPW5/S0ZuYMkuSkZrREVPH57Kbv19dR9/fTe0keIbtLC9FmZn2yRZdjKvjgGMIKeWsA==,iv:mwxEVj6bsghkXZ0v6IH6JA2JZfCoknjyOa8FTdOP2OQ=,tag:9N5EoUXsMU41hICaWMtVaw==,type:str] -maubot-admin-password: ENC[AES256_GCM,data:iJ3lZKaPWIOoVrkC0qx5tzxOdbks7c3J9WrR1c3KgpSrtzfiZtl36PPZFqs=,iv:P708rzoWfrcUWqDdU1Vw80xGQVYwwRYI1g/i6rDhOMU=,tag:x/Xjq7w/k7Qr4hcuuSEHYg==,type:str] -maubot-secret-key: ENC[AES256_GCM,data:8GZjOJo/Txl7aQf/jlHgctcmwk47CFP75tZyLfnbnlcgsEjCbclxxKJEv0zYZ4z270pQ9ieMx6JNGD6z61iSpw==,iv:3H9DJYAZiNaW4DSbREajaLnUXufxo1h9BUm2gYFPW/Y=,tag:y+7hQDlszhopnTaoHZ48yg==,type:str] -slack-bot-token: ENC[AES256_GCM,data:JRV8Fw2I9YMXttXWqPTlm1/2chF4m8KOilzsPuIyX8V7BYYb4uXlgW53MgfVcScXrgp981q7jL0=,iv:xJGHpI1WkZmRt5n9ZJmiu8IbdazrQJAb/ztHw5v7mXA=,tag:ajjAtWsOkXLV1iNQOf/h1g==,type:str] -forgejo-admin-password: ENC[AES256_GCM,data:+ckJhKS7ive6h/dxru7IZ2fjW7MRnMW2,iv:om0MfDwFRTqPgVETcelnmmKX4BtZfbO1feBseZ2kO0s=,tag:FirgrQwCM+fUab85liBWiw==,type:str] -forgejo-api-token: ENC[AES256_GCM,data:a7IZXAnzg6CS+GHS/grqaw5InbMoTs9igDI66sMe4z+A3tH4xDCtWQ==,iv:u+cSLF5w4MxO4yWblHscfEi1KzJnbSqONL5LMYBpQE4=,tag:XaxX1t3gcxfN2Z/xKfN8SA==,type:str] +matrix-registration-token: ENC[AES256_GCM,data:SiF+6nXlxpPddr83/CR0o1rs3jde6ewxKxMXpJ9t4vz6x0xaC4tFEIWh6u8xyrX/6ORqqPZr0mWsbWwNJM+MfA==,iv:Z6/irfWu8YCuSvtYmVGSkz8GhVUeuN5fP7qsVXlNUdE=,tag:fS8NfiT5ErhVMDOkKyac1w==,type:str] +acme-email: ENC[AES256_GCM,data:A97cinBoMWHpCpAM9A==,iv:VrROWl9HfVKZT4aq1T23puCUkbeoCbDRJbCqpOzCKG8=,tag:eXQ3IiMjn2njYgue1NvQog==,type:str] +slack-app-token: ENC[AES256_GCM,data:eXNpliGcPwrnc3pqRlaLmbjjVTKmJOQlchKt7uDlEEH4mWnzsm87VrRdi3usrVAJT+il7zapd+VQK5fEOcJwP1b4hGp1PvEFdiN1oofDKenRGNyptzolRA3cyhdQsCZVZA==,iv:Qc9aYgm5g7T6kG4hG9TGdK6RufosGnVbSV0IhguhHuk=,tag:fY9V4AKTcUvHKcMp/OoG2w==,type:str] +maubot-admin-password: ENC[AES256_GCM,data:C8s7rPrPI3V7NYksNVw8CW10QGR8iAnWo2yVO2i3Jv/3AU/dza3pwbu4bRQ=,iv:qVLpFC3BYQ48hem3I5msRt5s8nqf2WSGyeOIw1Ior70=,tag:nOXpD1pdb+GRBBPjhobqKg==,type:str] +maubot-secret-key: ENC[AES256_GCM,data:k8meKgyRYhYqR86GjpH0xCPJsrvKxy4LTHM4PJct0TmZnBatqpWoO6abqzPnsuyA8PHipAz4Yw7+VyXsUdifkQ==,iv:uUYuMja1X8U7FMYj1oOGVIZ/Opfoi/Zo/a7gQIS4FCQ=,tag:z8vVRohp5iREil3lW4ZoHg==,type:str] +slack-bot-token: ENC[AES256_GCM,data:2d1GNPvNwJBN8S2fBzL6E8fh6D2hGU8aFPEaNYHCfM+AhrzGctnzk3pgTOTpUWkXHDp5bCaxFGw=,iv:7lHPLQyL+GzH1siujx517BPQ+BlQXbuDbHMpaNH+MrQ=,tag:Qt/KiiFBHnbU5lz9mUWhvg==,type:str] +forgejo-admin-password: ENC[AES256_GCM,data:ih8u+gNJpYcmtWD/0voNBc+dyDf7aQaC,iv:WIrD3IgwUJZIR8BrO5ok1uL5YXdG4I+HgEzwvjnLaiw=,tag:Xon/rtoXQF1SbDtfDZYaiQ==,type:str] +forgejo-api-token: ENC[AES256_GCM,data:eOYykB65PbfMnFeW3U0l5HmV3yBtQ7pPlYdUfZsupRIO/8UTrCqfLw==,iv:8vw18fr/e8kSD2U09BN3GWzmJ7GzdSPssZDExazY6Jc=,tag:rMuAD+iqSZCfOtEWY1fJgA==,type:str] +restic: + password: ENC[AES256_GCM,data:ydVeGcvZThSYtXpMsgVxkVvykQDlvI5niy/YTHJ14h0=,iv:PLDgROTsRMbLXgnBkkPosfoJ/SV6Ejx46o8FFYZAGPQ=,tag:vxARTz5XAsaHr+jrtVjt9w==,type:str] + b2_account_id: ENC[AES256_GCM,data:Arz8ZZ8ahVNjvlPlVEbRbLzWMmLS3AXdgQ==,iv:YcOcAFIs3KjWEpMVOM7mtBGmdrh4IG47/esZIyxeUTg=,tag:eOi7Z4CW9DgzDFjjQwGsdg==,type:str] + b2_account_key: ENC[AES256_GCM,data:9jpuaRaGtGxz+Bp9wlOr5apECRorkpsC7+4+wrxUAQ==,iv:WFwKWc1yR8sU7xKVZ40RkCCGVzChGMxLLc5L6D12UBY=,tag:SwLzS6MwA0O1G6XwHo1LtA==,type:str] + b2_repo: ENC[AES256_GCM,data:JcvCGIJunXFq99mfo7+GQVjR,iv:ym7mu7yNwNZ2MuViKaEe6WXLuS58k++nFA42ZJHsbis=,tag:0mBsoVKY7aTzv4SxB8Jcyg==,type:str] sops: age: - recipient: age1vuxcwvdvzl2u7w6kudqvnnf45czrnhwv9aevjq9hyjjpa409jvkqhkz32q enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwMUhsUGl6dncxaGVScTVo - a3crTjdGaXpvNm5Xb2lHN3UwNWFzckRUempJCmNzK1E4dW5BVlZPMk5JS3ZtS21K - Nm40bGZzV0M3Z3JjaFA1Qit6S001dVkKLS0tIGQwODVJT1ZTeTNsSHZNd0IvSnNu - SmhqQ1RtUHloL1RwTWhlN0NxMTFrc0kKiJRArc6hfwRNQqI9zWnJjvgpD+RrYT8S - huj/komeDL3+gUJgXdbxvXKczLjtUf6bOjSm/BgwFiLG/dr1meWV/Q== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuTm1DNzZjM1VmSVU2NlA0 + RUFpZ2h0Snc0ZlVsc04zanBQZmcyaFh0aUc4CnNOQ0ZUS25xeTUwV1A0MzZFQzBT + NmhjVzVZR2gzb21iTFM2cDJRV0NDa3cKLS0tIG85RnRzYWQvNjNUMnN2bUJ6UUNn + Smp1ZkMwZ1RBb1NmWExYc280c20zYjAKwyChuUih0BTk7nYsit6aBkGDAzJV0xBa + gi2/bk5uLk7cW+JU46IrK9VPN6VhexDqN0k9ub3YMXNxfurn3wMNIg== -----END AGE ENCRYPTED FILE----- - recipient: age18ue40q4fw8uggdlfag7jf5nrawvfvsnv93nurschhuynus200yjsd775v3 enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBazZpazREcER4OGpRVEhl - UFE4My9paDdZMW45ZUQzbC9IT0s5NXc1NldNCm5GaHIvM29lN1ZiL0FobVJvWjBP - MEVQVFpMSHZWTkxOQkgyYkxzVjVYN00KLS0tIHVHeE5MU2QwSVNzcldaWURzdDlo - RU1TR24zOGxnd0syeVlvc3JHeGFhTTAKwg1TvZ6ixaDgBfz+3auoLVjdXnHuzvGv - pN+pUklEsaymDCun3rEUGiI0xA08WML1HAE7AyfqKa32wSJUOpmcMA== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkWlVpbGFPY04wbjVhQndX + cEpuVzliTkFocUhDMGhnNmNxek9pcXNLeEFRCmNWc1dHMFBpMWcxYkg2MEdYTjRs + YjVreWpHbng0dTBIQkM2QzBHM0EwMmMKLS0tIGFkYVprTytHSEc4TXR1NGc0V2VC + TFF5cmRNSFE2ODBjOVBTemFNUzQrREEKyzPRDrmR68VKmjDLoJ89Yz+9A0tQPMB9 + 1+0/F+3OAbk66FTycap4E2mIqxLFWifW2h7tOfP2exxXFktCQcgmyA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-01-09T01:45:06Z" - mac: ENC[AES256_GCM,data:gbYiINSWDI0Bhgrlv8A1ImUciQg17WT47RlN8VZfrbGfa6PKdnUQncrjuKBjMdVXhk2e6sIvLO2OGXJp2dznK8DEKJOeeHamKo6k5PnrAe81tLLI5wub6+q4vARwqV8SC5JJAMDT8+H+PQZ25ao98usJ4A9EJ4zD6EQ4Tbff4g8=,iv:CElQ1xzT7Z0VZW452hTR/DcSnfplW5GBfYOUalOP4nU=,tag:+nSgn2eEstS8aZD99jxuYw==,type:str] + lastmodified: "2026-01-10T21:48:44Z" + mac: ENC[AES256_GCM,data:HGGz77ONHpz/OjwJU1+F+D+MJyHJP/UrCytjrYKTRK1pirNsJWyCwWDSKkpXvLt3vgJBlnWLgzbCk9Bp7NpYOO+QooRETdIqaZHSpEGoQjcJjY1o/8j4/THxwTb0Yh5mVZKQg39tEGIFOIcYc8HLPBLGQEbh6JGN2F/4r5PseWI=,iv:70QniATCsMmRfpdPbDspUle35Okxj1y4AhEJvY1CpQI=,tag:HnsKH/qqC9bUsk2aHSkTZQ==,type:str] unencrypted_suffix: _unencrypted version: 3.11.0