# B2 backup module using restic # Backs up PostgreSQL dumps and application state to Backblaze B2 { config, pkgs, lib, ... }: with lib; let cfg = config.services.backup-b2; backupPaths = [ "/var/backup/postgresql" # DB dumps from services.postgresqlBackup "/var/lib/matrix-continuwuity" # Matrix homeserver (RocksDB) "/var/lib/forgejo" # Git hosting (repos, avatars) "/var/lib/mautrix-slack" # Bridge state "/var/lib/maubot" # Bot data "/var/lib/acme" # Let's Encrypt certificates "/home" # User home directories ]; excludePatterns = [ "*.sock" "*.pid" "*.log" "**/.git/objects/pack/*.tmp" # Git temp files # Home directory caches (can be rebuilt) "/home/*/.cache" "/home/*/.npm/_cacache" "/home/*/.bun/install/cache" "/home/*/node_modules" "/home/*/.nix-profile" "/home/*/.nix-defexpr" "/home/*/.local/share/Trash" ]; # Script to set up restic environment from sops secrets resticEnvScript = '' export RESTIC_PASSWORD_FILE="${config.sops.secrets."restic/password".path}" export B2_ACCOUNT_ID="$(cat "${config.sops.secrets."restic/b2_account_id".path}")" export B2_ACCOUNT_KEY="$(cat "${config.sops.secrets."restic/b2_account_key".path}")" export RESTIC_REPOSITORY="$(cat "${config.sops.secrets."restic/b2_repo".path}")" ''; in { options.services.backup-b2 = { enable = mkEnableOption "B2 backup service using restic"; }; config = mkIf cfg.enable { # Restic package available system-wide for manual operations environment.systemPackages = [ pkgs.restic ]; # sops secrets for restic (defined here, values in secrets.yaml) sops.secrets = { "restic/password" = { mode = "0400"; }; "restic/b2_account_id" = { mode = "0400"; }; "restic/b2_account_key" = { mode = "0400"; }; "restic/b2_repo" = { mode = "0400"; }; }; # Backup failure notification service systemd.services.backup-b2-failed = { description = "Handle backup failure notification"; serviceConfig = { Type = "oneshot"; User = "root"; }; script = '' echo "BACKUP FAILED at $(date)" | tee -a /var/log/backup-failures.log echo "Check: journalctl -u backup-b2 -n 50" # TODO: Add Matrix notification or healthchecks.io ping ''; }; # Backup service systemd.services.backup-b2 = { description = "Restic backup to Backblaze B2"; after = [ "network-online.target" "postgresql.service" ]; wants = [ "network-online.target" ]; onFailure = [ "backup-b2-failed.service" ]; # Don't require postgres - backup should still run even if DB is down # (will just skip the dump files if they don't exist) serviceConfig = { Type = "oneshot"; User = "root"; # Low priority IOSchedulingClass = "idle"; Nice = 19; # Timeout after 2 hours to prevent hung backups TimeoutStartSec = "2h"; }; path = [ pkgs.restic ]; script = '' set -euo pipefail ${resticEnvScript} # Initialize repo if it doesn't exist (idempotent) if ! restic snapshots &>/dev/null; then echo "Initializing restic repository..." restic init fi echo "Starting backup..." restic backup \ ${concatMapStringsSep " " (p: lib.escapeShellArg p) backupPaths} \ ${concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludePatterns} \ --tag ops-jrz1 \ --tag automated \ --verbose echo "Pruning old snapshots..." restic forget \ --keep-daily 7 \ --keep-weekly 4 \ --keep-monthly 6 \ --prune echo "Backup complete." restic snapshots --latest 3 ''; }; # Daily backup timer - 3 AM (after postgresql backup at 2 AM) systemd.timers.backup-b2 = { wantedBy = [ "timers.target" ]; timerConfig = { OnCalendar = "*-*-* 03:00:00"; Persistent = true; # Run if missed (e.g., server was down) RandomizedDelaySec = "5m"; }; }; # Weekly integrity check service systemd.services.backup-b2-check = { description = "Verify B2 backup integrity"; after = [ "network-online.target" ]; wants = [ "network-online.target" ]; onFailure = [ "backup-b2-failed.service" ]; serviceConfig = { Type = "oneshot"; User = "root"; IOSchedulingClass = "idle"; Nice = 19; # Timeout after 1 hour for integrity check TimeoutStartSec = "1h"; }; path = [ pkgs.restic ]; script = '' set -euo pipefail ${resticEnvScript} echo "Checking backup integrity (5% data sample)..." restic check --read-data-subset=5% echo "Integrity check passed." ''; }; # Weekly integrity check timer - Sunday 4 AM systemd.timers.backup-b2-check = { wantedBy = [ "timers.target" ]; timerConfig = { OnCalendar = "Sun *-*-* 04:00:00"; Persistent = true; }; }; }; }