Testing de Backup y Recuperación ante Desastres: Guía Completa para Validar RTO y RPO

# Ejemplo de objetivos de DR por nivel de sistema
tier_1_critico:
  sistemas: [procesamiento_pagos, autenticacion_usuarios]
  rto: 1 hora
  rpo: 15 minutos
  frecuencia_backup: replicacion_continua

tier_2_importante:
  sistemas: [gestion_pedidos, inventario]
  rto: 4 horas
  rpo: 1 hora
  frecuencia_backup: cada_hora

tier_3_estandar:
  sistemas: [reportes, analitica]
  rto: 24 horas
  rpo: 24 horas
  frecuencia_backup: diario

#!/bin/bash
# backup_validation.sh - Verificación automatizada de integridad de backup

set -e

BACKUP_PATH="/backups/daily"
CHECKSUM_FILE="/var/log/backup_checksums.txt"
SLACK_WEBHOOK="${SLACK_WEBHOOK_URL}"

# Verificar que el backup existe y es reciente
LATEST_BACKUP=$(ls -t ${BACKUP_PATH}/*.tar.gz 2>/dev/null | head -1)

if [ -z "$LATEST_BACKUP" ]; then
    echo "ERROR: No se encontró backup"
    curl -X POST "$SLACK_WEBHOOK" \
        -H "Content-Type: application/json" \
        -d '{"text": "ALERTA: No se encontró backup en '"${BACKUP_PATH}"'"}'
    exit 1
fi

# Verificar antigüedad del backup (debe ser menor a 25 horas)
BACKUP_AGE=$(( ($(date +%s) - $(stat -f %m "$LATEST_BACKUP")) / 3600 ))
if [ $BACKUP_AGE -gt 25 ]; then
    echo "ADVERTENCIA: El backup tiene ${BACKUP_AGE} horas de antigüedad"
    exit 1
fi

# Verificar integridad del backup
if ! tar -tzf "$LATEST_BACKUP" > /dev/null 2>&1; then
    echo "ERROR: El archivo de backup está corrupto"
    exit 1
fi

# Calcular y comparar checksums
CURRENT_CHECKSUM=$(sha256sum "$LATEST_BACKUP" | cut -d' ' -f1)
echo "$(date): $LATEST_BACKUP - $CURRENT_CHECKSUM" >> "$CHECKSUM_FILE"

echo "Validación de backup exitosa: $LATEST_BACKUP"

# db_restore_test.py - Testing automatizado de restauración de BD

import subprocess
import time
import psycopg2
from datetime import datetime

class DatabaseRestoreTest:
    def __init__(self, config):
        self.backup_path = config['backup_path']
        self.test_db = config['test_database']
        self.production_tables = config['critical_tables']
        self.rto_target = config['rto_seconds']

    def run_restore_test(self):
        start_time = time.time()
        results = {
            'timestamp': datetime.now().isoformat(),
            'success': False,
            'rto_met': False,
            'data_integrity': False
        }

        try:
            # Eliminar y recrear base de datos de test
            self._prepare_test_environment()

            # Restaurar desde backup
            restore_start = time.time()
            self._restore_backup()
            restore_duration = time.time() - restore_start

            # Validar integridad de datos
            results['data_integrity'] = self._validate_data()

            # Calcular RTO
            total_time = time.time() - start_time
            results['restore_duration_seconds'] = restore_duration
            results['total_duration_seconds'] = total_time
            results['rto_met'] = total_time <= self.rto_target
            results['success'] = results['data_integrity'] and results['rto_met']

        except Exception as e:
            results['error'] = str(e)

        return results

    def _restore_backup(self):
        cmd = f"pg_restore -h localhost -d {self.test_db} {self.backup_path}"
        subprocess.run(cmd, shell=True, check=True)

    def _validate_data(self):
        conn = psycopg2.connect(database=self.test_db)
        cursor = conn.cursor()

        for table in self.production_tables:
            cursor.execute(f"SELECT COUNT(*) FROM {table}")
            count = cursor.fetchone()[0]
            if count == 0:
                return False

        conn.close()
        return True

if __name__ == "__main__":
    config = {
        'backup_path': '/backups/latest/db.dump',
        'test_database': 'restore_test',
        'critical_tables': ['users', 'orders', 'payments'],
        'rto_seconds': 3600  # 1 hora de RTO
    }

    test = DatabaseRestoreTest(config)
    results = test.run_restore_test()
    print(f"Resultados del test de restauración: {results}")

# velero-restore-test.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
  name: velero-restore-test
  namespace: velero
spec:
  schedule: "0 3 * * 0"  # Semanal los domingos a las 3 AM
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: velero
          containers:

          - name: restore-test
            image: velero/velero:v1.12.0
            command:

            - /bin/sh
            - -c
            - |
              # Obtener último backup
              BACKUP_NAME=$(velero backup get -o json | jq -r '.items[-1].metadata.name')

              # Crear restauración en namespace de test
              velero restore create test-restore-${BACKUP_NAME} \
                --from-backup ${BACKUP_NAME} \
                --namespace-mappings production:restore-test \
                --wait

              # Validar restauración
              RESTORE_STATUS=$(velero restore get test-restore-${BACKUP_NAME} -o json | jq -r '.status.phase')

              if [ "$RESTORE_STATUS" = "Completed" ]; then
                echo "Test de restauración exitoso"
                # Ejecutar tests de validación
                kubectl -n restore-test get pods
                kubectl -n restore-test get pvc
              else
                echo "Test de restauración fallido: $RESTORE_STATUS"
                exit 1
              fi

              # Limpieza
              kubectl delete namespace restore-test --wait=true
          restartPolicy: OnFailure

# chaos_dr_test.py - Testing de caos automatizado para validación de DR

from chaoslib.experiment import run_experiment
import json

experiment = {
    "title": "Test de failover de base de datos",
    "description": "Validar failover automático a réplica",
    "steady-state-hypothesis": {
        "title": "La aplicación responde normalmente",
        "probes": [
            {
                "type": "probe",
                "name": "app-responds",
                "tolerance": 200,
                "provider": {
                    "type": "http",
                    "url": "https://api.example.com/health"
                }
            }
        ]
    },
    "method": [
        {
            "type": "action",
            "name": "kill-primary-database",
            "provider": {
                "type": "python",
                "module": "chaosaws.rds.actions",
                "func": "reboot_db_instance",
                "arguments": {
                    "db_instance_identifier": "prod-primary"
                }
            }
        },
        {
            "type": "probe",
            "name": "wait-for-failover",
            "provider": {
                "type": "python",
                "module": "chaosaws.rds.probes",
                "func": "instance_status",
                "arguments": {
                    "db_instance_identifier": "prod-replica"
                }
            },
            "tolerance": "available"
        }
    ],
    "rollbacks": [
        {
            "type": "action",
            "name": "verify-recovery",
            "provider": {
                "type": "http",
                "url": "https://api.example.com/health"
            }
        }
    ]
}

# Ejecutar experimento
result = run_experiment(experiment)
print(json.dumps(result, indent=2))

# dr_test_infrastructure.tf - Configuración de test DR multi-región

provider "aws" {
  alias  = "primary"
  region = "us-east-1"
}

provider "aws" {
  alias  = "dr"
  region = "us-west-2"
}

# Ambiente de test DR en región secundaria
resource "aws_instance" "dr_test" {
  provider      = aws.dr
  ami           = var.dr_ami_id
  instance_type = "t3.large"

  tags = {
    Name        = "dr-test-instance"
    Environment = "dr-test"
    AutoDelete  = "true"
  }

  lifecycle {
    # Prevenir impacto accidental en producción
    prevent_destroy = false
  }
}

# Lambda de validación de DR automatizado
resource "aws_lambda_function" "dr_validator" {
  provider      = aws.dr
  function_name = "dr-validation-test"
  runtime       = "python3.11"
  handler       = "validator.handler"
  timeout       = 900  # 15 minutos

  environment {
    variables = {
      PRIMARY_REGION = "us-east-1"
      RTO_TARGET     = "3600"
      RPO_TARGET     = "900"
    }
  }
}

# Test de DR programado
resource "aws_cloudwatch_event_rule" "monthly_dr_test" {
  provider            = aws.dr
  name                = "monthly-dr-test"
  schedule_expression = "cron(0 2 1 * ? *)"  # Primer día del mes a las 2 AM
}

# application_dr_test.yaml - Workflow de GitHub Actions

name: DR Test

on:
  schedule:

    - cron: '0 4 * * 0'  # Semanal Domingo 4 AM
  workflow_dispatch:

jobs:
  dr-test:
    runs-on: ubuntu-latest
    steps:

      - uses: actions/checkout@v4

      - name: Setup DR environment
        run: |
          # Desplegar en región DR
          terraform -chdir=terraform/dr init
          terraform -chdir=terraform/dr apply -auto-approve

      - name: Restore from backup
        id: restore
        run: |
          START_TIME=$(date +%s)

          # Iniciar restauración de backup
          aws backup start-restore-job \
            --recovery-point-arn ${{ secrets.LATEST_BACKUP_ARN }} \
            --iam-role-arn ${{ secrets.DR_ROLE_ARN }} \
            --metadata '{"targetInstanceId": "dr-test-instance"}'

          # Esperar completación de restauración
          while true; do
            STATUS=$(aws backup describe-restore-job --restore-job-id $JOB_ID --query 'Status' --output text)
            if [ "$STATUS" = "COMPLETED" ]; then break; fi
            if [ "$STATUS" = "FAILED" ]; then exit 1; fi
            sleep 30
          done

          END_TIME=$(date +%s)
          DURATION=$((END_TIME - START_TIME))
          echo "restore_duration=$DURATION" >> $GITHUB_OUTPUT

      - name: Validate application
        run: |
          # Ejecutar smoke tests contra ambiente DR
          npm run test:smoke -- --env=dr

          # Verificar integridad de datos
          npm run test:data-integrity -- --env=dr

      - name: Check RTO compliance
        run: |
          DURATION=${{ steps.restore.outputs.restore_duration }}
          RTO_TARGET=3600

          if [ $DURATION -gt $RTO_TARGET ]; then
            echo "RTO EXCEDIDO: ${DURATION}s > ${RTO_TARGET}s"
            exit 1
          fi
          echo "RTO CUMPLIDO: ${DURATION}s <= ${RTO_TARGET}s"

      - name: Cleanup DR environment
        if: always()
        run: |
          terraform -chdir=terraform/dr destroy -auto-approve

# dependency_aware_restore.yaml
restore_order:

  - name: infraestructura_red
    includes: [vpc, subnets, security_groups]
    validation: connectivity_test

  - name: capa_datos
    depends_on: [infraestructura_red]
    includes: [rds, elasticache, s3]
    validation: data_integrity_test

  - name: capa_aplicacion
    depends_on: [capa_datos]
    includes: [ecs_services, lambda_functions]
    validation: smoke_test

  - name: capa_ingreso
    depends_on: [capa_aplicacion]
    includes: [alb, route53]
    validation: e2e_test