Data Platform Disaster Recovery: RPO, RTO und Runbooks, die wirklich funktionieren

Tier 1 — Kritisch         | RPO: 1h   | RTO: 2h   | z.B. Customer-Billing
Tier 2 — Wichtig          | RPO: 4h   | RTO: 8h   | z.B. Operational-Dashboards
Tier 3 — Standard         | RPO: 24h  | RTO: 24h  | z.B. Historic Analytics
Tier 4 — Archiv           | RPO: 7d   | RTO: 48h  | z.B. Compliance-Archive

resource "aws_s3_bucket" "data_lake_primary" {
  bucket = "company-data-lake-eu-central-1"
  versioning { enabled = true }
}

resource "aws_s3_bucket" "data_lake_replica" {
  provider = aws.eu_west_1
  bucket   = "company-data-lake-eu-west-1"
  versioning { enabled = true }
}

resource "aws_iam_role" "replication_role" {
  name = "s3-replication-role"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Action    = "sts:AssumeRole"
      Effect    = "Allow"
      Principal = { Service = "s3.amazonaws.com" }
    }]
  })
}

resource "aws_s3_bucket_replication_configuration" "data_lake" {
  role   = aws_iam_role.replication_role.arn
  bucket = aws_s3_bucket.data_lake_primary.id

  rule {
    id     = "replicate-tier1-data"
    status = "Enabled"
    filter { prefix = "tier1/" }

    destination {
      bucket        = aws_s3_bucket.data_lake_replica.arn
      storage_class = "STANDARD"
      replication_time {
        status = "Enabled"
        time { minutes = 15 }
      }
      metrics {
        status = "Enabled"
        event_threshold { minutes = 15 }
      }
    }

    delete_marker_replication { status = "Enabled" }
  }
}

#!/bin/bash
BUCKET="company-data-lake-eu-central-1"
PREFIX="tier1/orders/2024/"
TARGET_TIME="2024-03-15T10:00:00Z"
RESTORE_BUCKET="company-data-lake-restore-eu-central-1"

aws s3api list-object-versions \
  --bucket $BUCKET \
  --prefix $PREFIX \
  --query "Versions[?LastModified<='${TARGET_TIME}']|[?IsLatest==\`true\`].[Key,VersionId]" \
  --output text | while read key version_id; do
    aws s3api copy-object \
      --copy-source "$BUCKET/$key?versionId=$version_id" \
      --bucket $RESTORE_BUCKET \
      --key "$key"
    echo "Restored: $key (version: $version_id)"
done

apiVersion: kafka.strimzi.io/v1beta2
kind: KafkaMirrorMaker2
metadata:
  name: mm2-dr-replication
  namespace: kafka
spec:
  version: 3.7.0
  replicas: 3
  connectCluster: dr-region

  clusters:
    - alias: primary
      bootstrapServers: kafka.primary-region.internal:9093
      tls:
        trustedCertificates:
          - secretName: primary-cluster-ca-cert
            certificate: ca.crt
      authentication:
        type: tls
        certificateAndKey:
          secretName: mm2-primary-user
          certificate: user.crt
          key: user.key

    - alias: dr-region
      bootstrapServers: kafka.dr-region.internal:9093

  mirrors:
    - sourceCluster: primary
      targetCluster: dr-region
      sourceConnector:
        tasksMax: 10
        config:
          replication.factor: 3
          offset-syncs.topic.replication.factor: 3
          replication.policy.class: org.apache.kafka.connect.mirror.IdentityReplicationPolicy
      checkpointConnector:
        config:
          checkpoints.topic.replication.factor: 3
          sync.group.offsets.enabled: "true"
          sync.group.offsets.interval.seconds: "60"
      topicsPattern: "tier1.*|tier2.*"

#!/bin/bash
set -euo pipefail

echo "=== KAFKA FAILOVER RUNBOOK ==="
echo "Timestamp: $(date -u)"

# 1. MirrorMaker2 Offset-Sync prüfen
kubectl exec -n kafka deploy/kafka-toolbox -- \
  kafka-consumer-groups.sh \
    --bootstrap-server kafka.dr-region.internal:9092 \
    --group data-platform-consumer \
    --describe | grep -E "TOPIC|LAG"

read -p "Lag akzeptabel? (y/n): " lag_ok
[[ $lag_ok != "y" ]] && echo "ABORT: Lag zu hoch" && exit 1

# 2. Producer auf primary stoppen
curl -X PATCH https://config.internal/v1/features/kafka_producer_enabled \
  -d '{"region":"primary","value":false}'

# 3. In-Flight Messages replizieren lassen
sleep 60

# 4. Consumer-Offsets übersetzen
kubectl exec -n kafka deploy/kafka-toolbox -- \
  kafka-groups.sh \
    --bootstrap-server kafka.dr-region.internal:9092 \
    --reset-offsets \
    --group data-platform-consumer \
    --from-file /checkpoints/primary.data-platform-consumer.offsets \
    --execute

# 5. DNS updaten
vault kv put secret/kafka/bootstrap \
  servers="kafka.dr-region.internal:9093" \
  region="dr" \
  failover_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)"

echo "=== FAILOVER COMPLETE ==="

#!/bin/bash
TABLES=("orders" "customers" "products" "transactions")
PROJECT="company-prod"
DATASET="analytics"
DR_BUCKET="gs://company-bq-dr-europe-west4"
DATE=$(date +%Y/%m/%d)

for TABLE in "${TABLES[@]}"; do
  bq extract \
    --destination_format PARQUET \
    --compression SNAPPY \
    "${PROJECT}:${DATASET}.${TABLE}" \
    "${DR_BUCKET}/${TABLE}/${DATE}/${TABLE}_*.parquet"
  echo "Exported ${TABLE} to ${DR_BUCKET}/${TABLE}/${DATE}/"
done