Cloud Storage Tiering für Data Lakes: Kosten halbieren ohne Performance-Verlust

flowchart LR
    subgraph Landing Zone
        L[Raw Ingest
Retention: 7 Tage
Tier: Standard]
    end
    subgraph Bronze
        B[Cleaned Raw
Retention: 90 Tage hot
Tier: Standard → IA @ 30d]
    end
    subgraph Silver
        S[Kuratiert / transformiert
Retention: 1 Jahr hot
Tier: Standard → IA @ 90d]
    end
    subgraph Gold
        G[Aggregiert / serving
Retention: 3 Jahre
Tier: Standard immer]
    end
    subgraph Archive
        A[Compliance / Historie
Retention: 7+ Jahre
Tier: Glacier Deep Archive]
    end
    L --> B --> S --> G
    S --> A
    B --> A

resource "aws_s3_bucket_lifecycle_configuration" "data_lake" {
  bucket = aws_s3_bucket.data_lake.id

  # Bronze: nach 30 Tagen IA, nach 90 Glacier
  rule {
    id     = "bronze-tiering"
    status = "Enabled"

    filter {
      prefix = "bronze/"
    }

    transition {
      days          = 30
      storage_class = "STANDARD_IA"
    }

    transition {
      days          = 90
      storage_class = "GLACIER_INSTANT_RETRIEVAL"
    }

    expiration {
      days = 365
    }
  }

  # Silver: IA nach 90 Tagen, Glacier nach 365
  rule {
    id     = "silver-tiering"
    status = "Enabled"

    filter {
      prefix = "silver/"
    }

    transition {
      days          = 90
      storage_class = "STANDARD_IA"
    }

    transition {
      days          = 365
      storage_class = "GLACIER_INSTANT_RETRIEVAL"
    }
  }

  # Archive: sofort Glacier Deep Archive
  rule {
    id     = "archive-deep"
    status = "Enabled"

    filter {
      prefix = "archive/"
    }

    transition {
      days          = 1
      storage_class = "DEEP_ARCHIVE"
    }

    noncurrent_version_expiration {
      noncurrent_days = 30
    }
  }

  # Landing: nach 7 Tagen löschen
  rule {
    id     = "landing-cleanup"
    status = "Enabled"

    filter {
      prefix = "landing/"
    }

    expiration {
      days = 7
    }
  }
}

# Autoclass auf einem GCS-Bucket aktivieren
gcloud storage buckets update gs://company-data-lake-bronze \
  --enable-autoclass \
  --autoclass-terminal-storage-class=ARCHIVE

# Für Buckets, wo Archive zu aggressiv ist (z. B. Silver):
gcloud storage buckets update gs://company-data-lake-silver \
  --enable-autoclass \
  --autoclass-terminal-storage-class=NEARLINE

resource "azurerm_storage_management_policy" "data_lake" {
  storage_account_id = azurerm_storage_account.data_lake.id

  rule {
    name    = "bronze-tiering"
    enabled = true

    filters {
      prefix_match = ["bronze/"]
      blob_types   = ["blockBlob"]
    }

    actions {
      base_blob {
        tier_to_cool_after_days_since_modification_greater_than    = 30
        tier_to_cold_after_days_since_modification_greater_than    = 90
        tier_to_archive_after_days_since_modification_greater_than = 365
        delete_after_days_since_modification_greater_than          = 730
      }
    }
  }

  rule {
    name    = "archive-immediate"
    enabled = true

    filters {
      prefix_match = ["archive/"]
      blob_types   = ["blockBlob"]
    }

    actions {
      base_blob {
        tier_to_archive_after_days_since_modification_greater_than = 1
      }
    }
  }
}

# PySpark / Databricks: Delta-Tabellen-Wartung
from delta.tables import DeltaTable
from pyspark.sql import SparkSession

spark = SparkSession.builder.appName("DeltaMaintenance").getOrCreate()

# OPTIMIZE: kleine Files kompaktieren (Bin-Pack)
delta_table = DeltaTable.forPath(spark, "s3://data-lake/silver/orders/")
delta_table.optimize().executeCompaction()

# Z-ORDER für häufig gefilterte Spalten (verbessert Data-Skipping)
delta_table.optimize().executeZOrderBy("order_date", "customer_id")

# VACUUM: Files älter als Retention-Window entfernen
# Default: 7 Tage; explizit setzen für Compliance
delta_table.vacuum(retentionHours=168)  # 7 Tage

# Databricks Workflow: Weekly Delta Maintenance
name: delta-maintenance-weekly
schedule:
  quartz_cron_expression: "0 0 2 ? * SUN"
  timezone_id: "UTC"
tasks:
  - task_key: optimize_silver
    notebook_task:
      notebook_path: /ops/delta_maintenance
      base_parameters:
        zone: silver
        z_order_cols: order_date,customer_id
    job_cluster_key: maintenance_cluster

clusters:
  - job_cluster_key: maintenance_cluster
    new_cluster:
      spark_version: 14.3.x-scala2.12
      node_type_id: m5.xlarge
      num_workers: 4

# Transaction-Log-Größe prüfen
aws s3 ls s3://data-lake/silver/orders/_delta_log/ --recursive | wc -l

# Delta Auto-Checkpoint alle 10 Transaktionen (konfigurierbar)
# Für High-Frequency-Tabellen niedriger setzen:
ALTER TABLE silver.orders SET TBLPROPERTIES (
  'delta.checkpointInterval' = '5',
  'delta.logRetentionDuration' = 'interval 30 days',
  'delta.deletedFileRetentionDuration' = 'interval 7 days'
);

# Einfacher Storage-Kosten-Rechner
def calculate_tiering_savings(
    total_gb: float,
    hot_pct: float,      # Anteil in Standard
    warm_pct: float,     # Anteil in IA / Nearline
    cold_pct: float,     # Anteil in Glacier / Archive
    standard_cost: float = 0.023,
    ia_cost: float = 0.0125,
    glacier_cost: float = 0.004,
    retrieval_cost_per_gb: float = 0.01,
    monthly_retrieval_gb: float = 0.0
) -> dict:

    hot_gb = total_gb * hot_pct
    warm_gb = total_gb * warm_pct
    cold_gb = total_gb * cold_pct

    baseline_cost = total_gb * standard_cost

    tiered_cost = (
        hot_gb * standard_cost +
        warm_gb * ia_cost +
        cold_gb * glacier_cost +
        monthly_retrieval_gb * retrieval_cost_per_gb
    )

    savings = baseline_cost - tiered_cost
    savings_pct = (savings / baseline_cost) * 100

    return {
        "baseline_monthly_usd": round(baseline_cost, 2),
        "tiered_monthly_usd": round(tiered_cost, 2),
        "monthly_savings_usd": round(savings, 2),
        "savings_percent": round(savings_pct, 1)
    }

# Beispiel: 500-TB-Data-Lake
result = calculate_tiering_savings(
    total_gb=500_000,
    hot_pct=0.15,
    warm_pct=0.35,
    cold_pct=0.50,
    monthly_retrieval_gb=5_000  # 5 TB warm retrieval/Monat
)
print(result)
# {'baseline_monthly_usd': 11500.0, 'tiered_monthly_usd': 3512.5,
#  'monthly_savings_usd': 7987.5, 'savings_percent': 69.5}