Infrastructure as Code für Data-Plattformen: Praxis-Guide

graph TB
    L4[Layer 4: Data Products<br/>dbt models, ML pipelines, notebooks]
    L3[Layer 3: Platform Services<br/>Airflow DAGs, Spark configs, catalog schemas]
    L2[Layer 2: Data Infrastructure<br/>Warehouses, lakes, streaming brokers]
    L1[Layer 1: Cloud Resources<br/>VPCs, IAM, storage, compute]
    L4 --> L3 --> L2 --> L1
    style L1 fill:#1e3a5f,color:#fff
    style L2 fill:#1a5276,color:#fff
    style L3 fill:#154360,color:#fff
    style L4 fill:#0e2f44,color:#fff

terraform/
├── modules/
│   ├── data-lake/
│   │   ├── main.tf
│   │   ├── variables.tf
│   │   ├── outputs.tf
│   │   └── README.md
│   ├── streaming-platform/
│   ├── data-warehouse/
│   └── governance/
├── environments/
│   ├── dev/
│   │   ├── main.tf
│   │   └── terraform.tfvars
│   ├── staging/
│   └── prod/
└── shared/
    ├── backend.tf
    └── providers.tf

# modules/data-lake/main.tf

locals {
  bucket_name = "${var.project_prefix}-${var.environment}-lakehouse"
  common_tags = merge(var.tags, {
    Module      = "data-lake"
    Environment = var.environment
    ManagedBy   = "terraform"
  })
}

resource "aws_s3_bucket" "lakehouse" {
  bucket = local.bucket_name
  tags   = local.common_tags
}

resource "aws_s3_bucket_versioning" "lakehouse" {
  bucket = aws_s3_bucket.lakehouse.id
  versioning_configuration {
    status = "Enabled"
  }
}

resource "aws_s3_bucket_lifecycle_configuration" "lakehouse" {
  bucket = aws_s3_bucket.lakehouse.id

  rule {
    id     = "bronze-to-glacier"
    status = "Enabled"
    filter { prefix = "bronze/" }
    transition {
      days          = 90
      storage_class = "GLACIER_IR"
    }
  }

  rule {
    id     = "silver-intelligent-tiering"
    status = "Enabled"
    filter { prefix = "silver/" }
    transition {
      days          = 30
      storage_class = "INTELLIGENT_TIERING"
    }
  }
}

# Lake Formation permissions
resource "aws_lakeformation_resource" "lakehouse" {
  arn      = aws_s3_bucket.lakehouse.arn
  role_arn = aws_iam_role.lakeformation_service.arn
}

resource "aws_lakeformation_permissions" "analyst_access" {
  for_each = toset(var.analyst_role_arns)

  principal   = each.value
  permissions = ["SELECT", "DESCRIBE"]

  table {
    database_name = aws_glue_catalog_database.silver.name
    wildcard      = true
  }
}

# modules/data-lake/catalog.tf

resource "aws_glue_catalog_database" "bronze" {
  name        = "${var.project_prefix}_bronze"
  description = "Raw ingested data — immutable, append-only"

  create_table_default_permission {
    permissions = ["ALL"]
    principal {
      data_lake_principal_identifier = "IAM_ALLOWED_PRINCIPALS"
    }
  }
}

resource "aws_glue_catalog_database" "silver" {
  name        = "${var.project_prefix}_silver"
  description = "Cleaned, validated, conformed data"
}

resource "aws_glue_catalog_database" "gold" {
  name        = "${var.project_prefix}_gold"
  description = "Business-ready aggregates and data products"
}

# modules/databricks-workspace/main.tf

terraform {
  required_providers {
    databricks = {
      source  = "databricks/databricks"
      version = "~> 1.38"
    }
  }
}

resource "databricks_cluster_policy" "data_engineering" {
  name = "data-engineering-${var.environment}"
  definition = jsonencode({
    "spark_version" : {
      "type" : "allowlist",
      "values" : ["13.3.x-scala2.12", "14.3.x-scala2.12"],
      "defaultValue" : "14.3.x-scala2.12"
    },
    "node_type_id" : {
      "type" : "allowlist",
      "values" : ["m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge"]
    },
    "autotermination_minutes" : {
      "type" : "fixed",
      "value" : 60,
      "hidden" : false
    },
    "custom_tags.team" : {
      "type" : "fixed",
      "value" : var.team_tag
    }
  })
}

resource "databricks_sql_warehouse" "main" {
  name             = "${var.project_prefix}-${var.environment}"
  cluster_size     = var.environment == "prod" ? "Medium" : "X-Small"
  auto_stop_mins   = 5
  min_num_clusters = 1
  max_num_clusters = var.environment == "prod" ? 3 : 1

  tags {
    custom_tags {
      key   = "environment"
      value = var.environment
    }
  }
}

resource "databricks_catalog" "main" {
  name    = var.catalog_name
  comment = "Main Unity Catalog for ${var.environment}"

  properties = {
    purpose = "data-platform"
  }
}

dbt_project/
├── dbt_project.yml
├── profiles.yml          # managed via Vault / environment vars
├── models/
│   ├── staging/          # raw -> typed, renamed
│   ├── intermediate/     # business logic
│   └── marts/            # data products for consumers
├── tests/
│   ├── generic/          # reusable test macros
│   └── singular/         # one-off assertions
├── macros/
├── seeds/                # small reference tables, version-controlled
└── analyses/             # ad-hoc, not materialised

# models/marts/finance/schema.yml
version: 2

models:
  - name: fct_revenue
    description: "Daily revenue fact table — SLA: 99.9% freshness within 2h of close"
    config:
      contract:
        enforced: true           # dbt will fail if types don't match
    columns:
      - name: revenue_id
        data_type: varchar
        constraints:
          - type: not_null
          - type: unique
      - name: amount_usd
        data_type: numeric(18,4)
        constraints:
          - type: not_null
      - name: transaction_date
        data_type: date
        constraints:
          - type: not_null
    tests:
      - dbt_expectations.expect_column_values_to_be_between:
          column_name: amount_usd
          min_value: 0
          max_value: 10000000

flowchart LR
    Dev[Developer<br/>local branch] -->|PR| GH[GitHub / GitLab]
    GH -->|CI trigger| CI{CI Pipeline}
    CI -->|terraform plan| TFP[Terraform Plan<br/>Preview]
    CI -->|dbt compile| DBT[dbt Compile<br/>+ lint]
    CI -->|schema tests| ST[Schema<br/>Regression Tests]
    CI -->|contract tests| CT[API Contract<br/>Tests via Harbinger]
    TFP & DBT & ST & CT -->|all pass| Merge[Merge to main]
    Merge -->|CD trigger| CD{CD Pipeline}
    CD -->|terraform apply dev| DEV[Dev Environment]
    DEV -->|smoke tests pass| STG[Staging]
    STG -->|integration tests pass| PROD[Production]

# .github/workflows/data-platform-ci.yml
name: Data Platform CI

on:
  pull_request:
    paths:
      - 'terraform/**'
      - 'dbt_project/**'
      - 'airflow/dags/**'

jobs:
  terraform-plan:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: terraform/environments/staging
    steps:
      - uses: actions/checkout@v4

      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: "1.7.x"

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_CICD_ROLE_ARN }}
          aws-region: eu-west-1

      - name: Terraform Init
        run: terraform init -backend-config="key=staging/terraform.tfstate"

      - name: Terraform Plan
        run: terraform plan -out=tfplan.binary

      - name: Convert Plan to JSON
        run: terraform show -json tfplan.binary > tfplan.json

      - name: Validate Plan (no deletes in prod tables)
        run: |
          python scripts/validate_plan.py tfplan.json             --no-destroy-pattern "aws_glue_catalog_table"             --no-destroy-pattern "aws_s3_bucket.lakehouse"

  dbt-ci:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dbt
        run: pip install dbt-spark dbt-expectations

      - name: dbt deps
        run: dbt deps
        working-directory: dbt_project

      - name: dbt compile (syntax check)
        run: dbt compile --profiles-dir . --target ci
        working-directory: dbt_project

      - name: dbt test (dev schema)
        run: dbt test --profiles-dir . --target ci --store-failures
        working-directory: dbt_project

graph LR
    CODE[Code / Config] -->|auto-deploy| DEV[dev<br/>shared cluster<br/>sample data]
    DEV -->|PR merge| STG[staging<br/>prod-size cluster<br/>anonymised prod data]
    STG -->|release tag| PROD[prod<br/>autoscaling<br/>full data]
    style PROD fill:#c0392b,color:#fff
    style STG fill:#d35400,color:#fff
    style DEV fill:#27ae60,color:#fff

# environments/prod/terraform.tfvars
environment        = "prod"
databricks_cluster_size = "Large"
min_workers        = 2
max_workers        = 20
enable_spot        = true
spot_bid_price_pct = 80
backup_retention   = 30
alert_email        = "data-platform-oncall@company.com"

# shared/backend.tf
terraform {
  backend "s3" {
    bucket         = "company-terraform-state"
    key            = "${var.environment}/data-platform/terraform.tfstate"
    region         = "eu-west-1"
    dynamodb_table = "terraform-state-locks"
    encrypt        = true
    kms_key_id     = "arn:aws:kms:eu-west-1:123456789:key/mrk-xxx"
  }
}

data "vault_generic_secret" "databricks_token" {
  path = "secret/data-platform/${var.environment}/databricks"
}

resource "databricks_token" "pipeline_sa" {
  comment          = "CI/CD service account — managed by Terraform"
  lifetime_seconds = 7776000  # 90 days, rotated by Vault
}

resource "vault_generic_secret" "databricks_token_output" {
  path = "secret/data-platform/${var.environment}/databricks"
  data_json = jsonencode({
    token = databricks_token.pipeline_sa.token_value
  })
}