# Required Tags für alle Resources definieren
resource "aws_config_config_rule" "required_tags" {
  name = "required-tags-data-platform"

  source {
    owner             = "AWS"
    source_identifier = "REQUIRED_TAGS"
  }

  input_parameters = jsonencode({
    tag1Key   = "team"
    tag2Key   = "product"
    tag3Key   = "environment"
    tag4Key   = "cost-center"
    tag5Key   = "managed-by"
  })

  scope {
    compliance_resource_types = [
      "AWS::S3::Bucket",
      "AWS::RDS::DBInstance",
      "AWS::EMR::Cluster",
      "AWS::Glue::Job",
      "AWS::Redshift::Cluster"
    ]
  }
}

# Tag-Policy in AWS Organizations
resource "aws_organizations_policy" "mandatory_tags" {
  name = "mandatory-tags-data-platform"
  type = "TAG_POLICY"

  content = jsonencode({
    tags = {
      team = {
        tag_value = {
          "@@assign" = ["data-platform", "marketing-analytics", "ml-platform", "finance-analytics"]
        }
      }
      environment = {
        tag_value = {
          "@@assign" = ["prod", "staging", "dev"]
        }
      }
    }
  })
}

# Base-Tags an jedes AWS-Resource
provider "aws" {
  region = var.aws_region

  default_tags {
    tags = {
      managed-by    = "terraform"
      environment   = var.environment
      repository    = "github.com/myorg/data-platform"
      last-modified = timestamp()
    }
  }
}

# AWS Cost Explorer: Kosten nach team-Tag
aws ce get-cost-and-usage \
  --time-period Start=$(date -d 'last month' +%Y-%m-01),End=$(date +%Y-%m-01) \
  --granularity MONTHLY \
  --metrics BlendedCost \
  --group-by Type=TAG,Key=team \
  --filter '{
    "And": [
      {"Dimensions": {"Key": "SERVICE", "Values": ["Amazon S3", "Amazon EMR", "Amazon Redshift", "AWS Glue"]}},
      {"Tags": {"Key": "environment", "Values": ["prod"]}}
    ]
  }' \
  --output table

# EMR-Cluster mit Spot-Instances
resource "aws_emr_cluster" "etl_cluster" {
  name          = "silver-transform-${var.environment}"
  release_label = "emr-6.13.0"
  applications  = ["Spark", "Hadoop"]

  ec2_attributes {
    subnet_id        = aws_subnet.private[0].id
    instance_profile = aws_iam_instance_profile.emr.arn
  }

  master_instance_group {
    instance_type = "m5.xlarge"  # On-demand für Master (nicht Spot)
  }

  core_instance_group {
    instance_type  = "r5.2xlarge"
    instance_count = 2
    # Core-Nodes On-demand für Stabilität
  }

  # Task-Nodes auf Spot — 60-80% günstiger
  task_instance_group {
    instance_type  = "r5.2xlarge"
    instance_count = 4

    bid_price = "0.15"  # Max-Bid: $0,15/h vs $0,504 On-demand

    ebs_config {
      size                 = 100
      type                 = "gp3"
      volumes_per_instance = 1
    }
  }

  auto_termination_policy {
    idle_timeout = 3600  # Cluster nach 1h Idle killen
  }

  tags = {
    team        = "data-platform"
    pipeline    = "silver-etl"
    environment = var.environment
  }
}

resource "aws_spot_fleet_request" "ml_training" {
  iam_fleet_role  = aws_iam_role.spot_fleet.arn
  target_capacity = 10
  allocation_strategy = "diversified"  # Nicht alle Eier in einen Pool

  launch_specification {
    instance_type = "r5.4xlarge"
    ami           = data.aws_ami.amazon_linux.id
    subnet_id     = aws_subnet.private[0].id
    spot_price    = "0.30"
  }

  launch_specification {
    instance_type = "r5a.4xlarge"
    ami           = data.aws_ami.amazon_linux.id
    subnet_id     = aws_subnet.private[1].id
    spot_price    = "0.28"
  }

  launch_specification {
    instance_type = "m5.8xlarge"
    ami           = data.aws_ami.amazon_linux.id
    subnet_id     = aws_subnet.private[2].id
    spot_price    = "0.35"
  }

  valid_until = timeadd(timestamp(), "720h")
}

resource "aws_s3_bucket_intelligent_tiering_configuration" "data_lake" {
  bucket = aws_s3_bucket.data_lake["silver"].id
  name   = "EntireBucket"

  tiering {
    access_tier = "DEEP_ARCHIVE_ACCESS"
    days        = 180
  }

  tiering {
    access_tier = "ARCHIVE_ACCESS"
    days        = 90
  }
}

# Lifecycle-Rules: Temp-/Staging-Daten aggressiv löschen
resource "aws_s3_bucket_lifecycle_configuration" "data_lake" {
  bucket = aws_s3_bucket.data_lake["bronze"].id

  rule {
    id     = "expire-staging-data"
    status = "Enabled"

    filter {
      prefix = "staging/"
    }

    expiration {
      days = 7
    }

    noncurrent_version_expiration {
      noncurrent_days = 3
    }
  }

  rule {
    id     = "transition-bronze-to-ia"
    status = "Enabled"

    filter {
      prefix = "orders/"
    }

    transition {
      days          = 30
      storage_class = "STANDARD_IA"
    }

    transition {
      days          = 90
      storage_class = "GLACIER_IR"
    }

    expiration {
      days = 2555  # 7 Jahre Retention für Compliance
    }
  }
}

# Konvertieren zu Parquet mit ZSTD (30-40% kleiner als Snappy)
spark-submit --class com.myorg.CompressJob my-job.jar \
  --input s3://my-platform-silver/orders/ \
  --output s3://my-platform-silver/orders-compressed/ \
  --format parquet \
  --compression zstd \
  --zstd-level 3  # Balance zwischen Speed und Ratio

# Echte Größen messen
aws s3 ls --recursive s3://my-platform-silver/orders/ --summarize | tail -2
aws s3 ls --recursive s3://my-platform-silver/orders-compressed/ --summarize | tail -2

# Athena-Queries für Cost-Attribution taggen
aws athena start-query-execution \
  --query-string "SELECT * FROM gold.orders WHERE dt = '2024-01-15'" \
  --work-group marketing-analytics \
  --query-execution-context Database=gold \
  --result-configuration OutputLocation=s3://my-platform-athena-results/marketing/

# Athena-Workgroup mit Per-Query Data-Scanned-Limit
resource "aws_athena_workgroup" "marketing" {
  name = "marketing-analytics"

  configuration {
    enforce_workgroup_configuration    = true
    publish_cloudwatch_metrics_enabled = true

    result_configuration {
      output_location = "s3://${aws_s3_bucket.athena_results.bucket}/marketing/"

      encryption_configuration {
        encryption_option = "SSE_KMS"
        kms_key            = aws_kms_key.data_platform.arn
      }
    }

    bytes_scanned_cutoff_per_query = 10737418240  # 10 GB max pro Query
  }

  tags = {
    team = "marketing-analytics"
  }
}

-- SCHLECHT: Full-Table-Scan ($$$)
SELECT COUNT(*) FROM gold.events
WHERE event_timestamp >= '2024-01-01';

-- GUT: Partition-Pruned (scant nur Jan-2024 Partitions)
SELECT COUNT(*) FROM gold.events
WHERE year = '2024' AND month = '01';

-- Check ob Partition-Pruning greift
EXPLAIN SELECT COUNT(*) FROM gold.events
WHERE year = '2024' AND month = '01';
-- "partition count" im Output prüfen vs total Partitions

resource "aws_budgets_budget" "data_platform" {
  name         = "data-platform-monthly-${var.environment}"
  budget_type  = "COST"
  limit_amount = "50000"
  limit_unit   = "USD"
  time_unit    = "MONTHLY"

  cost_filter {
    name = "TagKeyValue"
    values = ["user:environment$${var.environment}"]
  }

  notification {
    comparison_operator        = "GREATER_THAN"
    threshold                  = 80
    threshold_type             = "PERCENTAGE"
    notification_type          = "ACTUAL"
    subscriber_email_addresses = ["data-platform-lead@myorg.com"]
  }

  notification {
    comparison_operator        = "GREATER_THAN"
    threshold                  = 100
    threshold_type             = "PERCENTAGE"
    notification_type          = "FORECASTED"
    subscriber_email_addresses = ["data-platform-lead@myorg.com", "cto@myorg.com"]
  }
}

# AWS Cost Anomaly Detection
resource "aws_ce_anomaly_monitor" "data_platform" {
  name              = "data-platform-anomaly-monitor"
  monitor_type      = "DIMENSIONAL"
  monitor_dimension = "SERVICE"
}

resource "aws_ce_anomaly_subscription" "data_platform" {
  name      = "data-platform-anomaly-alerts"
  frequency = "DAILY"

  monitor_arn_list = [aws_ce_anomaly_monitor.data_platform.arn]

  subscriber {
    address = "data-platform-lead@myorg.com"
    type    = "EMAIL"
  }

  threshold_expression {
    and {
      dimension {
        key           = "ANOMALY_TOTAL_IMPACT_PERCENTAGE"
        match_options = ["GREATER_THAN_OR_EQUAL"]
        values        = ["20"]
      }
    }
    and {
      dimension {
        key           = "ANOMALY_TOTAL_IMPACT_ABSOLUTE"
        match_options = ["GREATER_THAN_OR_EQUAL"]
        values        = ["500"]
      }
    }
  }
}

# Quick Cost-Audit: Top 10 Glue-Jobs nach DPU-Stunden diesen Monat
aws glue get-job-runs --job-name silver-orders-transform \
  --query 'JobRuns[?CompletedOn>=`2024-01-01`].[JobName,ExecutionTime,MaxCapacity]' \
  --output table | sort -k3 -rn | head -10