initial commit

2026-02-13 15:44:24 +08:00
parent 66be8224f4
commit 09ce4c881a
570 changed files with 61807 additions and 0 deletions
@@ -0,0 +1,61 @@
+<!-- This readme file is generated with terraform-docs -->
+
+This module configure CloudwatchLog and stream logs to s3 bucket via Kinesis Firehose
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| terraform | ~> 1.3.0 |
+| aws | >= 5.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| aws | >= 5.0 |
+| random | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_cloudwatch_log_group.firehose-log](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
+| [aws_cloudwatch_log_subscription_filter.cwl-sub-filter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_subscription_filter) | resource |
+| [aws_iam_policy.cwlog-role-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.firehose-role-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_role.cwlog-stream-role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role.firehose-stream-iam-role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.cwlog-role-policy-attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.firehose-role-policy-attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_kinesis_firehose_delivery_stream.cwl-s3-firehose-stream](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
+| [random_id.rid](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource |
+| [aws_caller_identity.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| cwl-region | AWS region where Cloudwatch LogGroup resides. Needed for setting up cwlog-stream-role | `string` | n/a | yes |
+| dest-bucket-arn | Destination S3 bucket ARN | `string` | n/a | yes |
+| dest-bucket-kmskey-arn | KMS key ARN for destination bucket | `string` | n/a | yes |
+| dest-bucket-prefix | S3 object prefix for this stream. Please do not start with / end with a /. For example, r53-log/acme.local/ | `string` | n/a | yes |
+| enable-firehose-errorlog | Enable firehose errorlog | `bool` | `false` | no |
+| firehose-kmskey-arn | KMS Key arn for Firehose | `string` | n/a | yes |
+| source-cwlgroup-name | Name of source CloudwatchLog group | `string` | n/a | yes |
+| stream-name | Name of Kinesis Data Firehose delivery stream | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cloudwatchstream-iam-role-arn | n/a |
+| firehose-iam-role-arn | n/a |
+ 
+---
+## Authorship
+This module was developed by Rackspace.
@@ -0,0 +1,162 @@
+resource "aws_kinesis_firehose_delivery_stream" "cwl-s3-firehose-stream" {
+  name        = var.stream-name
+  destination = "extended_s3"
+
+  extended_s3_configuration {
+    role_arn            = aws_iam_role.firehose-stream-iam-role.arn
+    bucket_arn          = var.dest-bucket-arn
+    prefix              = trimprefix(var.dest-bucket-prefix, "/")
+    error_output_prefix = "FirehoseErrors/"
+    kms_key_arn         = var.dest-bucket-kmskey-arn
+    compression_format  = "GZIP"
+    cloudwatch_logging_options {
+      enabled         = var.enable-firehose-errorlog
+      log_group_name  = try(aws_cloudwatch_log_group.firehose-log[0].name, null)
+      log_stream_name = "DestinationDelivery"
+    }
+  }
+  server_side_encryption {
+    enabled  = true
+    key_type = "CUSTOMER_MANAGED_CMK"
+    key_arn  = var.firehose-kmskey-arn
+  }
+}
+
+resource "aws_cloudwatch_log_group" "firehose-log" {
+  count             = var.enable-firehose-errorlog ? 1 : 0
+  name              = "/aws/kinesisfirehose/${var.stream-name}"
+  retention_in_days = 365
+}
+
+resource "aws_cloudwatch_log_subscription_filter" "cwl-sub-filter" {
+  log_group_name  = var.source-cwlgroup-name
+  name            = "stream-to-s3"
+  role_arn        = aws_iam_role.cwlog-stream-role.arn
+  filter_pattern  = ""
+  destination_arn = aws_kinesis_firehose_delivery_stream.cwl-s3-firehose-stream.arn
+}
+
+resource "random_id" "rid" {
+  byte_length = 4
+}
+
+resource "aws_iam_role" "firehose-stream-iam-role" {
+  name        = "firehose-stream-role-${var.stream-name}-${random_id.rid.dec}"
+  description = "Kinesis Firehose IAM role for streaming logs from CloudwatchLog to S3"
+  assume_role_policy = jsonencode(
+    {
+      "Version" : "2012-10-17",
+      "Statement" : [
+        {
+          "Sid" : "FirehoseStreaming",
+          "Effect" : "Allow",
+          "Principal" : {
+            "Service" : "firehose.amazonaws.com"
+          },
+          "Action" : "sts:AssumeRole"
+        }
+      ]
+    }
+  )
+}
+
+resource "aws_iam_role_policy_attachment" "firehose-role-policy-attachment" {
+  role       = aws_iam_role.firehose-stream-iam-role.name
+  policy_arn = aws_iam_policy.firehose-role-policy.arn
+}
+
+resource "aws_iam_policy" "firehose-role-policy" {
+  name        = "kinesis-firehose-log-stream-${var.stream-name}-${random_id.rid.dec}"
+  description = "Policy for Kinesis Firehose streaming logs to s3"
+  policy = jsonencode(
+    {
+      "Version" : "2012-10-17",
+      "Statement" : [
+        {
+          "Effect" : "Allow",
+          "Action" : [
+            "s3:AbortMultipartUpload",
+            "s3:GetBucketLocation",
+            "s3:GetObject",
+            "s3:ListBucket",
+            "s3:ListBucketMultipartUploads",
+            "s3:PutObject"
+          ],
+          "Resource" : [
+            var.dest-bucket-arn,
+            "${var.dest-bucket-arn}/*"
+          ]
+        },
+        {
+          "Effect" : "Allow",
+          "Action" : [
+            "kms:Decrypt",
+            "kms:GenerateDataKey"
+          ],
+          "Resource" : [
+            var.dest-bucket-kmskey-arn
+          ]
+        },
+        {
+          "Effect" : "Allow",
+          "Action" : [
+            "logs:PutLogEvents",
+            "logs:PutLogEventsBatch",
+            "logs:CreateLogStream"
+          ],
+          "Resource" : [
+            "arn:aws:logs:*:*:log-group:/aws/kinesisfirehose/${var.stream-name}/*"
+          ]
+        }
+      ]
+    }
+  )
+}
+
+
+resource "aws_iam_role" "cwlog-stream-role" {
+  name        = "cloudwatchlog-stream-role-${var.stream-name}-${random_id.rid.dec}"
+  description = "CloudwatchLog role for streaming to firehose"
+  assume_role_policy = jsonencode(
+    {
+      "Version" : "2012-10-17",
+      "Statement" : [
+        {
+          "Sid" : "CloudwatchLogStreaming",
+          "Effect" : "Allow",
+          "Principal" : {
+            "Service" : "logs.${var.cwl-region}.amazonaws.com"
+          },
+          "Action" : "sts:AssumeRole"
+        }
+      ]
+    }
+  )
+}
+
+
+resource "aws_iam_role_policy_attachment" "cwlog-role-policy-attachment" {
+  role       = aws_iam_role.cwlog-stream-role.name
+  policy_arn = aws_iam_policy.cwlog-role-policy.arn
+}
+
+resource "aws_iam_policy" "cwlog-role-policy" {
+  name        = "cloudwatchlog-stream-${var.stream-name}-${random_id.rid.dec}"
+  description = "Policy for CloudWatch Logs streaming to Kinesis Firehose"
+  policy = jsonencode(
+    {
+      "Version" : "2012-10-17",
+      "Statement" : [
+        {
+          "Effect" : "Allow",
+          "Action" : ["firehose:PutRecord"],
+          "Resource" : [
+            "arn:aws:firehose:${var.cwl-region}:${data.aws_caller_identity.this.account_id}:deliverystream/${var.stream-name}"
+          ]
+        }
+      ]
+    }
+  )
+}
+
+data "aws_caller_identity" "this" {}
@@ -0,0 +1,7 @@
+output firehose-iam-role-arn {
+  value = aws_iam_role.firehose-stream-iam-role.arn
+}
+
+output cloudwatchstream-iam-role-arn {
+  value = aws_iam_role.cwlog-stream-role.arn
+}
@@ -0,0 +1,40 @@
+variable "stream-name" {
+  type        = string
+  description = "Name of Kinesis Data Firehose delivery stream"
+}
+
+variable "firehose-kmskey-arn" {
+  type        = string
+  description = "KMS Key arn for Firehose"
+}
+
+variable "dest-bucket-arn" {
+  type        = string
+  description = "Destination S3 bucket ARN"
+}
+
+variable "dest-bucket-prefix" {
+  type        = string
+  description = "S3 object prefix for this stream. Please do not start with / end with a /. For example, r53-log/acme.local/"
+}
+
+variable "dest-bucket-kmskey-arn" {
+  type        = string
+  description = "KMS key ARN for destination bucket"
+}
+
+variable "source-cwlgroup-name" {
+  type        = string
+  description = "Name of source CloudwatchLog group"
+}
+
+variable "cwl-region" {
+  type        = string
+  description = "AWS region where Cloudwatch LogGroup resides. Needed for setting up cwlog-stream-role"
+}
+
+variable "enable-firehose-errorlog" {
+  type        = bool
+  description = "Enable firehose errorlog"
+  default     = false
+}
@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~> 1.3.0"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+  }
+}