296 lines
8.5 KiB
Terraform
296 lines
8.5 KiB
Terraform
/**
|
|
* # eks-ipv6-nginxpod
|
|
*
|
|
* ## Features
|
|
* - Use terraform-aws-eks to deploy eks cluster and a nodegroup using spot instances
|
|
* - Use Ipv6 for eks cluster
|
|
* - Dependent VPC and roles are created
|
|
* - use pod identity for EBS abd loadbalancer controller
|
|
* - Create a bastion to manage EKS cluster
|
|
*
|
|
*
|
|
*/
|
|
data "aws_region" "this" {}
|
|
|
|
# Eks Vpc on IPv6
|
|
resource "random_pet" "pet" {
|
|
length = 1
|
|
}
|
|
|
|
locals {
|
|
vpc_cidr = "10.18.0.0/16"
|
|
# ensure there is room for future expansion
|
|
private_net_start = cidrsubnet(local.vpc_cidr, 2, 1)
|
|
public_net_start = cidrsubnet(local.vpc_cidr, 2, 2)
|
|
}
|
|
|
|
data "aws_availability_zones" "this" {
|
|
state = "available"
|
|
}
|
|
|
|
resource "random_shuffle" "Select2Az" {
|
|
input = data.aws_availability_zones.this.names
|
|
result_count = 2
|
|
}
|
|
|
|
module "vpc" {
|
|
source = "terraform-aws-modules/vpc/aws"
|
|
version = "6.6.0"
|
|
|
|
name = "lab-vpc"
|
|
cidr = local.vpc_cidr
|
|
|
|
azs = random_shuffle.Select2Az.result
|
|
enable_ipv6 = true
|
|
public_subnet_assign_ipv6_address_on_creation = true
|
|
private_subnet_assign_ipv6_address_on_creation = true
|
|
# private_subnet_ipv6_native = true # EKS requires free IPv4 addresses. see README
|
|
private_subnets = cidrsubnets(local.private_net_start, 4, 4) # EKS requires free IPv4 addresses. see README
|
|
public_subnets = cidrsubnets(local.public_net_start, 8, 8) # 2 AZ required by eks lbc
|
|
public_subnet_ipv6_prefixes = [0, 1]
|
|
private_subnet_ipv6_prefixes = [10, 11]
|
|
public_subnet_tags = {
|
|
"kubernetes.io/role/elb" = 1
|
|
}
|
|
|
|
enable_dns_hostnames = true
|
|
enable_dns_support = true
|
|
|
|
# nat gateway and eigw (vpc module creates the dns64 /64 route to NGW)
|
|
enable_nat_gateway = true # AWS public endpoints do not support IPv6
|
|
single_nat_gateway = true
|
|
create_egress_only_igw = true
|
|
|
|
enable_flow_log = false
|
|
create_flow_log_cloudwatch_log_group = false
|
|
create_flow_log_cloudwatch_iam_role = false
|
|
manage_default_network_acl = false
|
|
}
|
|
|
|
# EKS resources
|
|
locals {
|
|
userdata = <<EOT
|
|
MIME-Version: 1.0
|
|
Content-Type: multipart/mixed; boundary="//"
|
|
|
|
--//
|
|
Content-Type: application/node.eks.aws
|
|
|
|
---
|
|
apiVersion: node.eks.aws/v1alpha1
|
|
kind: NodeConfig
|
|
spec:
|
|
cluster:
|
|
apiServerEndpoint: ${module.eks.cluster_endpoint}
|
|
certificateAuthority: ${module.eks.cluster_certificate_authority_data}
|
|
cidr: ${module.eks.cluster_service_cidr}
|
|
name: ${module.eks.cluster_name}
|
|
kubelet:
|
|
config:
|
|
maxPods: 110
|
|
clusterDNS:
|
|
- ${replace(module.eks.cluster_service_cidr, "/\\/.*/", "a")}
|
|
|
|
--//--
|
|
EOT
|
|
}
|
|
|
|
resource "aws_launch_template" "node_lt" {
|
|
name = "eks135-node-template"
|
|
description = "Launch template for eks 1.35"
|
|
vpc_security_group_ids = [module.eks.node_security_group_id]
|
|
update_default_version = true
|
|
|
|
# Critical: Set hop limit to 2 for pod IMDS access, required for aws lbc
|
|
metadata_options {
|
|
http_endpoint = "enabled"
|
|
http_tokens = "required" # IMDSv2 required
|
|
http_put_response_hop_limit = 2 # Allows pods to reach IMDS
|
|
instance_metadata_tags = "enabled"
|
|
}
|
|
|
|
block_device_mappings {
|
|
device_name = "/dev/xvda"
|
|
ebs {
|
|
volume_size = 20
|
|
volume_type = "gp3"
|
|
encrypted = true
|
|
kms_key_id = module.KmsKeys.cmks.eksebs.arn
|
|
}
|
|
}
|
|
# must not specify this # image_id = data.aws_ami.eks_worker.id
|
|
user_data = base64encode(local.userdata)
|
|
tag_specifications {
|
|
resource_type = "instance"
|
|
tags = {
|
|
Name = "${module.eks.cluster_name}-worker"
|
|
}
|
|
}
|
|
tag_specifications {
|
|
resource_type = "volume"
|
|
tags = {
|
|
Name = "${module.eks.cluster_name}-worker"
|
|
}
|
|
}
|
|
}
|
|
|
|
# eks optimized ami
|
|
# data "aws_ami" "eks_worker" {
|
|
# name_regex = "amazon-eks-node-al2023-x86_64-standard-1\\.35.*"
|
|
# owners = ["800184023465"]
|
|
# most_recent = true
|
|
# }
|
|
|
|
module "eks" {
|
|
source = "terraform-aws-modules/eks/aws"
|
|
# version = "20.34.0"
|
|
create_iam_role = true
|
|
name = "${var.eks_cluster_name}-${random_pet.pet.id}"
|
|
kubernetes_version = "1.35"
|
|
# enabled_log_types = ["api", "audit", "authenticator", "controllerManager", "scheduler"]
|
|
create_security_group = true
|
|
security_group_additional_rules = {
|
|
bastion_access = {
|
|
description = "Allow access from bastion"
|
|
protocol = "tcp"
|
|
from_port = 443
|
|
to_port = 443
|
|
type = "ingress"
|
|
source_security_group_id = module.bastion-sg.id
|
|
}
|
|
}
|
|
vpc_id = module.vpc.vpc_id
|
|
subnet_ids = module.vpc.private_subnets
|
|
ip_family = "ipv6"
|
|
create_cni_ipv6_iam_policy = true
|
|
create_kms_key = false
|
|
create_cloudwatch_log_group = true
|
|
cloudwatch_log_group_kms_key_id = module.KmsKeys.cmks.log.arn
|
|
cloudwatch_log_group_retention_in_days = 14
|
|
endpoint_private_access = true
|
|
endpoint_public_access = false
|
|
enable_irsa = false
|
|
create_node_security_group = true
|
|
# authentication_mode = "API_AND_CONFIG_MAP" # use access entries and leave this to default
|
|
upgrade_policy = {
|
|
support_type = "STANDARD"
|
|
}
|
|
|
|
encryption_config = {
|
|
resources = ["secrets"]
|
|
provider_key_arn = module.KmsKeys.cmks.secret.arn
|
|
}
|
|
|
|
addons = {
|
|
coredns = {}
|
|
eks-pod-identity-agent = {
|
|
preserve = false
|
|
before_compute = true
|
|
}
|
|
kube-proxy = {}
|
|
aws-ebs-csi-driver = {
|
|
preserve = false
|
|
pod_identity_association = [{
|
|
role_arn = module.CsiPodIdentity.role-arn
|
|
service_account = "ebs-csi-controller-sa"
|
|
}]
|
|
}
|
|
vpc-cni = {
|
|
before_compute = true
|
|
configuration_values = jsonencode({
|
|
env = {
|
|
ENABLE_POD_ENI = "true",
|
|
POD_SECURITY_GROUP_ENFORCING_MODE = "strict",
|
|
# in prefix mode, ipv6 will have /80 and ipv4 will have /28
|
|
ENABLE_PREFIX_DELEGATION = "true"
|
|
},
|
|
init = {
|
|
env = {
|
|
DISABLE_TCP_EARLY_DEMUX = "true"
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
node_iam_role_additional_policies = {
|
|
SsmManaged = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
|
|
}
|
|
|
|
eks_managed_node_groups = {
|
|
EksNodeGroup1 = {
|
|
# required for setting hop limit to 2 for pod IMDS access, required for aws lbc
|
|
create_launch_template = false
|
|
use_custom_launch_template = true
|
|
launch_template_id = aws_launch_template.node_lt.id
|
|
launch_template_version = aws_launch_template.node_lt.latest_version
|
|
|
|
min_size = 2
|
|
max_size = 2
|
|
desired_size = 2
|
|
|
|
instance_types = ["t3.large"]
|
|
capacity_type = "SPOT"
|
|
subnet_ids = module.vpc.private_subnets
|
|
}
|
|
}
|
|
|
|
access_entries = {
|
|
ClusterAdminRole = {
|
|
principal_arn = "arn:aws:iam::040216112220:role/rackLE"
|
|
policy_associations = {
|
|
ClusterAdminPolicy = {
|
|
policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
|
|
access_scope = {
|
|
type = "cluster"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
BastionRole = {
|
|
principal_arn = module.BastionRole.role-arn
|
|
policy_associations = {
|
|
ClusterAdminPolicy = {
|
|
policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
|
|
access_scope = {
|
|
type = "cluster"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
# Allow http traffic from ALB to eks node
|
|
resource "aws_security_group_rule" "eks_node_alb_ingress" {
|
|
type = "ingress"
|
|
from_port = 80
|
|
to_port = 80
|
|
protocol = "tcp"
|
|
security_group_id = module.eks.node_security_group_id
|
|
ipv6_cidr_blocks = [module.vpc.vpc_ipv6_cidr_block]
|
|
description = "ALB to nginx pods port 80"
|
|
}
|
|
|
|
|
|
# CMK for encryption
|
|
|
|
module "KmsKeys" {
|
|
source = "../modules/security_identity_compliance/CustomerManagedKmsKeys"
|
|
|
|
rotation_period_in_days = 180
|
|
enable_key_rotation = true
|
|
deletion_window_in_days = 7
|
|
name-prefix = "${var.environment}-${var.project}"
|
|
create_asg_role = false
|
|
create-allpurpose-key = false
|
|
create-backup-key = false
|
|
create-database-key = false
|
|
create-eksebs-key = true
|
|
create-log-key = true
|
|
create-notify-key = false
|
|
create-secret-key = true
|
|
create-storage-key = false
|
|
} |