chore: renamed layers

2026-02-13 17:48:17 +08:00
parent a11eaf25cc
commit 958e4977e9
15 changed files with 4 additions and 0 deletions
@@ -0,0 +1,64 @@
+# Post-install steps
+
+## Create lbc service account
+kubectl apply -f 1-lbc.yaml
+
+## Install AWS Load Balancer Controller in EKS
+helm repo add eks https://aws.github.io/eks-charts
+helm repo update
+
+helm install aws-load-balancer-controller eks/aws-load-balancer-controller \
+-n kube-system \
+--set clusterName=xpk-eks01-sunbird \
+--set serviceAccount.create=false \
+--set serviceAccount.name=aws-load-balancer-controller-sa
+
+kubectl -n kube-system get deployment aws-load-balancer-controller
+
+kubectl logs -n kube-system deployment/aws-load-balancer-controller -f
+
+## Allow web traffic to nodes
+Port 80 needs to be allowed on eks node's SGs. Then ALB can successfully register targets. This is now done in main.tf.
+
+## Testing
+ALB correctly sending traffic to nginx pods!
+
+```bash
+curl k8s-default-nginxing-a42064aa7e-1786392641.ap-east-1.elb.amazonaws.com
+<h1>Web Server nginx-web-f5988bf66-9lghc - Unique ID: </h1><p>Deployed on EKS Wed Feb 11 09:46:41 UTC 2026</p>
+
+curl k8s-default-nginxing-a42064aa7e-1786392641.ap-east-1.elb.amazonaws.com
+<h1>Web Server nginx-web-f5988bf66-6ptff - Unique ID: </h1><p>Deployed on EKS Wed Feb 11 09:46:41 UTC 2026</p>
+
+curl k8s-default-nginxing-a42064aa7e-1786392641.ap-east-1.elb.amazonaws.com
+<h1>Web Server nginx-web-f5988bf66-tw6rr - Unique ID: </h1><p>Deployed on EKS Wed Feb 11 09:46:45 UTC 2026</p>
+
+```
+
+## Notes on IPv6
+EKS could not be deployed on ipv6-only private subnets. It appears AWS requires at least 2 free IPv4 addresses in the subnet.
+I tried and the following error was returned.
+
+```
+Error: creating EKS Cluster (xpk-eks01-akita): operation error EKS: CreateCluster, https response error StatusCode: 400, 
+RequestID: b25794cc-3220-4393-a435-c92e2f8aafdd, InvalidParameterException: Atleast one subnet in each AZ should have 2 free IPs. 
+Invalid AZs: { [ap-east-1c, ap-east-1b] }, provided subnets: { subnet-02aaf75a3e4700f74, subnet-02071b29e2883d5b1 }
+```
+
+## Notes on KMS key
+I tried using aws-managed key for EKS, but it failed to deploy with an error.
+
+```hcl
+  encryption_config = {
+    provider_key_arn = "arn:aws:kms:${data.aws_region.this.id}:${data.aws_caller_identity.current.account_id}:alias/aws/secretsmanager"
+    resources        = ["secrets"]
+  }
+```
+
+```
+ Error: creating EKS Cluster (xpk-eks01-vervet): operation error EKS: CreateCluster, https response error StatusCode: 400, RequestID: 
+ 0b866e07-352a-439c-9196-f7a671bdd0ee, api error InvalidRequestException: User not authorized to perform kms:CreateGrant operation
+```
+
+When I used ```create_kms_key = true```, EKS was created successfully. I can see that the EKS cluster role is explicitly allowed
+in the key policy.
@@ -0,0 +1,146 @@
+module "BastionRole" {
+  source                  = "../../modules/security_identity_compliance/iam-role-v2"
+  description             = "EKS bastion instance profile"
+  role-name               = "BastionInstanceProfile"
+  trusted-entity          = "ec2.amazonaws.com"
+  create-instance-profile = true
+  policies = {
+    EksAdmin = {
+      description = "Eks read permissions required for kubectl"
+      policy = jsonencode(
+        {
+          "Statement" : [
+            {
+              "Sid" : "EksRead",
+              "Action" : [
+                "eks:Describe*",
+                "eks:List*"
+              ],
+              "Effect" : "Allow",
+              "Resource" : "*"
+            }
+          ],
+          "Version" : "2012-10-17"
+        }
+      )
+    }
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "BastionProfilePermissions" {
+  role       = module.BastionRole.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+}
+
+module "eks-bastion" {
+  depends_on = [module.eks] # essential for initializing kubectl in userdata
+  source     = "../../modules/compute/ec2"
+
+  additional-tags  = {}
+  ami-id           = data.aws_ami.this.id
+  asso-eip         = false
+  asso-public-ip   = true
+  use-ipv6         = true
+  data-volumes     = {}
+  ebs-encrypted    = true
+  instance-name    = "${var.environment}-eks-bastion-${random_pet.pet.id}"
+  instance-type    = "t4g.micro"
+  key-name         = aws_key_pair.kp.key_name
+  kms-key-id       = ""
+  root-volume-size = "8"
+  # security-groups  = [module.bastion-sg.id, module.eks.cluster_primary_security_group_id]
+  security-groups  = [module.bastion-sg.id]
+  subnet-id        = module.vpc.public_subnets[0]
+  instance-profile = module.BastionRole.profile-name[0]
+  spot-max-price   = 0.0116 # t4g.micro
+  user-data        = <<EOF
+#!/bin/bash
+# eks bastion setup
+## Install git
+dnf -y install git
+
+## Install kubectl
+curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/arm64/kubectl
+chmod +x kubectl
+mv kubectl /usr/local/bin/
+
+## Install helm
+cd /tmp
+wget -O/tmp/helm.tgz https://get.helm.sh/helm-v4.1.1-linux-arm64.tar.gz
+tar zxf /tmp/helm.tgz
+mv /tmp/linux-arm64/helm /usr/local/bin/helm
+chmod +x /usr/local/bin/helm
+
+## Install eksctl
+cd /tmp
+ARCH=arm64
+PLATFORM=$(uname -s)_$ARCH
+curl -sLO "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_$PLATFORM.tar.gz"
+tar zxf eksctl_Linux_arm64.tar.gz
+mv eksctl /usr/local/bin
+chmod +x /usr/local/bin/eksctl
+
+## Create kube config
+echo Create kube config...
+/usr/bin/aws eks update-kubeconfig --name ${var.eks_cluster_name}-${random_pet.pet.id}
+# echo Sleep for 5 minutes and wait for fargate profile to come up
+# /usr/bin/sleep 300
+#
+# ## Grant EKS console access to IAM role: must be executed with cluster creator's identity. cluster role as instance profile won't do it
+# echo Patching configmap/aws-auth...
+# ROLE="    - rolearn: arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/rackLE\n      username: build\n      groups:\n        - system:masters"
+# /usr/local/bin/kubectl --kubeconfig=/root/.kube/config get -n kube-system configmap/aws-auth -o yaml | awk "/mapRoles: \|/{print;print \"$ROLE\";next}1" > /tmp/aws-auth-patch.yml
+# /usr/local/bin/kubectl --kubeconfig=/root/.kube/config patch configmap/aws-auth -n kube-system --patch "$(cat /tmp/aws-auth-patch.yml)"
+# /usr/local/bin/kubectl --kubeconfig=/root/.kube/config get -n kube-system configmap/aws-auth -o yaml
+EOF
+}
+
+data "aws_ami" "this" {
+  most_recent = true
+  name_regex  = "^al2023-ami-2023.*-kernel-6.1-arm64"
+  owners      = ["amazon"]
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "architecture"
+    values = ["arm64"]
+  }
+}
+
+resource "tls_private_key" "sshkey" {
+  algorithm = "ED25519"
+}
+
+resource "aws_key_pair" "kp" {
+  key_name   = "${var.environment}-eks-bastion-${random_pet.pet.id}-key"
+  public_key = tls_private_key.sshkey.public_key_openssh
+}
+
+module "bastion-sg" {
+  source = "../../modules/compute/security_group"
+
+  description = "${var.environment}-eks-bastion-${random_pet.pet.id}-sg"
+  egress = {
+    r1 = "-1,-1,-1,0.0.0.0/0,Allow egress"
+  }
+  ingress = {
+    r1 = "tcp,22,22,0.0.0.0/0,ssh"
+  }
+  name   = "eks-bastion-${random_pet.pet.id}-sg"
+  vpc-id = module.vpc.vpc_id
+}
+
+# my security_group module does not support ipv6_cidr_blocks
+resource "aws_security_group_rule" "ipv6_egress" {
+  security_group_id = module.bastion-sg.id
+  type              = "egress"
+  from_port         = -1
+  to_port           = -1
+  protocol          = "all"
+  ipv6_cidr_blocks  = ["::/0"]
+  description       = "Allow ipv6 egress"
+}
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aws-load-balancer-controller-sa
+  namespace: kube-system
@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-web
+spec:
+  replicas: 10
+  selector:
+    matchLabels:
+      app: nginx-web
+  template:
+    metadata:
+      labels:
+        app: nginx-web
+      annotations:
+        # Require dedicated ENI per pod
+        vpc.cni.amazonaws.com/network-mode: "IPV4"
+        vpc.cni.amazonaws.com/eniMode: "per-pod"      # One ENI per pod
+        vpc.cni.amazonaws.com/eniPrefixMode: "GLOBAL"  # Prefix mode for efficiency
+    spec:
+      initContainers:
+        - name: unique-index
+          image: busybox:1.35
+          command: ['sh', '-c']
+          args:
+            - |
+              echo "<h1>Web Server $(POD_NAME)</h1><p>Deployed at $(date)</p>" > /usr/share/nginx/html/index.html
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          volumeMounts:
+            - name: nginx-html
+              mountPath: /usr/share/nginx/html
+      containers:
+        - name: nginx
+          image: nginx:1.27-alpine
+          ports:
+            - containerPort: 80
+          volumeMounts:
+            - name: nginx-html
+              mountPath: /usr/share/nginx/html
+              readOnly: true
+      volumes:
+        - name: nginx-html
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+spec:
+  selector:
+    app: nginx-web
+  ports:
+    - port: 80
+      targetPort: 80
+  type: ClusterIP
@@ -0,0 +1,21 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: aln-ingress-nginx-service
+  annotations:
+    alb.ingress.kubernetes.io/scheme: internet-facing
+    alb.ingress.kubernetes.io/ip-address-type: dualstack
+    alb.ingress.kubernetes.io/healthcheck-path: /
+    alb.ingress.kubernetes.io/target-type: ip
+spec:
+  ingressClassName: alb
+  rules:
+    - http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: nginx-service
+                port:
+                  number: 80
@@ -0,0 +1,297 @@
+/**
+* # eks-ipv6-nginxpod
+*
+* ## Features
+* - Use terraform-aws-eks to deploy eks cluster and a nodegroup using spot instances
+* - Use Ipv6 for eks cluster
+* - Dependent VPC and roles are created
+* - use pod identity for EBS abd loadbalancer controller
+* - Create a bastion to manage EKS cluster
+*
+*
+ */
+data "aws_region" "this" {}
+
+# Eks Vpc on IPv6
+resource "random_pet" "pet" {
+  length = 1
+}
+
+locals {
+  vpc_cidr = "10.18.0.0/16"
+  # ensure there is room for future expansion
+  private_net_start = cidrsubnet(local.vpc_cidr, 2, 1)
+  public_net_start  = cidrsubnet(local.vpc_cidr, 2, 2)
+}
+
+data "aws_availability_zones" "this" {
+  state = "available"
+}
+
+resource "random_shuffle" "Select2Az" {
+  input        = data.aws_availability_zones.this.names
+  result_count = 2
+}
+
+module "vpc" {
+  source  = "terraform-aws-modules/vpc/aws"
+  version = "6.6.0"
+
+  name = "lab-vpc"
+  cidr = local.vpc_cidr
+
+  azs                                            = random_shuffle.Select2Az.result
+  enable_ipv6                                    = true
+  public_subnet_assign_ipv6_address_on_creation  = true
+  private_subnet_assign_ipv6_address_on_creation = true
+  # private_subnet_ipv6_native                     = true # EKS requires free IPv4 addresses. see README
+  private_subnets              = cidrsubnets(local.private_net_start, 4, 4) # EKS requires free IPv4 addresses. see README
+  public_subnets               = cidrsubnets(local.public_net_start, 8, 8)  # 2 AZ required by eks lbc
+  public_subnet_ipv6_prefixes  = [0, 1]
+  private_subnet_ipv6_prefixes = [10, 11]
+  public_subnet_tags = {
+    "kubernetes.io/role/elb" = 1
+  }
+
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+
+  # nat gateway and eigw (vpc module creates the dns64 /64 route to NGW)
+  enable_nat_gateway     = true # AWS public endpoints do not support IPv6
+  single_nat_gateway     = true
+  create_egress_only_igw = true
+
+  enable_flow_log                      = false
+  create_flow_log_cloudwatch_log_group = false
+  create_flow_log_cloudwatch_iam_role  = false
+  manage_default_network_acl           = false
+}
+
+# EKS resources
+module "CsiPodIdentity" {
+  source      = "../../modules/security_identity_compliance/iam-role-v2"
+  description = "EKSCSIDriverRole"
+  role-name   = "AmazonEBSCSIDriverRole"
+  trusted-entity = jsonencode(
+    {
+      "Version" : "2012-10-17",
+      "Statement" : [
+        {
+          "Effect" : "Allow",
+          "Principal" : {
+            "Service" : "pods.eks.amazonaws.com"
+          },
+          "Action" : [
+            "sts:AssumeRole",
+            "sts:TagSession"
+          ]
+        }
+      ]
+    }
+  )
+}
+
+# 2 policies are required for the ebs csi to work
+resource "aws_iam_role_policy_attachment" "CsiPodIdentity" {
+  for_each = toset([
+    "arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess",
+    "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"
+  ])
+  role       = module.CsiPodIdentity.name
+  policy_arn = each.value
+}
+
+locals {
+  userdata = <<EOT
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="//"
+
+--//
+Content-Type: application/node.eks.aws
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    apiServerEndpoint: ${module.eks.cluster_endpoint}
+    certificateAuthority: ${module.eks.cluster_certificate_authority_data}
+    cidr: ${module.eks.cluster_service_cidr}
+    name: ${module.eks.cluster_name}
+  kubelet:
+    config:
+      maxPods: 110
+      clusterDNS:
+      - ${replace(module.eks.cluster_service_cidr, "/\\/.*/", "a")}
+
+--//--
+EOT
+}
+
+resource "aws_launch_template" "node_lt" {
+  name                   = "eks135-node-template"
+  description            = "Launch template for eks 1.35"
+  vpc_security_group_ids = [module.eks.node_security_group_id]
+  update_default_version = true
+
+  # Critical: Set hop limit to 2 for pod IMDS access, required for aws lbc
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_tokens                 = "required" # IMDSv2 required
+    http_put_response_hop_limit = 2          # Allows pods to reach IMDS
+    instance_metadata_tags      = "enabled"
+  }
+
+  block_device_mappings {
+    device_name = "/dev/xvda"
+    ebs {
+      volume_size = 20
+      volume_type = "gp3"
+    }
+  }
+  # must not specify this # image_id  = data.aws_ami.eks_worker.id
+  user_data = base64encode(local.userdata)
+  tag_specifications {
+    resource_type = "instance"
+    tags = {
+      Name = "${module.eks.cluster_name}-worker"
+    }
+  }
+  tag_specifications {
+    resource_type = "volume"
+    tags = {
+      Name = "${module.eks.cluster_name}-worker"
+    }
+  }
+}
+
+# eks optimized ami
+# data "aws_ami" "eks_worker" {
+#   name_regex  = "amazon-eks-node-al2023-x86_64-standard-1\\.35.*"
+#   owners      = ["800184023465"]
+#   most_recent = true
+# }
+
+module "eks" {
+  source = "terraform-aws-modules/eks/aws"
+  # version                         = "20.34.0"
+  create_iam_role    = true
+  name               = "${var.eks_cluster_name}-${random_pet.pet.id}"
+  kubernetes_version = "1.35"
+  # enabled_log_types           = ["api", "audit", "authenticator", "controllerManager", "scheduler"]
+  create_security_group = true
+  security_group_additional_rules = {
+    bastion_access = {
+      description              = "Allow access from bastion"
+      protocol                 = "tcp"
+      from_port                = 443
+      to_port                  = 443
+      type                     = "ingress"
+      source_security_group_id = module.bastion-sg.id
+    }
+  }
+  vpc_id                      = module.vpc.vpc_id
+  subnet_ids                  = module.vpc.private_subnets
+  ip_family                   = "ipv6"
+  create_cni_ipv6_iam_policy  = true
+  create_kms_key              = true
+  endpoint_private_access     = true
+  endpoint_public_access      = false
+  enable_irsa                 = false
+  create_cloudwatch_log_group = false
+  create_node_security_group  = true
+  # authentication_mode         = "API_AND_CONFIG_MAP" # use access entries and leave this to default
+  upgrade_policy = {
+    support_type = "STANDARD"
+  }
+
+  addons = {
+    coredns = {}
+    eks-pod-identity-agent = {
+      before_compute = true
+    }
+    kube-proxy = {}
+    aws-ebs-csi-driver = {
+      pod_identity_association = [{
+        role_arn        = module.CsiPodIdentity.role-arn
+        service_account = "ebs-csi-controller-sa"
+      }]
+    }
+    vpc-cni = {
+      before_compute = true
+      configuration_values = jsonencode({
+        env = {
+          ENABLE_POD_ENI                    = "true",
+          POD_SECURITY_GROUP_ENFORCING_MODE = "strict",
+          # in prefix mode, ipv6 will have /80 and ipv4 will have /28
+          ENABLE_PREFIX_DELEGATION = "true"
+        },
+        init = {
+          env = {
+            DISABLE_TCP_EARLY_DEMUX = "true"
+          }
+        }
+      })
+    }
+  }
+
+  node_iam_role_additional_policies = {
+    SsmManaged = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+  }
+
+  eks_managed_node_groups = {
+    EksNodeGroup1 = {
+      # required for setting hop limit to 2 for pod IMDS access, required for aws lbc
+      create_launch_template     = false
+      use_custom_launch_template = true
+      launch_template_id         = aws_launch_template.node_lt.id
+      launch_template_version    = aws_launch_template.node_lt.latest_version
+
+      min_size     = 2
+      max_size     = 2
+      desired_size = 2
+
+      instance_types = ["t3.large"]
+      capacity_type  = "SPOT"
+      subnet_ids     = module.vpc.private_subnets
+    }
+  }
+
+  access_entries = {
+    ClusterAdminRole = {
+      principal_arn = "arn:aws:iam::040216112220:role/rackLE"
+      policy_associations = {
+        ClusterAdminPolicy = {
+          policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
+          access_scope = {
+            type = "cluster"
+          }
+        }
+      }
+    }
+    BastionRole = {
+      principal_arn = module.BastionRole.role-arn
+      policy_associations = {
+        ClusterAdminPolicy = {
+          policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
+          access_scope = {
+            type = "cluster"
+          }
+        }
+      }
+    }
+  }
+}
+
+
+
+# Allow http traffic from ALB to eks node
+resource "aws_security_group_rule" "eks_node_alb_ingress" {
+  type              = "ingress"
+  from_port         = 80
+  to_port           = 80
+  protocol          = "tcp"
+  security_group_id = module.eks.node_security_group_id
+  ipv6_cidr_blocks  = [module.vpc.vpc_ipv6_cidr_block]
+  description       = "ALB to nginx pods port 80"
+}
@@ -0,0 +1,14 @@
+# # https://github.com/terraform-aws-modules/terraform-aws-eks-pod-identity
+module "aws_lb_controller_pod_identity" {
+  source = "terraform-aws-modules/eks-pod-identity/aws"
+
+  name                            = "aws-loadbalancer-controller"
+  attach_aws_lb_controller_policy = true
+  associations = {
+    this = {
+      cluster_name    = module.eks.cluster_name
+      namespace       = "kube-system"
+      service_account = "aws-load-balancer-controller-sa"
+    }
+  }
+}
@@ -0,0 +1,35 @@
+provider "aws" {
+  region = var.aws-region
+
+  default_tags {
+    tags = {
+      ServiceProvider = "RackspaceTechnology"
+      Environment     = var.environment
+      Project         = var.project
+      Application     = var.application
+      TerraformDir    = join("/", reverse(slice(reverse(split("/", path.cwd)), 0, 2)))
+    }
+  }
+}
+
+terraform {
+  required_version = "~> 1.13.0"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.0"
+    }
+  }
+
+  /*
+  backend "s3" {
+    bucket         = "whk1-bea-sys-ss-prd-tfgen2-state1"
+    key            = "terraform_state/LandingZone/master-payer/sso.tfstate"
+    region         = "ap-east-1"
+    dynamodb_table = "whk1-bea-sys-ss-prd-tfgen2-lock"
+    encrypt        = true
+  }
+  */
+}
+
+data aws_caller_identity current {}
@@ -0,0 +1,12 @@
+variable "aws-region" {}
+variable "aws-region-short" {}
+variable "customer-name" {}
+variable "environment" {}
+variable "project" {}
+variable "application" {}
+variable "eks_master_user_arn" {}
+
+variable "eks_cluster_name" {
+  type    = string
+  default = "xpk-eks01"
+}