feat: experimenting with ThreadPoolExecutor

aws/AwsEnvReview.py

@@ -18,6 +18,7 @@ from datetime import date
 from mdutils.mdutils import MdUtils
 import os
 import json
+import concurrent.futures
 
 
 def printTitle(level: int, title: str):
@@ -40,7 +41,7 @@ def getAgeFromDate(inputDate):
 
 def printResult(content: list, header: str):
     if len(content) <= 0:
-        mdFile.new_paragraph("👏 No issue found.")
+        mdFile.new_paragraph("✅ No issue found.")
         return
     header = "Item," + header
     table = header.split(",")
@@ -53,19 +54,30 @@ def printResult(content: list, header: str):
     return
 
 
-print("Script started. It may take 7+ minutes to run. Report will be saved to AwsReviewReport.md. Please be patient...")
+def botoSession(region:str) -> boto3.Session:
+    session = boto3.Session(region_name=region)
+    return session
 
+
+# initialize md file output
 mdFile = MdUtils(file_name='AwsReviewReport.md', title='Aws Review ' + str(date.today()))
-sts = boto3.client("sts")
-aid = sts.get_caller_identity().get("Account")
-client = boto3.client('ec2', region_name="us-east-1")
-regions = getAllRegions(client)
-
-printTitle(3, f"Primary region: {os.environ['AWS_DEFAULT_REGION']}\n")
-
 mdFile.write("-" * 5)
 outTable = []
 
+print("Script started. It may take 7+ minutes to run. Report will be saved to AwsReviewReport.md. Please be patient...")
+printTitle(3, f"Primary region: {os.environ['AWS_DEFAULT_REGION']}\n")
+
+# create sessions
+globalSession = botoSession("us-east-1")
+localSession = botoSession(os.environ['AWS_DEFAULT_REGION'])
+
+# create clients
+sts = globalSession.client("sts")
+ec2Client = localSession.client("ec2")
+
+aid = sts.get_caller_identity().get("Account")
+regions = getAllRegions(ec2Client)
+
 """Check instances stopped for long time"""
 printTitle(1, "Ec2 service review")
 printTitle(2, "[Cost Optimization] Instances stopped for over 14 days")
@@ -73,7 +85,8 @@ printTitle(3, "Consider backing up and terminate instances "
               "or use AutoScalingGroup to spin up and down instances as needed.")
 
 for r in regions:
-    client = boto3.client('ec2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("ec2")
     response = client.describe_instances()
     if len(response.get("Reservations")) > 0:
         for i in jmespath.search("Reservations[*].Instances[*]", response):
@@ -88,7 +101,8 @@ printTitle(3, "Consider requiring IDMSv2. For more information, "
 outTable = []
 
 for r in regions:
-    client = boto3.client('ec2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("ec2")
     response = client.describe_instances()
     if len(response.get("Reservations")) > 0:
         for i in jmespath.search("Reservations[*].Instances[*]", response):
@@ -106,7 +120,8 @@ printTitle(2,"[Sustainability] Use of early generation instance type")
 printTitle(3, "Consider using current generation instances")
 outTable = []
 for r in regions:
-    client = boto3.client('ec2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("ec2")
     response = client.describe_instances()
     if len(response.get("Reservations")) > 0:
         for i in jmespath.search("Reservations[*].Instances[*]", response):
@@ -125,7 +140,8 @@ printTitle(2, "[Cost Optimization] Unattached EBS volumes")
 printTitle(3, "Consider backing up the volumes and delete them")
 outTable = []
 for r in regions:
-    client = boto3.client('ec2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("ec2")
     response = client.describe_volumes(
         Filters=[
             {
@@ -143,7 +159,8 @@ printTitle(2, "[Cost Optimization] EBS snapshots more than 365 days old")
 printTitle(3,"Consider removing snapshots if no longer needed")
 outTable = []
 for r in regions:
-    client = boto3.client('ec2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("ec2")
     response = client.describe_snapshots(
         OwnerIds=[aid]
     )
@@ -163,7 +180,8 @@ printTitle(3, "Consider replacing volume with encrypted ones. "
               "and snapshots afterwards.")
 outTable = []
 for r in regions:
-    client = boto3.client('ec2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("ec2")
     response = client.describe_volumes(
         Filters=[
             {
@@ -186,7 +204,8 @@ printTitle(3, "Consider deleting unused EIP")
 outTable = []
 
 for r in regions:
-    client = boto3.client('ec2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("ec2")
     response = client.describe_addresses()
     for i in response.get("Addresses"):
         if i.get("AssociationId") is None:
@@ -200,7 +219,8 @@ printTitle(3, "Consider setting more restrictive rules allowing access from spec
 outTable = []
 
 for r in regions:
-    client = boto3.client('ec2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("ec2")
     response = client.describe_security_group_rules()
     for sgr in jmespath.search("SecurityGroupRules[?IsEgress==`false`]", response):
         if (not sgr.get("IsEgress")
@@ -220,7 +240,8 @@ printTitle(3, "Consider encrypting RDS instances. For more detail, see "
               "https://docs.aws.amazon.com/prescriptive-guidance/latest/patterns/encrypt-an-existing-amazon-rds-for-postgresql-db-instance.html")
 outTable = []
 for r in regions:
-    client = boto3.client('rds', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("rds")
     response = client.describe_db_instances()
     for i in response.get("DBInstances"):
         if i.get("StorageEncrypted") == "False":
@@ -236,7 +257,8 @@ printTitle(2, "[Reliability] RDS instance running in single availability zone")
 printTitle(3, "Consider enabling multi-az for production use.")
 outTable = []
 for r in regions:
-    client = boto3.client('rds', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("rds")
     response = client.describe_db_instances()
     for i in response.get("DBInstances"):
         if not i.get("MultiAZ"):
@@ -247,13 +269,15 @@ for r in regions:
             outTable.append([r, aid, i.get("DBClusterIdentifier"), i.get("Engine")])
 printResult(outTable, "Region, AccountID, DBIdentifier, Engine")
 
+"""Check outdated lambda runtime"""
 printTitle(1, "Lambda service review")
 printTitle(2, "[Security] Outdated Lambda runtime")
 printTitle(3, "Consider changing to currently supported Lambda runtime versions, "
               "listed on https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html")
 outTable = []
 for r in regions:
-    client = boto3.client('lambda', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("lambda")
     response = client.list_functions()
     for i in response.get("Functions"):
         if i.get("Runtime") is not None:
@@ -267,7 +291,7 @@ printTitle(2, "[Security] Iam user access key not rotated for 180 days")
 printTitle(3, "Consider rotating access key")
 outTable = []
 
-client = boto3.client('iam', region_name="us-east-1")
+client = globalSession.client("iam")
 listUsers = client.list_users()
 users = jmespath.search("Users[*].UserName", listUsers)
 for u in users:
@@ -284,7 +308,7 @@ printTitle(3, "Consider granting minimum privileges "
   "https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_job-functions.html")
 outTable = []
 
-client = boto3.client('iam', region_name="us-east-1")
+client = globalSession.client("iam")
 entityResp = client.list_entities_for_policy(
     PolicyArn='arn:aws:iam::aws:policy/AdministratorAccess'
 )
@@ -303,7 +327,7 @@ printTitle(3, "Typically these roles should not have admin or iam permissions.")
 outTable = []
 
 # Get a list of roles
-client = boto3.client('lambda')
+client = localSession.client("lambda")
 roles = set()
 paginator = client.get_paginator('list_functions')
 for page in paginator.paginate():
@@ -322,7 +346,7 @@ for page in paginator.paginate():
             roles.add(role['RoleName'])
 
 # Need to remove non-existent roles
-iam_client = boto3.client('iam')
+iam_client = globalSession.client("iam")
 confirmed_roles = set()
 for role in roles:
     try:
@@ -351,7 +375,7 @@ high_risk_actions = {
 }
 
 # Check inline policies for each role
-client = boto3.client('iam', region_name="us-east-1")
+client = globalSession.client("iam")
 for role in roles:
     inline_policy_names = client.list_role_policies(RoleName=role)['PolicyNames']
     for policy_name in inline_policy_names:
@@ -402,7 +426,8 @@ printTitle(3, "Consider setting retention")
 outTable = []
 
 for r in regions:
-    client = boto3.client('logs', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("logs")
     response = client.describe_log_groups()
     for i in response.get("logGroups"):
         if i.get("retentionInDays") is None:
@@ -415,7 +440,8 @@ printTitle(3, "Consider encrypting LogGroups")
 outTable = []
 
 for r in regions:
-    client = boto3.client('logs', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("logs")
     response = client.describe_log_groups()
     for i in response.get("logGroups"):
         if i.get("kmsKeyId") is None:
@@ -428,7 +454,8 @@ printTitle(2, "[Reliability] Ec2/Rds instances found but AWSBackup plan missing"
 printTitle(3, "Consider setting up AWSBackup plans to backup AWS resources.")
 outTable = []
 for r in regions:
-    client = boto3.client('backup', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("backup")
     response = client.list_backup_plans()
     if len(response.get("BackupPlansList")) <= 0:
         ec2client = boto3.client("ec2", region_name=r)
@@ -448,7 +475,7 @@ printTitle(2, "[Security] S3 bucket policy missing")
 printTitle(3, "Consider creating bucket policy and restrict access to bucket")
 outTable = []
 
-client = boto3.client('s3', region_name="us-east-1")
+client = globalSession.client("s3")
 response = client.list_buckets()
 for i in jmespath.search("Buckets[*].Name", response):
     try:
@@ -462,7 +489,7 @@ printTitle(2, "[Security] S3 public access block")
 printTitle(3, "Blocking public access prevents accidental data leak due to misconfigurations in bucket policy or acl")
 
 # get account id
-sts = boto3.client("sts")
+sts = globalSession.client("sts")
 account_id = sts.get_caller_identity()["Account"]
 s3control = boto3.client("s3control")
 try:
@@ -482,7 +509,8 @@ printTitle(3, "Consider Graviton instances such as t4g/r7g to optimize your infr
 outTable = []
 
 for r in regions:
-    client = boto3.client('elasticache', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("elasticache")
     response = client.describe_cache_clusters()
     for i in response.get("CacheClusters"):
         if re.search("[0-9]g.", i.get("CacheNodeType")) is None:
@@ -496,7 +524,8 @@ printTitle(3, "Consider removing empty target groups")
 outTable = []
 
 for r in regions:
-    client = boto3.client('elbv2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("elbv2")
     response = client.describe_target_groups()
     for i in response.get("TargetGroups"):
         tgResp = client.describe_target_health(TargetGroupArn=i.get("TargetGroupArn"))
@@ -513,7 +542,8 @@ printTitle(3, "Consider enabling auto key rotation. When a key is rotated, previ
 outTable = []
 
 for r in regions:
-    client = boto3.client('kms', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("kms")
     response = client.list_keys()
     for i in jmespath.search("Keys[*].KeyId", response):
         try:
@@ -538,7 +568,8 @@ printTitle(3, "Consider restricting access to private API with a "
 outTable = []
 
 for r in regions:
-    client = boto3.client('apigateway', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("apigateway")
     response = client.get_rest_apis()
     for i in response.get("items"):
         if "PRIVATE" in i.get("endpointConfiguration").get("types") and len(i.get("policy")) <= 0:
@@ -552,7 +583,7 @@ printTitle(3, "Consider enabling encryption for cloudtrail")
 outTable = []
 
 #for r in regions:
-client = boto3.client('cloudtrail')
+client = localSession.client('cloudtrail')
 response = client.describe_trails()
 for i in response.get("trailList"):
     if i.get("KmsKeyId") is None:
@@ -566,7 +597,8 @@ outTable = []
 multiRegionTrailCount = 0
 
 for r in regions:
-    client = boto3.client('cloudtrail', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("cloudtrail")
     response = client.describe_trails()
     for i in response.get("trailList"):
         if i.get("IsMultiRegionTrail"):
@@ -583,7 +615,8 @@ printTitle(2, "[Security] VPC flow log not enabled")
 printTitle(3, "Consider enabling VPC flowlog for audit purpose or delete the default VPCs if not in use")
 
 for r in regions:
-    client = boto3.client('ec2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("ec2")
     response = client.describe_vpcs()
     vpc_ids = [vpc['VpcId'] for vpc in response['Vpcs']]
     for vpc_id in vpc_ids:
@@ -607,7 +640,8 @@ printTitle(3, "Consider deleting the default VPCs if not in use")
 outTable = []
 
 for r in regions:
-    client = boto3.client('ec2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("ec2")
     response = client.describe_vpcs(
         Filters=[{'Name': 'isDefault', 'Values': ['true']}]
     )
@@ -617,16 +651,26 @@ for r in regions:
 
 printResult(outTable, "Region, AccountID, DefaultVpcId")
 
-"""Check VPN endpoints"""
+"""Check VPC endpoints"""
+def check_vpc_endpoint(region: str):
+    fSession = botoSession(region=region)
+    fClient = fSession.client("ec2")
+    fResponse = fClient.describe_vpc_endpoints()
+    if len(fResponse['VpcEndpoints']) <= 0:
+        return [region, aid, "Vpc endpoint not found"]
+    else:
+        return None
+
 printTitle(2, "[Security] Use of VPC endpoints")
 printTitle(3, "Consider deploying VPC endpoints and connect to AWS api endpoints privately")
 outTable = []
 
-for r in regions:
-    client = boto3.client('ec2', region_name=r)
-    response = client.describe_vpc_endpoints()
-    if len(response['VpcEndpoints']) <= 0:
-       outTable.append([r, aid, "Vpc endpoint not found"])
+with concurrent.futures.ThreadPoolExecutor() as executor:
+    # Map returns results in the same order as regions
+    results = executor.map(check_vpc_endpoint, regions)
+    for result in results:
+        if result:
+            outTable.append(result)
 
 printResult(outTable, "Region, AccountID, VpcEndpointCount")
 
@@ -638,7 +682,8 @@ printTitle(3, "Consider having 2 tunnels for each site VPN connection. "
 outTable = []
 
 for r in regions:
-    client = boto3.client('ec2', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("ec2")
     response = client.describe_vpn_connections()
     for i in response.get("VpnConnections"):
         if len(jmespath.search("Options.TunnelOptions[*].OutsideIpAddress", i)) < 2:
@@ -651,7 +696,7 @@ printTitle(2, "[Security] Cloudfront origins allow public access")
 printTitle(3, "Your ALB is exposed to public, which bypass edge and WAF protection. Consider restricting access from Cloudfront only")
 outTable = []
 
-client = boto3.client('elbv2')
+client = localSession.client("elbv2")
 response = client.describe_load_balancers()
 alb_sgs = {}
 for alb in response['LoadBalancers']:
@@ -686,7 +731,8 @@ printTitle(3, "Consider using AmazonLinux2023. "
 outTable = []
 
 for r in regions:
-    client = boto3.client('eks', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("eks")
     response = client.list_clusters()
     for cluster in response.get("clusters"):
         ngsResp = client.list_nodegroups(clusterName=cluster)
@@ -708,7 +754,8 @@ printTitle(3, "Consider using upgrading Eks cluster. "
 outTable = []
 
 for r in regions:
-    client = boto3.client('eks', region_name=r)
+    newSession = botoSession(region=r)
+    client = newSession.client("eks")
     response = client.list_clusters()
     for cluster in response.get("clusters"):
         clusterResp = client.describe_cluster(name=cluster)