From eb68f146ed7a1bda8f3dc7f467bb66a274cd2166 Mon Sep 17 00:00:00 2001 From: KenF Date: Fri, 23 Jan 2026 08:20:08 +0800 Subject: [PATCH] refector: optimized concurrent.futures --- aws-ddb.py | 19 ++++++++----------- aws-ec2.py | 21 +++++++++------------ aws-ecs.py | 19 ++++++++----------- aws-efs.py | 19 ++++++++----------- aws-eks.py | 19 ++++++++----------- aws-elasticache.py | 21 +++++++++------------ aws-emr.py | 21 +++++++++------------ aws-lambda.py | 21 +++++++++------------ aws-lb.py | 25 +++++++++++-------------- aws-logs.py | 21 +++++++++------------ aws-opensearch.py | 21 +++++++++------------ aws-rds.py | 21 +++++++++------------ aws-subnets.py | 21 +++++++++------------ 13 files changed, 115 insertions(+), 154 deletions(-) diff --git a/aws-ddb.py b/aws-ddb.py index 8d52797..69b27f5 100755 --- a/aws-ddb.py +++ b/aws-ddb.py @@ -25,8 +25,7 @@ def getResources(region_name: str) -> list[list[str | int]]: for t in table_names: resp2 = client.describe_table(TableName=t) i = resp2['Table'] - if i: - return_data.append([i['TableName'], i['TableStatus'], i['TableSizeBytes'], i['ItemCount'], region_name]) + return_data.append([i['TableName'], i['TableStatus'], i['TableSizeBytes'], i['ItemCount'], region_name]) return return_data # Main function @@ -35,16 +34,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("DynamoDB") - final_data = [] ws.append(["TableName", "TableStatus", "TableSizeBytes", "ItemCount", "Region"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(getResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - for row in final_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-ec2.py b/aws-ec2.py index f015b3f..4a6c5a9 100755 --- a/aws-ec2.py +++ b/aws-ec2.py @@ -16,7 +16,7 @@ def getRegions(all_regions=False): response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('ec2', region_name=region_name) response = client.describe_instances() @@ -27,8 +27,7 @@ def printResources(region_name: str) -> list[list[str | int]]: if tag['Key'] == "Name": name_tag = tag['Value'] break - if i: - return_data.append([i['InstanceId'], name_tag, i.get('PlatformDetails'), i['InstanceType'], i['PrivateIpAddress'], i['Placement']['AvailabilityZone']]) + return_data.append([i['InstanceId'], name_tag, i.get('PlatformDetails'), i['InstanceType'], i['PrivateIpAddress'], i['Placement']['AvailabilityZone']]) return return_data # Main function @@ -37,16 +36,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("Ec2") - final_data = [] ws.append(["InstanceId","NameTag","Platform","InstanceType","PrivateIpAddress","AZ"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - for row in final_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-ecs.py b/aws-ecs.py index dbcb5db..50259ff 100755 --- a/aws-ecs.py +++ b/aws-ecs.py @@ -17,7 +17,7 @@ def getRegions(all_regions=False): response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('ecs', region_name=region_name) response = client.list_clusters() @@ -33,17 +33,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("Ecs") - final_data = [] ws.append(["clusterName", "runningTasksCount", "capacityProviders", "Region"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - clean_data = [row for row in final_data if row] - for row in clean_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-efs.py b/aws-efs.py index 9c87b06..cd281bc 100755 --- a/aws-efs.py +++ b/aws-efs.py @@ -17,7 +17,7 @@ def getRegions(all_regions=False): response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('efs', region_name=region_name) response = client.describe_file_systems() @@ -31,17 +31,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("EFS") - final_data = [] ws.append(["FileSystemId", "Name", "PerformanceMode", "SizeMB", "AZ"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - clean_data = [row for row in final_data if row] - for row in clean_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-eks.py b/aws-eks.py index 225b835..1c9d8d5 100755 --- a/aws-eks.py +++ b/aws-eks.py @@ -17,7 +17,7 @@ def getRegions(all_regions=False): response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('eks', region_name=region_name) response = client.list_clusters() @@ -32,17 +32,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("EKS") - final_data = [] ws.append(["name", "version", "Region"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - clean_data = [row for row in final_data if row] - for row in clean_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-elasticache.py b/aws-elasticache.py index 6bcb7e8..a9303c2 100755 --- a/aws-elasticache.py +++ b/aws-elasticache.py @@ -17,13 +17,12 @@ def getRegions(all_regions=False): response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('elasticache', region_name=region_name) response = client.describe_cache_clusters() for i in response['CacheClusters']: - if i: - return_data.append([i['CacheClusterId'], i['CacheNodeType'], i['Engine'], i['EngineVersion'], i['NumCacheNodes'], i['PreferredAvailabilityZone']]) + return_data.append([i['CacheClusterId'], i['CacheNodeType'], i['Engine'], i['EngineVersion'], i['NumCacheNodes'], i['PreferredAvailabilityZone']]) return return_data # Main function @@ -32,16 +31,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("Elasticache") - final_data = [] ws.append(["CacheClusterId", "CacheNodeType", "Engine", "EngineVersion", "NumCacheNodes", "AZ"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - for row in final_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-emr.py b/aws-emr.py index 82b256f..ea6c499 100755 --- a/aws-emr.py +++ b/aws-emr.py @@ -16,13 +16,12 @@ def getRegions(all_regions=False): response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('emr', region_name=region_name) response = client.list_clusters() for i in response['Clusters']: - if i: - return_data.append([i['Name'], i['Status'], i['NormalizedInstanceHours'], region_name]) + return_data.append([i['Name'], i['Status'], i['NormalizedInstanceHours'], region_name]) return return_data # Main function @@ -31,16 +30,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("EMR") - final_data = [] ws.append(["Name", "Status", "NormalizedInstanceHours", "Region"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - for row in final_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-lambda.py b/aws-lambda.py index d61a2c5..dd2a8c9 100755 --- a/aws-lambda.py +++ b/aws-lambda.py @@ -16,15 +16,14 @@ def getRegions(all_regions=False): response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('lambda', region_name=region_name) paginator = client.get_paginator('list_functions') for page in paginator.paginate(): functions = page.get('Functions', []) for i in functions: - if i: - return_data.append([i['FunctionName'], i['Runtime'], i['Architectures'][0], region_name]) + return_data.append([i['FunctionName'], i['Runtime'], i['Architectures'][0], region_name]) return return_data # Main function @@ -33,16 +32,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("Lambda") - final_data = [] ws.append(["FunctionName","Runtime","Architectures","region"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - for row in final_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-lb.py b/aws-lb.py index 4c96d8a..b102936 100755 --- a/aws-lb.py +++ b/aws-lb.py @@ -12,18 +12,17 @@ from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') +def getRegions(all_regions=False) -> list[str]: + ec2 = boto3.client('ec2', region_name='us-east-1') response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('elbv2', region_name=region_name) response = client.describe_load_balancers() for i in response['LoadBalancers']: - if i: - return_data.append([i['LoadBalancerName'], i['Scheme'], i['Type'], region_name]) + return_data.append([i['LoadBalancerName'], i['Scheme'], i['Type'], region_name]) return return_data # Main function @@ -32,16 +31,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("LoadBalancers") - final_data = [] ws.append(["LoadBalancerName", "Scheme", "Type", "Region"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - for row in final_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-logs.py b/aws-logs.py index f566948..bf8352e 100755 --- a/aws-logs.py +++ b/aws-logs.py @@ -16,13 +16,12 @@ def getRegions(all_regions=False): response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('logs', region_name=region_name) response = client.describe_log_groups() for logGroup in response['logGroups']: - if logGroup: - return_data.append([logGroup['logGroupName'], logGroup.get('retentionInDays'), logGroup['storedBytes'], region_name]) + return_data.append([logGroup['logGroupName'], logGroup.get('retentionInDays'), logGroup['storedBytes'], region_name]) return return_data # Main function @@ -31,16 +30,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("CloudwatchLogs") - final_data = [] ws.append(["logGroupName","retentionInDays","storedBytes","region"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - for row in final_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') # Call main function diff --git a/aws-opensearch.py b/aws-opensearch.py index c7e0253..d1d1b43 100755 --- a/aws-opensearch.py +++ b/aws-opensearch.py @@ -16,13 +16,12 @@ def getRegions(all_regions=False): response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('opensearch', region_name=region_name) response = client.list_domain_names() for i in response['DomainNames']: - if i: - return_data.append([i['DomainName'], i['EngineType'], region_name]) + return_data.append([i['DomainName'], i['EngineType'], region_name]) return return_data # Main function @@ -31,16 +30,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("OpenSearch") - final_data = [] ws.append(["DomainName","EngineType","region"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - for row in final_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-rds.py b/aws-rds.py index a73ad46..26ad628 100755 --- a/aws-rds.py +++ b/aws-rds.py @@ -17,13 +17,12 @@ def getRegions(all_regions=False): response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('rds', region_name=region_name) response = client.describe_db_instances() for i in response['DBInstances']: - if i: - return_data.append([i['DBInstanceIdentifier'], i['DBInstanceClass'], i['Engine'], i['EngineVersion'], i['MultiAZ'], region_name]) + return_data.append([i['DBInstanceIdentifier'], i['DBInstanceClass'], i['Engine'], i['EngineVersion'], i['MultiAZ'], region_name]) return return_data # Main function @@ -32,16 +31,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("RDS") - final_data = [] ws.append(["DBInstanceIdentifier", "DBInstanceClass", "Engine", "EngineVersion", "MultiAZ", "Region"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - for row in final_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-subnets.py b/aws-subnets.py index 60baa58..b294df4 100755 --- a/aws-subnets.py +++ b/aws-subnets.py @@ -16,15 +16,14 @@ def getRegions(all_regions=False): response = ec2.describe_regions(AllRegions=all_regions) return [region['RegionName'] for region in response['Regions']] -def printResources(region_name: str) -> list[list[str | int]]: +def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('ec2', region_name=region_name) response = client.describe_subnets() for i in response['Subnets']: default_vpc_check = client.describe_vpcs(VpcIds=[i['VpcId']]) for v in default_vpc_check['Vpcs']: - if i: - return_data.append([i['SubnetId'], i['VpcId'], i['CidrBlock'], i['AvailabilityZone'], v['IsDefault']]) + return_data.append([i['SubnetId'], i['VpcId'], i['CidrBlock'], i['AvailabilityZone'], v['IsDefault']]) return return_data # Main function @@ -33,16 +32,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("Subnets") - final_data = [] ws.append(["SubnetId", "VpcId", "CidrBlock", "AvailabilityZone", "InDefaultVpc"]) - with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor: - futures = [executor.submit(printResources, region_name=r) for r in getRegions()] - for future in concurrent.futures.as_completed(futures): - region_data = future.result() - final_data.extend(region_data) - - for row in final_data: - ws.append(row) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx')