refector: optimized concurrent.futures

This commit is contained in:
KenF
2026-01-23 08:20:08 +08:00
parent 675f0f6ef3
commit eb68f146ed
13 changed files with 115 additions and 154 deletions
+8 -11
View File
@@ -25,8 +25,7 @@ def getResources(region_name: str) -> list[list[str | int]]:
for t in table_names:
resp2 = client.describe_table(TableName=t)
i = resp2['Table']
if i:
return_data.append([i['TableName'], i['TableStatus'], i['TableSizeBytes'], i['ItemCount'], region_name])
return_data.append([i['TableName'], i['TableStatus'], i['TableSizeBytes'], i['ItemCount'], region_name])
return return_data
# Main function
@@ -35,16 +34,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("DynamoDB")
final_data = []
ws.append(["TableName", "TableStatus", "TableSizeBytes", "ItemCount", "Region"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(getResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
for row in final_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
+9 -12
View File
@@ -16,7 +16,7 @@ def getRegions(all_regions=False):
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('ec2', region_name=region_name)
response = client.describe_instances()
@@ -27,8 +27,7 @@ def printResources(region_name: str) -> list[list[str | int]]:
if tag['Key'] == "Name":
name_tag = tag['Value']
break
if i:
return_data.append([i['InstanceId'], name_tag, i.get('PlatformDetails'), i['InstanceType'], i['PrivateIpAddress'], i['Placement']['AvailabilityZone']])
return_data.append([i['InstanceId'], name_tag, i.get('PlatformDetails'), i['InstanceType'], i['PrivateIpAddress'], i['Placement']['AvailabilityZone']])
return return_data
# Main function
@@ -37,16 +36,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("Ec2")
final_data = []
ws.append(["InstanceId","NameTag","Platform","InstanceType","PrivateIpAddress","AZ"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
for row in final_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
+8 -11
View File
@@ -17,7 +17,7 @@ def getRegions(all_regions=False):
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('ecs', region_name=region_name)
response = client.list_clusters()
@@ -33,17 +33,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("Ecs")
final_data = []
ws.append(["clusterName", "runningTasksCount", "capacityProviders", "Region"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
clean_data = [row for row in final_data if row]
for row in clean_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
+8 -11
View File
@@ -17,7 +17,7 @@ def getRegions(all_regions=False):
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('efs', region_name=region_name)
response = client.describe_file_systems()
@@ -31,17 +31,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("EFS")
final_data = []
ws.append(["FileSystemId", "Name", "PerformanceMode", "SizeMB", "AZ"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
clean_data = [row for row in final_data if row]
for row in clean_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
+8 -11
View File
@@ -17,7 +17,7 @@ def getRegions(all_regions=False):
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('eks', region_name=region_name)
response = client.list_clusters()
@@ -32,17 +32,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("EKS")
final_data = []
ws.append(["name", "version", "Region"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
clean_data = [row for row in final_data if row]
for row in clean_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
+9 -12
View File
@@ -17,13 +17,12 @@ def getRegions(all_regions=False):
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('elasticache', region_name=region_name)
response = client.describe_cache_clusters()
for i in response['CacheClusters']:
if i:
return_data.append([i['CacheClusterId'], i['CacheNodeType'], i['Engine'], i['EngineVersion'], i['NumCacheNodes'], i['PreferredAvailabilityZone']])
return_data.append([i['CacheClusterId'], i['CacheNodeType'], i['Engine'], i['EngineVersion'], i['NumCacheNodes'], i['PreferredAvailabilityZone']])
return return_data
# Main function
@@ -32,16 +31,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("Elasticache")
final_data = []
ws.append(["CacheClusterId", "CacheNodeType", "Engine", "EngineVersion", "NumCacheNodes", "AZ"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
for row in final_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
+9 -12
View File
@@ -16,13 +16,12 @@ def getRegions(all_regions=False):
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('emr', region_name=region_name)
response = client.list_clusters()
for i in response['Clusters']:
if i:
return_data.append([i['Name'], i['Status'], i['NormalizedInstanceHours'], region_name])
return_data.append([i['Name'], i['Status'], i['NormalizedInstanceHours'], region_name])
return return_data
# Main function
@@ -31,16 +30,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("EMR")
final_data = []
ws.append(["Name", "Status", "NormalizedInstanceHours", "Region"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
for row in final_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
+9 -12
View File
@@ -16,15 +16,14 @@ def getRegions(all_regions=False):
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('lambda', region_name=region_name)
paginator = client.get_paginator('list_functions')
for page in paginator.paginate():
functions = page.get('Functions', [])
for i in functions:
if i:
return_data.append([i['FunctionName'], i['Runtime'], i['Architectures'][0], region_name])
return_data.append([i['FunctionName'], i['Runtime'], i['Architectures'][0], region_name])
return return_data
# Main function
@@ -33,16 +32,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("Lambda")
final_data = []
ws.append(["FunctionName","Runtime","Architectures","region"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
for row in final_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
+11 -14
View File
@@ -12,18 +12,17 @@ from openpyxl import load_workbook
from openpyxl.worksheet.worksheet import Worksheet
def getRegions(all_regions=False):
ec2 = boto3.client('ec2')
def getRegions(all_regions=False) -> list[str]:
ec2 = boto3.client('ec2', region_name='us-east-1')
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('elbv2', region_name=region_name)
response = client.describe_load_balancers()
for i in response['LoadBalancers']:
if i:
return_data.append([i['LoadBalancerName'], i['Scheme'], i['Type'], region_name])
return_data.append([i['LoadBalancerName'], i['Scheme'], i['Type'], region_name])
return return_data
# Main function
@@ -32,16 +31,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("LoadBalancers")
final_data = []
ws.append(["LoadBalancerName", "Scheme", "Type", "Region"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
for row in final_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
+9 -12
View File
@@ -16,13 +16,12 @@ def getRegions(all_regions=False):
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('logs', region_name=region_name)
response = client.describe_log_groups()
for logGroup in response['logGroups']:
if logGroup:
return_data.append([logGroup['logGroupName'], logGroup.get('retentionInDays'), logGroup['storedBytes'], region_name])
return_data.append([logGroup['logGroupName'], logGroup.get('retentionInDays'), logGroup['storedBytes'], region_name])
return return_data
# Main function
@@ -31,16 +30,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("CloudwatchLogs")
final_data = []
ws.append(["logGroupName","retentionInDays","storedBytes","region"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
for row in final_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
# Call main function
+9 -12
View File
@@ -16,13 +16,12 @@ def getRegions(all_regions=False):
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('opensearch', region_name=region_name)
response = client.list_domain_names()
for i in response['DomainNames']:
if i:
return_data.append([i['DomainName'], i['EngineType'], region_name])
return_data.append([i['DomainName'], i['EngineType'], region_name])
return return_data
# Main function
@@ -31,16 +30,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("OpenSearch")
final_data = []
ws.append(["DomainName","EngineType","region"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
for row in final_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
+9 -12
View File
@@ -17,13 +17,12 @@ def getRegions(all_regions=False):
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('rds', region_name=region_name)
response = client.describe_db_instances()
for i in response['DBInstances']:
if i:
return_data.append([i['DBInstanceIdentifier'], i['DBInstanceClass'], i['Engine'], i['EngineVersion'], i['MultiAZ'], region_name])
return_data.append([i['DBInstanceIdentifier'], i['DBInstanceClass'], i['Engine'], i['EngineVersion'], i['MultiAZ'], region_name])
return return_data
# Main function
@@ -32,16 +31,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("RDS")
final_data = []
ws.append(["DBInstanceIdentifier", "DBInstanceClass", "Engine", "EngineVersion", "MultiAZ", "Region"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
for row in final_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')
+9 -12
View File
@@ -16,15 +16,14 @@ def getRegions(all_regions=False):
response = ec2.describe_regions(AllRegions=all_regions)
return [region['RegionName'] for region in response['Regions']]
def printResources(region_name: str) -> list[list[str | int]]:
def getResources(region_name: str) -> list[list[str | int]]:
return_data = []
client = boto3.client('ec2', region_name=region_name)
response = client.describe_subnets()
for i in response['Subnets']:
default_vpc_check = client.describe_vpcs(VpcIds=[i['VpcId']])
for v in default_vpc_check['Vpcs']:
if i:
return_data.append([i['SubnetId'], i['VpcId'], i['CidrBlock'], i['AvailabilityZone'], v['IsDefault']])
return_data.append([i['SubnetId'], i['VpcId'], i['CidrBlock'], i['AvailabilityZone'], v['IsDefault']])
return return_data
# Main function
@@ -33,16 +32,14 @@ def main() -> None:
wb = load_workbook('aws-inventory.xlsx')
ws = wb.create_sheet("Subnets")
final_data = []
ws.append(["SubnetId", "VpcId", "CidrBlock", "AvailabilityZone", "InDefaultVpc"])
with concurrent.futures.ProcessPoolExecutor(max_workers=6) as executor:
futures = [executor.submit(printResources, region_name=r) for r in getRegions()]
for future in concurrent.futures.as_completed(futures):
region_data = future.result()
final_data.extend(region_data)
for row in final_data:
ws.append(row)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
results = executor.map(getResources, getRegions())
for region_rows in results:
# append to worksheet only if resoruces are found in the region
if region_rows:
for row in region_rows:
ws.append(row)
wb.save('aws-inventory.xlsx')