From a39c39a63688acd1d00aaef0b4d5c4178b187373 Mon Sep 17 00:00:00 2001 From: KenF Date: Fri, 23 Jan 2026 10:23:45 +0800 Subject: [PATCH] feat: added caching to aws region query --- .gitignore | 1 + aws-apigw.py | 25 +++++++++---- aws-ddb.py | 8 ++--- aws-ec2.py | 7 ++-- aws-ecs.py | 8 ++--- aws-efs.py | 8 ++--- aws-eks.py | 8 ++--- aws-elasticache.py | 8 ++--- aws-emr.py | 7 ++-- aws-lambda.py | 7 ++-- aws-lb.py | 8 ++--- aws-logs.py | 7 ++-- aws-opensearch.py | 7 ++-- aws-rds.py | 8 ++--- aws-subnets.py | 7 ++-- projectlib/__pycache__/aws.cpython-314.pyc | Bin 0 -> 1831 bytes projectlib/aws.py | 40 +++++++++++++++++++++ run-inventory-scripts.sh | 6 +++- 18 files changed, 91 insertions(+), 79 deletions(-) create mode 100644 projectlib/__pycache__/aws.cpython-314.pyc create mode 100644 projectlib/aws.py diff --git a/.gitignore b/.gitignore index f0014ae..fa0970d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ aws-inventory.xlsx +cache.pkl \ No newline at end of file diff --git a/aws-apigw.py b/aws-apigw.py index d016d7a..94d0285 100755 --- a/aws-apigw.py +++ b/aws-apigw.py @@ -8,6 +8,17 @@ License: This program is released under the MIT License # Imports import boto3 from openpyxl import load_workbook +import concurrent.futures +import projectlib.aws + + +def getResources(region_name: str) -> list[list[str | int]]: + return_data = [] + client = boto3.client('apigateway', region_name=region_name) + response = client.get_rest_apis() + for i in response['items']: + return_data.append([i["name"], i["endpointConfiguration"]["types"][0], region_name]) + return return_data # Main function def main() -> None: @@ -15,12 +26,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("ApiGateway") - client = boto3.client('apigateway') - response = client.get_rest_apis() - - ws.append(["RestAPIName","Scope"]) - for i in response['items']: - ws.append([i["name"], i["endpointConfiguration"]["types"][0]]) + ws.append(["RestAPIName","Scope","Region"]) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, projectlib.aws.getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-ddb.py b/aws-ddb.py index 69b27f5..e771eaa 100755 --- a/aws-ddb.py +++ b/aws-ddb.py @@ -10,11 +10,7 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet - -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] +import projectlib.aws def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -36,7 +32,7 @@ def main() -> None: ws.append(["TableName", "TableStatus", "TableSizeBytes", "ItemCount", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-ec2.py b/aws-ec2.py index 4a6c5a9..8129a26 100755 --- a/aws-ec2.py +++ b/aws-ec2.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -38,7 +35,7 @@ def main() -> None: ws.append(["InstanceId","NameTag","Platform","InstanceType","PrivateIpAddress","AZ"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-ecs.py b/aws-ecs.py index 50259ff..4e6dfea 100755 --- a/aws-ecs.py +++ b/aws-ecs.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('ecs', region_name=region_name) @@ -35,7 +31,7 @@ def main() -> None: ws.append(["clusterName", "runningTasksCount", "capacityProviders", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-efs.py b/aws-efs.py index cd281bc..77f3ad5 100755 --- a/aws-efs.py +++ b/aws-efs.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('efs', region_name=region_name) @@ -33,7 +29,7 @@ def main() -> None: ws.append(["FileSystemId", "Name", "PerformanceMode", "SizeMB", "AZ"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-eks.py b/aws-eks.py index 1c9d8d5..6fbcbaf 100755 --- a/aws-eks.py +++ b/aws-eks.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('eks', region_name=region_name) @@ -34,7 +30,7 @@ def main() -> None: ws.append(["name", "version", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-elasticache.py b/aws-elasticache.py index a9303c2..eb08bcd 100755 --- a/aws-elasticache.py +++ b/aws-elasticache.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('elasticache', region_name=region_name) @@ -33,7 +29,7 @@ def main() -> None: ws.append(["CacheClusterId", "CacheNodeType", "Engine", "EngineVersion", "NumCacheNodes", "AZ"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-emr.py b/aws-emr.py index ea6c499..a81c58c 100755 --- a/aws-emr.py +++ b/aws-emr.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -32,7 +29,7 @@ def main() -> None: ws.append(["Name", "Status", "NormalizedInstanceHours", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-lambda.py b/aws-lambda.py index dd2a8c9..7ff9c17 100755 --- a/aws-lambda.py +++ b/aws-lambda.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -34,7 +31,7 @@ def main() -> None: ws.append(["FunctionName","Runtime","Architectures","region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-lb.py b/aws-lb.py index b102936..dc7008e 100755 --- a/aws-lb.py +++ b/aws-lb.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False) -> list[str]: - ec2 = boto3.client('ec2', region_name='us-east-1') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('elbv2', region_name=region_name) @@ -33,7 +29,7 @@ def main() -> None: ws.append(["LoadBalancerName", "Scheme", "Type", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-logs.py b/aws-logs.py index bf8352e..20567e2 100755 --- a/aws-logs.py +++ b/aws-logs.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -32,7 +29,7 @@ def main() -> None: ws.append(["logGroupName","retentionInDays","storedBytes","region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-opensearch.py b/aws-opensearch.py index d1d1b43..54b7164 100755 --- a/aws-opensearch.py +++ b/aws-opensearch.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -32,7 +29,7 @@ def main() -> None: ws.append(["DomainName","EngineType","region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-rds.py b/aws-rds.py index 26ad628..db8a776 100755 --- a/aws-rds.py +++ b/aws-rds.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('rds', region_name=region_name) @@ -33,7 +29,7 @@ def main() -> None: ws.append(["DBInstanceIdentifier", "DBInstanceClass", "Engine", "EngineVersion", "MultiAZ", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-subnets.py b/aws-subnets.py index b294df4..496731e 100755 --- a/aws-subnets.py +++ b/aws-subnets.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -34,7 +31,7 @@ def main() -> None: ws.append(["SubnetId", "VpcId", "CidrBlock", "AvailabilityZone", "InDefaultVpc"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/projectlib/__pycache__/aws.cpython-314.pyc b/projectlib/__pycache__/aws.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..601f2e7f11ee871f5421fc2786e9ae5b27c846b5 GIT binary patch literal 1831 zcma)6-D@0G6hC)nc6Mhr`%zQe*f!0i8(Wrqtc{_yTMpu=iGDd zy}xtrJ*PjJkOAoybLqHCdiC%SO#J3d+o~=_!tDFS%M(LDn@aL-Pz> z32M6GD*m#eygfdnwCv@nHL0wXmyIE(VwDL(VvP~Q$=XfmqnC03h zIs9DbDb(qm?4-HlyYC?215V}HH^OccsRy!f5cbZ^C!1P4jYV(|_2LV}mnfcW;dO*r zVH!`{OS7OxPqzGbBC`=Swi`_4)EckD(%)zaT5su2@Zo3b;P+$o;du! zMos>oQQN;49Y8P44YybK{38al(JUy?5AB;rkVB%f%pcjs4 z{)hbx?$}(Bi?3B=HOv7|(JV{B2aLJ7gvsj-C}9ffidj<}vs|$ZWzodWw-xP@ZR%*} z82G|<<;>hWXlq&CkSrG*jioqBrPe+LxzNnAf=uJMe|w&C?!px1jq>oC6nKMJ4*bF6 zn0)c(x>8N68Ys$VEK6-7qR}>km>I??G@;zeLM~1@+oOV``O8!^E@LWtG(J6kc4D$v z8XKQ1Qo(kN8Wo#)3k1QW!GFVo&yy0pz7pei28*$v{{%+ z6=Ri>MH&fXU^%|zcuqHmpS??3SQ#I1?DzUwxS9dy?vHM3DWX1LEl$1J?&`2J8JB@`?J{t<+1KsqBMP zcKxIKsqCZF>H7Jt;lj749uF5ECC45N7uUs`7FwVZh5EVs$+0Jim+yBd_2Raa__T6o zWK&W$B<1dz?O5_g{CfOmaU<5<;6d(vDndtReX7xo2P4wIeosT{9BGKCdMqg|?VsLZ z^z}QTZxkZmAKy3HDg4lr8SRSPgZOA#y4MvS%}F^jmJ?~TRMPD-meGT_zygh literal 0 HcmV?d00001 diff --git a/projectlib/aws.py b/projectlib/aws.py new file mode 100644 index 0000000..76f6c08 --- /dev/null +++ b/projectlib/aws.py @@ -0,0 +1,40 @@ +#!/usr/bin/python3 +r""" +Documentation + +License: This program is released under the MIT License +""" + +# Imports +import boto3 +import pickle +import os + + +PICKLE_FILE = 'cache.pkl' + +def getRegions(all_regions=False) -> list[str]: + """ + Returns all aws regions, cached in pickle file to avoid repeated AWS calls. + + :param all_regions: + :return: + """ + cache_key = "aws_regions" + + if os.path.exists(PICKLE_FILE): + with open(PICKLE_FILE, 'rb') as f: + cache = pickle.load(f) + if cache_key in cache: + return cache[cache_key] + + # Fetch from AWS and cache it + ec2 = boto3.client('ec2', region_name='us-east-1') + response = ec2.describe_regions(AllRegions=all_regions) + regions = [region['RegionName'] for region in response['Regions']] + + cache = {cache_key: regions} + with open(PICKLE_FILE, 'wb') as f: + pickle.dump(cache, f) + + return regions \ No newline at end of file diff --git a/run-inventory-scripts.sh b/run-inventory-scripts.sh index dd45a9a..e327200 100755 --- a/run-inventory-scripts.sh +++ b/run-inventory-scripts.sh @@ -1,8 +1,12 @@ #!/bin/bash -rm -f aws-inventory.xlsx +# Remove previous inventory and cache file +rm -f aws-inventory.xlsx cache.pkl + +# Create a new workbook ./init_workbook.py +# Execute individual inventory scripts for i in aws*.py; do echo "# $i" ./"$i" | mlr --c2t cat | column -t