feat: added caching to aws region query

.gitignore

@@ -1 +1,2 @@
 aws-inventory.xlsx
+cache.pkl

aws-apigw.py

@@ -8,6 +8,17 @@ License: This program is released under the MIT License
 # Imports
 import boto3
 from openpyxl import load_workbook
+import concurrent.futures
+import projectlib.aws
+
+
+def getResources(region_name: str) -> list[list[str | int]]:
+    return_data = []
+    client = boto3.client('apigateway', region_name=region_name)
+    response = client.get_rest_apis()
+    for i in response['items']:
+        return_data.append([i["name"], i["endpointConfiguration"]["types"][0], region_name])
+    return return_data
 
 # Main function
 def main() -> None:
@@ -15,12 +26,14 @@ def main() -> None:
     wb = load_workbook('aws-inventory.xlsx')
     ws = wb.create_sheet("ApiGateway")
 
-    client = boto3.client('apigateway')
-    response = client.get_rest_apis()
-
-    ws.append(["RestAPIName","Scope"])
-    for i in response['items']:
-        ws.append([i["name"], i["endpointConfiguration"]["types"][0]])
+    ws.append(["RestAPIName","Scope","Region"])
+    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
+        results = executor.map(getResources, projectlib.aws.getRegions())
+        for region_rows in results:
+            # append to worksheet only if resoruces are found in the region
+            if region_rows:
+                for row in region_rows:
+                    ws.append(row)
 
     wb.save('aws-inventory.xlsx')
 

aws-ddb.py

@@ -10,11 +10,7 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
-
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
+import projectlib.aws
 
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
@@ -36,7 +32,7 @@ def main() -> None:
 
     ws.append(["TableName", "TableStatus", "TableSizeBytes", "ItemCount", "Region"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-ec2.py

@@ -10,11 +10,8 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
 
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
@@ -38,7 +35,7 @@ def main() -> None:
 
     ws.append(["InstanceId","NameTag","Platform","InstanceType","PrivateIpAddress","AZ"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-ecs.py

@@ -10,13 +10,9 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
 
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
-
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
     client = boto3.client('ecs', region_name=region_name)
@@ -35,7 +31,7 @@ def main() -> None:
 
     ws.append(["clusterName", "runningTasksCount", "capacityProviders", "Region"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-efs.py

@@ -10,13 +10,9 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
 
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
-
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
     client = boto3.client('efs', region_name=region_name)
@@ -33,7 +29,7 @@ def main() -> None:
 
     ws.append(["FileSystemId", "Name", "PerformanceMode", "SizeMB", "AZ"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-eks.py

@@ -10,13 +10,9 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
 
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
-
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
     client = boto3.client('eks', region_name=region_name)
@@ -34,7 +30,7 @@ def main() -> None:
 
     ws.append(["name", "version", "Region"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-elasticache.py

@@ -10,13 +10,9 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
 
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
-
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
     client = boto3.client('elasticache', region_name=region_name)
@@ -33,7 +29,7 @@ def main() -> None:
 
     ws.append(["CacheClusterId", "CacheNodeType", "Engine", "EngineVersion", "NumCacheNodes", "AZ"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-emr.py

@@ -10,11 +10,8 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
 
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
@@ -32,7 +29,7 @@ def main() -> None:
 
     ws.append(["Name", "Status", "NormalizedInstanceHours", "Region"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-lambda.py

@@ -10,11 +10,8 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
 
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
@@ -34,7 +31,7 @@ def main() -> None:
 
     ws.append(["FunctionName","Runtime","Architectures","region"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-lb.py

@@ -10,13 +10,9 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
 
-def getRegions(all_regions=False) -> list[str]:
-    ec2 = boto3.client('ec2', region_name='us-east-1')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
-
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
     client = boto3.client('elbv2', region_name=region_name)
@@ -33,7 +29,7 @@ def main() -> None:
 
     ws.append(["LoadBalancerName", "Scheme", "Type", "Region"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-logs.py

@@ -10,11 +10,8 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
 
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
@@ -32,7 +29,7 @@ def main() -> None:
 
     ws.append(["logGroupName","retentionInDays","storedBytes","region"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-opensearch.py

@@ -10,11 +10,8 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
 
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
@@ -32,7 +29,7 @@ def main() -> None:
 
     ws.append(["DomainName","EngineType","region"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-rds.py

@@ -10,13 +10,9 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
 
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
-
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
     client = boto3.client('rds', region_name=region_name)
@@ -33,7 +29,7 @@ def main() -> None:
 
     ws.append(["DBInstanceIdentifier", "DBInstanceClass", "Engine", "EngineVersion", "MultiAZ", "Region"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

aws-subnets.py

@@ -10,11 +10,8 @@ import boto3
 import concurrent.futures
 from openpyxl import load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
+import projectlib.aws
 
-def getRegions(all_regions=False):
-    ec2 = boto3.client('ec2')
-    response = ec2.describe_regions(AllRegions=all_regions)
-    return [region['RegionName'] for region in response['Regions']]
 
 def getResources(region_name: str) -> list[list[str | int]]:
     return_data = []
@@ -34,7 +31,7 @@ def main() -> None:
 
     ws.append(["SubnetId", "VpcId", "CidrBlock", "AvailabilityZone", "InDefaultVpc"])
     with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
-        results = executor.map(getResources, getRegions())
+        results = executor.map(getResources, projectlib.aws.getRegions())
         for region_rows in results:
             # append to worksheet only if resoruces are found in the region
             if region_rows:

projectlib/aws.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python3
+r"""
+Documentation
+
+License: This program is released under the MIT License
+"""
+
+# Imports
+import boto3
+import pickle
+import os
+
+
+PICKLE_FILE = 'cache.pkl'
+
+def getRegions(all_regions=False) -> list[str]:
+    """
+    Returns all aws regions, cached in pickle file to avoid repeated AWS calls.
+
+    :param all_regions:
+    :return:
+    """
+    cache_key = "aws_regions"
+
+    if os.path.exists(PICKLE_FILE):
+        with open(PICKLE_FILE, 'rb') as f:
+            cache = pickle.load(f)
+            if cache_key in cache:
+                return cache[cache_key]
+
+    # Fetch from AWS and cache it
+    ec2 = boto3.client('ec2', region_name='us-east-1')
+    response = ec2.describe_regions(AllRegions=all_regions)
+    regions = [region['RegionName'] for region in response['Regions']]
+
+    cache = {cache_key: regions}
+    with open(PICKLE_FILE, 'wb') as f:
+        pickle.dump(cache, f)
+
+    return regions

run-inventory-scripts.sh

@@ -1,8 +1,12 @@
 #!/bin/bash
 
-rm -f aws-inventory.xlsx
+# Remove previous inventory and cache file
+rm -f aws-inventory.xlsx cache.pkl
+
+# Create a new workbook
 ./init_workbook.py
 
+# Execute individual inventory scripts
 for i in aws*.py; do
 	echo "# $i"
 	./"$i" | mlr --c2t cat | column -t