diff --git a/.gitignore b/.gitignore index f0014ae..fa0970d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ aws-inventory.xlsx +cache.pkl \ No newline at end of file diff --git a/aws-apigw.py b/aws-apigw.py index d016d7a..94d0285 100755 --- a/aws-apigw.py +++ b/aws-apigw.py @@ -8,6 +8,17 @@ License: This program is released under the MIT License # Imports import boto3 from openpyxl import load_workbook +import concurrent.futures +import projectlib.aws + + +def getResources(region_name: str) -> list[list[str | int]]: + return_data = [] + client = boto3.client('apigateway', region_name=region_name) + response = client.get_rest_apis() + for i in response['items']: + return_data.append([i["name"], i["endpointConfiguration"]["types"][0], region_name]) + return return_data # Main function def main() -> None: @@ -15,12 +26,14 @@ def main() -> None: wb = load_workbook('aws-inventory.xlsx') ws = wb.create_sheet("ApiGateway") - client = boto3.client('apigateway') - response = client.get_rest_apis() - - ws.append(["RestAPIName","Scope"]) - for i in response['items']: - ws.append([i["name"], i["endpointConfiguration"]["types"][0]]) + ws.append(["RestAPIName","Scope","Region"]) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(getResources, projectlib.aws.getRegions()) + for region_rows in results: + # append to worksheet only if resoruces are found in the region + if region_rows: + for row in region_rows: + ws.append(row) wb.save('aws-inventory.xlsx') diff --git a/aws-ddb.py b/aws-ddb.py index 69b27f5..e771eaa 100755 --- a/aws-ddb.py +++ b/aws-ddb.py @@ -10,11 +10,7 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet - -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] +import projectlib.aws def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -36,7 +32,7 @@ def main() -> None: ws.append(["TableName", "TableStatus", "TableSizeBytes", "ItemCount", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-ec2.py b/aws-ec2.py index 4a6c5a9..8129a26 100755 --- a/aws-ec2.py +++ b/aws-ec2.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -38,7 +35,7 @@ def main() -> None: ws.append(["InstanceId","NameTag","Platform","InstanceType","PrivateIpAddress","AZ"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-ecs.py b/aws-ecs.py index 50259ff..4e6dfea 100755 --- a/aws-ecs.py +++ b/aws-ecs.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('ecs', region_name=region_name) @@ -35,7 +31,7 @@ def main() -> None: ws.append(["clusterName", "runningTasksCount", "capacityProviders", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-efs.py b/aws-efs.py index cd281bc..77f3ad5 100755 --- a/aws-efs.py +++ b/aws-efs.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('efs', region_name=region_name) @@ -33,7 +29,7 @@ def main() -> None: ws.append(["FileSystemId", "Name", "PerformanceMode", "SizeMB", "AZ"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-eks.py b/aws-eks.py index 1c9d8d5..6fbcbaf 100755 --- a/aws-eks.py +++ b/aws-eks.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('eks', region_name=region_name) @@ -34,7 +30,7 @@ def main() -> None: ws.append(["name", "version", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-elasticache.py b/aws-elasticache.py index a9303c2..eb08bcd 100755 --- a/aws-elasticache.py +++ b/aws-elasticache.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('elasticache', region_name=region_name) @@ -33,7 +29,7 @@ def main() -> None: ws.append(["CacheClusterId", "CacheNodeType", "Engine", "EngineVersion", "NumCacheNodes", "AZ"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-emr.py b/aws-emr.py index ea6c499..a81c58c 100755 --- a/aws-emr.py +++ b/aws-emr.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -32,7 +29,7 @@ def main() -> None: ws.append(["Name", "Status", "NormalizedInstanceHours", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-lambda.py b/aws-lambda.py index dd2a8c9..7ff9c17 100755 --- a/aws-lambda.py +++ b/aws-lambda.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -34,7 +31,7 @@ def main() -> None: ws.append(["FunctionName","Runtime","Architectures","region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-lb.py b/aws-lb.py index b102936..dc7008e 100755 --- a/aws-lb.py +++ b/aws-lb.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False) -> list[str]: - ec2 = boto3.client('ec2', region_name='us-east-1') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('elbv2', region_name=region_name) @@ -33,7 +29,7 @@ def main() -> None: ws.append(["LoadBalancerName", "Scheme", "Type", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-logs.py b/aws-logs.py index bf8352e..20567e2 100755 --- a/aws-logs.py +++ b/aws-logs.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -32,7 +29,7 @@ def main() -> None: ws.append(["logGroupName","retentionInDays","storedBytes","region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-opensearch.py b/aws-opensearch.py index d1d1b43..54b7164 100755 --- a/aws-opensearch.py +++ b/aws-opensearch.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -32,7 +29,7 @@ def main() -> None: ws.append(["DomainName","EngineType","region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-rds.py b/aws-rds.py index 26ad628..db8a776 100755 --- a/aws-rds.py +++ b/aws-rds.py @@ -10,13 +10,9 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] - def getResources(region_name: str) -> list[list[str | int]]: return_data = [] client = boto3.client('rds', region_name=region_name) @@ -33,7 +29,7 @@ def main() -> None: ws.append(["DBInstanceIdentifier", "DBInstanceClass", "Engine", "EngineVersion", "MultiAZ", "Region"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/aws-subnets.py b/aws-subnets.py index b294df4..496731e 100755 --- a/aws-subnets.py +++ b/aws-subnets.py @@ -10,11 +10,8 @@ import boto3 import concurrent.futures from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet +import projectlib.aws -def getRegions(all_regions=False): - ec2 = boto3.client('ec2') - response = ec2.describe_regions(AllRegions=all_regions) - return [region['RegionName'] for region in response['Regions']] def getResources(region_name: str) -> list[list[str | int]]: return_data = [] @@ -34,7 +31,7 @@ def main() -> None: ws.append(["SubnetId", "VpcId", "CidrBlock", "AvailabilityZone", "InDefaultVpc"]) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - results = executor.map(getResources, getRegions()) + results = executor.map(getResources, projectlib.aws.getRegions()) for region_rows in results: # append to worksheet only if resoruces are found in the region if region_rows: diff --git a/projectlib/__pycache__/aws.cpython-314.pyc b/projectlib/__pycache__/aws.cpython-314.pyc new file mode 100644 index 0000000..601f2e7 Binary files /dev/null and b/projectlib/__pycache__/aws.cpython-314.pyc differ diff --git a/projectlib/aws.py b/projectlib/aws.py new file mode 100644 index 0000000..76f6c08 --- /dev/null +++ b/projectlib/aws.py @@ -0,0 +1,40 @@ +#!/usr/bin/python3 +r""" +Documentation + +License: This program is released under the MIT License +""" + +# Imports +import boto3 +import pickle +import os + + +PICKLE_FILE = 'cache.pkl' + +def getRegions(all_regions=False) -> list[str]: + """ + Returns all aws regions, cached in pickle file to avoid repeated AWS calls. + + :param all_regions: + :return: + """ + cache_key = "aws_regions" + + if os.path.exists(PICKLE_FILE): + with open(PICKLE_FILE, 'rb') as f: + cache = pickle.load(f) + if cache_key in cache: + return cache[cache_key] + + # Fetch from AWS and cache it + ec2 = boto3.client('ec2', region_name='us-east-1') + response = ec2.describe_regions(AllRegions=all_regions) + regions = [region['RegionName'] for region in response['Regions']] + + cache = {cache_key: regions} + with open(PICKLE_FILE, 'wb') as f: + pickle.dump(cache, f) + + return regions \ No newline at end of file diff --git a/run-inventory-scripts.sh b/run-inventory-scripts.sh index dd45a9a..e327200 100755 --- a/run-inventory-scripts.sh +++ b/run-inventory-scripts.sh @@ -1,8 +1,12 @@ #!/bin/bash -rm -f aws-inventory.xlsx +# Remove previous inventory and cache file +rm -f aws-inventory.xlsx cache.pkl + +# Create a new workbook ./init_workbook.py +# Execute individual inventory scripts for i in aws*.py; do echo "# $i" ./"$i" | mlr --c2t cat | column -t