feat: output in markdown table

aws.stepfunction/ListOrgAccounts.json

@@ -0,0 +1,51 @@
+{
+  "Comment": "A description of my state machine",
+  "StartAt": "ListAccounts",
+  "States": {
+    "ListAccounts": {
+      "Type": "Task",
+      "Parameters": {},
+      "Resource": "arn:aws:states:::aws-sdk:organizations:listAccounts",
+      "OutputPath": "$.Accounts[*].Id",
+      "Next": "Map"
+    },
+    "Map": {
+      "Type": "Map",
+      "ItemProcessor": {
+        "ProcessorConfig": {
+          "Mode": "INLINE"
+        },
+        "StartAt": "Lambda Invoke",
+        "States": {
+          "Lambda Invoke": {
+            "Type": "Task",
+            "Resource": "arn:aws:states:::lambda:invoke",
+            "OutputPath": "$.Payload",
+            "Parameters": {
+              "Payload.$": "$",
+              "FunctionName": "arn:aws:lambda:ap-southeast-1:111122223333:function:xpk-test:$LATEST"
+            },
+            "Retry": [
+              {
+                "ErrorEquals": [
+                  "Lambda.ServiceException",
+                  "Lambda.AWSLambdaException",
+                  "Lambda.SdkClientException",
+                  "Lambda.TooManyRequestsException"
+                ],
+                "IntervalSeconds": 1,
+                "MaxAttempts": 3,
+                "BackoffRate": 2,
+                "JitterStrategy": "FULL"
+              }
+            ],
+            "End": true
+          }
+        }
+      },
+      "End": true,
+      "ItemsPath": "$"
+    }
+  },
+  "QueryLanguage": "JSONPath"
+}

aws/cloudfront-usage.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+r"""
+Documentation
+
+License: This program is released under the MIT License
+"""
+
+# Imports
+import boto3
+from datetime import datetime, timedelta
+from dateutil.relativedelta import relativedelta
+import humanize
+
+# Main function
+def main() -> None:
+    # get distributions
+    cw = boto3.client('cloudwatch', region_name='us-east-1')
+    cf = boto3.client('cloudfront', region_name='us-east-1')
+    distros = cf.list_distributions()['DistributionList']['Items']
+    dist_ids = [d['Id'] for d in distros if d['Status'] == 'Deployed']
+    print(f"Distributions discovered: {len(dist_ids)}")
+
+    # print markdown table headers
+    print("| DistId | Month | NoRequests | Download |")
+    print("| ------ | ----- | ---------- | -------- |") 
+
+    for dist_id in dist_ids:
+        for month_offset in range(-3, 0):  # Past 3 months
+            now = datetime.now()
+            first_of_current_month = now.replace(day=1)
+            start = (first_of_current_month + relativedelta(months=month_offset)).replace(day=1)
+            end = (start + relativedelta(months=1)) - relativedelta(seconds=1)  # Last day
+
+            # BytesDownloaded
+            bytes_resp = cw.get_metric_statistics(
+                Namespace='AWS/CloudFront', MetricName='BytesDownloaded',
+                Dimensions=[{'Name': 'DistributionId', 'Value': dist_id}, {'Name': 'Region', 'Value': 'Global'}],
+                StartTime=start, EndTime=end, Period=2678400, Statistics=['Sum']
+            )
+            bytes_total = bytes_resp['Datapoints'][0]['Sum'] if bytes_resp['Datapoints'] else 0
+
+            # Requests
+            reqs_resp = cw.get_metric_statistics(
+                Namespace='AWS/CloudFront', MetricName='Requests',
+                Dimensions=[{'Name': 'DistributionId', 'Value': dist_id}, {'Name': 'Region', 'Value': 'Global'}],
+                StartTime=start, EndTime=end, Period=2678400, Statistics=['Sum']
+            )
+            reqs_total = reqs_resp['Datapoints'][0]['Sum'] if reqs_resp['Datapoints'] else 0
+
+            # only interested in distro with high traffic (>10G)
+            # if bytes_total > 10 * 1024 ** 3:
+            print(f"| {dist_id} | {start:%Y-%b} | {reqs_total:,.0f} | {humanize.naturalsize(bytes_total)} |")
+
+
+# Call main function
+if __name__ == '__main__':
+    main()

aws/deleteSecurityGroup.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+SG=$1
+
+aws ec2 describe-security-group-rules --filters "Name=group-id,Values=$SG" | jq -cr '.SecurityGroupRules[] | select(.IsEgress == true) | .SecurityGroupRuleId' | parallel 'aws ec2 revoke-security-group-egress --group-id $SG --security-group-rule-ids {}'
+
+aws ec2 describe-security-group-rules --filters "Name=group-id,Values=$SG" | jq -cr '.SecurityGroupRules[] | select(.IsEgress == false) | .SecurityGroupRuleId' | parallel 'aws ec2 revoke-security-group-ingress --group-id $SG --security-group-rule-ids {}'
+
+aws ec2 delete-security-group --group-id $SG

aws/ssm-patch-status.sh

@@ -0,0 +1 @@
+aws ssm describe-instance-patch-states --instance-ids $(aws ec2 describe-instances --query Reservations[].Instances[].InstanceId --output text) | jq -cr '["InstanceId", "PatchGroup", "BaselineId", "CriticalNonCompliantCount", "SecurityNonCompliantCount", "OtherNonCompliantCount"],(.InstancePatchStates[] | [.InstanceId, .PatchGroup, .BaselineId, .CriticalNonCompliantCount, .SecurityNonCompliantCount, .OtherNonCompliantCount]) | @csv'

py/dict.py

@@ -1,13 +0,0 @@
-#!/usr/bin/env python3
-# Python data types
-# list = [value1, value2, value3,...valueN]
-# set = {value1, value2, value3,...valueN}
-# dict = { key1:value1, key2:value2,...keyN:valueN }
-
-# Sample use of list of dict
-
-datagroup = [{'name': '203.60.15.113/32', 'data': ''}, {'name': '222.186.30.174/32', 'data': ''},{'name': '120.136.32.106/32', 'data': ''}]
-newrecord = {'name': '1.2.3.4/32', 'data': ''}
-
-datagroup.append(newrecord)
-print(datagroup)

py/examples_tutorials/.gitignore

@@ -0,0 +1 @@
+downloaded_files

py/examples_tutorials/concurrency_threading.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+# Imports
+import threading
+import multiprocessing
+import concurrent.futures
+import time
+
+def task(name):
+    """
+    Dummy function which pretends to do some work
+    """
+    print(f"Thread {name}: Starting...")
+    time.sleep(1)
+    print(f"Thread {name}: Finishing.")
+
+def threading_example():
+    """
+    threading: not truly concurrent as GIL (Global Interpreter Lock) limits 1 process for each bytecode execution.
+    it does allow the process to do more work while other threads are not busy.
+    threading is relatively light-weight
+    """
+    threads = []
+    for i in range(3):
+        threads.append(threading.Thread(target=task, args=(i,)))
+
+    for i in threads:
+        i.start()
+
+    for i in threads:
+        i.join()
+
+    print("threading_example: All threads completed.")
+
+def multiprocessing_example():
+    """
+    multiprocessing: True parallel execution on multiple CPU cores. Tasks are ran on independent processes.
+    More resource expensive compared to threading
+    """
+    mp = []
+    for i in range(3):
+        mp.append(multiprocessing.Process(target=task, args=(i,)))
+
+    for i in mp:
+        i.start()
+
+    for i in mp:
+        i.join()
+
+    print("multiprocessing_example: Done with all calculations!")
+
+def concurrent_futures_example():
+    """
+    high-level implementation of threading. facilitate result consolidation
+    for high-level implemetnation of multiprocessing, use ProcessPoolExecutor
+    """
+    with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
+        for i in range(3):
+            executor.submit(task, i)
+    print("concurrent_futures_example: All threads completed.")
+
+# Main function
+def main() -> None:
+    threading_example()
+    multiprocessing_example()
+    concurrent_futures_example()
+
+# Call main function
+if __name__ == '__main__':
+    main()

py/examples_tutorials/dict.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+"""
+Python data types
+list = [value1, value2, value3,...valueN]
+set = {value1, value2, value3,...valueN}
+dict = { key1:value1, key2:value2,...keyN:valueN }
+"""
+
+list_of_dict = [{'name': '203.60.15.113/32', 'data': ''}, {'name': '222.186.30.174/32', 'data': ''},{'name': '120.136.32.106/32', 'data': ''}]
+new_record = {'name': '1.2.3.4/32', 'data': ''}
+
+list_of_dict.append(new_record)
+print(list_of_dict)

py/examples_tutorials/duck.csv

@@ -0,0 +1,4 @@
+name,age
+tom, 22
+sam, 32
+mary, 19

py/examples_tutorials/duckdb_demo.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import duckdb
+
+# Create an in-memory DuckDB connection
+con = duckdb.connect(':memory:')
+
+# first query which selects a number
+r1 = con.sql("SELECT 42 AS i")
+con.sql("SELECT i * 2 AS k FROM r1").show()
+
+# create a table. insert a row and query the table
+con.sql("CREATE TABLE test (i INTEGER)")
+con.sql("INSERT INTO test VALUES (42)")
+con.table("test").show()
+
+# read a csv into duckdb
+csvt = con.read_csv("duck.csv")
+con.sql("SELECT * FROM csvt WHERE name = 'tom'").show()
+
+# explicitly close the connection
+con.close()

py/examples_tutorials/enums_demo.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+r"""
+Documentation
+
+License: This program is released under the MIT License
+"""
+
+# Imports
+from enum import Enum
+
+class Color(Enum):
+    RED = '#FF0000'
+    GREEN = '#00FF00'
+    BLUE = '#0000FF'
+
+# function which uses the Enum
+def paint_wall(color: Color) -> None:
+    match color:
+        case Color.RED:
+            print("Red wall, are you serious?")
+        case Color.GREEN:
+            print("Green wall, very foresty.")
+        case Color.BLUE:
+            print("Blue wall, I like it.")
+        case _:
+            print("Other colors are not preferred.")
+
+# Main function
+def main() -> None:
+    # Check Enum name and value
+    print(f"Enum: {Color.RED} Name: {Color.RED.name} Value: {Color.RED.value}")
+    # function that uses Enum
+    paint_wall(Color.RED)
+    # print all Enum values
+    print([x.value for x in Color])
+
+    # check if value is a member of the Enum, kind of pointless as IDE will report the problem ahead
+    # print(Color.BLACK in Color)
+
+# Call main function
+if __name__ == '__main__':
+    main()

py/examples_tutorials/fstring_format.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+# Main function
+def main() -> None:
+    """
+    Number formatting in f-string using format specifiers
+    The following prints will output
+
+     314%
+     3.14
+     12,345
+     00012345
+         12345
+    len(str(var1)) = 8
+    """
+    var1: float = 3.141516
+    var2: float = 12345
+    print(f"{var1: .0%}")
+    print(f"{var1: .2f}")
+    print(f"{var2: ,}")
+    print(f"{var2: 09}")
+    print(f"{var2: >10}")
+    print(f"{len(str(var1)) = }")
+
+# Call main function
+if __name__ == '__main__':
+    main()

py/examples_tutorials/lambda.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+def square(x: int) -> int:
+    return x ** 2
+
+# Main function
+def main() -> None:
+    """
+    lambda: a one-liner anonymous function. in its simplest form, it just a function.
+    for example, the following can be rewritten with a add_1 function which returns x + 1
+    """
+    add_1 = lambda x: x + 1
+    result = add_1(1)
+    print(result)
+
+    """
+    map function: apply a function to every element in an iterable
+    returns a map object which can then be casted to a list
+    """
+    numbers = range(1,10)
+    # square is the function and numbers is the iterable where elements will be sent from
+    results = list(map(square, numbers))
+    print(results)
+
+    """
+    implement the same map function with lambda
+    """
+    results2 = list(map(lambda x: x ** 2, numbers))
+    print(results2)
+
+    """
+    filter function: apply function to every element. if true, keep the element. if false, reject it
+    """
+    even_numbers = list(filter(lambda x: x % 2 == 0, numbers))
+    print(even_numbers)
+
+# Call main function
+if __name__ == '__main__':
+    main()

py/examples_tutorials/loops_and_logging.py

@@ -1,19 +1,21 @@
-#!/usr/bin/python3
-r"""
-Python loops demo
+#!/usr/bin/env python3
 
-License: This program is released under the MIT License
-"""
 import logging
 logging.basicConfig(level=logging.INFO, format="%(funcName)s %(levelname)s: %(message)s")
 
 def while_loop() -> None:
+    """
+    while loop which requires a counter
+    """
     counter: int = 1
     while counter <= 5:
         logging.info(counter)
         counter += 1
 
 def for_loop() -> None:
+    """
+    for loop which puts the counter inline
+    """
     for i in range(5):
         logging.info(i)
 
@@ -21,7 +23,6 @@ def for_loop() -> None:
 def main() -> None:
     """
     Both functions will log a message 5 times, but the for loop is so much simpler
-
     """
     while_loop()
     for_loop()

py/examples_tutorials/numpy_demo.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import numpy as np
+
+celsius_temps = np.array([70, 75, 80])
+fahrenheit_temps = (celsius_temps * 9/5) + 32
+
+print(fahrenheit_temps)  # [158. 167. 176.]
+
+
+revenues = np.array([1000, 1500, 800, 2000, 1200])
+costs = np.array([600, 900, 500, 1100, 700])
+tax_rates = np.array([0.15, 0.18, 0.12, 0.20, 0.16])
+
+gross_profits = revenues - costs
+net_profits = gross_profits * (1 - tax_rates)
+
+print(net_profits) # [340. 492. 264. 720. 420.]

py/examples_tutorials/parallel-asyncio.py

@@ -1,9 +1,6 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 """
-Documentation
 Demonstrate how to use asyncio
-
-License: This program is released under the MIT License
 """
 
 # Imports

py/examples_tutorials/seleniumbase_demo.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+from seleniumbase import SB
+from seleniumbase.common.exceptions import WebDriverException
+import re
+
+def main():
+        try:
+            with SB(uc=True, headless=True) as sb:
+                """
+                UC mode is designed to make browser automation appear human and thus evade detection by anti-bot systems.
+                CDP mode uses the Chrome DevTools Protocol to allow more direct and lower-level control over the browser.
+                don't need this # sb.activate_cdp_mode(url)
+                """
+
+                sb.open("https://www.openssh.org/goals.html")
+
+                # page source
+                print(sb.get_page_source())
+                print("-" * 80)
+
+                # enumerate all elements... causes TimeoutError
+                # all_elements = sb.find_elements("xpath", "/html/body/*")
+                # print(f"Total elements found: {len(all_elements)}")
+                # for elem in all_elements:
+                #     print(elem.tag_name)
+                # print("-" * 80)
+
+                # specific html element and element id
+                print(sb.get_text("/html/body/h2[@id='OpenBSD']"))
+                print("-" * 80)
+
+                # similar to the above, but line breaks are differently presented
+                print(sb.get_element("xpath", "//h2[@id='OpenBSD']").get_attribute('innerText'))
+                print("-" * 80)
+
+        except WebDriverException as e:
+            print(f"Failed to get page: {e}")
+        except Exception as e:
+            print(e)
+
+
+if __name__ == '__main__':
+    main()

py/examples_tutorials/terminal_table.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+"""
+Use terminaltables to draw a nice table
+"""
+from terminaltables import SingleTable
+
+def main():
+    table_data = [
+        ["Heading1", "Heading2"],
+        ["row1 column1", "row1 column2"],
+        ["row2 column1", "row2 column2"],
+        ["row3 column1", "row3 column2"],
+    ]
+    t1 = SingleTable(table_data)
+    print(t1.table)
+
+if __name__ == '__main__':
+    main()

py/goldenRatio.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+r"""
+Documentation
+
+License: This program is released under the MIT License
+"""
+
+# Imports
+import sys
+from terminaltables import SingleTable
+
+# Main function
+def main() -> None:
+    difference = float(sys.argv[2]) - float(sys.argv[1]);
+
+    if float(sys.argv[2]) > float(sys.argv[1]):
+        label1, label2 = "LO", "HI"
+    else:
+        label1, label2 = "HI", "LO"
+
+    table_data = [
+        ['Position', 'Value'],
+        [label1, f"{int(sys.argv[1]):,}"],
+        ['0.382', f"{int(difference * 0.382 + float(sys.argv[1])):,}"],
+        ['0.5', f"{int(difference * 0.5 + float(sys.argv[1])):,}"],
+        ['0.618', f"{int(difference * 0.618 + float(sys.argv[1])):,}"],
+        ['0.764', f"{int(difference * 0.764 + float(sys.argv[1])):,}"],
+        [label2, f"{int(sys.argv[2]):,}"]
+    ]
+
+    t1 = SingleTable(table_data)
+    print(t1.table)
+
+# Call main function
+if __name__ == '__main__':
+    main()

py/log-test.py

@@ -1,11 +0,0 @@
-#!/usr/bin/python3
-import logging
-
-logging.basicConfig(format='%(levelname)s: %(message)s',level=logging.DEBUG)
-
-logging.debug('Debug message')
-logging.info('Info message')
-logging.warning('Warning message')
-logging.error('Error message')
-logging.critical('Critical message')
-

py/pwgen.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python3
+import sys
 
 import petname
 import secrets
+from itertools import chain
 
 def generate_password(names: int) -> str:
     """
@@ -11,11 +13,38 @@ def generate_password(names: int) -> str:
     :return: A simple yet secure password
     """
     results = []
-    for _ in range(names):
-        results.append(petname.Name().capitalize())
+    while len(results) < names:
+        candidate = petname.Name().capitalize()
+        results.append(candidate) if len(candidate) > 3 else None
     results.append(str(secrets.randbelow(8999)+1000))
     return ".".join(results)
 
+def generate_insane_password(length: int) -> str:
+    """
+    Generate an insane password using extended ascii characters
+
+    :param length:
+    :return:
+    """
+    ascii_pool: str = ''.join(chr(i) for i in chain(range(40, 127), range(180, 256)))
+    return ''.join(secrets.choice(ascii_pool) for _ in range(length))
+
+def password_qc(pw: str) -> bool:
+    specials = set("!@#$%^&*()-_=+[]{};:'\"|,.<>/?`~\\")
+
+    has_lower = any(c.islower() for c in pw)
+    has_upper = any(c.isupper() for c in pw)
+    has_digit = any(c.isdigit() for c in pw)
+    has_special = any(c in specials for c in pw)
+
+    return has_lower and has_upper and has_digit and has_special
 
 if __name__ == '__main__':
-    print(generate_password(2))
+    if len(sys.argv) <= 1:
+        print(generate_password(3))
+    elif sys.argv[1] == '-insane' or sys.argv[1] == '-i':
+        pw = generate_insane_password(20)
+        while not password_qc(pw):
+            pw = generate_insane_password(20)
+        else:
+            print(pw)

py/string-test.py

@@ -1,7 +0,0 @@
-#!/usr/bin/python3
-
-msg = ""
-msg += "Line 1"
-msg += "Line 2"
-
-print(msg)

py/table.py

@@ -1,11 +0,0 @@
-from prettytable import PrettyTable
-x = PrettyTable()
-x.field_names = ["City name", "Area", "Population", "Annual Rainfall"]
-x.add_row(["Adelaide", 1295, 1158259, 600.5])
-x.add_row(["Brisbane", 5905, 1857594, 1146.4])
-x.add_row(["Darwin", 112, 120900, 1714.7])
-x.add_row(["Hobart", 1357, 205556, 619.5])
-x.add_row(["Sydney", 2058, 4336374, 1214.8])
-x.add_row(["Melbourne", 1566, 3806092, 646.9])
-x.add_row(["Perth", 5386, 1554769, 869.4])
-print(x);

py/termtable.py

@@ -1,10 +0,0 @@
-from terminaltables import SingleTable
-
-table_data = [
-    ["Heading1", "Heading2"],
-    ["row1 column1", "row1 column2"],
-    ["row2 column1", "row2 column2"],
-    ["row3 column1", "row3 column2"],
-]
-t1 = SingleTable(table_data)
-print(t1.table)

py/test.py

@@ -1,8 +0,0 @@
-#!/usr/bin/env python
-import os, time
-
-fileage = os.path.getmtime('/var/log/ufw.log');
-
-if time.time() - fileage > 7200:
-    print ('File is > 2 hours old')
-