From a141fa76f1b17887d059ab68d5da627d55629361 Mon Sep 17 00:00:00 2001
From: Nicholas Kalar <nick@kalar.codes>
Date: Mon, 3 Nov 2025 23:16:11 -0500
Subject: [PATCH] fixed logging

---
 src/extract.py      |  9 +++++----
 src/load.py         | 18 ++++++++++++++----
 src/orchestrator.py | 10 +++++++---
 src/transform.py    |  8 ++++++--
 4 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/src/extract.py b/src/extract.py
index e193894..e714d35 100644
--- a/src/extract.py
+++ b/src/extract.py
@@ -6,15 +6,16 @@ import json
 from dotenv import load_dotenv
 from datetime import date, datetime
 
-load_dotenv
+load_dotenv()
+
+log_file = os.getenv('LOG_FILE')
+logging.basicConfig(filename=log_file, level=logging.INFO)
+logger = logging.getLogger('extract.py')
 
 google_api_key = os.getenv('GOOGLE_API_KEY')
 google_header = {'key': google_api_key}
 open_lib_header = {'User-Agent': 'Kalar-LMS nick@kalar.codes'}
 
-logger = logging.getLogger('extract.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
-
 today = date.today()
 
 def extract_book_json(url, header=[]):
diff --git a/src/load.py b/src/load.py
index db106c7..504a9c2 100644
--- a/src/load.py
+++ b/src/load.py
@@ -1,4 +1,5 @@
 import os
+import time
 import logging
 import json
 import psycopg
@@ -9,23 +10,32 @@ import random
 
 load_dotenv()
 
-logger = logging.getLogger('load.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
+log_file = os.getenv('LOG_FILE')
+logging.basicConfig(filename=log_file, level=logging.INFO)
+logger = logging.getLogger('extract.py')
 
 db_name = os.getenv('DB_NAME')
 db_user = os.getenv('DB_USER')
 db_password = os.getenv('DB_PASSWORD')
+db_host = os.getenv('DB_HOST')
+db_port = os.getenv('DB_PORT')
 
 today = date.today()
 
 def start():
-    with psycopg.connect(f'dbname={db_name} user={db_user} password={db_password}') as conn, \
+    time.sleep(.25)
+    logger.info(f"{datetime.now()}: Attempting connection...")
+    db_connection_string = f'dbname={db_name} user={db_user} password={db_password} host={db_host} port={db_port}'
+    with psycopg.connect(db_connection_string) as conn, \
         open(f'output/transformed_{today}.json', 'r') as transformed_books:
+            logger.info(f'{datetime.now()}: Connection established')
             books = json.loads(transformed_books.read())
             with conn.cursor() as cur:
-                cur.execute(f'DROP TABLE IF EXISTS Collection_Item') # TODO: REMOVE WHEN TESTING COMPLETED
+                cur.execute(f'DROP TABLE IF EXISTS collection_item') # TODO: REMOVE WHEN TESTING COMPLETED
                 cur.execute(sql_statements.collection_item_table_creation)
+                logger.info(f'{datetime.now()}: Table Created')
                 load_transformed_books(cur, books)
+                logger.info(f'{datetime.now()}: Books loaded')
             
 
 def load_transformed_books(cursor, books):
diff --git a/src/orchestrator.py b/src/orchestrator.py
index 0a09d92..3246c53 100644
--- a/src/orchestrator.py
+++ b/src/orchestrator.py
@@ -5,9 +5,13 @@ import transform
 import load
 import logging
 from datetime import date, datetime
+from dotenv import load_dotenv
 
-logger = logging.getLogger('orchestrator.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
+load_dotenv()
+
+log_file = os.getenv('LOG_FILE')
+logging.basicConfig(filename=log_file, level=logging.INFO)
+logger = logging.getLogger('extract.py')
 
 today = date.today()
 
@@ -39,4 +43,4 @@ if __name__ == '__main__':
     os.remove(f'output/raw_google_books_{today}.json')
     os.remove(f'output/raw_open_lib_books_{today}.json')
     os.remove(f'output/transformed_{today}.json')
-    logger.info(f'{datetime.now()}:Orchestration complete. ETL Pipeline executed without errors.')
+    logger.info(f'{datetime.now()}:Orchestration complete. ETL Pipeline executed without errors.')
\ No newline at end of file
diff --git a/src/transform.py b/src/transform.py
index 3afb9d5..479e108 100644
--- a/src/transform.py
+++ b/src/transform.py
@@ -2,9 +2,13 @@ import os
 import json
 import logging
 from datetime import date, datetime
+from dotenv import load_dotenv
 
-logger = logging.getLogger('transform.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
+load_dotenv()
+
+log_file = os.getenv('LOG_FILE')
+logging.basicConfig(filename=log_file, level=logging.INFO)
+logger = logging.getLogger('extract.py')
 
 today = date.today()