fixed logging

src/extract.py

@@ -6,15 +6,16 @@ import json
 from dotenv import load_dotenv
 from datetime import date, datetime
 
-load_dotenv
+load_dotenv()
+
+log_file = os.getenv('LOG_FILE')
+logging.basicConfig(filename=log_file, level=logging.INFO)
+logger = logging.getLogger('extract.py')
 
 google_api_key = os.getenv('GOOGLE_API_KEY')
 google_header = {'key': google_api_key}
 open_lib_header = {'User-Agent': 'Kalar-LMS nick@kalar.codes'}
 
-logger = logging.getLogger('extract.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
-
 today = date.today()
 
 def extract_book_json(url, header=[]):

src/load.py

@@ -1,4 +1,5 @@
 import os
+import time
 import logging
 import json
 import psycopg
@@ -9,23 +10,32 @@ import random
 
 load_dotenv()
 
-logger = logging.getLogger('load.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
+log_file = os.getenv('LOG_FILE')
+logging.basicConfig(filename=log_file, level=logging.INFO)
+logger = logging.getLogger('extract.py')
 
 db_name = os.getenv('DB_NAME')
 db_user = os.getenv('DB_USER')
 db_password = os.getenv('DB_PASSWORD')
+db_host = os.getenv('DB_HOST')
+db_port = os.getenv('DB_PORT')
 
 today = date.today()
 
 def start():
-    with psycopg.connect(f'dbname={db_name} user={db_user} password={db_password}') as conn, \
+    time.sleep(.25)
+    logger.info(f"{datetime.now()}: Attempting connection...")
+    db_connection_string = f'dbname={db_name} user={db_user} password={db_password} host={db_host} port={db_port}'
+    with psycopg.connect(db_connection_string) as conn, \
         open(f'output/transformed_{today}.json', 'r') as transformed_books:
+            logger.info(f'{datetime.now()}: Connection established')
             books = json.loads(transformed_books.read())
             with conn.cursor() as cur:
-                cur.execute(f'DROP TABLE IF EXISTS Collection_Item') # TODO: REMOVE WHEN TESTING COMPLETED
+                cur.execute(f'DROP TABLE IF EXISTS collection_item') # TODO: REMOVE WHEN TESTING COMPLETED
                 cur.execute(sql_statements.collection_item_table_creation)
+                logger.info(f'{datetime.now()}: Table Created')
                 load_transformed_books(cur, books)
+                logger.info(f'{datetime.now()}: Books loaded')
             
 
 def load_transformed_books(cursor, books):

src/orchestrator.py

@@ -5,9 +5,13 @@ import transform
 import load
 import logging
 from datetime import date, datetime
+from dotenv import load_dotenv
 
-logger = logging.getLogger('orchestrator.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
+load_dotenv()
+
+log_file = os.getenv('LOG_FILE')
+logging.basicConfig(filename=log_file, level=logging.INFO)
+logger = logging.getLogger('extract.py')
 
 today = date.today()
 
@@ -39,4 +43,4 @@ if __name__ == '__main__':
     os.remove(f'output/raw_google_books_{today}.json')
     os.remove(f'output/raw_open_lib_books_{today}.json')
     os.remove(f'output/transformed_{today}.json')
-    logger.info(f'{datetime.now()}:Orchestration complete. ETL Pipeline executed without errors.')
+    logger.info(f'{datetime.now()}:Orchestration complete. ETL Pipeline executed without errors.')

src/transform.py

@@ -2,9 +2,13 @@ import os
 import json
 import logging
 from datetime import date, datetime
+from dotenv import load_dotenv
 
-logger = logging.getLogger('transform.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
+load_dotenv()
+
+log_file = os.getenv('LOG_FILE')
+logging.basicConfig(filename=log_file, level=logging.INFO)
+logger = logging.getLogger('extract.py')
 
 today = date.today()