NickKalar/LMS-DB-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tweaks and INSERT logic

Browse Source
This commit is contained in:
Nick Kalar
2025-06-19 16:46:46 -04:00
parent 7f517c647f
commit 977ac07974
1 changed files with 25 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
src/load.py
View File

@@ -1,13 +1,14 @@
import os import os
from dotenv import load_dotenv
import psycopg
from datetime import date
import logging import logging
import json
import psycopg
from dotenv import load_dotenv
from datetime import date, datetime
load_dotenv() load_dotenv()
logger = logging.getLogger('load.py') logger = logging.getLogger('load.py')
logging.basicConfig(filename='lms-etl.log', level=logging.DEBUG) logging.basicConfig(filename=os.getenv('LOG_FILE'), level=os.getenv('LOGGING_LEVEL'))
db_name = os.getenv('DB_NAME') db_name = os.getenv('DB_NAME')
db_user = os.getenv('DB_USER') db_user = os.getenv('DB_USER')
@@ -15,21 +16,21 @@ db_password = os.getenv('DB_PASSWORD')
today = date.today() today = date.today()
collections_table_creation = """ collections_table_creation = '''
CREATE TABLE IF NOT EXISTS "Collection_Item"( CREATE TABLE IF NOT EXISTS Collection_Item(
"id" BIGINT PRIMARY KEY, "id" BIGSERIAL PRIMARY KEY,
"title" VARCHAR(255) NULL, "title" VARCHAR(255) NULL,
"author" VARCHAR(255) NULL, "author" VARCHAR(255) NULL,
"publisher" VARCHAR(255) NULL, "publisher" VARCHAR(255) NULL,
"publishing_date" DATE NULL, "publishing_date" VARCHAR(255) NULL,
"loc_number" VARCHAR(255) NULL, "loc_number" VARCHAR(255) NULL,
"dewey_decimal_number" VARCHAR(255) NULL, "dewey_decimal_number" VARCHAR(255) NULL,
"isbn" BIGINT NULL,
"sort_title" VARCHAR(255) NULL, "sort_title" VARCHAR(255) NULL,
"format" VARCHAR(255) NULL, "format" VARCHAR(255) NULL,
"language" VARCHAR(255) NULL, "language" VARCHAR(255) NULL,
"page_count" BIGINT NULL, "page_count" BIGINT NULL,
"genre" VARCHAR(255) NULL, "categories" VARCHAR(255) NULL,
"subject" VARCHAR(255) NULL,
"description" BIGINT NULL, "description" BIGINT NULL,
"price_in_cents" BIGINT NULL, "price_in_cents" BIGINT NULL,
"cover_image_uri" VARCHAR(255) NULL, "cover_image_uri" VARCHAR(255) NULL,
@@ -38,18 +39,29 @@ collections_table_creation = """
"is_lost" BOOLEAN NULL, "is_lost" BOOLEAN NULL,
"lost_date" DATE NULL "lost_date" DATE NULL
) )
""" '''
def start(): def start():
with psycopg.connect(f'dbname={db_name} user={db_user} password={db_password}') as conn, \ with psycopg.connect(f'dbname={db_name} user={db_user} password={db_password}') as conn, \
open(f"output/transformed_{today}.json", 'r'): open(f'output/transformed_{today}.json', 'r') as transformed_books:
with conn.cursor() as cur: with conn.cursor() as cur:
cur.execute(f'DROP TABLE IF EXISTS Collection_Item') # TODO: REMOVE WHEN TESTING COMPLETED
cur.execute(collections_table_creation) cur.execute(collections_table_creation)
books = json.loads(transformed_books.read())
for book in books['books']:
cur.execute(f'INSERT INTO Collection_Item ' \
'(title, author, publisher, publishing_date, isbn, sort_title, format, language, categories, page_count, is_checked_in, is_archived, is_lost) ' \
f'VALUES (\'{book['title']}\',\'{book['author']}\',\'{book['publisher']}\',\'{book['publishing_date']}\',{book['isbn']},\'{book['sort_title']}\','
f'\'{book['format']}\',\'{book['language']}\',\'{book['categories']}\',{book['page_count']},{book['is_checked_in']},{book['is_archived']},{book['is_lost']});')
logger.info(f'{datetime.now()}:Book {book['title']} loaded.')
def load_transformed_books(): def load_transformed_books():
pass pass
if __name__ == '__main__': if __name__ == '__main__':
print('Loading Started')
logger.info(f'{datetime.now()}:Loading Started')
start() start()
print('Loading Done') print('Loading Done')
logger.info(f'{datetime.now()}:Loading Done')