refactored functions
This commit is contained in:
23
src/load.py
23
src/load.py
@@ -17,23 +17,30 @@ db_password = os.getenv('DB_PASSWORD')
|
|||||||
|
|
||||||
today = date.today()
|
today = date.today()
|
||||||
|
|
||||||
collections_table_creation = sql_statements.collections_table_creation
|
|
||||||
|
|
||||||
def start():
|
def start():
|
||||||
with psycopg.connect(f'dbname={db_name} user={db_user} password={db_password}') as conn, \
|
with psycopg.connect(f'dbname={db_name} user={db_user} password={db_password}') as conn, \
|
||||||
open(f'output/transformed_{today}.json', 'r') as transformed_books:
|
open(f'output/transformed_{today}.json', 'r') as transformed_books:
|
||||||
|
books = json.loads(transformed_books.read())
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
cur.execute(f'DROP TABLE IF EXISTS Collection_Item') # TODO: REMOVE WHEN TESTING COMPLETED
|
cur.execute(f'DROP TABLE IF EXISTS Collection_Item') # TODO: REMOVE WHEN TESTING COMPLETED
|
||||||
cur.execute(collections_table_creation)
|
cur.execute(sql_statements.collections_table_creation)
|
||||||
books = json.loads(transformed_books.read())
|
load_transformed_books(cur, books)
|
||||||
|
|
||||||
|
|
||||||
|
def load_transformed_books(cursor, books):
|
||||||
|
'''
|
||||||
|
Takes a pyscopg connection cursor and a dictionary of books and inserts
|
||||||
|
the books into a PostgreSQL database.
|
||||||
|
|
||||||
|
Keyword arguments:
|
||||||
|
cursor - a psycopg.connect.cursor object
|
||||||
|
books - a dictionary of transformed books following the schema for the
|
||||||
|
`collection_item` SQL table
|
||||||
|
'''
|
||||||
for book in books['books']:
|
for book in books['books']:
|
||||||
cur.execute(sql_statements.collection_insert_statement(book))
|
cursor.execute(sql_statements.collection_insert_statement(book))
|
||||||
logger.info(f'{datetime.now()}:Book {book['title']} loaded.')
|
logger.info(f'{datetime.now()}:Book {book['title']} loaded.')
|
||||||
|
|
||||||
def load_transformed_books():
|
|
||||||
pass
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
print('Loading Started')
|
print('Loading Started')
|
||||||
logger.info(f'{datetime.now()}:Loading Started')
|
logger.info(f'{datetime.now()}:Loading Started')
|
||||||
|
|||||||
Reference in New Issue
Block a user