NickKalar/LMS-DB-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up

Browse Source
This commit is contained in:
Nick Kalar
2025-06-19 16:46:59 -04:00
parent 977ac07974
commit 005461d726
1 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/orchestrator.py
View File

@@ -1,14 +1,14 @@
import os
import sys
import extract import extract
import transform import transform
import load import load
import logging import logging
import os
import sys
from datetime import date, datetime from datetime import date, datetime
logger = logging.getLogger('orchestrator.py') logger = logging.getLogger('orchestrator.py')
logging.basicConfig(filename=os.getenv('LOG_FILE'), level=os.getenv('LOGGING_LEVEL'))
logging.basicConfig(filename='lms-etl.log', level=logging.DEBUG)
today = date.today() today = date.today()
if __name__ == '__main__': if __name__ == '__main__':
@@ -36,7 +36,7 @@ if __name__ == '__main__':
sys.exit(1) sys.exit(1)
logger.info(f'{datetime.now()}:Loading completed.') logger.info(f'{datetime.now()}:Loading completed.')
# os.remove(f'output/raw_google_books_{today}.json') os.remove(f'output/raw_google_books_{today}.json')
# os.remove(f'output/raw_open_lib_books_{today}.json') os.remove(f'output/raw_open_lib_books_{today}.json')
# os.remove(f'output/transformed_{today}.json') os.remove(f'output/transformed_{today}.json')
logger.info(f'{datetime.now()}:Orchestration complete. ETL Pipeline executed without errors.') logger.info(f'{datetime.now()}:Orchestration complete. ETL Pipeline executed without errors.')