Merge pull request #1 from Kalarsoft/Create-Orchestrator-Script

Create orchestrator script

README.md

@@ -7,8 +7,6 @@ Environmental Variables:
 `DB_USER`        - The authorized user for the database.  
 `DB_PASSWORD`    - The Password to access the database.  
 `LOG_FILE`       - The file location for logs to be saved to.  
-`LOGGING_LEVEL`  - The logging level desired for operation. `logging.INFO` is standard,    
-                   but `logging.DEBUG` can be used for more insight and `logging.ERROR` if only issues are needed.  
 
 ## extract.py
 The extract.py file contains functions to pull data related to books from different APIs.

orchestrator.sh

@@ -0,0 +1,21 @@
+#!/bin/zsh
+source venv/bin/activate;
+
+pip install -r requirements.txt;
+
+python3 src/extract.py &>> lms-etl.log;
+
+ret=$?
+if [[ $ret -ne 1 ]]; then
+python3 src/transform.py &>> lms-etl.log;
+fi
+
+ret=$?
+if [ $ret -ne 1 ]; then
+    python3 src/load.py &>> lms-etl.log;
+fi
+
+ret=$?
+if [ $ret -ne 1 ]; then
+    rm  output/*;
+fi

src/extract.py

@@ -14,7 +14,7 @@ google_header = {'key': google_api_key}
 open_lib_header = {'User-Agent': 'Kalar-LMS nick@kalar.codes'}
 
 logger = logging.getLogger('extract.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=os.getenv('LOGGING_LEVEL'))
+logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
 
 today = date.today()
 

src/load.py

@@ -8,7 +8,7 @@ from datetime import date, datetime
 load_dotenv()
 
 logger = logging.getLogger('load.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=os.getenv('LOGGING_LEVEL'))
+logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
 
 db_name = os.getenv('DB_NAME')
 db_user = os.getenv('DB_USER')

src/orchestrator.py

@@ -7,7 +7,7 @@ import logging
 from datetime import date, datetime
 
 logger = logging.getLogger('orchestrator.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=os.getenv('LOGGING_LEVEL'))
+logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
 
 today = date.today()
 

src/transform.py

@@ -4,7 +4,7 @@ import logging
 from datetime import date, datetime
 
 logger = logging.getLogger('transform.py')
-logging.basicConfig(filename=os.getenv('LOG_FILE'), level=os.getenv('LOGGING_LEVEL'))
+logging.basicConfig(filename=os.getenv('LOG_FILE'), level=logging.INFO)
 
 today = date.today()