Experimentation

.gitignore

@@ -194,4 +194,5 @@ cython_debug/
 .cursorindexingignore
 
 .DS_Store
-output/*
+output/*
+.vscode/settings.json

Dockerfile

@@ -6,4 +6,6 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . .
 
+VOLUME ["/etl/logs"]
+
 CMD ["python3", "./src/load.py"]

README.md

@@ -42,7 +42,7 @@ match the column names of the database schema.
 
 ## load.py
 Takes the JSON file created by transform.py and loads the data into a PostgreSQL database for  
-retreival later.
+retrieval later.
 
 ## orchestrator.py
 Handles the orchestration of each program being ran one after the other. Ensures each  

compose.yaml

@@ -9,7 +9,7 @@ services:
     expose:
       - "5432"
     ports:
-      - "8081:5432"
+      - "5432:5432"
     shm_size: 128mb
     environment:
       - POSTGRES_PASSWORD=${DB_PASSWORD}
@@ -26,7 +26,9 @@ services:
     restart: always
   
   etl:
-    image: etl-pipeline:0.3
+    container_name: etl_pipeline
+    image: etl-pipeline
+    build: .
     depends_on:
       - db
     networks:
@@ -34,9 +36,7 @@ services:
     command: "python3 ./src/orchestrator.py"
     restart: no
     volumes:
-      - postgres_data:/var/lib/etl
-    pre_stop:
-      - command: docker cp CONTAINER:/var/lib/etl/lms-etl.log /tmp/app_logs
+      - /tmp/etl/logs/:/var/lib/etl/logs
 
 networks:
   backend:

src/load.py

@@ -50,7 +50,7 @@ def load_transformed_books(cursor, books):
                  `collection_item` SQL table
     '''
     for book in books['books']:
-        # This for i in range statment simulates a library buying multiple copies of a book.
+        # This for i in range statement simulates a library buying multiple copies of a book.
         try:
             for i in range(random.randrange(1, 10)):
                 cursor.execute(sql_statements.collection_insert_statement(book))