release: merge dev branch to main of Dec 2022

.devcontainer/devcontainer.json

@@ -0,0 +1,10 @@
+{
+	"name": "PySpark Sample",
+	"dockerFile": "../Dockerfile",
+	"context": "..",
+	"extensions": [
+		"ms-python.python",
+		"njpwerner.autodocstring"
+	],
+	"onCreateCommand": "pip install -r ./requirements_dev.txt"
+}

.devcontainer/settings.vscode.json

@@ -0,0 +1,3 @@
+{
+    "python.pythonPath": "/usr/local/bin/python"
+}

.github/workflows/CI.yml

@@ -0,0 +1,44 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: CI
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ "dev", "feat/**", "ci/**" ]
+  pull_request:
+    branches: [ "main","dev" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest
+        if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names (ignore F821)
+        flake8 . --count --select=E9,F63,F7,F82 --ignore=F821 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        # pytest and show the report in term and save it as html files
+        pytest --cov=src/ tests/ --ignore=integration --cov-report term --cov-report html:code-coverage
+    - name: Archive code coverage results
+      uses: actions/upload-artifact@v3
+      with:
+        name: code-coverage-report
+        path: ./code-coverage

.gitignore

@@ -130,3 +130,6 @@ dmypy.json
 spark-warehouse/
 data/storage/
 tmp/
+FileStore/
+stg_data1/
+.DS_Store

.vscode/launch.json

@@ -10,7 +10,25 @@
       "request": "launch",
       "program": "src/main.py",
       "console": "integratedTerminal",
-      "args": ["--config-path", "./example/pipeline_fruit.json", "--working-dir", "./tmp", "--show-result", "True", "--build-landing-zone", "True"],
+      "args": ["--config-path", "./example/pipeline_fruit.json", "--working-dir", "./tmp", "--show-result", "--build-landing-zone"],
+      "justMyCode": true
+    },
+    {
+      "name": "Python: main.py parking sensor",
+      "type": "python",
+      "request": "launch",
+      "program": "src/main.py",
+      "console": "integratedTerminal",
+      "args": ["--config-path", "./example/pipeline_parking_sensors.json", "--working-dir", "./tmp", "--show-result", "--cleanup-database"],
+      "justMyCode": true
+    },
+    {
+      "name": "Python: main.py ADLS",
+      "type": "python",
+      "request": "launch",
+      "program": "src/main.py",
+      "console": "integratedTerminal",
+      "args": ["--config-path", "./example/pipeline_fruit_batch_ADLS.json", "--working-dir", "./tmp", "--show-result", "--cleanup-database"],
       "justMyCode": true
     },
     {
@@ -19,7 +37,7 @@
       "request": "launch",
       "program": "src/main.py",
       "console": "integratedTerminal",
-      "args": ["--config-path", "./example/pipeline_fruit_parallel.json", "--working-dir", "./tmp", "--show-result", "True", "--await-termination", "30", "--build-landing-zone", "True", "--cleanup-database", "True"],
+      "args": ["--config-path", "./example/pipeline_fruit_parallel.json", "--working-dir", "./tmp", "--show-result", "--await-termination", "30", "--build-landing-zone", "--cleanup-database"],
       "justMyCode": true
     },
     {
@@ -34,6 +52,18 @@
       "args": ["run", "--no-debugger", "--no-reload"],
       "jinja": true,
       "justMyCode": true
+    },
+    {
+      "name": "Python: pytest",
+      "type": "python",
+      "request": "launch",
+      "module": "pytest",
+      "cwd": "${workspaceRoot}",
+      "env": {
+      },
+      "args": ["tests"],
+      "jinja": true,
+      "justMyCode": true
     }
   ]
 }

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.9.13
+
+# Install pylint
+RUN pip install --upgrade pip && \
+    pip install pylint
+
+# Install git, process tools
+RUN apt-get update && apt-get -y install git procps
+
+# Install OpenJDK-11
+RUN apt-get update && \
+    apt-get install -y openjdk-11-jdk && \
+    apt-get install -y ant && \
+    apt-get clean;
+
+
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64/
+RUN export JAVA_HOME
+
+WORKDIR /usr/src/app
+ENV FLASK_APP=./src/app.py
+ENV FLASK_RUN_HOST=0.0.0.0
+#Server will reload itself on file changes if in dev mode
+ENV FLASK_ENV=development 
+# Add path for pytest
+ENV PYTHONPATH /workspaces/config-driven-data-pipeline/src
+
+COPY requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+COPY ./src ./src
+COPY ./web ./web
+CMD ["flask", "run"]

README.md

@@ -1,6 +1,10 @@
 # Config-Driven Data Pipeline
 
-[![pypi](https://img.shields.io/pypi/v/cddp.svg)](https://pypi.org/project/cddp)
+[![pypi](https://img.shields.io/pypi/v/cddp.svg)](https://pypi.org/project/cddp) 
+[![CI](https://github.com/Azure/config-driven-data-pipeline/actions/workflows/CI.yml/badge.svg?branch=dev)](https://github.com/Azure/config-driven-data-pipeline/actions/workflows/CI.yml)
+[![Docker pulls](https://img.shields.io/docker/pulls/mayemsft/cddp-docker.svg)](https://hub.docker.com/r/mayemsft/cddp-docker)
+
+
 
 ## Why this solution
 
@@ -67,15 +71,15 @@ Spark SQL are used in the standardization block and the serving block, one is me
 Run the batch mode pipeline in local PySpark environment:
 
 ```bash
-python src/main.py --config-path ./example/pipeline_fruit_batch.json --working-dir ./tmp --show-result True --build-landing-zone True --cleanup-database True
+python src/main.py --config-path ./example/pipeline_fruit_batch.json --working-dir ./tmp --show-result --build-landing-zone --cleanup-database
 ```
 
 Here is [another example](example/pipeline_fruit_streaming.json) of streaming based data pipeline. 
 
 Run the streaming mode pipeline in local PySpark environment:
 
 ```bash
-python src/main.py --config-path ./example/pipeline_fruit_streaming.json --working-dir ./tmp --await-termination 60 --show-result True  --build-landing-zone True --cleanup-database True
+python src/main.py --config-path ./example/pipeline_fruit_streaming.json --working-dir ./tmp --await-termination 60 --show-result  --build-landing-zone --cleanup-database
 ```
 
 After running the pipeline, the result will show in the console.
@@ -89,7 +93,19 @@ After running the pipeline, the result will show in the console.
    3|     Orange| 28.0
    6|     Banana| 17.0
    2|      Peach| 39.0
-
+
+## Run the CDDP UI with Docker
+
+```bash
+docker pull mayemsft/cddp-docker:latest
+```
+
+```bash
+docker run -d -p 8080:5000 mayemsft/cddp-docker:latest
+```
+
+
+
 ## Reference
 
 - [Medallion Architecture – Databricks](https://www.databricks.com/glossary/medallion-architecture)