SK-1772: Beta Release for Python SDK V2

.github/workflows/beta-release.yml

@@ -0,0 +1,18 @@
+name: Public Beta Release
+
+on:
+  push:
+    tags: '*.*.*b*'
+    paths-ignore:
+      - "setup.py"
+      - "*.yml"
+      - "*.md"
+      - "skyflow/utils/_version.py"
+
+jobs:
+  build-and-deploy:
+    uses: ./.github/workflows/shared-build-and-deploy.yml
+    with:
+      ref: main
+      pypi-token: ${{ secrets.PYPI_PUBLISH_TOKEN }}
+      is-internal: false

.github/workflows/ci.yml

@@ -16,46 +16,9 @@ jobs:
           checkAllCommitMessages: 'true'
           accessToken: ${{ secrets.PAT_ACTIONS }}
           error: 'One of your your commit messages is not matching the format with JIRA ID Ex: ( SDK-123 commit message )'
-  Test:
-    runs-on: ubuntu-latest
-    steps:   
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: '3.8'
-
-    - name: create-json
-      id: create-json
-      uses: jsdaniell/create-json@1.1.2
-      with:
-        name: "credentials.json"
-        json: ${{ secrets.CREDENTIALS_FILE_STR }}
-
-    - name: 'Setup .env'
-      run: |
-        touch .env
-        echo VAULT_ID=${{secrets.VAULT_ID}} >> .env
-        echo VAULT_URL=${{secrets.VAULT_URL}} >> .env
-        echo DETOKENIZE_TEST_TOKEN=${{secrets.DETOKENIZE_TEST_TOKEN}} >> .env
-        echo DETOKENIZE_TEST_VALUE=${{secrets.DETOKENIZE_TEST_VALUE}} >> .env
-        echo CREDENTIALS_FILE_PATH=./credentials.json >> .env
-        echo CVV_GEN_CONNECTION_URL=${{secrets.CVV_GEN_CONNECTION_URL}} >> .env
-        echo VISA_CONNECTION_BASIC_AUTH=${{secrets.VISA_CONNECTION_BASIC_AUTH}} >> .env
-        echo SKYFLOW_ID1=${{secrets.SKYFLOW_ID1}} >> .env
-        echo SKYFLOW_ID2=${{secrets.SKYFLOW_ID2}} >> .env
-        echo SKYFLOW_ID3=${{secrets.SKYFLOW_ID3}} >> .env
 
-    - name: 'Run Tests'
-      run: |
-        python -m pip install --upgrade pip
-        pip install requests pyjwt datetime aiohttp cryptography python-dotenv coverage
-        coverage run --source skyflow -m unittest discover
-    - name: coverage
-      run: coverage xml -o test-coverage.xml
-    - name: Codecov
-      uses: codecov/codecov-action@v2.1.0
-      with:
-        token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
-        files: test-coverage.xml
-        name: codecov-skyflow-python
-        verbose: true
+  test:
+    uses: ./.github/workflows/shared-tests.yml
+    with:
+      python-version: '3.8'
+    secrets: inherit

.github/workflows/internal-release.yml

@@ -0,0 +1,23 @@
+name: Internal Release
+
+on:
+  push:
+    tags-ignore:
+      - '*.*'
+    paths-ignore:
+      - "setup.py"
+      - "*.yml"
+      - "*.md"
+      - "skyflow/utils/_version.py"
+      - "samples/**"
+    branches:
+      - release/*
+
+jobs:
+  build-and-deploy:
+    uses: ./.github/workflows/shared-build-and-deploy.yml
+    with:
+      ref: ${{ github.ref_name }}
+      is-internal: true
+    secrets: inherit
+

.github/workflows/main.yml

@@ -6,46 +6,7 @@ on:
       - main
 
 jobs:
-  Test:
-    runs-on: ubuntu-latest
-    steps:   
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: '3.8'
-
-    - name: create-json
-      id: create-json
-      uses: jsdaniell/create-json@1.1.2
-      with:
-        name: "credentials.json"
-        json: ${{ secrets.CREDENTIALS_FILE_STR }}
-
-    - name: 'Setup .env'
-      run: |
-        touch .env
-        echo VAULT_ID=${{secrets.VAULT_ID}} >> .env
-        echo VAULT_URL=${{secrets.VAULT_URL}} >> .env
-        echo DETOKENIZE_TEST_TOKEN=${{secrets.DETOKENIZE_TEST_TOKEN}} >> .env
-        echo DETOKENIZE_TEST_VALUE=${{secrets.DETOKENIZE_TEST_VALUE}} >> .env
-        echo CREDENTIALS_FILE_PATH=./credentials.json >> .env
-        echo CVV_GEN_CONNECTION_URL=${{secrets.CVV_GEN_CONNECTION_URL}} >> .env
-        echo VISA_CONNECTION_BASIC_AUTH=${{secrets.VISA_CONNECTION_BASIC_AUTH}} >> .env
-        echo SKYFLOW_ID1=${{secrets.SKYFLOW_ID1}} >> .env
-        echo SKYFLOW_ID2=${{secrets.SKYFLOW_ID2}} >> .env
-        echo SKYFLOW_ID3=${{secrets.SKYFLOW_ID3}} >> .env
-
-    - name: 'Run Tests'
-      run: |
-        python -m pip install --upgrade pip
-        pip install requests pyjwt datetime aiohttp cryptography python-dotenv coverage
-        coverage run --source skyflow -m unittest discover
-    - name: coverage
-      run: coverage xml -o test-coverage.xml
-    - name: Codecov
-      uses: codecov/codecov-action@v2.1.0
-      with:
-        token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
-        files: test-coverage.xml
-        name: codecov-skyflow-python
-        verbose: true
+  test:
+    uses: ./.github/workflows/shared-tests.yml
+    with:
+      python-version: '3.8'

.github/workflows/release.yml

@@ -7,47 +7,12 @@ on:
       - "setup.py"
       - "*.yml"
       - "*.md"
-      - "skyflow/version.py"
+      - "skyflow/utils/_version.py"
 
 jobs:
   build-and-deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.PAT_ACTIONS }}
-          ref: main
-          fetch-depth: 0
-      - uses: actions/setup-python@v2
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-
-      - name: Get Previous tag
-        id: previoustag
-        uses: WyriHaximus/github-action-get-previous-tag@v1
-        with:
-          fallback: 1.0.0
-
-      - name: Bump Version
-        run: |
-          chmod +x ./ci-scripts/bump_version.sh
-          ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}"
-
-      - name: Commit changes
-        run: |
-          git config user.name ${{ github.actor }}
-          git config user.email ${{ github.actor }}@users.noreply.github.com
-          git add setup.py
-          git add skyflow/version.py
-          git commit -m "[AUTOMATED] Public Release - ${{ steps.previoustag.outputs.tag }}"
-          git push origin
-
-      - name: Build and publish
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_TOKEN }}
-        run: |
-          python setup.py sdist bdist_wheel
-          twine upload dist/*
+    uses: ./.github/workflows/shared-build-and-deploy.yml
+    with:
+      ref: main
+      is-internal: false
+    secrets: inherit

.github/workflows/shared-build-and-deploy.yml

@@ -0,0 +1,84 @@
+name: Shared Build and Deploy
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git reference to use (e.g., main or branch name)'
+        required: true
+        type: string
+
+      is-internal:
+        description: 'Flag for internal release'
+        required: true
+        type: boolean
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+
+      - name: Get Previous tag
+        id: previoustag
+        uses: WyriHaximus/github-action-get-previous-tag@v1
+        with:
+          fallback: 1.0.0
+
+      - name: Bump Version
+        run: |
+          chmod +x ./ci-scripts/bump_version.sh
+          if ${{ inputs.is-internal }}; then
+            ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}" "$(git rev-parse --short "$GITHUB_SHA")"
+          else
+            ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}"
+          fi
+
+      - name: Commit changes
+        run: |
+          git config user.name "${{ github.actor }}"
+          git config user.email "${{ github.actor }}@users.noreply.github.com"
+          git add setup.py
+          git add skyflow/utils/_version.py
+
+          if [ "${{ inputs.is-internal }}" = "true" ]; then
+            VERSION="${{ steps.previoustag.outputs.tag }}.dev0+$(git rev-parse --short $GITHUB_SHA)"
+            COMMIT_MESSAGE="[AUTOMATED] Private Release $VERSION"
+            git commit -m "$COMMIT_MESSAGE"
+            git push origin ${{ github.ref_name }} -f
+          else
+            COMMIT_MESSAGE="[AUTOMATED] Public Release - ${{ steps.previoustag.outputs.tag }}"
+            git commit -m "$COMMIT_MESSAGE"
+            git push origin
+          fi
+
+      - name: Build and Publish Package
+        if: ${{ !inputs.is-internal }}
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_TOKEN }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*
+
+      - name: Build and Publish to JFrog Artifactory
+        if: ${{ inputs.is-internal }}
+        env:
+          TWINE_USERNAME: ${{ secrets.JFROG_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.JFROG_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload --repository-url https://prekarilabs.jfrog.io/artifactory/api/pypi/skyflow-python/ dist/*
+
+
+

.github/workflows/shared-tests.yml

@@ -0,0 +1,41 @@
+name: Shared Test Steps
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        description: 'Python version to use'
+        required: true
+        type: string
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/create-json@1.1.2
+        with:
+          name: "credentials.json"
+          json: ${{ secrets.VALID_SKYFLOW_CREDS_TEST }}
+
+      - name: 'Run Tests'
+        run: |
+          pip install -r requirements.txt
+          python -m coverage run --source=skyflow --omit=skyflow/generated/*,skyflow/utils/validations/*,skyflow/vault/data/*,skyflow/vault/tokens/*,skyflow/vault/connection/*,skyflow/error/*,skyflow/utils/enums/*,skyflow/vault/controller/_audit.py,skyflow/vault/controller/_bin_look_up.py -m unittest discover
+
+      - name: coverage
+        run: coverage xml -o test-coverage.xml
+
+      - name: Codecov
+        uses: codecov/codecov-action@v2.1.0
+        with:
+          token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
+          files: test-coverage.xml
+          name: codecov-skyflow-python
+          verbose: true