refactor

.github/workflows/test.yaml

@@ -13,94 +13,115 @@ jobs:
           - windows-latest
           - macos-latest
         python-version:
-          - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"
           - "3.12"
           - "3.13"
-        exclude:
-          - os: macos-latest
-            python-version: "3.8"
-          - os: macos-latest
-            python-version: "3.13"            
     runs-on: ${{matrix.os}}
-    name: 'Run Tests on ${{matrix.os}} with Python ${{matrix.python-version}}'
+    name: '${{matrix.os}} Python ${{matrix.python-version}}'
 
     steps:
-      - uses: actions/checkout@v4.1.1
+      - name: Checkout
+        uses: actions/checkout@v4.2.2
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5.1.0
+        uses: actions/setup-python@v5.6.0
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies from requirements.txt
+      - name: Upgrade pip
         shell: bash
         run: |
           python3 -m pip install --upgrade pip
-          pip install -r requirements.txt
 
-      # Windows
-      - name: Install psycopg
-        if: matrix.os == 'windows-latest'
+      - name: Install pystackql with all dependencies
+        run: |
+          pip install -e .
+
+      - name: Install test dependencies
         run: |
-          pip install psycopg[binary]
-        shell: powershell
-      # End Windows
+          pip install pytest>=6.2.5 pytest-cov>=2.12.0 nose>=1.3.7
 
-      # Linux
-      - name: Install PostgreSQL dev libraries on Ubuntu
-        if: matrix.os == 'ubuntu-latest'
+      - name: setup-stackql
+        uses: stackql/setup-stackql@v2.2.3
+        with:
+          use_wrapper: true
+
+      - name: Show stackql version (Linux/macOS)
+        if: matrix.os != 'windows-latest'
+        shell: bash
         run: |
-          sudo apt-get update
-          pip install psycopg
-      # End Linux
+          stackql --version
 
-      # macOS
-      - name: Install PostgreSQL dev libraries on macOS
-        if: matrix.os == 'macos-latest'
+      - name: Show stackql version (Windows)
+        if: matrix.os == 'windows-latest'
+        shell: cmd
         run: |
-          brew install postgresql@14
-          pip install psycopg
-      # End macOS
+          stackql-bin.exe --version
 
-      - name: Install pystackql
+      - name: Move stackql binary to temp dir (Linux/macOS)
+        if: matrix.os != 'windows-latest'
+        shell: bash
+        run: |
+          STACKQL_PATH=$(which stackql)
+          mkdir -p /tmp || true
+          cp "$STACKQL_PATH" /tmp/stackql
+          echo "StackQL binary moved from ${STACKQL_PATH} to /tmp/stackql"
+
+      - name: Move stackql binary to temp dir (Windows)
+        if: matrix.os == 'windows-latest'
+        shell: pwsh
         run: |
-          pip install .
+          $bin = Join-Path $Env:STACKQL_CLI_PATH 'stackql-bin.exe'
+          if (-Not (Test-Path $bin)) {
+            throw "Binary not found at $bin"
+          }
+          Copy-Item $bin -Destination "C:\Temp\stackql.exe" -Force
+          Write-Host "Moved real StackQL binary to C:\Temp\stackql.exe"
 
-      - name: Run tests
+      - name: Run non-server tests
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }}
-          STACKQL_GITHUB_USERNAME: ${{ secrets.STACKQL_GITHUB_USERNAME }}
-          STACKQL_GITHUB_PASSWORD: ${{ secrets.STACKQL_GITHUB_PASSWORD }}
-          CUSTOM_STACKQL_GITHUB_USERNAME: ${{ secrets.CUSTOM_STACKQL_GITHUB_USERNAME }}
-          CUSTOM_STACKQL_GITHUB_PASSWORD: ${{ secrets.CUSTOM_STACKQL_GITHUB_PASSWORD }}
-          AWS_REGION: ${{ vars.AWS_REGION }}
-          AWS_REGIONS: ${{ vars.AWS_REGIONS }}
-          GCP_PROJECT: ${{ vars.GCP_PROJECT }}
-          GCP_ZONE: ${{ vars.GCP_ZONE }}
+          GITHUB_ACTIONS: 'true'
         run: |
-          python3 -m tests.pystackql_tests
-        shell: bash
-        if: matrix.os != 'windows-latest'
+          python3 run_tests.py
 
-      - name: Run tests on Windows
+      - name: Start StackQL server and run tests (Linux/macOS)
+        if: matrix.os != 'windows-latest'
+        shell: bash      
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }}
-          STACKQL_GITHUB_USERNAME: ${{ secrets.STACKQL_GITHUB_USERNAME }}
-          STACKQL_GITHUB_PASSWORD: ${{ secrets.STACKQL_GITHUB_PASSWORD }}
-          CUSTOM_STACKQL_GITHUB_USERNAME: ${{ secrets.CUSTOM_STACKQL_GITHUB_USERNAME }}
-          CUSTOM_STACKQL_GITHUB_PASSWORD: ${{ secrets.CUSTOM_STACKQL_GITHUB_PASSWORD }}
-          AWS_REGION: ${{ vars.AWS_REGION }}
-          AWS_REGIONS: ${{ vars.AWS_REGIONS }}
-          GCP_PROJECT: ${{ vars.GCP_PROJECT }}
-          GCP_ZONE: ${{ vars.GCP_ZONE }}
+          GITHUB_ACTIONS: 'true'
         run: |
-          python3 -m tests.pystackql_tests
+            nohup /tmp/stackql -v --pgsrv.port=5466 srv &
+            sleep 5
+            python3 run_server_tests.py
+
+      - name: Start StackQL server (Windows)
+        if: matrix.os == 'windows-latest'
         shell: pwsh
+        run: |
+          Start-Process -FilePath "C:\Temp\stackql.exe" `
+                        -ArgumentList "-v", "--pgsrv.port=5466", "srv"
+          Start-Sleep -Seconds 5
+
+      - name: Stop StackQL server (Linux/macOS)
+        if: matrix.os != 'windows-latest'
+        shell: bash
+        run: |
+          echo "Stopping StackQL server on Unix/macOS..."
+          PID=$(pgrep -f "/tmp/stackql.*srv" || pgrep -f "stackql.*srv" || echo "")
+          if [ -z "$PID" ]; then
+              echo "No stackql server process found."
+          else
+              echo "stopping stackql server (PID: $PID)..."
+              kill -9 $PID
+              echo "stackql server stopped."
+          fi
+
+      - name: Stop StackQL server (Windows)
         if: matrix.os == 'windows-latest'
+        shell: cmd
+        run: |
+          echo "Stopping StackQL server on Windows..."
+          taskkill /F /IM stackql.exe 2>nul || echo "No stackql.exe process found"
+          echo "StackQL server stopped (Windows)"

.gitignore

@@ -1,11 +1,21 @@
 # Miscellaneous
-/.stackql
-/.stackql/*
 .pypirc
-
 /.vscode
 /.vscode/*
 
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# stackql
+.stackql/
+stackql
+stackql-*.sh
+.env
+nohup.out
+
 # Byte-compiled / optimized / DLL files
 __pycache__
 *.py[cod]

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## v3.8.0 (2025-06-04)
+
+### Updates
+
+- Refactor
+- Enhanced test coverage
+
 ## v3.7.2 (2024-11-19)
 
 ### Updates

launch_venv.sh

@@ -0,0 +1,140 @@
+#!/bin/bash
+# launch_venv.sh - Script to create, set up and activate a Python virtual environment for PyStackQL
+
+# Use simpler code without colors when running with sh
+if [ -n "$BASH_VERSION" ]; then
+    # Color definitions for bash
+    GREEN='\033[0;32m'
+    YELLOW='\033[1;33m'
+    BLUE='\033[0;34m'
+    RED='\033[0;31m'
+    NC='\033[0m' # No Color
+
+    # Function to print colored text in bash
+    cecho() {
+        printf "%b%s%b\n" "$1" "$2" "$NC"
+    }
+else
+    # No colors for sh
+    cecho() {
+        echo "$2"
+    }
+fi
+
+# Default virtual environment name
+VENV_NAME=".venv"
+REQUIREMENTS_FILE="requirements.txt"
+
+# Function to check if command exists
+command_exists() {
+    command -v "$1" >/dev/null 2>&1
+}
+
+# Banner
+cecho "$BLUE" "======================================="
+cecho "$BLUE" "  PyStackQL Development Environment   "
+cecho "$BLUE" "======================================="
+echo ""
+
+# Check for Python
+if ! command_exists python3; then
+    cecho "$RED" "Error: Python 3 is not installed."
+    echo "Please install Python 3 and try again."
+    exit 1
+fi
+
+# Print Python version
+cecho "$YELLOW" "Using Python:"
+python3 --version
+echo ""
+
+# Create virtual environment if it doesn't exist
+if [ ! -d "$VENV_NAME" ]; then
+    cecho "$YELLOW" "Creating virtual environment in ${VENV_NAME}..."
+    python3 -m venv "$VENV_NAME"
+    if [ $? -ne 0 ]; then
+        cecho "$RED" "Error: Failed to create virtual environment."
+        exit 1
+    fi
+    cecho "$GREEN" "Virtual environment created successfully."
+else
+    cecho "$YELLOW" "Using existing virtual environment in ${VENV_NAME}"
+fi
+
+# Determine the activate script based on OS
+case "$OSTYPE" in
+    msys*|win*|cygwin*)
+        # Windows
+        ACTIVATE_SCRIPT="$VENV_NAME/Scripts/activate"
+        ;;
+    *)
+        # Unix-like (Linux, macOS)
+        ACTIVATE_SCRIPT="$VENV_NAME/bin/activate"
+        ;;
+esac
+
+# Check if activation script exists
+if [ ! -f "$ACTIVATE_SCRIPT" ]; then
+    cecho "$RED" "Error: Activation script not found at $ACTIVATE_SCRIPT"
+    echo "The virtual environment may be corrupt. Try removing the $VENV_NAME directory and running this script again."
+    exit 1
+fi
+
+# Source the activation script
+cecho "$YELLOW" "Activating virtual environment..."
+. "$ACTIVATE_SCRIPT"
+if [ $? -ne 0 ]; then
+    cecho "$RED" "Error: Failed to activate virtual environment."
+    exit 1
+fi
+
+# Install/upgrade pip, setuptools, and wheel
+cecho "$YELLOW" "Upgrading pip, setuptools, and wheel..."
+pip install --upgrade pip setuptools wheel
+if [ $? -ne 0 ]; then
+    cecho "$RED" "Warning: Failed to upgrade pip, setuptools, or wheel. Continuing anyway."
+fi
+
+# Check if requirements.txt exists
+if [ ! -f "$REQUIREMENTS_FILE" ]; then
+    cecho "$RED" "Error: $REQUIREMENTS_FILE not found."
+    echo "Please make sure the file exists in the current directory."
+    cecho "$YELLOW" "Continuing with an activated environment without installing dependencies."
+else
+    # Install requirements
+    cecho "$YELLOW" "Installing dependencies from $REQUIREMENTS_FILE..."
+    pip install -r "$REQUIREMENTS_FILE"
+    if [ $? -ne 0 ]; then
+        cecho "$RED" "Warning: Some dependencies may have failed to install."
+    else
+        cecho "$GREEN" "Dependencies installed successfully."
+    fi
+fi
+
+# Install the package in development mode if setup.py exists
+if [ -f "setup.py" ]; then
+    cecho "$YELLOW" "Installing PyStackQL in development mode..."
+    pip install .
+    if [ $? -ne 0 ]; then
+        cecho "$RED" "Warning: Failed to install package in development mode."
+    else
+        cecho "$GREEN" "Package installed in development mode."
+    fi
+fi
+
+# Success message
+echo ""
+cecho "$GREEN" "Virtual environment is now set up and activated!"
+cecho "$YELLOW" "You can use PyStackQL and run tests."
+echo ""
+cecho "$BLUE" "To run tests:"
+echo "  python run_tests.py"
+echo ""
+cecho "$BLUE" "To deactivate the virtual environment when done:"
+echo "  deactivate"
+echo ""
+cecho "$BLUE" "======================================="
+
+# Keep the terminal open with the activated environment
+# The script will be source'd, so the environment stays active
+exec "${SHELL:-bash}"

pystackql/__init__.py

@@ -1,3 +1,17 @@
-from .stackql import StackQL
-from .magic import StackqlMagic
-from .magics import StackqlServerMagic
+# pystackql/__init__.py
+
+"""
+PyStackQL - Python wrapper for StackQL
+
+This package provides a Python interface to the StackQL query language
+for cloud resource querying.
+"""
+
+# Import the core StackQL class
+from .core import StackQL
+
+# Import the magic classes for Jupyter integration
+from .magic_ext import StackqlMagic, StackqlServerMagic
+
+# Define the public API
+__all__ = ['StackQL', 'StackqlMagic', 'StackqlServerMagic']