Add static site generator for dataset browsing

- Added dark-themed static site with dataset pages
- Parse markdown documentation and frontmatter (YAML)
- Display column information and data samples (first 10 rows)
- Show example queries from markdown frontmatter
- Generate individual pages for each dataset with CSV/JSON/Parquet links
- Updated GitHub Actions to deploy site to GitHub Pages
- Added dependencies: markdown and pyyaml
- Fixed input directory to point to queries/ folder

Author: gankoji

.github/workflows/datasets.yml

@@ -9,6 +9,17 @@ on:
       - main
   workflow_dispatch: # Allow manual trigger
 
+# Allow GitHub Actions to deploy to Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow one concurrent deployment
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
 jobs:
   generate-datasets:
     runs-on: ubuntu-latest
@@ -37,7 +48,7 @@ jobs:
       - name: Run sql2csv
         run: |
           sql2csv \
-            --input-sql-dir ${{ github.workspace }} \
+            --input-sql-dir ${{ github.workspace }}/queries \
             --output-dir /tmp/output \
             --pgsql-creds-file /tmp/creds/.pgpass \
             --aws-creds-file /tmp/creds/.aws-creds
@@ -46,3 +57,15 @@ jobs:
         if: always()
         run: |
           rm -rf /tmp/creds
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+
+      - name: Upload site artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: '/tmp/output/site'
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

sql2datasets/pyproject.toml

@@ -18,7 +18,7 @@ classifiers = [
     "Topic :: Database",
     "Topic :: Utilities",
 ]
-dependencies = ["boto3", "python-dotenv", "psycopg2-binary", "pandas", "pyarrow"]
+dependencies = ["boto3", "python-dotenv", "psycopg2-binary", "pandas", "pyarrow", "markdown", "pyyaml"]
 readme = "README.md"
 
 [project.scripts]