diff --git a/.gitattributes b/.gitattributes index dfe0770..80cfa88 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ # Auto detect text files and perform LF normalization -* text=auto +* text=auto eol=lf +*.gleam text eol=lf \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1cf7819 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,69 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + name: Build and Test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start ClickHouse + run: | + docker compose up -d clickhouse + echo "Waiting for ClickHouse to be ready..." + for i in $(seq 1 60); do + if curl -sSf http://localhost:8123/ >/dev/null 2>&1; then + echo "ClickHouse is ready" + break + fi + sleep 2 + done + + - name: Setup Erlang/OTP and Gleam + uses: erlef/setup-beam@v1 + with: + otp-version: "27.0" + gleam-version: "1.13.0" + + - name: Show Gleam version + run: gleam --version + + - name: Cache Gleam deps and build + uses: actions/cache@v4 + with: + path: | + ~/.cache/gleam + ~/.gleam + ./_gleam_deps + ./build + key: ${{ runner.os }}-gleam-1.13.0-otp27-cache-v1-${{ hashFiles('**/gleam.toml') }} + restore-keys: | + ${{ runner.os }}-gleam-1.13.0-otp27-cache-v1- + + - name: Install dependencies + run: gleam deps download + + - name: Check formatting + run: gleam format --check src test + + - name: Build project + run: gleam build + + - name: Run all tests + env: + CLICKHOUSE_USER: test_user + CLICKHOUSE_PASSWORD: test_password + CLICKHOUSE_DB: test_db + CLICKHOUSE_URL: http://localhost:8123 + run: gleam test + + - name: Cleanup ClickHouse + if: always() + run: docker compose down -v diff --git a/.github/workflows/smoke-integration.yml b/.github/workflows/smoke-integration.yml new file mode 100644 index 0000000..5881ff0 --- /dev/null +++ b/.github/workflows/smoke-integration.yml @@ -0,0 +1,65 @@ +name: Smoke Integration (PR) + +on: + pull_request: + branches: ["main"] + +jobs: + smoke-integration: + name: Smoke Integration (ClickHouse) + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start ClickHouse (docker-compose) + env: + CLICKHOUSE_IMAGE: clickhouse/clickhouse-server:23.7 + run: | + echo "Using CLICKHOUSE_IMAGE=$CLICKHOUSE_IMAGE" + docker compose pull clickhouse || true + docker compose up -d --remove-orphans clickhouse + + - name: Wait for ClickHouse HTTP + run: | + echo "Waiting for ClickHouse HTTP on localhost:8123..." + for i in $(seq 1 40); do + if curl -sSf http://localhost:8123/ >/dev/null 2>&1; then + echo "ClickHouse is up" + break + fi + sleep 2 + done + + - name: Smoke test - SELECT 1 + run: | + set -e + OUT=$(curl -sS -u test_user:test_password "http://localhost:8123/?query=SELECT%201%20as%20result%20FORMAT%20JSONEachRow&database=test_db") + echo "Got: $OUT" + if [ "$OUT" != '{"result":1}' ] && [ "$OUT" != '{"result":1}\n' ]; then + echo "Unexpected response: $OUT" + exit 2 + fi + + - name: Smoke test - create table, insert and select + run: | + set -e + DDL="CREATE TABLE IF NOT EXISTS smoke_test (id UInt32, name String) ENGINE = MergeTree() ORDER BY id" + curl -sS -u test_user:test_password -d "$DDL" "http://localhost:8123/?database=test_db" + + INSERT="INSERT INTO smoke_test (id,name) VALUES (1,'smoke')" + curl -sS -u test_user:test_password -d "$INSERT" "http://localhost:8123/?database=test_db" + + OUT=$(curl -sS -u test_user:test_password "http://localhost:8123/?query=SELECT%20*%20FROM%20smoke_test%20WHERE%20id%3D1%20FORMAT%20JSONEachRow&database=test_db") + echo "Select returned: $OUT" + if [ -z "$OUT" ]; then + echo "Select returned empty" + exit 3 + fi + + - name: Cleanup ClickHouse + if: always() + run: | + docker compose down -v || true diff --git a/README.md b/README.md index ed6d532..59a35b8 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Sparkling logo

-[![License](https://img.shields.io/badge/license-Apache%202.0-yellow.svg)](LICENSE) [![Built with Gleam](https://img.shields.io/badge/Built%20with-Gleam-ffaff3)](https://gleam.run) +[![CI](https://github.com/lupodevelop/sparkling/actions/workflows/ci.yml/badge.svg)](https://github.com/lupodevelop/sparkling/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/license-Apache%202.0-yellow.svg)](LICENSE) [![Built with Gleam](https://img.shields.io/badge/Built%20with-Gleam-ffaff3)](https://gleam.run) [![Gleam Version](https://img.shields.io/badge/gleam-%3E%3D1.13.0-ffaff3)](https://gleam.run) **Sparkling** is a *lightweight*, **type-safe** data layer for **ClickHouse** written in Gleam. It provides a small, focused API for defining schemas, building queries, and encoding/decoding ClickHouse formats. @@ -12,6 +12,37 @@ > Why "Sparkling"? One rainy Tuesday a tiny inflatable rubber duck stole a shooting pink star and decided to become a freelance data wrangler, and it now guides queries through the night, humming 8-bit lullabies. Totally plausible. +## Quick start + +See the extracted quick start example: `docs/quickstart.md` it contains a short walkthrough (define schema, build a query, execute it with a repo). + +Minimal example: + +```gleam +import sparkling/repo + +let r = repo.new("http://localhost:8123") + |> repo.with_database("mydb") + +case r.execute_sql(r, "SELECT 1 as result FORMAT JSONEachRow") { + Ok(body) -> io.println(body) + Error(_) -> io.println("query failed") +} +``` + +## What you'll find here + +- `sparkling/schema` — typed table & column definitions +- `sparkling/query` — immutable query builder (to_sql) +- `sparkling/repo` — HTTP executor with retry hooks +- `sparkling/encode` / `sparkling/decode` — format handlers (JSONEachRow default) +- `sparkling/types` — helpers for Decimal, DateTime64, UUID, LowCardinality + +For more examples see `docs/examples/` and `docs/quickstart.md`. + +**Design note:** Sparkling's API and composable query builder were partly inspired by *Ecto*; +many ideas about schema definition and query composition borrow from its approach while keeping a small, Gleam-friendly surface. + ## Development Run tests and format/check locally: diff --git a/test/TESTS.md b/test/TESTS.md new file mode 100644 index 0000000..74b9060 --- /dev/null +++ b/test/TESTS.md @@ -0,0 +1,62 @@ +# Tests — Quick reference + +This document explains how tests are organised in the repository and how to run them locally and in CI. + +Directory layout + +- `test/sparkling/` — unit and component tests that do not require external services (fast). +- `test/smoke/` — smoke/sanity tests. Quick checks that the test runner and a minimal API behave as expected. +- `test/integration/` — integration tests that require external services (e.g. ClickHouse). These are slower and should not run on every PR. + +Running tests locally + +- Run the full test suite (if you have required services available): + +```bash +# from the project root +gleam test +``` + +- Run unit tests only (recommended for PRs): + +```bash +# if your test runner accepts paths: +gleam test test/sparkling +``` + +If your runner does not accept paths, run `gleam test` locally and ensure integration tests are not executed by default (integration tests should be placed under `test/integration/`). + +- Run integration tests (requires ClickHouse or other external services): + +```bash +# Start ClickHouse (example using Docker) +docker run -d --name clickhouse-server -p 8123:8123 -p 9000:9000 clickhouse/clickhouse-server:latest + +# Run integration tests +gleam test test/integration + +# When finished, stop/remove the container +docker stop clickhouse-server && docker rm clickhouse-server +``` + +Environment variables + +- Use environment variables to configure integration endpoints (example): + +```bash +export CLICKHOUSE_URL=http://localhost:8123 +``` + +Document required variables in `test/integration/README.md`. + +CI guidance + +- Pull Requests: run unit tests only. +- Integration tests: run in separate CI jobs (manual trigger, nightly, or on release tags). Ensure the CI environment has access to required services and secrets before enabling integration jobs. + +Best practices and PR checklist + +- [ ] Unit tests pass locally (`gleam test`). +- [ ] Integration tests are documented in `test/integration/README.md` with clear prerequisites and run instructions. +- [ ] Do not add CI workflows that perform sensitive operations (publishing) in the import PR. Add automation in a separate PR after the code is stable. +- [ ] Keep a lightweight smoke test under `test/smoke/` to validate the test runner and minimal API surface. \ No newline at end of file diff --git a/test/integration/README.md b/test/integration/README.md new file mode 100644 index 0000000..e4cca4e --- /dev/null +++ b/test/integration/README.md @@ -0,0 +1,98 @@ +# Integration tests + +Integration tests that exercise the code against a real ClickHouse instance. + +This document explains how to run integration tests locally and how to wire them into CI in a safe way. + +Prerequisites + +- Docker (used for the example below). +- docker-compose (optional, if you prefer a compose-based setup). + +Recommended local setup (Docker Compose) + +1. Start ClickHouse with docker-compose (example): + +```bash +# from the repository root, if you have a docker-compose.yml configured +docker-compose up -d +``` + +2. Wait for the server to be ready: + +```bash +docker-compose ps +``` + +3. Run only the integration tests: + +```bash +# from the repository root +gleam test test/integration +``` + +4. Tear down the environment when finished: + +```bash +docker-compose down -v +``` + +Quick single-container alternative (no compose) + +```bash +docker run -d --name clickhouse-server -p 8123:8123 -p 9000:9000 clickhouse/clickhouse-server:latest +# wait for readiness, then run tests +gleam test test/integration +docker stop clickhouse-server && docker rm clickhouse-server +``` + +Connection details (local defaults) + +- URL: http://localhost:8123 +- Database: test_db (tests may create/drop their own DBs) +- User: test_user +- Password: test_password + +If your tests require different credentials or endpoints, set the appropriate environment variables (see the `Environment variables` section below). + +Implemented integration tests + +- `basic_connectivity_test.gleam` — verifies connection and basic queries +- `complex_types_roundtrip_test.gleam` — round-trip tests for complex ClickHouse types +- `format_compatibility_test.gleam` — tests different I/O formats (JSONEachRow, CSV, TabSeparated) +- `performance_test.gleam` — optional performance/load checks (may be slow) + +Environment variables + +Use environment variables to configure endpoints and credentials for integration tests. Example: + +```bash +export CLICKHOUSE_URL=http://localhost:8123 +export CLICKHOUSE_USER=test_user +export CLICKHOUSE_PASSWORD=test_password +``` + +Document required environment variables and secrets clearly before enabling integration jobs in CI. + +CI guidance + +- Integration tests should not run by default on every PR. Run them in separate CI jobs, e.g. on manual dispatch, nightly builds, or release tags. +- Example GitHub Actions job (run in a dedicated workflow or as a separate job): + +```yaml +- name: Integration tests (optional) + run: | + docker-compose up -d + # optionally wait / healthcheck + sleep 10 + gleam test test/integration + docker-compose down -v + # do not fail the whole pipeline if integration env is missing + continue-on-error: true +``` + +Notes and recommendations + +- Keep integration tests isolated and idempotent: tests should create and drop any resources they need. +- Mark expensive or flaky tests (e.g. `performance_test.gleam`) so CI can skip them unless explicitly requested. +- Before enabling integration tests in CI, ensure any required secrets or credentials are set in the CI environment. diff --git a/test/sparkling/encode_decode_roundtrip_test.gleam b/test/sparkling/encode_decode_roundtrip_test.gleam deleted file mode 100644 index d07c3e6..0000000 --- a/test/sparkling/encode_decode_roundtrip_test.gleam +++ /dev/null @@ -1,390 +0,0 @@ -/// Encoding tests for complex ClickHouse types -/// Verifies correct JSON serialization without precision loss -import gleam/dict -import gleam/json -import gleam/option.{None, Some} -import gleam/string -import gleeunit/should -import sparkling/encode -import sparkling/types - -// ============================================================================ -// Decimal encoding tests -// ============================================================================ - -pub fn decimal_encode_test() { - let assert Ok(dec) = types.decimal("123.456") - let encoded = encode.decimal(dec) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("123.456") - |> should.be_true() -} - -pub fn decimal_large_precision_test() { - let assert Ok(dec) = types.decimal("999999999999999.123456789") - let encoded = encode.decimal(dec) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("999999999999999.123456789") - |> should.be_true() -} - -pub fn decimal_negative_test() { - let assert Ok(dec) = types.decimal("-123.45") - let encoded = encode.decimal(dec) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("-123.45") - |> should.be_true() -} - -// ============================================================================ -// DateTime64 encoding tests -// ============================================================================ - -pub fn datetime64_roundtrip_test() { - // DateTime64 with precision and timezone - case types.datetime64("2024-11-07 15:30:45.123", 3, option.Some("UTC")) { - Ok(dt) -> { - let encoded = encode.datetime64(dt) - let json_str = json.to_string(encoded) - - // Verify encoding produces valid string - json_str - |> string.contains("2024-11-07") - |> should.be_true - } - Error(e) -> panic as { "Failed to create DateTime64: " <> e } - } -} - -pub fn datetime64_with_timezone_test() { - case types.datetime64("2024-01-15 10:30:45", 0, Some("UTC")) { - Ok(dt) -> { - let encoded = encode.datetime64(dt) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("2024-01-15") - |> should.be_true() - } - Error(e) -> panic as { "Failed to create DateTime64: " <> e } - } -} - -pub fn datetime64_from_epoch_test() { - case types.datetime64_from_epoch(1_705_315_845, 0, None) { - Ok(dt) -> { - let encoded = encode.datetime64(dt) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("1705315845") - |> should.be_true() - } - Error(e) -> panic as { "Failed to create DateTime64: " <> e } - } -} - -// ============================================================================ -// UUID encoding tests -// ============================================================================ - -pub fn uuid_encode_test() { - let assert Ok(uuid) = types.uuid("550e8400-e29b-41d4-a716-446655440000") - let encoded = encode.uuid(uuid) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("550e8400-e29b-41d4-a716-446655440000") - |> should.be_true() -} - -// ============================================================================ -// Array encoding tests -// ============================================================================ - -pub fn array_int_encode_test() { - let arr = [1, 2, 3, 4, 5] - let encoded = encode.array(arr, json.int) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("1") - |> should.be_true() - - json_str - |> string.contains("5") - |> should.be_true() -} - -pub fn array_string_encode_test() { - let arr = ["hello", "world", "test"] - let encoded = encode.array(arr, json.string) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("hello") - |> should.be_true() - - json_str - |> string.contains("world") - |> should.be_true() -} - -pub fn array_nested_encode_test() { - let arr = [[1, 2], [3, 4], [5, 6]] - let encoded = encode.array(arr, fn(inner) { encode.array(inner, json.int) }) - let json_str = json.to_string(encoded) - - // Should contain nested structure - json_str - |> string.contains("[") - |> should.be_true() -} - -// ============================================================================ -// Map encoding tests -// ============================================================================ - -pub fn map_string_int_encode_test() { - let map = - dict.from_list([ - #("one", json.int(1)), - #("two", json.int(2)), - #("three", json.int(3)), - ]) - - let encoded = encode.clickhouse_map_from_dict(map) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("one") - |> should.be_true() - - json_str - |> string.contains("1") - |> should.be_true() -} - -pub fn map_string_string_encode_test() { - let map = - dict.from_list([ - #("name", json.string("Alice")), - #("city", json.string("Rome")), - #("country", json.string("Italy")), - ]) - - let encoded = encode.clickhouse_map_from_dict(map) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("Alice") - |> should.be_true() - - json_str - |> string.contains("Rome") - |> should.be_true() -} - -// ============================================================================ -// Tuple encoding tests -// ============================================================================ - -pub fn tuple_mixed_types_test() { - let elements = [ - json.int(42), - json.string("hello"), - json.bool(True), - json.float(3.14), - ] - - let encoded = encode.clickhouse_tuple(elements) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("42") - |> should.be_true() - - json_str - |> string.contains("hello") - |> should.be_true() -} - -// ============================================================================ -// Nested structure encoding tests -// ============================================================================ - -pub fn nested_structure_encode_test() { - let nested = - dict.from_list([ - #("id", json.int(1)), - #("name", json.string("Test")), - #("active", json.bool(True)), - #( - "metadata", - json.object([ - #("version", json.int(2)), - #("tags", json.array(["a", "b", "c"], json.string)), - ]), - ), - ]) - - let encoded = encode.nested_from_dict(nested) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("Test") - |> should.be_true() - - json_str - |> string.contains("metadata") - |> should.be_true() -} - -// ============================================================================ -// LowCardinality encoding tests -// ============================================================================ - -pub fn low_cardinality_encode_test() { - let lc = types.low_cardinality_string("active") - let encoded = encode.low_cardinality_string(lc) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("active") - |> should.be_true() -} - -// ============================================================================ -// Nullable encoding tests -// ============================================================================ - -pub fn nullable_some_value_test() { - let nullable = Some(json.int(42)) - let encoded = encode.nullable(nullable) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("42") - |> should.be_true() -} - -pub fn nullable_none_value_test() { - let nullable = None - let encoded = encode.nullable(nullable) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("null") - |> should.be_true() -} - -// ============================================================================ -// Enum encoding tests -// ============================================================================ - -pub fn enum8_encode_test() { - // Enum8 just holds mappings, we encode the integer value directly - let encoded = encode.enum_value(1) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("1") - |> should.be_true() -} - -pub fn enum16_encode_test() { - // Enum16 just holds mappings, we encode the integer value directly - let encoded = encode.enum_value(200) - let json_str = json.to_string(encoded) - - json_str - |> string.contains("200") - |> should.be_true() -} - -// ============================================================================ -// Complex composite encoding test -// ============================================================================ - -pub fn complex_record_encode_test() { - // Simulate a complex record with multiple types - let assert Ok(decimal_price) = types.decimal("99.99") - let assert Ok(uuid_id) = types.uuid("550e8400-e29b-41d4-a716-446655440000") - let assert Ok(created_at) = types.datetime64("2024-01-15 10:30:45", 0, None) - let status = types.low_cardinality_string("active") - - let record = - dict.from_list([ - #("id", encode.uuid(uuid_id)), - #("price", encode.decimal(decimal_price)), - #("created_at", encode.datetime64(created_at)), - #("status", encode.low_cardinality_string(status)), - #("tags", encode.array(["tag1", "tag2"], json.string)), - #( - "metadata", - json.object([ - #("version", json.int(1)), - ]), - ), - ]) - - let encoded = encode.encode_record(record) - - // Verify all fields present - encoded - |> string.contains("550e8400") - |> should.be_true() - - encoded - |> string.contains("99.99") - |> should.be_true() - - encoded - |> string.contains("active") - |> should.be_true() - - encoded - |> string.contains("tag1") - |> should.be_true() -} - -// ============================================================================ -// JSONEachRow batch encoding test -// ============================================================================ - -pub fn json_each_row_batch_encode_test() { - let assert Ok(dec1) = types.decimal("10.50") - let assert Ok(dec2) = types.decimal("20.75") - - let records = [ - dict.from_list([ - #("id", json.int(1)), - #("price", encode.decimal(dec1)), - ]), - dict.from_list([ - #("id", json.int(2)), - #("price", encode.decimal(dec2)), - ]), - ] - - let encoded = encode.encode_json_each_row(records) - - // Should have newline-separated rows - encoded - |> string.contains("10.50") - |> should.be_true() - - encoded - |> string.contains("20.75") - |> should.be_true() - - encoded - |> string.contains("\n") - |> should.be_true() -}