Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
DUCKDB_VERSION := v1.4.0
DUCKDB_VERSION := v1.2.0
EXTENSION_NAME := duckarrow
# Extract version from git tag, fall back to "dev" for local builds
EXTENSION_VERSION := $(shell git describe --tags --exact-match 2>/dev/null || echo "dev")
Expand Down
5 changes: 1 addition & 4 deletions config_function.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,10 +150,7 @@ func duckarrow_configure_callback(info C.duckdb_function_info, input C.duckdb_da
SetDuckArrowConfig(uri, username, password, skipVerify)

// Return a confirmation message
result := "DuckArrow configured successfully"
resultCStr := C.CString(result)
C.duckdb_vector_assign_string_element(output, i, resultCStr)
C.free(unsafe.Pointer(resultCStr))
duckdb.AssignStringToVector(duckdb.Vector{Ptr: unsafe.Pointer(output)}, int(i), "DuckArrow configured successfully")
}
}

Expand Down
2 changes: 1 addition & 1 deletion duckarrow-local.sql
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
LOAD './build/duckarrow.duckdb_extension';

-- Configure duckarrow with credentials (enables replacement scan)
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'duckarrow_user', 'duckarrow_password');
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password');

-- Query remote tables directly using duckarrow."TableName" syntax:
-- SELECT * FROM duckarrow."Order" LIMIT 5;
Expand Down
2 changes: 1 addition & 1 deletion duckdb-go-api
Submodule duckdb-go-api updated 1 files
+116 −13 functions.go
93 changes: 16 additions & 77 deletions table_function.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,7 @@ func duckarrow_bind_wrapper(info C.duckdb_bind_info) {
ctx := context.Background()
connResult, err := flight.GetConnection(ctx, cfg)
if err != nil {
errStr := C.CString(fmt.Sprintf("connection failed: %v", err))
C.duckdb_bind_set_error(info, errStr)
C.free(unsafe.Pointer(errStr))
duckdb.SetBindError(duckdb.BindInfo{Ptr: unsafe.Pointer(info)}, "connection failed: %v", err)
return
}

Expand All @@ -130,9 +128,7 @@ func duckarrow_bind_wrapper(info C.duckdb_bind_info) {
} else {
connResult.Client.Close()
}
errStr := C.CString(fmt.Sprintf("query failed: %v", err))
C.duckdb_bind_set_error(info, errStr)
C.free(unsafe.Pointer(errStr))
duckdb.SetBindError(duckdb.BindInfo{Ptr: unsafe.Pointer(info)}, "query failed: %v", err)
return
}

Expand Down Expand Up @@ -334,9 +330,7 @@ func duckarrow_init_wrapper(info C.duckdb_init_info) {
for i := C.idx_t(0); i < columnCount; i++ {
colIdx := C.duckdb_init_get_column_index(info, i)
if int(colIdx) >= len(bindData.AllColumns) {
errStr := C.CString(fmt.Sprintf("column index %d out of range (max %d)", colIdx, len(bindData.AllColumns)-1))
C.duckdb_init_set_error(info, errStr)
C.free(unsafe.Pointer(errStr))
duckdb.SetInitError(duckdb.InitInfo{Ptr: unsafe.Pointer(info)}, "column index %d out of range (max %d)", colIdx, len(bindData.AllColumns)-1)
return
}
projectedColumns[i] = bindData.AllColumns[colIdx]
Expand All @@ -349,9 +343,7 @@ func duckarrow_init_wrapper(info C.duckdb_init_info) {
ctx := context.Background()
result, err := bindData.Client.Query(ctx, query)
if err != nil {
errStr := C.CString(fmt.Sprintf("query execution failed: %v", err))
C.duckdb_init_set_error(info, errStr)
C.free(unsafe.Pointer(errStr))
duckdb.SetInitError(duckdb.InitInfo{Ptr: unsafe.Pointer(info)}, "query execution failed: %v", err)
return
}

Expand All @@ -370,9 +362,7 @@ func duckarrow_scan_wrapper(info C.duckdb_function_info, output C.duckdb_data_ch
bindHandle := cgo.Handle(uintptr(bindPtr))
bindData, ok := bindHandle.Value().(*BindData)
if !ok {
errStr := C.CString("internal error: invalid bind data type")
C.duckdb_function_set_error(info, errStr)
C.free(unsafe.Pointer(errStr))
duckdb.SetFunctionError(duckdb.FunctionInfo{Ptr: unsafe.Pointer(info)}, "internal error: invalid bind data type")
return
}

Expand All @@ -381,9 +371,7 @@ func duckarrow_scan_wrapper(info C.duckdb_function_info, output C.duckdb_data_ch
stateHandle := cgo.Handle(uintptr(statePtr))
state, ok := stateHandle.Value().(*ScanState)
if !ok {
errStr := C.CString("internal error: invalid scan state type")
C.duckdb_function_set_error(info, errStr)
C.free(unsafe.Pointer(errStr))
duckdb.SetFunctionError(duckdb.FunctionInfo{Ptr: unsafe.Pointer(info)}, "internal error: invalid scan state type")
return
}

Expand Down Expand Up @@ -414,13 +402,8 @@ func scanHardcodedData(info C.duckdb_function_info, output C.duckdb_data_chunk,
nameVec := duckdb.DataChunkGetVector(duckdb.DataChunk{Ptr: unsafe.Pointer(output)}, 1)

for i, row := range rows {
idStr := C.CString(row.id)
C.duckdb_vector_assign_string_element(C.duckdb_vector(idVec.Ptr), C.idx_t(i), idStr)
C.free(unsafe.Pointer(idStr))

nameStr := C.CString(row.name)
C.duckdb_vector_assign_string_element(C.duckdb_vector(nameVec.Ptr), C.idx_t(i), nameStr)
C.free(unsafe.Pointer(nameStr))
duckdb.AssignStringToVector(idVec, i, row.id)
duckdb.AssignStringToVector(nameVec, i, row.name)
}

C.duckdb_data_chunk_set_size(output, C.idx_t(len(rows)))
Expand All @@ -444,9 +427,7 @@ func scanArrowData(info C.duckdb_function_info, output C.duckdb_data_chunk, bind

if !bindData.Reader.Next() {
if err := bindData.Reader.Err(); err != nil {
errStr := C.CString(err.Error())
C.duckdb_function_set_error(info, errStr)
C.free(unsafe.Pointer(errStr))
duckdb.SetFunctionError(duckdb.FunctionInfo{Ptr: unsafe.Pointer(info)}, "%v", err)
}
atomic.StoreInt32(&state.Done, 1)
C.duckdb_data_chunk_set_size(output, 0)
Expand Down Expand Up @@ -474,9 +455,7 @@ func scanArrowData(info C.duckdb_function_info, output C.duckdb_data_chunk, bind
duckVec := duckdb.DataChunkGetVector(duckdb.DataChunk{Ptr: unsafe.Pointer(output)}, uint64(colIdx))

if err := convertArrowToDuckDB(arrowCol, duckVec, int(state.BatchPosition), rowsToEmit); err != nil {
errStr := C.CString(fmt.Sprintf("convert col %d: %v", colIdx, err))
C.duckdb_function_set_error(info, errStr)
C.free(unsafe.Pointer(errStr))
duckdb.SetFunctionError(duckdb.FunctionInfo{Ptr: unsafe.Pointer(info)}, "convert col %d: %v", colIdx, err)
return
}
}
Expand All @@ -500,9 +479,7 @@ func convertArrowToDuckDB(arrowCol arrow.Array, duckVec duckdb.Vector, offset, c
duckdb.ValiditySetRowInvalid(validity, uint64(i))
continue
}
cStr := C.CString(col.Value(srcIdx))
C.duckdb_vector_assign_string_element(C.duckdb_vector(duckVec.Ptr), C.idx_t(i), cStr)
C.free(unsafe.Pointer(cStr))
duckdb.AssignStringToVector(duckVec, i, col.Value(srcIdx))
}

case *array.LargeString:
Expand All @@ -512,9 +489,7 @@ func convertArrowToDuckDB(arrowCol arrow.Array, duckVec duckdb.Vector, offset, c
duckdb.ValiditySetRowInvalid(validity, uint64(i))
continue
}
cStr := C.CString(col.Value(srcIdx))
C.duckdb_vector_assign_string_element(C.duckdb_vector(duckVec.Ptr), C.idx_t(i), cStr)
C.free(unsafe.Pointer(cStr))
duckdb.AssignStringToVector(duckVec, i, col.Value(srcIdx))
}

case *array.Int64:
Expand Down Expand Up @@ -763,23 +738,7 @@ func convertArrowToDuckDB(arrowCol arrow.Array, duckVec duckdb.Vector, offset, c
duckdb.ValiditySetRowInvalid(validity, uint64(i))
continue
}
data := col.Value(srcIdx)
if len(data) == 0 {
// Empty binary - assign empty string
C.duckdb_vector_assign_string_element_len(
C.duckdb_vector(duckVec.Ptr),
C.idx_t(i),
nil,
0,
)
continue
}
C.duckdb_vector_assign_string_element_len(
C.duckdb_vector(duckVec.Ptr),
C.idx_t(i),
(*C.char)(unsafe.Pointer(&data[0])),
C.idx_t(len(data)),
)
duckdb.AssignBytesToVector(duckVec, i, col.Value(srcIdx))
}

case *array.LargeBinary:
Expand All @@ -790,23 +749,7 @@ func convertArrowToDuckDB(arrowCol arrow.Array, duckVec duckdb.Vector, offset, c
duckdb.ValiditySetRowInvalid(validity, uint64(i))
continue
}
data := col.Value(srcIdx)
if len(data) == 0 {
// Empty binary - assign empty string
C.duckdb_vector_assign_string_element_len(
C.duckdb_vector(duckVec.Ptr),
C.idx_t(i),
nil,
0,
)
continue
}
C.duckdb_vector_assign_string_element_len(
C.duckdb_vector(duckVec.Ptr),
C.idx_t(i),
(*C.char)(unsafe.Pointer(&data[0])),
C.idx_t(len(data)),
)
duckdb.AssignBytesToVector(duckVec, i, col.Value(srcIdx))
}

case *array.FixedSizeBinary:
Expand All @@ -826,9 +769,7 @@ func convertArrowToDuckDB(arrowCol arrow.Array, duckVec duckdb.Vector, offset, c
} else {
val = fmt.Sprintf("%x", data)
}
cStr := C.CString(val)
C.duckdb_vector_assign_string_element(C.duckdb_vector(duckVec.Ptr), C.idx_t(i), cStr)
C.free(unsafe.Pointer(cStr))
duckdb.AssignStringToVector(duckVec, i, val)
}

case *array.Decimal128:
Expand Down Expand Up @@ -1162,9 +1103,7 @@ func convertArrowToDuckDB(arrowCol arrow.Array, duckVec duckdb.Vector, offset, c
val = fmt.Sprintf("%v", arrowCol.GetOneForMarshal(srcIdx))
}

cStr := C.CString(val)
C.duckdb_vector_assign_string_element(C.duckdb_vector(duckVec.Ptr), C.idx_t(i), cStr)
C.free(unsafe.Pointer(cStr))
duckdb.AssignStringToVector(duckVec, i, val)
}
}

Expand Down
4 changes: 2 additions & 2 deletions tests/connection_pool_test.sql
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ LOAD './build/duckarrow.duckdb_extension';

-- Configure
SELECT '=== Configure ===' as test;
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'duckarrow_user', 'duckarrow_password', true);
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);

-- First query (cold connection - expect ~150-200ms)
SELECT '=== Query 1 (cold connection) ===' as test;
Expand All @@ -32,7 +32,7 @@ SELECT id, name FROM duckarrow."Order" LIMIT 3;

-- Test reconfiguration creates new connection (same credentials, new pool entry)
SELECT '=== Reconfigure (same server, fresh connection) ===' as test;
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'duckarrow_user', 'duckarrow_password', true);
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);

-- First query after reconfig (should use existing pool)
SELECT '=== Query 6 (after reconfig) ===' as test;
Expand Down
2 changes: 1 addition & 1 deletion tests/connection_test.sql
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
.bail on
LOAD './build/duckarrow.duckdb_extension';
-- Configure credentials
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'duckarrow_user', 'duckarrow_password', true);
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);
SELECT * FROM duckarrow_query(
'grpc+tls://localhost:31337',
'SELECT 1 as test_value'
Expand Down
2 changes: 1 addition & 1 deletion tests/data_types_test.sql
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

-- Load extension and configure
LOAD './build/duckarrow.duckdb_extension';
SELECT duckarrow_configure('grpc://localhost:31337', 'duckarrow_user', 'duckarrow_password');
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);

.print '=== Data Type Tests ==='

Expand Down
10 changes: 5 additions & 5 deletions tests/error_handling_test.sql
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ LOAD './build/duckarrow.duckdb_extension';
.bail off

-- First configure correctly
SELECT duckarrow_configure('grpc://localhost:31337', 'duckarrow_user', 'duckarrow_password');
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);

-- This should fail with a syntax error from the remote server
SELECT * FROM duckarrow."Order" WHERE SELECTT;
Expand All @@ -30,7 +30,7 @@ SELECT * FROM duckarrow."Order" WHERE SELECTT;
.print '--- Test 2: Non-existent Table ---'

-- Reconfigure (in case previous test affected state)
SELECT duckarrow_configure('grpc://localhost:31337', 'duckarrow_user', 'duckarrow_password');
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);

-- This should fail with table not found
SELECT * FROM duckarrow."ThisTableDefinitelyDoesNotExist12345";
Expand Down Expand Up @@ -94,7 +94,7 @@ SELECT 1 FROM duckarrow."Order" LIMIT 1;
.print '--- Test 7: Invalid Credentials ---'

-- Configure with wrong credentials
SELECT duckarrow_configure('grpc://localhost:31337', 'wrong_user', 'wrong_pass');
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'wrong_user', 'wrong_pass');

-- This should fail with auth error
SELECT * FROM duckarrow."Order" LIMIT 1;
Expand All @@ -108,7 +108,7 @@ SELECT * FROM duckarrow."Order" LIMIT 1;
.print '--- Test 8: SQL Injection Prevention ---'

-- Restore valid config first
SELECT duckarrow_configure('grpc://localhost:31337', 'duckarrow_user', 'duckarrow_password');
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);

-- These should be blocked by table name validation
-- Semicolon injection attempt
Expand Down Expand Up @@ -151,7 +151,7 @@ SELECT * FROM duckarrow."malicious*/Order";
.bail on

-- Restore valid configuration
SELECT duckarrow_configure('grpc://localhost:31337', 'duckarrow_user', 'duckarrow_password');
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);

-- This should work after all the error tests
SELECT COUNT(*) as recovery_test FROM duckarrow."Order";
Expand Down
2 changes: 1 addition & 1 deletion tests/full_test.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
LOAD './build/duckarrow.duckdb_extension';

-- Configure credentials
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'duckarrow_user', 'duckarrow_password', true);
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);

-- Test 1: Basic query with real data and types
SELECT '=== Test 1: Basic query with types ===' as test;
Expand Down
2 changes: 1 addition & 1 deletion tests/integration_test.sql
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

-- Load extension and configure
LOAD './build/duckarrow.duckdb_extension';
SELECT duckarrow_configure('grpc://localhost:31337', 'duckarrow_user', 'duckarrow_password');
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);

.print '=== Edge Case Integration Tests ==='

Expand Down
4 changes: 2 additions & 2 deletions tests/replacement_scan_test.sql
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ LOAD './build/duckarrow.duckdb_extension';

-- Test 1: Configure duckarrow URI
SELECT '=== Test 1: Configure duckarrow URI ===' as test;
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'duckarrow_user', 'duckarrow_password', true);
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);

-- Test 2: Basic replacement scan query
SELECT '=== Test 2: Basic replacement scan ===' as test;
Expand Down Expand Up @@ -73,7 +73,7 @@ SELECT duckarrow_configure('grpc://localhost:1234', 'user', 'pass');

-- Test 16: Reconfigure back to TLS for remaining tests
SELECT '=== Test 16: Reconfigure to TLS ===' as test;
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'duckarrow_user', 'duckarrow_password', true);
SELECT duckarrow_configure('grpc+tls://localhost:31337', 'gizmosql_user', 'gizmosql_password', true);

-- Test 17: NULL handling
SELECT '=== Test 17: NULL handling ===' as test;
Expand Down
5 changes: 1 addition & 4 deletions version_function.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,8 @@ func duckarrow_version_callback(info C.duckdb_function_info, input C.duckdb_data
}

// Return the version string for each row
versionCStr := C.CString(Version)
defer C.free(unsafe.Pointer(versionCStr))

for i := range inputSize {
C.duckdb_vector_assign_string_element(output, C.idx_t(i), versionCStr)
duckdb.AssignStringToVector(duckdb.Vector{Ptr: unsafe.Pointer(output)}, i, Version)
}
}

Expand Down
Loading