From 2ef65c778b734f19dba1a8d313fcd7061bf75960 Mon Sep 17 00:00:00 2001 From: Brage Date: Wed, 4 Feb 2026 12:40:52 +0100 Subject: [PATCH 1/2] feat: performance test. cli tensor query w/ default accept header --- .gitignore | 1 + .../app/query-profiles/default.xml | 4 + .../cli_feed_client/cli_feed_client.rb | 116 +++++++++++++++++- .../performance/cli_feed_client/paragraph.sd | 22 ++++ 4 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 tests/performance/cli_feed_client/app/query-profiles/default.xml create mode 100644 tests/performance/cli_feed_client/paragraph.sd diff --git a/.gitignore b/.gitignore index 5f880a796..e90670e22 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ Gemfile.lock _work/ *.iml target/ +tests/performance/cli_feed_client/data/ diff --git a/tests/performance/cli_feed_client/app/query-profiles/default.xml b/tests/performance/cli_feed_client/app/query-profiles/default.xml new file mode 100644 index 000000000..139c2999d --- /dev/null +++ b/tests/performance/cli_feed_client/app/query-profiles/default.xml @@ -0,0 +1,4 @@ + + 10000 + 10000 + \ No newline at end of file diff --git a/tests/performance/cli_feed_client/cli_feed_client.rb b/tests/performance/cli_feed_client/cli_feed_client.rb index b4f5aa645..eaf65516f 100644 --- a/tests/performance/cli_feed_client/cli_feed_client.rb +++ b/tests/performance/cli_feed_client/cli_feed_client.rb @@ -7,19 +7,28 @@ class CliFeedClientTest < PerformanceTest - DOCUMENTS = 2000000 + DOCUMENTS = 2_000_000 TINY = 10 - LARGE = 10000 + LARGE = 10_000 DUMMY_ROUTE = 'null/default' + N_TENSOR_QUERIES = 10 + def timeout_seconds 1800 end def setup - set_owner('hmusum') + set_owner('hmusum, bragehk') set_description('Benchmarking of the Vespa CLI feed client and vespa-feed-client (Java implementation)') + @vespa_destination_pid = nil + end + + def test_tensor_query_throughput + container_node = deploy_tensor_app + feed_tensor_documents(container_node, 3200) + run_tensor_query_benchmark(container_node, N_TENSOR_QUERIES) end def test_throughput @@ -153,4 +162,105 @@ def deploy_test_app gw end + private + def feed_tensor_documents(container_node, num_docs) + tensor_docs_s3 = "mips-data/paragraph_docs.400k.json" + # Use a permanent local cache directory for the data file + local_data_dir = "#{selfdir}/data" + vespa.adminserver.execute("mkdir -p #{local_data_dir}") + full_file = "#{local_data_dir}/paragraph_docs.400k.json" + + # Download from S3 only if the file doesn't exist locally + if vespa.adminserver.execute("test -f #{full_file}; echo $?").to_i != 0 + downloaded_file = download_file_from_s3(tensor_docs_s3, vespa.adminserver, "nearest-neighbor") + vespa.adminserver.execute("cp #{downloaded_file} #{full_file}") + end + + subset_file = "#{dirs.tmpdir}/paragraph_docs.#{num_docs}.json" + + lines_to_extract = num_docs + 1 + vespa.adminserver.execute("(head -#{lines_to_extract} #{full_file} | head -c -2; echo ''; echo ']') > #{subset_file}", :exceptiononfailure => true) + + endpoint = "https://#{container_node.hostname}:#{Environment.instance.vespa_web_service_port}/" + feed_cmd = "env " + + "VESPA_CLI_DATA_PLANE_TRUST_ALL=true " + + "VESPA_CLI_DATA_PLANE_CA_CERT_FILE=#{tls_env.ca_certificates_file} " + + "VESPA_CLI_DATA_PLANE_CERT_FILE=#{tls_env.certificate_file} " + + "VESPA_CLI_DATA_PLANE_KEY_FILE=#{tls_env.private_key_file} " + + "vespa feed " + + "--target=#{endpoint} " + + "#{subset_file}" + vespa.adminserver.execute(feed_cmd) + end + + private + def run_tensor_query_benchmark(container_node, num_queries) + endpoint = "https://#{container_node.hostname}:#{Environment.instance.vespa_web_service_port}/" + label = "vespa-cli-tensor-query" + cpu_monitor = Perf::System.new(container_node) + cpu_monitor.start + profiler_start + + start_time = Time.now + num_queries.times do + run_single_tensor_query(endpoint) + end + end_time = Time.now + duration_seconds = end_time - start_time + + profiler_report(label) + cpu_monitor.end + + qps = num_queries / duration_seconds + write_report( + [ + Proc.new do |result| + result.add_metric('tensor.query.qps', qps.to_s) + result.add_metric('tensor.query.count', num_queries.to_s) + result.add_metric('tensor.query.duration_seconds', duration_seconds.to_s) + end, + parameter_filler('label', label), + cpu_monitor.fill + ] + ) + puts("Tensor query benchmark: #{num_queries} queries in #{duration_seconds}s (#{qps} QPS") + end + + private + def run_single_tensor_query(endpoint) + out_file = "#{dirs.tmpdir}/tensor_query.out" + err_file = "#{dirs.tmpdir}/tensor_query.err" + query_cmd = "env " + + "VESPA_CLI_DATA_PLANE_TRUST_ALL=true " + + "VESPA_CLI_DATA_PLANE_CA_CERT_FILE=#{tls_env.ca_certificates_file} " + + "VESPA_CLI_DATA_PLANE_CERT_FILE=#{tls_env.certificate_file} " + + "VESPA_CLI_DATA_PLANE_KEY_FILE=#{tls_env.private_key_file} " + + "/home/bragehk/git/vespa/client/go/bin/vespa query " + + "--target=#{endpoint} " + + "--verbose " + + "'yql=select * from paragraph where id >= 0' " + + "'hits=3200' " + + "> #{out_file} 2> #{err_file}" + + vespa.adminserver.execute(query_cmd) + end + + private + def deploy_tensor_app + container_cluster = Container.new("dpcluster1"). + jvmoptions('-Xms16g -Xmx16g -XX:NewRatio=1'). + search(Searching.new). + documentapi(ContainerDocumentApi.new). + component(AccessLog.new("disabled")) + output = deploy_app(SearchApp.new. + sd(selfdir + 'paragraph.sd'). + search_dir(selfdir + 'app'). + container(container_cluster). + threads_per_search(1)) + start + gw = @vespa.container.values.first + wait_for_application(gw, output) + gw + end + end diff --git a/tests/performance/cli_feed_client/paragraph.sd b/tests/performance/cli_feed_client/paragraph.sd new file mode 100644 index 000000000..b90db89cf --- /dev/null +++ b/tests/performance/cli_feed_client/paragraph.sd @@ -0,0 +1,22 @@ +# Copyright Vespa.ai. All rights reserved. + +schema paragraph { + document paragraph { + + field id type long { + indexing: attribute | summary + } + + field embedding type tensor(x[768]) { + indexing: attribute | summary + } + + } + + rank-profile default { + first-phase { + expression: attribute(id) + } + } + +} From 8b6d1578d70050befd872edda1983cb5bfe835d4 Mon Sep 17 00:00:00 2001 From: Brage Date: Wed, 4 Feb 2026 12:43:46 +0100 Subject: [PATCH 2/2] fix: remove abs path --- tests/performance/cli_feed_client/cli_feed_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/cli_feed_client/cli_feed_client.rb b/tests/performance/cli_feed_client/cli_feed_client.rb index eaf65516f..79e0060bd 100644 --- a/tests/performance/cli_feed_client/cli_feed_client.rb +++ b/tests/performance/cli_feed_client/cli_feed_client.rb @@ -235,7 +235,7 @@ def run_single_tensor_query(endpoint) "VESPA_CLI_DATA_PLANE_CA_CERT_FILE=#{tls_env.ca_certificates_file} " + "VESPA_CLI_DATA_PLANE_CERT_FILE=#{tls_env.certificate_file} " + "VESPA_CLI_DATA_PLANE_KEY_FILE=#{tls_env.private_key_file} " + - "/home/bragehk/git/vespa/client/go/bin/vespa query " + + "vespa query " + "--target=#{endpoint} " + "--verbose " + "'yql=select * from paragraph where id >= 0' " +