Skip to content

feat: update openai tests to work with both clients #2442

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hardikjshah merged 1 commit into from
Jun 12, 2025
Merged

feat: update openai tests to work with both clients #2442

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions llama_stack/providers/utils/memory/openai_vector_store_mixin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,11 +109,6 @@ async def openai_create_vector_store(
provider_id=provider_id,
provider_resource_id=provider_vector_db_id,
)
from rich.pretty import pprint

print("VECTOR DB")
pprint(vector_db)

# Register the vector DB
await self.register_vector_db(vector_db)

Expand Down
54 changes: 29 additions & 25 deletions tests/integration/inference/test_openai_embeddings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,15 @@ def skip_if_model_doesnt_support_variable_dimensions(model_id):
pytest.skip("{model_id} does not support variable output embedding dimensions")


def skip_if_model_doesnt_support_openai_embeddings(client_with_models, model_id):
if isinstance(client_with_models, LlamaStackAsLibraryClient):
pytest.skip("OpenAI embeddings are not supported when testing with library client yet.")
@pytest.fixture(params=["openai_client", "llama_stack_client"])
def compat_client(request, client_with_models):
if request.param == "openai_client" and isinstance(client_with_models, LlamaStackAsLibraryClient):
pytest.skip("OpenAI client tests not supported with library client")
return request.getfixturevalue(request.param)

provider = provider_from_model(client_with_models, model_id)

def skip_if_model_doesnt_support_openai_embeddings(client, model_id):
provider = provider_from_model(client, model_id)
if provider.provider_type in (
"inline::meta-reference",
"remote::bedrock",
Expand All @@ -58,13 +62,13 @@ def openai_client(client_with_models):
return OpenAI(base_url=base_url, api_key="fake")


def test_openai_embeddings_single_string(openai_client, client_with_models, embedding_model_id):
def test_openai_embeddings_single_string(compat_client, client_with_models, embedding_model_id):
"""Test OpenAI embeddings endpoint with a single string input."""
skip_if_model_doesnt_support_openai_embeddings(client_with_models, embedding_model_id)

input_text = "Hello, world!"

response = openai_client.embeddings.create(
response = compat_client.embeddings.create(
model=embedding_model_id,
input=input_text,
encoding_format="float",
Expand All @@ -80,13 +84,13 @@ def test_openai_embeddings_single_string(openai_client, client_with_models, embe
assert all(isinstance(x, float) for x in response.data[0].embedding)


def test_openai_embeddings_multiple_strings(openai_client, client_with_models, embedding_model_id):
def test_openai_embeddings_multiple_strings(compat_client, client_with_models, embedding_model_id):
"""Test OpenAI embeddings endpoint with multiple string inputs."""
skip_if_model_doesnt_support_openai_embeddings(client_with_models, embedding_model_id)

input_texts = ["Hello, world!", "How are you today?", "This is a test."]

response = openai_client.embeddings.create(
response = compat_client.embeddings.create(
model=embedding_model_id,
input=input_texts,
)
Expand All @@ -103,13 +107,13 @@ def test_openai_embeddings_multiple_strings(openai_client, client_with_models, e
assert all(isinstance(x, float) for x in embedding_data.embedding)


def test_openai_embeddings_with_encoding_format_float(openai_client, client_with_models, embedding_model_id):
def test_openai_embeddings_with_encoding_format_float(compat_client, client_with_models, embedding_model_id):
"""Test OpenAI embeddings endpoint with float encoding format."""
skip_if_model_doesnt_support_openai_embeddings(client_with_models, embedding_model_id)

input_text = "Test encoding format"

response = openai_client.embeddings.create(
response = compat_client.embeddings.create(
model=embedding_model_id,
input=input_text,
encoding_format="float",
Expand All @@ -121,15 +125,15 @@ def test_openai_embeddings_with_encoding_format_float(openai_client, client_with
assert all(isinstance(x, float) for x in response.data[0].embedding)


def test_openai_embeddings_with_dimensions(openai_client, client_with_models, embedding_model_id):
def test_openai_embeddings_with_dimensions(compat_client, client_with_models, embedding_model_id):
"""Test OpenAI embeddings endpoint with custom dimensions parameter."""
skip_if_model_doesnt_support_openai_embeddings(client_with_models, embedding_model_id)
skip_if_model_doesnt_support_variable_dimensions(embedding_model_id)

input_text = "Test dimensions parameter"
dimensions = 16

response = openai_client.embeddings.create(
response = compat_client.embeddings.create(
model=embedding_model_id,
input=input_text,
dimensions=dimensions,
Expand All @@ -142,14 +146,14 @@ def test_openai_embeddings_with_dimensions(openai_client, client_with_models, em
assert len(response.data[0].embedding) > 0


def test_openai_embeddings_with_user_parameter(openai_client, client_with_models, embedding_model_id):
def test_openai_embeddings_with_user_parameter(compat_client, client_with_models, embedding_model_id):
"""Test OpenAI embeddings endpoint with user parameter."""
skip_if_model_doesnt_support_openai_embeddings(client_with_models, embedding_model_id)

input_text = "Test user parameter"
user_id = "test-user-123"

response = openai_client.embeddings.create(
response = compat_client.embeddings.create(
model=embedding_model_id,
input=input_text,
user=user_id,
Expand All @@ -161,41 +165,41 @@ def test_openai_embeddings_with_user_parameter(openai_client, client_with_models
assert len(response.data[0].embedding) > 0


def test_openai_embeddings_empty_list_error(openai_client, client_with_models, embedding_model_id):
def test_openai_embeddings_empty_list_error(compat_client, client_with_models, embedding_model_id):
"""Test that empty list input raises an appropriate error."""
skip_if_model_doesnt_support_openai_embeddings(client_with_models, embedding_model_id)

with pytest.raises(Exception): # noqa: B017
openai_client.embeddings.create(
compat_client.embeddings.create(
model=embedding_model_id,
input=[],
)


def test_openai_embeddings_invalid_model_error(openai_client, client_with_models, embedding_model_id):
def test_openai_embeddings_invalid_model_error(compat_client, client_with_models, embedding_model_id):
"""Test that invalid model ID raises an appropriate error."""
skip_if_model_doesnt_support_openai_embeddings(client_with_models, embedding_model_id)

with pytest.raises(Exception): # noqa: B017
openai_client.embeddings.create(
compat_client.embeddings.create(
model="invalid-model-id",
input="Test text",
)


def test_openai_embeddings_different_inputs_different_outputs(openai_client, client_with_models, embedding_model_id):
def test_openai_embeddings_different_inputs_different_outputs(compat_client, client_with_models, embedding_model_id):
"""Test that different inputs produce different embeddings."""
skip_if_model_doesnt_support_openai_embeddings(client_with_models, embedding_model_id)

input_text1 = "This is the first text"
input_text2 = "This is completely different content"

response1 = openai_client.embeddings.create(
response1 = compat_client.embeddings.create(
model=embedding_model_id,
input=input_text1,
)

response2 = openai_client.embeddings.create(
response2 = compat_client.embeddings.create(
model=embedding_model_id,
input=input_text2,
)
Expand All @@ -208,15 +212,15 @@ def test_openai_embeddings_different_inputs_different_outputs(openai_client, cli
assert embedding1 != embedding2


def test_openai_embeddings_with_encoding_format_base64(openai_client, client_with_models, embedding_model_id):
def test_openai_embeddings_with_encoding_format_base64(compat_client, client_with_models, embedding_model_id):
"""Test OpenAI embeddings endpoint with base64 encoding format."""
skip_if_model_doesnt_support_openai_embeddings(client_with_models, embedding_model_id)
skip_if_model_doesnt_support_variable_dimensions(embedding_model_id)

input_text = "Test base64 encoding format"
dimensions = 12

response = openai_client.embeddings.create(
response = compat_client.embeddings.create(
model=embedding_model_id,
input=input_text,
encoding_format="base64",
Expand All @@ -241,13 +245,13 @@ def test_openai_embeddings_with_encoding_format_base64(openai_client, client_wit
assert all(isinstance(x, float) for x in embedding_floats)


def test_openai_embeddings_base64_batch_processing(openai_client, client_with_models, embedding_model_id):
def test_openai_embeddings_base64_batch_processing(compat_client, client_with_models, embedding_model_id):
"""Test OpenAI embeddings endpoint with base64 encoding for batch processing."""
skip_if_model_doesnt_support_openai_embeddings(client_with_models, embedding_model_id)

input_texts = ["First text for base64", "Second text for base64", "Third text for base64"]

response = openai_client.embeddings.create(
response = compat_client.embeddings.create(
model=embedding_model_id,
input=input_texts,
encoding_format="base64",
Expand Down
5 changes: 1 addition & 4 deletions tests/integration/vector_io/test_openai_vector_stores.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@


def skip_if_provider_doesnt_support_openai_vector_stores(client_with_models):
if isinstance(client_with_models, LlamaStackAsLibraryClient):
pytest.skip("OpenAI vector stores are not supported when testing with library client yet.")

vector_io_providers = [p for p in client_with_models.providers.list() if p.api == "vector_io"]
for p in vector_io_providers:
if p.provider_type in ["inline::faiss", "inline::sqlite-vec"]:
Expand All @@ -34,7 +31,7 @@ def openai_client(client_with_models):
return OpenAI(base_url=base_url, api_key="fake")


@pytest.fixture(params=["openai_client"]) # , "llama_stack_client"])
@pytest.fixture(params=["openai_client", "llama_stack_client"])
def compat_client(request, client_with_models):
if request.param == "openai_client" and isinstance(client_with_models, LlamaStackAsLibraryClient):
pytest.skip("OpenAI client tests not supported with library client")
Expand Down