From 0868fad0570a1a04010a00acb3f4eab7644b95fc Mon Sep 17 00:00:00 2001 From: Jacob Simionato Date: Wed, 6 Aug 2025 09:53:39 +0930 Subject: [PATCH 1/9] Move system instruction param to ai_client --- examples/generic_chat/lib/main.dart | 2 +- examples/travel_app/lib/main.dart | 2 +- pkgs/flutter_genui/lib/src/ai_client/ai_client.dart | 11 ++++++----- .../lib/src/ai_client/llm_connection.dart | 1 - pkgs/flutter_genui/lib/src/core/genui_manager.dart | 7 +++---- pkgs/flutter_genui/test/core/genui_manager_test.dart | 5 +++-- pkgs/flutter_genui/test/fake_ai_client.dart | 4 +++- 7 files changed, 17 insertions(+), 15 deletions(-) diff --git a/examples/generic_chat/lib/main.dart b/examples/generic_chat/lib/main.dart index 5947912b1..fefba3c7e 100644 --- a/examples/generic_chat/lib/main.dart +++ b/examples/generic_chat/lib/main.dart @@ -61,13 +61,13 @@ class _MyHomePageState extends State { void initState() { super.initState(); final aiClient = AiClient( + systemInstruction: Content.system(systemPrompt), loggingCallback: (severity, message) { debugPrint('[$severity] $message'); }, ); _genUiManager = GenUiManager.conversation( catalog: coreCatalog, - instruction: systemPrompt, llmConnection: aiClient, ); } diff --git a/examples/travel_app/lib/main.dart b/examples/travel_app/lib/main.dart index de24421d7..78141e4b6 100644 --- a/examples/travel_app/lib/main.dart +++ b/examples/travel_app/lib/main.dart @@ -55,13 +55,13 @@ class _MyHomePageState extends State { void initState() { super.initState(); aiClient = AiClient( + systemInstruction: Content.system(prompt), loggingCallback: (severity, message) { debugPrint('[$severity] $message'); }, ); _genUiManager = GenUiManager.conversation( catalog: catalog, - instruction: prompt, llmConnection: aiClient, ); } diff --git a/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart b/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart index 4070911be..da184ab52 100644 --- a/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart +++ b/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart @@ -79,6 +79,7 @@ class AiClient implements LlmConnection { this.loggingCallback, this.tools = const [], this.outputToolName = 'provideFinalOutput', + this.systemInstruction, }) : model = ValueNotifier(model) { final duplicateToolNames = tools.map((t) => t.name).toSet(); if (duplicateToolNames.length != tools.length) { @@ -105,6 +106,7 @@ class AiClient implements LlmConnection { this.loggingCallback, this.tools = const [], this.outputToolName = 'provideFinalOutput', + this.systemInstruction, }) : model = ValueNotifier(model) { final duplicateToolNames = tools.map((t) => t.name).toSet(); if (duplicateToolNames.length != tools.length) { @@ -213,6 +215,9 @@ class AiClient implements LlmConnection { /// Defaults to 'provideFinalOutput'. final String outputToolName; + /// The system instruction to use for the AI. + final Content? systemInstruction; + /// The total number of input tokens used by this client. int inputTokenUsage = 0; @@ -257,12 +262,11 @@ class AiClient implements LlmConnection { List conversation, Schema outputSchema, { Iterable additionalTools = const [], - Content? systemInstruction, }) async { return await _generateContentWithRetries(conversation, outputSchema, [ ...tools, ...additionalTools, - ], systemInstruction); + ]); } /// The default factory function for creating a [GenerativeModel]. @@ -310,7 +314,6 @@ class AiClient implements LlmConnection { List contents, Schema outputSchema, List availableTools, - Content? systemInstruction, ) async { var attempts = 0; var delay = initialDelay; @@ -338,7 +341,6 @@ class AiClient implements LlmConnection { contents, outputSchema, availableTools, - systemInstruction, // Reset the delay and attempts on success. () { delay = initialDelay; @@ -372,7 +374,6 @@ class AiClient implements LlmConnection { List contents, Schema outputSchema, List availableTools, - Content? systemInstruction, void Function() onSuccess, ) async { // Create an "output" tool that copies its args into the output. diff --git a/pkgs/flutter_genui/lib/src/ai_client/llm_connection.dart b/pkgs/flutter_genui/lib/src/ai_client/llm_connection.dart index b5aa4b035..402d27b10 100644 --- a/pkgs/flutter_genui/lib/src/ai_client/llm_connection.dart +++ b/pkgs/flutter_genui/lib/src/ai_client/llm_connection.dart @@ -11,6 +11,5 @@ abstract interface class LlmConnection { List conversation, Schema outputSchema, { Iterable additionalTools = const [], - Content? systemInstruction, }); } diff --git a/pkgs/flutter_genui/lib/src/core/genui_manager.dart b/pkgs/flutter_genui/lib/src/core/genui_manager.dart index fe382dd19..fba5e9e4c 100644 --- a/pkgs/flutter_genui/lib/src/core/genui_manager.dart +++ b/pkgs/flutter_genui/lib/src/core/genui_manager.dart @@ -15,14 +15,14 @@ class GenUiManager { GenUiManager.conversation({ LlmConnection? llmConnection, this.catalog = const Catalog([]), - this.instruction = '', + String instruction = '', }) { - this.llmConnection = llmConnection ?? AiClient(); + this.llmConnection = + llmConnection ?? AiClient(systemInstruction: Content.system(instruction)); _eventManager = UiEventManager(callback: handleEvents); } final Catalog catalog; - final String instruction; late final LlmConnection llmConnection; late final UiEventManager _eventManager; @@ -115,7 +115,6 @@ class GenUiManager { final response = await llmConnection.generateContent( conversation, outputSchema, - systemInstruction: Content.system(instruction), ); if (response == null) { return; diff --git a/pkgs/flutter_genui/test/core/genui_manager_test.dart b/pkgs/flutter_genui/test/core/genui_manager_test.dart index c166707c5..51ef8c64d 100644 --- a/pkgs/flutter_genui/test/core/genui_manager_test.dart +++ b/pkgs/flutter_genui/test/core/genui_manager_test.dart @@ -19,10 +19,11 @@ void main() { late FakeAiClient fakeAiClient; setUp(() { - fakeAiClient = FakeAiClient(); + fakeAiClient = FakeAiClient( + systemInstruction: Content.system('You are a helpful assistant.'), + ); manager = GenUiManager.conversation( catalog: coreCatalog, - instruction: 'You are a helpful assistant.', llmConnection: fakeAiClient, ); }); diff --git a/pkgs/flutter_genui/test/fake_ai_client.dart b/pkgs/flutter_genui/test/fake_ai_client.dart index 49e5d89e0..db36fe2fb 100644 --- a/pkgs/flutter_genui/test/fake_ai_client.dart +++ b/pkgs/flutter_genui/test/fake_ai_client.dart @@ -9,18 +9,20 @@ import 'package:flutter_genui/src/ai_client/llm_connection.dart'; import 'package:flutter_genui/src/ai_client/tools.dart'; class FakeAiClient implements LlmConnection { + FakeAiClient({this.systemInstruction}); + Object? response; Exception? exception; int generateContentCallCount = 0; List lastConversation = []; Future Function()? preGenerateContent; + Content? systemInstruction; @override Future generateContent( List conversation, Schema outputSchema, { Iterable additionalTools = const [], - Content? systemInstruction, }) async { await preGenerateContent?.call(); generateContentCallCount++; From c84707b5fc92cef2a7895ef4e547ef9387ce9998 Mon Sep 17 00:00:00 2001 From: Jacob Simionato Date: Wed, 6 Aug 2025 09:56:20 +0930 Subject: [PATCH 2/9] Make systemInstruction private --- .../flutter_genui/lib/src/ai_client/ai_client.dart | 14 ++++++++------ .../test/core/genui_manager_test.dart | 5 ++--- pkgs/flutter_genui/test/fake_ai_client.dart | 3 --- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart b/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart index da184ab52..f63ecfa90 100644 --- a/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart +++ b/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart @@ -79,8 +79,9 @@ class AiClient implements LlmConnection { this.loggingCallback, this.tools = const [], this.outputToolName = 'provideFinalOutput', - this.systemInstruction, - }) : model = ValueNotifier(model) { + Content? systemInstruction, + }) : _systemInstruction = systemInstruction, + model = ValueNotifier(model) { final duplicateToolNames = tools.map((t) => t.name).toSet(); if (duplicateToolNames.length != tools.length) { final duplicateTools = tools.where((t) { @@ -106,8 +107,9 @@ class AiClient implements LlmConnection { this.loggingCallback, this.tools = const [], this.outputToolName = 'provideFinalOutput', - this.systemInstruction, - }) : model = ValueNotifier(model) { + Content? systemInstruction, + }) : _systemInstruction = systemInstruction, + model = ValueNotifier(model) { final duplicateToolNames = tools.map((t) => t.name).toSet(); if (duplicateToolNames.length != tools.length) { final duplicateTools = tools.where((t) { @@ -216,7 +218,7 @@ class AiClient implements LlmConnection { final String outputToolName; /// The system instruction to use for the AI. - final Content? systemInstruction; + final Content? _systemInstruction; /// The total number of input tokens used by this client. int inputTokenUsage = 0; @@ -429,7 +431,7 @@ class AiClient implements LlmConnection { final model = modelCreator( configuration: this, - systemInstruction: systemInstruction, + systemInstruction: _systemInstruction, tools: generativeAiTools, toolConfig: ToolConfig( functionCallingConfig: FunctionCallingConfig.any( diff --git a/pkgs/flutter_genui/test/core/genui_manager_test.dart b/pkgs/flutter_genui/test/core/genui_manager_test.dart index 51ef8c64d..c166707c5 100644 --- a/pkgs/flutter_genui/test/core/genui_manager_test.dart +++ b/pkgs/flutter_genui/test/core/genui_manager_test.dart @@ -19,11 +19,10 @@ void main() { late FakeAiClient fakeAiClient; setUp(() { - fakeAiClient = FakeAiClient( - systemInstruction: Content.system('You are a helpful assistant.'), - ); + fakeAiClient = FakeAiClient(); manager = GenUiManager.conversation( catalog: coreCatalog, + instruction: 'You are a helpful assistant.', llmConnection: fakeAiClient, ); }); diff --git a/pkgs/flutter_genui/test/fake_ai_client.dart b/pkgs/flutter_genui/test/fake_ai_client.dart index db36fe2fb..b8600b4a1 100644 --- a/pkgs/flutter_genui/test/fake_ai_client.dart +++ b/pkgs/flutter_genui/test/fake_ai_client.dart @@ -9,14 +9,11 @@ import 'package:flutter_genui/src/ai_client/llm_connection.dart'; import 'package:flutter_genui/src/ai_client/tools.dart'; class FakeAiClient implements LlmConnection { - FakeAiClient({this.systemInstruction}); - Object? response; Exception? exception; int generateContentCallCount = 0; List lastConversation = []; Future Function()? preGenerateContent; - Content? systemInstruction; @override Future generateContent( From 78cf0c86178d24b310821bc43312284bc738f134 Mon Sep 17 00:00:00 2001 From: Jacob Simionato Date: Wed, 6 Aug 2025 09:58:55 +0930 Subject: [PATCH 3/9] Make llmConnection a required param --- doc/USAGE.md | 2 +- pkgs/flutter_genui/lib/src/core/genui_manager.dart | 7 ++----- pkgs/flutter_genui/test/core/genui_manager_test.dart | 1 - pkgs/spikes/usage_test/lib/main.dart | 3 ++- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/doc/USAGE.md b/doc/USAGE.md index 1285804f1..91c89b853 100644 --- a/doc/USAGE.md +++ b/doc/USAGE.md @@ -56,7 +56,7 @@ to configure your Flutter project. you want the generated user interface to render: ```dart - final GenUiManager _genUiManager = GenUiManager.conversation(); + final GenUiManager _genUiManager = GenUiManager.conversation(llmConnection: AiClient()); ``` 1. Update the `build` method of the widget to render `_genUiManager.widget()`. diff --git a/pkgs/flutter_genui/lib/src/core/genui_manager.dart b/pkgs/flutter_genui/lib/src/core/genui_manager.dart index fba5e9e4c..99fbba1d1 100644 --- a/pkgs/flutter_genui/lib/src/core/genui_manager.dart +++ b/pkgs/flutter_genui/lib/src/core/genui_manager.dart @@ -13,17 +13,14 @@ import 'conversation_widget.dart'; class GenUiManager { GenUiManager.conversation({ - LlmConnection? llmConnection, + required this.llmConnection, this.catalog = const Catalog([]), - String instruction = '', }) { - this.llmConnection = - llmConnection ?? AiClient(systemInstruction: Content.system(instruction)); _eventManager = UiEventManager(callback: handleEvents); } final Catalog catalog; - late final LlmConnection llmConnection; + final LlmConnection llmConnection; late final UiEventManager _eventManager; // Context used for future LLM inferences diff --git a/pkgs/flutter_genui/test/core/genui_manager_test.dart b/pkgs/flutter_genui/test/core/genui_manager_test.dart index c166707c5..02b86ccc5 100644 --- a/pkgs/flutter_genui/test/core/genui_manager_test.dart +++ b/pkgs/flutter_genui/test/core/genui_manager_test.dart @@ -22,7 +22,6 @@ void main() { fakeAiClient = FakeAiClient(); manager = GenUiManager.conversation( catalog: coreCatalog, - instruction: 'You are a helpful assistant.', llmConnection: fakeAiClient, ); }); diff --git a/pkgs/spikes/usage_test/lib/main.dart b/pkgs/spikes/usage_test/lib/main.dart index b73a74f18..6051c3ead 100644 --- a/pkgs/spikes/usage_test/lib/main.dart +++ b/pkgs/spikes/usage_test/lib/main.dart @@ -34,7 +34,8 @@ class MyHomePage extends StatefulWidget { } class _MyHomePageState extends State { - final GenUiManager _genUiManager = GenUiManager.conversation(); + final GenUiManager _genUiManager = + GenUiManager.conversation(llmConnection: AiClient()); @override void initState() { From fe578a54490660a08991d6f1639e6f57b313f843 Mon Sep 17 00:00:00 2001 From: Jacob Simionato Date: Wed, 6 Aug 2025 10:03:57 +0930 Subject: [PATCH 4/9] Make the parameter a string --- examples/generic_chat/lib/main.dart | 2 +- examples/travel_app/lib/main.dart | 2 +- pkgs/flutter_genui/lib/src/ai_client/ai_client.dart | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/generic_chat/lib/main.dart b/examples/generic_chat/lib/main.dart index fefba3c7e..35ec056b5 100644 --- a/examples/generic_chat/lib/main.dart +++ b/examples/generic_chat/lib/main.dart @@ -61,7 +61,7 @@ class _MyHomePageState extends State { void initState() { super.initState(); final aiClient = AiClient( - systemInstruction: Content.system(systemPrompt), + systemInstruction: systemPrompt, loggingCallback: (severity, message) { debugPrint('[$severity] $message'); }, diff --git a/examples/travel_app/lib/main.dart b/examples/travel_app/lib/main.dart index 78141e4b6..7cfc7d296 100644 --- a/examples/travel_app/lib/main.dart +++ b/examples/travel_app/lib/main.dart @@ -55,7 +55,7 @@ class _MyHomePageState extends State { void initState() { super.initState(); aiClient = AiClient( - systemInstruction: Content.system(prompt), + systemInstruction: prompt, loggingCallback: (severity, message) { debugPrint('[$severity] $message'); }, diff --git a/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart b/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart index f63ecfa90..fb4f9036e 100644 --- a/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart +++ b/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart @@ -79,7 +79,7 @@ class AiClient implements LlmConnection { this.loggingCallback, this.tools = const [], this.outputToolName = 'provideFinalOutput', - Content? systemInstruction, + String? systemInstruction, }) : _systemInstruction = systemInstruction, model = ValueNotifier(model) { final duplicateToolNames = tools.map((t) => t.name).toSet(); @@ -107,7 +107,7 @@ class AiClient implements LlmConnection { this.loggingCallback, this.tools = const [], this.outputToolName = 'provideFinalOutput', - Content? systemInstruction, + String? systemInstruction, }) : _systemInstruction = systemInstruction, model = ValueNotifier(model) { final duplicateToolNames = tools.map((t) => t.name).toSet(); @@ -218,7 +218,7 @@ class AiClient implements LlmConnection { final String outputToolName; /// The system instruction to use for the AI. - final Content? _systemInstruction; + final String? _systemInstruction; /// The total number of input tokens used by this client. int inputTokenUsage = 0; @@ -431,7 +431,8 @@ class AiClient implements LlmConnection { final model = modelCreator( configuration: this, - systemInstruction: _systemInstruction, + systemInstruction: + _systemInstruction == null ? null : Content.system(_systemInstruction!), tools: generativeAiTools, toolConfig: ToolConfig( functionCallingConfig: FunctionCallingConfig.any( From fc1134a62d597f97b22446ecc779ddc488b65277 Mon Sep 17 00:00:00 2001 From: Jacob Simionato Date: Wed, 6 Aug 2025 10:53:44 +0930 Subject: [PATCH 5/9] Improvements to conversation widget and gen ui manager --- .../lib/src/core/conversation_widget.dart | 77 +++++++++++++------ .../lib/src/core/genui_manager.dart | 16 ++-- .../lib/src/model/chat_message.dart | 9 --- .../test/core/genui_manager_test.dart | 52 ++++++++++--- .../test/core/surface_widget_test.dart | 62 ++++++++++----- 5 files changed, 143 insertions(+), 73 deletions(-) diff --git a/pkgs/flutter_genui/lib/src/core/conversation_widget.dart b/pkgs/flutter_genui/lib/src/core/conversation_widget.dart index 70ce0cbd3..cb2d436bc 100644 --- a/pkgs/flutter_genui/lib/src/core/conversation_widget.dart +++ b/pkgs/flutter_genui/lib/src/core/conversation_widget.dart @@ -9,17 +9,27 @@ import '../model/chat_message.dart'; import '../model/surface_widget.dart'; import '../model/ui_models.dart'; +typedef SystemMessageBuilder = + Widget Function(BuildContext context, SystemMessage message); + +typedef UserPromptBuilder = + Widget Function(BuildContext context, UserPrompt message); + class ConversationWidget extends StatelessWidget { const ConversationWidget({ super.key, required this.messages, required this.catalog, required this.onEvent, + this.systemMessageBuilder, + this.userPromptBuilder, }); final List messages; final void Function(Map event) onEvent; final Catalog catalog; + final SystemMessageBuilder? systemMessageBuilder; + final UserPromptBuilder? userPromptBuilder; @override Widget build(BuildContext context) { @@ -28,30 +38,49 @@ class ConversationWidget extends StatelessWidget { itemBuilder: (context, index) { final message = messages[index]; return switch (message) { - SystemMessage() => Card( - elevation: 2.0, - margin: const EdgeInsets.symmetric(horizontal: 8.0, vertical: 4.0), - child: ListTile( - title: Text(message.text), - leading: const Icon(Icons.smart_toy_outlined), - ), - ), - TextResponse() => Card( - elevation: 2.0, - margin: const EdgeInsets.symmetric(horizontal: 8.0, vertical: 4.0), - child: ListTile( - title: Text(message.text), - leading: const Icon(Icons.smart_toy_outlined), - ), - ), - UserPrompt() => Card( - elevation: 2.0, - margin: const EdgeInsets.symmetric(horizontal: 8.0, vertical: 4.0), - child: ListTile( - title: Text(message.text, textAlign: TextAlign.right), - trailing: const Icon(Icons.person), - ), - ), + SystemMessage() => + systemMessageBuilder != null + ? systemMessageBuilder!(context, message) + : Card( + elevation: 2.0, + margin: const EdgeInsets.symmetric( + horizontal: 8.0, + vertical: 4.0, + ), + child: ListTile( + title: Text(message.text), + leading: const Icon(Icons.smart_toy_outlined), + ), + ), + UserPrompt() => + userPromptBuilder != null + ? userPromptBuilder!(context, message) + : Padding( + padding: const EdgeInsets.symmetric( + vertical: 4.0, + horizontal: 8.0, + ), + child: Row( + mainAxisAlignment: MainAxisAlignment.end, + children: [ + Flexible( + child: Card( + shape: const RoundedRectangleBorder( + borderRadius: const BorderRadius.only( + topLeft: const Radius.circular(20.0), + bottomLeft: const Radius.circular(20.0), + bottomRight: const Radius.circular(20.0), + ), + ), + child: Padding( + padding: const EdgeInsets.all(12.0), + child: Text(message.text), + ), + ), + ), + ], + ), + ), UiResponse() => Padding( padding: const EdgeInsets.all(16.0), child: SurfaceWidget( diff --git a/pkgs/flutter_genui/lib/src/core/genui_manager.dart b/pkgs/flutter_genui/lib/src/core/genui_manager.dart index 99fbba1d1..00eff3b53 100644 --- a/pkgs/flutter_genui/lib/src/core/genui_manager.dart +++ b/pkgs/flutter_genui/lib/src/core/genui_manager.dart @@ -15,12 +15,16 @@ class GenUiManager { GenUiManager.conversation({ required this.llmConnection, this.catalog = const Catalog([]), + this.userPromptBuilder, + this.systemMessageBuilder, }) { _eventManager = UiEventManager(callback: handleEvents); } final Catalog catalog; final LlmConnection llmConnection; + final UserPromptBuilder? userPromptBuilder; + final SystemMessageBuilder? systemMessageBuilder; late final UiEventManager _eventManager; // Context used for future LLM inferences @@ -117,9 +121,6 @@ class GenUiManager { return; } final responseMap = response as Map; - if (responseMap['responseText'] case final String responseText) { - _chatHistory.add(TextResponse(text: responseText)); - } if (responseMap['actions'] case final List actions) { for (final actionMap in actions.cast>()) { final action = actionMap['action'] as String; @@ -196,12 +197,6 @@ class GenUiManager { /// is always valid according to the schema. Schema get outputSchema => Schema.object( properties: { - 'responseText': Schema.string( - description: - 'The text response to the user query. This should be used ' - 'when the query is fully satisfied and no more information is ' - 'needed.', - ), 'actions': Schema.array( description: 'A list of actions to be performed on the UI surfaces.', items: Schema.object( @@ -239,7 +234,6 @@ class GenUiManager { description: 'A schema for defining a simple UI tree to be rendered by ' 'Flutter.', - optionalProperties: ['actions', 'responseText'], ); Widget widget() { @@ -253,6 +247,8 @@ class GenUiManager { onEvent: (event) { _eventManager.add(UiEvent.fromMap(event)); }, + systemMessageBuilder: systemMessageBuilder, + userPromptBuilder: userPromptBuilder, ); }, ); diff --git a/pkgs/flutter_genui/lib/src/model/chat_message.dart b/pkgs/flutter_genui/lib/src/model/chat_message.dart index edd9295ed..f4e32e69c 100644 --- a/pkgs/flutter_genui/lib/src/model/chat_message.dart +++ b/pkgs/flutter_genui/lib/src/model/chat_message.dart @@ -27,15 +27,6 @@ class UserPrompt extends ChatMessage { final String text; } -/// A message representing a text response from the AI. -class TextResponse extends ChatMessage { - /// Creates a [TextResponse] with the given [text]. - const TextResponse({required this.text}); - - /// The text of the AI's response. - final String text; -} - /// A message representing a UI response from the AI. class UiResponse extends ChatMessage { /// Creates a [UiResponse] with the given UI [definition]. diff --git a/pkgs/flutter_genui/test/core/genui_manager_test.dart b/pkgs/flutter_genui/test/core/genui_manager_test.dart index 02b86ccc5..586754d32 100644 --- a/pkgs/flutter_genui/test/core/genui_manager_test.dart +++ b/pkgs/flutter_genui/test/core/genui_manager_test.dart @@ -34,7 +34,25 @@ void main() { 'sendUserPrompt adds message and calls AI, updates with response', () async { const prompt = 'Hello'; - fakeAiClient.response = {'responseText': 'Hi back'}; + fakeAiClient.response = { + 'actions': [ + { + 'action': 'add', + 'surfaceId': 's1', + 'definition': { + 'root': 'root', + 'widgets': [ + { + 'id': 'root', + 'widget': { + 'text': {'text': 'Hi back'}, + }, + }, + ], + }, + }, + ], + }; final chatHistoryCompleter = Completer>(); manager.uiDataStream.listen((data) { @@ -49,8 +67,7 @@ void main() { expect(chatHistory[0], isA()); expect((chatHistory[0] as UserPrompt).text, prompt); - expect(chatHistory[1], isA()); - expect((chatHistory[1] as TextResponse).text, 'Hi back'); + expect(chatHistory[1], isA()); expect(fakeAiClient.generateContentCallCount, 1); expect( @@ -279,14 +296,30 @@ void main() { eventType: 'onTap', timestamp: DateTime.now(), ); - fakeAiClient.response = {'responseText': 'event handled'}; + fakeAiClient.response = { + 'actions': [ + { + 'action': 'add', + 'surfaceId': 's2', + 'definition': { + 'root': 'root', + 'widgets': [ + { + 'id': 'root', + 'widget': { + 'text': {'text': 'event handled'}, + }, + }, + ], + }, + }, + ], + }; final eventCompleter = Completer>(); final eventSub = manager.uiDataStream.listen((data) { - // Wait for the text response from the event - if (data.isNotEmpty && - data.last is TextResponse && - (data.last as TextResponse).text == 'event handled') { + // Wait for the ui response from the event + if (data.whereType().length > 1) { if (!eventCompleter.isCompleted) { eventCompleter.complete(data); } @@ -307,8 +340,7 @@ void main() { contains('user has interacted with the UI'), ); - expect(chatHistory.last, isA()); - expect((chatHistory.last as TextResponse).text, 'event handled'); + expect(chatHistory.last, isA()); }); test('handles AI error gracefully', () async { diff --git a/pkgs/flutter_genui/test/core/surface_widget_test.dart b/pkgs/flutter_genui/test/core/surface_widget_test.dart index d18634bf2..8bae6365d 100644 --- a/pkgs/flutter_genui/test/core/surface_widget_test.dart +++ b/pkgs/flutter_genui/test/core/surface_widget_test.dart @@ -25,28 +25,11 @@ void main() { ), ); expect(find.text('Hello'), findsOneWidget); - expect(find.byIcon(Icons.person), findsOneWidget); - }); - - testWidgets('renders TextResponse correctly', (WidgetTester tester) async { - final messages = [const TextResponse(text: 'Hi there')]; - await tester.pumpWidget( - MaterialApp( - home: Scaffold( - body: ConversationWidget( - messages: messages, - catalog: coreCatalog, - onEvent: (_) {}, - ), - ), - ), - ); - expect(find.text('Hi there'), findsOneWidget); - expect(find.byIcon(Icons.smart_toy_outlined), findsOneWidget); + expect(find.byIcon(Icons.person), findsNothing); }); testWidgets('renders SystemMessage correctly', (WidgetTester tester) async { - final messages = [const SystemMessage(text: 'Error')]; + final messages = [const SystemMessage(text: 'Hi there')]; await tester.pumpWidget( MaterialApp( home: Scaffold( @@ -58,7 +41,7 @@ void main() { ), ), ); - expect(find.text('Error'), findsOneWidget); + expect(find.text('Hi there'), findsOneWidget); expect(find.byIcon(Icons.smart_toy_outlined), findsOneWidget); }); @@ -94,5 +77,44 @@ void main() { expect(find.byType(SurfaceWidget), findsOneWidget); expect(find.text('UI Content'), findsOneWidget); }); + + testWidgets('uses custom userPromptBuilder', (WidgetTester tester) async { + final messages = [const UserPrompt(text: 'Hello')]; + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: ConversationWidget( + messages: messages, + catalog: coreCatalog, + onEvent: (_) {}, + userPromptBuilder: (context, message) => + const Text('Custom User Prompt'), + ), + ), + ), + ); + expect(find.text('Custom User Prompt'), findsOneWidget); + expect(find.text('Hello'), findsNothing); + }); + + testWidgets('uses custom systemMessageBuilder', + (WidgetTester tester) async { + final messages = [const SystemMessage(text: 'Error')]; + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: ConversationWidget( + messages: messages, + catalog: coreCatalog, + onEvent: (_) {}, + systemMessageBuilder: (context, message) => + const Text('Custom System Message'), + ), + ), + ), + ); + expect(find.text('Custom System Message'), findsOneWidget); + expect(find.text('Error'), findsNothing); + }); }); } From 4c99f688e708378ee8837dea02b8c7fed8941935 Mon Sep 17 00:00:00 2001 From: Jacob Simionato Date: Wed, 6 Aug 2025 11:03:11 +0930 Subject: [PATCH 6/9] Update conversation widget appearance --- .../lib/src/core/conversation_widget.dart | 96 ++++++++++++------- .../test/core/surface_widget_test.dart | 2 +- 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/pkgs/flutter_genui/lib/src/core/conversation_widget.dart b/pkgs/flutter_genui/lib/src/core/conversation_widget.dart index cb2d436bc..5a5460f64 100644 --- a/pkgs/flutter_genui/lib/src/core/conversation_widget.dart +++ b/pkgs/flutter_genui/lib/src/core/conversation_widget.dart @@ -41,45 +41,18 @@ class ConversationWidget extends StatelessWidget { SystemMessage() => systemMessageBuilder != null ? systemMessageBuilder!(context, message) - : Card( - elevation: 2.0, - margin: const EdgeInsets.symmetric( - horizontal: 8.0, - vertical: 4.0, - ), - child: ListTile( - title: Text(message.text), - leading: const Icon(Icons.smart_toy_outlined), - ), + : _ChatMessage( + text: message.text, + icon: Icons.smart_toy_outlined, + alignment: MainAxisAlignment.start, ), UserPrompt() => userPromptBuilder != null ? userPromptBuilder!(context, message) - : Padding( - padding: const EdgeInsets.symmetric( - vertical: 4.0, - horizontal: 8.0, - ), - child: Row( - mainAxisAlignment: MainAxisAlignment.end, - children: [ - Flexible( - child: Card( - shape: const RoundedRectangleBorder( - borderRadius: const BorderRadius.only( - topLeft: const Radius.circular(20.0), - bottomLeft: const Radius.circular(20.0), - bottomRight: const Radius.circular(20.0), - ), - ), - child: Padding( - padding: const EdgeInsets.all(12.0), - child: Text(message.text), - ), - ), - ), - ], - ), + : _ChatMessage( + text: message.text, + icon: Icons.person, + alignment: MainAxisAlignment.end, ), UiResponse() => Padding( padding: const EdgeInsets.all(16.0), @@ -96,3 +69,56 @@ class ConversationWidget extends StatelessWidget { ); } } + +class _ChatMessage extends StatelessWidget { + const _ChatMessage({ + required this.text, + required this.icon, + required this.alignment, + }); + + final String text; + final IconData icon; + final MainAxisAlignment alignment; + + @override + Widget build(BuildContext context) { + final isStart = alignment == MainAxisAlignment.start; + return Padding( + padding: const EdgeInsets.symmetric(vertical: 4.0, horizontal: 8.0), + child: Row( + mainAxisAlignment: alignment, + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Flexible( + child: Card( + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.only( + topLeft: Radius.circular( + alignment == MainAxisAlignment.start ? 5 : 25, + ), + topRight: Radius.circular( + alignment == MainAxisAlignment.start ? 25 : 5, + ), + bottomLeft: const Radius.circular(25), + bottomRight: const Radius.circular(25), + ), + ), + child: Padding( + padding: const EdgeInsets.all(12.0), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + if (isStart) ...[Icon(icon), const SizedBox(width: 8.0)], + Flexible(child: Text(text)), + if (!isStart) ...[const SizedBox(width: 8.0), Icon(icon)], + ], + ), + ), + ), + ), + ], + ), + ); + } +} diff --git a/pkgs/flutter_genui/test/core/surface_widget_test.dart b/pkgs/flutter_genui/test/core/surface_widget_test.dart index 8bae6365d..30f6e436e 100644 --- a/pkgs/flutter_genui/test/core/surface_widget_test.dart +++ b/pkgs/flutter_genui/test/core/surface_widget_test.dart @@ -25,7 +25,7 @@ void main() { ), ); expect(find.text('Hello'), findsOneWidget); - expect(find.byIcon(Icons.person), findsNothing); + expect(find.byIcon(Icons.person), findsOneWidget); }); testWidgets('renders SystemMessage correctly', (WidgetTester tester) async { From 86bd769582b8e694e77db1a7e53272fb8d327b8d Mon Sep 17 00:00:00 2001 From: Jacob Simionato Date: Wed, 6 Aug 2025 11:45:11 +0930 Subject: [PATCH 7/9] Small update to remove flexible --- pkgs/flutter_genui/lib/src/core/conversation_widget.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/flutter_genui/lib/src/core/conversation_widget.dart b/pkgs/flutter_genui/lib/src/core/conversation_widget.dart index 5a5460f64..7212c25fb 100644 --- a/pkgs/flutter_genui/lib/src/core/conversation_widget.dart +++ b/pkgs/flutter_genui/lib/src/core/conversation_widget.dart @@ -110,7 +110,7 @@ class _ChatMessage extends StatelessWidget { mainAxisSize: MainAxisSize.min, children: [ if (isStart) ...[Icon(icon), const SizedBox(width: 8.0)], - Flexible(child: Text(text)), + Text(text), if (!isStart) ...[const SizedBox(width: 8.0), Icon(icon)], ], ), From a68cbc00e40c956bf8b579b456533cfaa19b8ab4 Mon Sep 17 00:00:00 2001 From: Jacob Simionato Date: Wed, 6 Aug 2025 12:57:18 +0930 Subject: [PATCH 8/9] Fix formatting --- pkgs/flutter_genui/lib/src/ai_client/ai_client.dart | 13 +++++++------ .../test/core/surface_widget_test.dart | 5 +++-- pkgs/spikes/usage_test/lib/main.dart | 5 +++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart b/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart index fb4f9036e..edeec4874 100644 --- a/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart +++ b/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart @@ -80,8 +80,8 @@ class AiClient implements LlmConnection { this.tools = const [], this.outputToolName = 'provideFinalOutput', String? systemInstruction, - }) : _systemInstruction = systemInstruction, - model = ValueNotifier(model) { + }) : _systemInstruction = systemInstruction, + model = ValueNotifier(model) { final duplicateToolNames = tools.map((t) => t.name).toSet(); if (duplicateToolNames.length != tools.length) { final duplicateTools = tools.where((t) { @@ -108,8 +108,8 @@ class AiClient implements LlmConnection { this.tools = const [], this.outputToolName = 'provideFinalOutput', String? systemInstruction, - }) : _systemInstruction = systemInstruction, - model = ValueNotifier(model) { + }) : _systemInstruction = systemInstruction, + model = ValueNotifier(model) { final duplicateToolNames = tools.map((t) => t.name).toSet(); if (duplicateToolNames.length != tools.length) { final duplicateTools = tools.where((t) { @@ -431,8 +431,9 @@ class AiClient implements LlmConnection { final model = modelCreator( configuration: this, - systemInstruction: - _systemInstruction == null ? null : Content.system(_systemInstruction!), + systemInstruction: _systemInstruction == null + ? null + : Content.system(_systemInstruction!), tools: generativeAiTools, toolConfig: ToolConfig( functionCallingConfig: FunctionCallingConfig.any( diff --git a/pkgs/flutter_genui/test/core/surface_widget_test.dart b/pkgs/flutter_genui/test/core/surface_widget_test.dart index 30f6e436e..b4ff426cd 100644 --- a/pkgs/flutter_genui/test/core/surface_widget_test.dart +++ b/pkgs/flutter_genui/test/core/surface_widget_test.dart @@ -97,8 +97,9 @@ void main() { expect(find.text('Hello'), findsNothing); }); - testWidgets('uses custom systemMessageBuilder', - (WidgetTester tester) async { + testWidgets('uses custom systemMessageBuilder', ( + WidgetTester tester, + ) async { final messages = [const SystemMessage(text: 'Error')]; await tester.pumpWidget( MaterialApp( diff --git a/pkgs/spikes/usage_test/lib/main.dart b/pkgs/spikes/usage_test/lib/main.dart index af42cedbd..67f98b4a3 100644 --- a/pkgs/spikes/usage_test/lib/main.dart +++ b/pkgs/spikes/usage_test/lib/main.dart @@ -35,8 +35,9 @@ class MyHomePage extends StatefulWidget { } class _MyHomePageState extends State { - final GenUiManager _genUiManager = - GenUiManager.conversation(llmConnection: AiClient()); + final GenUiManager _genUiManager = GenUiManager.conversation( + llmConnection: AiClient(), + ); @override void initState() { From afe64354aa57eaa89200bbef8493bac13ed5fe78 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Wed, 6 Aug 2025 09:57:46 -0700 Subject: [PATCH 9/9] Fix analyzer issue --- pkgs/flutter_genui/lib/src/ai_client/ai_client.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart b/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart index edeec4874..d63545755 100644 --- a/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart +++ b/pkgs/flutter_genui/lib/src/ai_client/ai_client.dart @@ -433,7 +433,7 @@ class AiClient implements LlmConnection { configuration: this, systemInstruction: _systemInstruction == null ? null - : Content.system(_systemInstruction!), + : Content.system(_systemInstruction), tools: generativeAiTools, toolConfig: ToolConfig( functionCallingConfig: FunctionCallingConfig.any(