From dd8d717ef030261e0ade96702f13ff0bf98eb029 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 03:00:54 +0000 Subject: [PATCH] feat: Add "Neon Sketch" drawing tool and improve UI This commit introduces a new "Neon Sketch" drawing tool that allows the Gemini agent to create and display drawings in the user interface. Key changes include: - **Drawing Tool Integration:** Added a `drawing` tool to the Gemini agent, enabling it to interpret natural language commands and generate drawing instructions. - **`DrawingPadApp` Component:** Implemented a `DrawingPadApp` component that can render drawings based on commands received from the agent. - **Improved User Feedback:** Added a loading indicator to the command input to provide visual feedback when the agent is processing a request. - **Refined Error Handling:** Implemented a more informative error window that displays the failed command and error message. --- runtime/backend/gemini-agent.js | 8 + runtime/frontend/src/App.css | 52 ++++ runtime/frontend/src/App.tsx | 85 ++++-- .../frontend/src/components/CommandInput.css | 12 + .../frontend/src/components/CommandInput.tsx | 1 + .../frontend/src/components/DrawingPadApp.css | 136 ++++----- .../frontend/src/components/DrawingPadApp.tsx | 257 ++++++++++-------- 7 files changed, 320 insertions(+), 231 deletions(-) diff --git a/runtime/backend/gemini-agent.js b/runtime/backend/gemini-agent.js index 5258018e796..68c3abd7f66 100644 --- a/runtime/backend/gemini-agent.js +++ b/runtime/backend/gemini-agent.js @@ -284,6 +284,12 @@ CRITICAL CONSTRAINTS: 4. Never remove, hide, or disable the command input text box rendered by CommandInput. 5. When embedding multi-line ASCII art or code, wrap it in a 
 element and call String.raw on the string literal so the JSX remains valid without escape errors.
 
+TOOLS:
+- You have a special "drawing" tool available.
+- If the user asks to draw something, use the drawing tool.
+- The tool takes a list of drawing commands.
+- Schema: { tool: 'drawing', commands: [{ shape: 'line' | 'circle' | 'rectangle', color: string (hex), positions: [x, y, x2, y2, ...] }] }
+
 REQUEST:
 The user said: "${userCommand}".
 Translate this into a compelling end product in GeneratedContent.tsx only. Remember you can create anything that can be rendered in a browser!
@@ -325,6 +331,8 @@ Please apply the requested update and include DONE at the very end.`;
     promptWithContext,
     '--output-format',
     'json',
+    '--tools',
+    'drawing',
   ];
 
   return new Promise((resolve, reject) => {
diff --git a/runtime/frontend/src/App.css b/runtime/frontend/src/App.css
index e8ad75b40d1..c4232ddedfc 100644
--- a/runtime/frontend/src/App.css
+++ b/runtime/frontend/src/App.css
@@ -92,3 +92,55 @@ body, html {
 .video-window__caption a:focus-visible {
   text-decoration: underline;
 }
+
+.error-window {
+  padding: 16px;
+}
+
+.error-window h3 {
+  color: #c93434;
+  margin: 0 0 8px;
+  font-size: 16px;
+}
+
+.error-window p {
+  color: #444;
+  font-size: 13px;
+  line-height: 1.5;
+  margin: 0;
+}
+
+.error-window-details {
+  background: rgba(201, 52, 52, 0.12);
+  border: 1px solid rgba(201, 52, 52, 0.2);
+  padding: 12px;
+  border-radius: 8px;
+  margin-top: 12px;
+  font-size: 12px;
+  line-height: 1.4;
+  color: #333;
+}
+
+.error-window-details strong {
+  display: block;
+  font-weight: 600;
+  margin-bottom: 4px;
+}
+
+.error-window-details p {
+  font-size: 12px;
+}
+
+.error-window-details p:not(:last-child) {
+  margin-bottom: 8px;
+}
+
+.error-window-details code {
+  font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier,
+    monospace;
+  background: rgba(201, 52, 52, 0.1);
+  padding: 2px 4px;
+  border-radius: 4px;
+  white-space: pre-wrap;
+  word-break: break-word;
+}
diff --git a/runtime/frontend/src/App.tsx b/runtime/frontend/src/App.tsx
index c87f5c59411..671e8ca1a15 100644
--- a/runtime/frontend/src/App.tsx
+++ b/runtime/frontend/src/App.tsx
@@ -10,7 +10,7 @@ import type { WindowData } from './components/Desktop';
 import CommandInput from './components/CommandInput';
 import GeneratedContent from './components/GeneratedContent';
 import GeminiStatsWindow from './components/GeminiStatsWindow';
-import DrawingPadApp from './components/DrawingPadApp';
+import DrawingPadApp, { type DrawingCommand } from './components/DrawingPadApp';
 import './App.css';
 
 const API_BASE_URL = 'http://localhost:3001';
@@ -97,21 +97,32 @@ function App() {
     return () => clearInterval(interval);
   }, [refreshStatus]);
 
-  const createCommandWindow = useCallback((request: string) => {
-    const newWindow: WindowData = {
-      id: `window-${Date.now()}`,
-      title: `Request: ${truncate(request, 24)}`,
-      content: (
-        

-          
Preview

-          
Request: {request}

-          
Gemini will replace this window with a tailored experience.

-        

-      ),
-    };
-
-    setWindows((prev) => [...prev, newWindow]);
-  }, []);
+  const createErrorWindow = useCallback(
+    (request: string, errorMessage: string) => {
+      const newWindow: WindowData = {
+        id: `error-window-${Date.now()}`,
+        title: `Error: ${truncate(request, 24)}`,
+        content: (
+          

+            
Agent Request Failed

+            
Your request could not be completed. See details below.

+            

+              Command:
+              

+                {request}
+              

+              Error:
+              

+                {errorMessage}
+              

+            

+          

+        ),
+      };
+      setWindows((prev) => [...prev, newWindow]);
+    },
+    [],
+  );
 
   const handleCloseWindow = useCallback((id: string) => {
     setWindows((prev) => prev.filter((window) => window.id !== id));
@@ -187,14 +198,17 @@ function App() {
     });
   }, [openStaticWindow]);
 
-  const handleOpenDrawingPad = useCallback(() => {
-    openStaticWindow({
-      id: 'drawing-pad',
-      title: 'Neon Sketch',
-      content: ,
-      position: { x: 520, y: 160 },
-    });
-  }, [openStaticWindow]);
+  const handleOpenDrawingPad = useCallback(
+    (initialCommands?: DrawingCommand[]) => {
+      openStaticWindow({
+        id: 'drawing-pad',
+        title: 'Neon Sketch',
+        content: ,
+        position: { x: 520, y: 160 },
+      });
+    },
+    [openStaticWindow],
+  );
 
   const handleOpenUsageStats = useCallback(() => {
     openStaticWindow({
@@ -236,6 +250,20 @@ function App() {
         const mode =
           (data.mode as AgentMode) ?? agentStatus.agentMode ?? 'UNKNOWN';
 
+        if (data.result?.output) {
+          try {
+            const agentJson = JSON.parse(data.result.output);
+            if (
+              agentJson.tool === 'drawing' &&
+              Array.isArray(agentJson.commands)
+            ) {
+              handleOpenDrawingPad(agentJson.commands);
+            }
+          } catch (e) {
+            console.warn('Could not parse agent JSON output', e);
+          }
+        }
+
         setLastActivity({
           command,
           message: data.message || 'Command processed.',
@@ -259,14 +287,19 @@ function App() {
               : null,
           timestamp: Date.now(),
         });
-        createCommandWindow(command);
+        createErrorWindow(command, message);
       } finally {
         setIsProcessing(false);
         setPendingCommand(null);
         refreshStatus();
       }
     },
-    [agentStatus.agentMode, createCommandWindow, refreshStatus],
+    [
+      agentStatus.agentMode,
+      createErrorWindow,
+      handleOpenDrawingPad,
+      refreshStatus,
+    ],
   );
 
   const commandHint = useMemo(() => {
diff --git a/runtime/frontend/src/components/CommandInput.css b/runtime/frontend/src/components/CommandInput.css
index f2d89937e1b..e50c9b9ea0e 100644
--- a/runtime/frontend/src/components/CommandInput.css
+++ b/runtime/frontend/src/components/CommandInput.css
@@ -247,6 +247,18 @@
   margin: 0;
 }
 
+.command-input-status.pending .loading-indicator {
+  display: inline-block;
+  width: 12px;
+  height: 12px;
+  border: 2px solid rgba(255, 255, 255, 0.4);
+  border-top-color: #fff;
+  border-radius: 50%;
+  animation: spin 1s linear infinite;
+  margin-right: 8px;
+  vertical-align: middle;
+}
+
 @keyframes command-submit-spin {
   to {
     transform: rotate(360deg);
diff --git a/runtime/frontend/src/components/CommandInput.tsx b/runtime/frontend/src/components/CommandInput.tsx
index 2e3403e0ace..c441d17a124 100644
--- a/runtime/frontend/src/components/CommandInput.tsx
+++ b/runtime/frontend/src/components/CommandInput.tsx
@@ -80,6 +80,7 @@ export default function CommandInput({
         

+          {statusTone === 'pending' && 
}
           {statusMessage}
           {authError && (
             {authError}
diff --git a/runtime/frontend/src/components/DrawingPadApp.css b/runtime/frontend/src/components/DrawingPadApp.css
index 5832001f3fd..8581d50c5e9 100644
--- a/runtime/frontend/src/components/DrawingPadApp.css
+++ b/runtime/frontend/src/components/DrawingPadApp.css
@@ -1,112 +1,72 @@
-.drawing-pad {
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+.drawing-pad-app {
   display: flex;
   flex-direction: column;
-  gap: 16px;
-  min-width: 420px;
-  max-width: 680px;
-}
-
-.drawing-pad__header h3 {
-  margin: 0 0 4px;
-  font-size: 20px;
+  background-color: #222;
+  border-radius: 8px;
+  padding: 16px;
+  box-shadow: 0 4px 12px rgba(0, 0, 0, 0.5);
 }
 
-.drawing-pad__header p {
-  margin: 0;
-  color: #4e4c63;
-  font-size: 14px;
+.drawing-canvas {
+  background-color: #000;
+  border-radius: 4px;
+  cursor: crosshair;
+  touch-action: none;
 }
 
-.drawing-pad__toolbar {
+.drawing-controls {
   display: flex;
-  flex-wrap: wrap;
+  justify-content: space-between;
   align-items: center;
-  gap: 14px;
-  padding: 12px 16px;
-  border: 1px solid #dad6ff;
-  border-radius: 12px;
-  background: linear-gradient(135deg, rgba(102, 126, 234, 0.1), rgba(118, 75, 162, 0.1));
-  box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.5);
+  margin-top: 12px;
 }
 
-.drawing-pad__tool {
+.color-palette {
   display: flex;
-  align-items: center;
   gap: 8px;
-  font-size: 12px;
-  font-weight: 600;
-  text-transform: uppercase;
-  letter-spacing: 0.08em;
-  color: #2d1b69;
-}
-
-.drawing-pad__tool input[type='range'] {
-  accent-color: #704bff;
-  width: 130px;
 }
 
-.drawing-pad__tool input[type='color'] {
-  width: 42px;
+.color-swatch {
+  width: 28px;
   height: 28px;
-  border: none;
-  border-radius: 6px;
-  padding: 0;
-  background: transparent;
-}
-
-.drawing-pad__value {
-  padding: 2px 8px;
-  border-radius: 999px;
-  background: rgba(45, 27, 105, 0.08);
-  color: #2d1b69;
-  font-size: 12px;
-}
-
-.drawing-pad__clear {
-  margin-left: auto;
-  padding: 8px 14px;
-  border-radius: 999px;
-  border: 1px solid rgba(45, 27, 105, 0.25);
-  background: rgba(255, 255, 255, 0.9);
-  color: #2d1b69;
-  font-weight: 600;
-  letter-spacing: 0.05em;
+  border-radius: 50%;
+  border: 2px solid #fff;
   cursor: pointer;
-  transition: transform 0.1s ease, box-shadow 0.1s ease;
+  transition:
+    transform 0.2s,
+    box-shadow 0.2s;
 }
 
-.drawing-pad__clear:hover {
-  transform: translateY(-1px);
-  box-shadow: 0 4px 8px rgba(45, 27, 105, 0.1);
+.color-swatch:hover {
+  transform: scale(1.1);
 }
 
-.drawing-pad__canvas {
-  width: 100%;
-  height: 420px;
-  border-radius: 14px;
-  border: 2px solid rgba(45, 27, 105, 0.25);
-  background: repeating-linear-gradient(
-      0deg,
-      rgba(126, 107, 197, 0.04),
-      rgba(126, 107, 197, 0.04) 24px,
-      rgba(255, 255, 255, 0.4) 24px,
-      rgba(255, 255, 255, 0.4) 48px
-    ),
-    #ffffff;
-  cursor: crosshair;
+.color-swatch.active {
+  box-shadow: 0 0 8px 2px var(--swatch-color);
+  transform: scale(1.15);
 }
 
-@media (max-width: 600px) {
-  .drawing-pad {
-    min-width: 0;
-  }
-
-  .drawing-pad__toolbar {
-    flex-direction: column;
-    align-items: flex-start;
-  }
+.clear-button {
+  background-color: #444;
+  color: #fff;
+  border: 1px solid #666;
+  border-radius: 4px;
+  padding: 8px 16px;
+  font-family: 'VT323', monospace;
+  font-size: 16px;
+  cursor: pointer;
+  transition:
+    background-color 0.2s,
+    box-shadow 0.2s;
+}
 
-  .drawing-pad__clear {
-    margin-left: 0;
-  }
+.clear-button:hover {
+  background-color: #555;
+  box-shadow: 0 0 8px rgba(255, 255, 255, 0.3);
 }
diff --git a/runtime/frontend/src/components/DrawingPadApp.tsx b/runtime/frontend/src/components/DrawingPadApp.tsx
index e5f52938518..9a81ea25fb9 100644
--- a/runtime/frontend/src/components/DrawingPadApp.tsx
+++ b/runtime/frontend/src/components/DrawingPadApp.tsx
@@ -4,155 +4,178 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { useEffect, useRef, useState } from 'react';
-import type { PointerEvent as ReactPointerEvent } from 'react';
-
+import { useRef, useEffect, useState } from 'react';
 import './DrawingPadApp.css';
 
-const DEFAULT_COLOR = '#2d1b69';
-const DEFAULT_BRUSH_SIZE = 6;
+export interface DrawingCommand {
+  shape: 'line' | 'circle' | 'rectangle';
+  color: string;
+  positions: number[];
+}
 
-export default function DrawingPadApp() {
-  const canvasRef = useRef(null);
-  const contextRef = useRef(null);
-  const colorRef = useRef(DEFAULT_COLOR);
-  const brushSizeRef = useRef(DEFAULT_BRUSH_SIZE);
-  const [isDrawing, setIsDrawing] = useState(false);
-  const [color, setColor] = useState(DEFAULT_COLOR);
-  const [brushSize, setBrushSize] = useState(DEFAULT_BRUSH_SIZE);
+interface DrawingPadAppProps {
+  initialCommands?: DrawingCommand[];
+}
 
-  useEffect(() => {
-    const canvas = canvasRef.current;
-    if (!canvas) return;
-
-    const resizeCanvas = () => {
-      const dpr = window.devicePixelRatio || 1;
-      const { offsetWidth, offsetHeight } = canvas;
-      canvas.width = offsetWidth * dpr;
-      canvas.height = offsetHeight * dpr;
-
-      const context = canvas.getContext('2d');
-      if (!context) return;
-
-      context.scale(dpr, dpr);
-      context.lineCap = 'round';
-      context.lineJoin = 'round';
-      context.strokeStyle = colorRef.current;
-      context.lineWidth = brushSizeRef.current;
-      contextRef.current = context;
-    };
+const COLORS = ['#00f6ff', '#ff00ff', '#5eff00', '#ffff00', '#ff8000'];
 
-    resizeCanvas();
+export default function DrawingPadApp({
+  initialCommands = [],
+}: DrawingPadAppProps) {
+  const canvasRef = useRef(null);
+  const [drawing, setDrawing] = useState(false);
+  const [color, setColor] = useState(COLORS[0]);
+  const [lastPos, setLastPos] = useState<{ x: number; y: number } | null>(null);
 
-    window.addEventListener('resize', resizeCanvas);
-    return () => {
-      window.removeEventListener('resize', resizeCanvas);
-    };
-  }, []);
+  const getCanvasContext = () => {
+    const canvas = canvasRef.current;
+    if (!canvas) return null;
+    return canvas.getContext('2d');
+  };
 
   useEffect(() => {
-    colorRef.current = color;
-    if (contextRef.current) {
-      contextRef.current.strokeStyle = color;
-    }
+    const ctx = getCanvasContext();
+    if (!ctx) return;
+
+    ctx.strokeStyle = color;
+    ctx.lineWidth = 5;
+    ctx.lineCap = 'round';
+    ctx.lineJoin = 'round';
+    ctx.shadowBlur = 10;
+    ctx.shadowColor = color;
   }, [color]);
 
   useEffect(() => {
-    brushSizeRef.current = brushSize;
-    if (contextRef.current) {
-      contextRef.current.lineWidth = brushSize;
+    const ctx = getCanvasContext();
+    if (!ctx || !initialCommands.length) return;
+
+    for (const command of initialCommands) {
+      ctx.strokeStyle = command.color;
+      ctx.shadowColor = command.color;
+      ctx.lineWidth = 5;
+      ctx.lineCap = 'round';
+      ctx.lineJoin = 'round';
+      ctx.shadowBlur = 10;
+
+      ctx.beginPath();
+      if (command.shape === 'line' && command.positions.length >= 4) {
+        const [x1, y1, ...rest] = command.positions;
+        ctx.moveTo(x1, y1);
+        for (let i = 0; i < rest.length; i += 2) {
+          ctx.lineTo(rest[i], rest[i + 1]);
+        }
+      } else if (command.shape === 'circle' && command.positions.length >= 3) {
+        const [x, y, radius] = command.positions;
+        ctx.arc(x, y, radius, 0, 2 * Math.PI);
+      } else if (
+        command.shape === 'rectangle' &&
+        command.positions.length >= 4
+      ) {
+        const [x, y, width, height] = command.positions;
+        ctx.rect(x, y, width, height);
+      }
+      ctx.stroke();
     }
-  }, [brushSize]);
 
-  const handlePointerDown = (event: ReactPointerEvent) => {
-    const { offsetX, offsetY } = event.nativeEvent;
-    const context = contextRef.current;
-    if (!context) return;
+    ctx.strokeStyle = color;
+    ctx.shadowColor = color;
+  }, [initialCommands, color]);
 
-    context.beginPath();
-    context.moveTo(offsetX, offsetY);
-    setIsDrawing(true);
+  const getPointerPosition = (
+    e: React.MouseEvent | React.TouchEvent,
+  ): { x: number; y: number } | null => {
+    const canvas = canvasRef.current;
+    if (!canvas) return null;
+
+    const rect = canvas.getBoundingClientRect();
+    const clientX = 'touches' in e ? e.touches[0].clientX : e.clientX;
+    const clientY = 'touches' in e ? e.touches[0].clientY : e.clientY;
+
+    return {
+      x: clientX - rect.left,
+      y: clientY - rect.top,
+    };
   };
 
-  const handlePointerUp = () => {
-    if (!contextRef.current) return;
-    contextRef.current.closePath();
-    setIsDrawing(false);
+  const startDrawing = (e: React.MouseEvent | React.TouchEvent) => {
+    const pos = getPointerPosition(e);
+    if (!pos) return;
+
+    setDrawing(true);
+    setLastPos(pos);
   };
 
-  const handlePointerMove = (event: ReactPointerEvent) => {
-    if (!isDrawing) return;
-    const { offsetX, offsetY } = event.nativeEvent;
-    const context = contextRef.current;
-    if (!context) return;
+  const draw = (e: React.MouseEvent | React.TouchEvent) => {
+    if (!drawing) return;
+
+    const ctx = getCanvasContext();
+    const pos = getPointerPosition(e);
+    if (!ctx || !pos || !lastPos) return;
+
+    ctx.beginPath();
+    ctx.moveTo(lastPos.x, lastPos.y);
+    ctx.lineTo(pos.x, pos.y);
+    ctx.stroke();
 
-    context.lineTo(offsetX, offsetY);
-    context.stroke();
+    setLastPos(pos);
   };
 
-  const handlePointerLeave = () => {
-    if (isDrawing) {
-      handlePointerUp();
-    }
+  const stopDrawing = () => {
+    setDrawing(false);
+    setLastPos(null);
   };
 
   const clearCanvas = () => {
     const canvas = canvasRef.current;
-    const context = contextRef.current;
-    if (!canvas || !context) return;
+    const ctx = getCanvasContext();
+    if (!canvas || !ctx) return;
 
-    context.clearRect(0, 0, canvas.width, canvas.height);
+    ctx.clearRect(0, 0, canvas.width, canvas.height);
   };
 
   return (
-    

-      

-        
Neon Sketch

-        

-          Paint freely, remix colors, and clear the slate whenever inspiration
-          strikes.
-        

-      

-
-      

-        
-
-        
-
-        
-      

-
+    

       
+      

+        

+          {COLORS.map((swatch) => (
+            
 setColor(swatch)}
+              onKeyDown={(e) => {
+                if (e.key === 'Enter' || e.key === ' ') {
+                  setColor(swatch);
+                }
+              }}
+              role="button"
+              tabIndex={0}
+              aria-label={`Set color to ${swatch}`}
+            />
+          ))}
+        

+        
+      

     

   );
 }