diff --git a/runtime/backend/gemini-agent.js b/runtime/backend/gemini-agent.js index 5258018e796..68c3abd7f66 100644 --- a/runtime/backend/gemini-agent.js +++ b/runtime/backend/gemini-agent.js @@ -284,6 +284,12 @@ CRITICAL CONSTRAINTS: 4. Never remove, hide, or disable the command input text box rendered by CommandInput. 5. When embedding multi-line ASCII art or code, wrap it in a
element and call String.raw on the string literal so the JSX remains valid without escape errors.
+TOOLS:
+- You have a special "drawing" tool available.
+- If the user asks to draw something, use the drawing tool.
+- The tool takes a list of drawing commands.
+- Schema: { tool: 'drawing', commands: [{ shape: 'line' | 'circle' | 'rectangle', color: string (hex), positions: [x, y, x2, y2, ...] }] }
+
REQUEST:
The user said: "${userCommand}".
Translate this into a compelling end product in GeneratedContent.tsx only. Remember you can create anything that can be rendered in a browser!
@@ -325,6 +331,8 @@ Please apply the requested update and include DONE at the very end.`;
promptWithContext,
'--output-format',
'json',
+ '--tools',
+ 'drawing',
];
return new Promise((resolve, reject) => {
diff --git a/runtime/frontend/src/App.css b/runtime/frontend/src/App.css
index e8ad75b40d1..c4232ddedfc 100644
--- a/runtime/frontend/src/App.css
+++ b/runtime/frontend/src/App.css
@@ -92,3 +92,55 @@ body, html {
.video-window__caption a:focus-visible {
text-decoration: underline;
}
+
+.error-window {
+ padding: 16px;
+}
+
+.error-window h3 {
+ color: #c93434;
+ margin: 0 0 8px;
+ font-size: 16px;
+}
+
+.error-window p {
+ color: #444;
+ font-size: 13px;
+ line-height: 1.5;
+ margin: 0;
+}
+
+.error-window-details {
+ background: rgba(201, 52, 52, 0.12);
+ border: 1px solid rgba(201, 52, 52, 0.2);
+ padding: 12px;
+ border-radius: 8px;
+ margin-top: 12px;
+ font-size: 12px;
+ line-height: 1.4;
+ color: #333;
+}
+
+.error-window-details strong {
+ display: block;
+ font-weight: 600;
+ margin-bottom: 4px;
+}
+
+.error-window-details p {
+ font-size: 12px;
+}
+
+.error-window-details p:not(:last-child) {
+ margin-bottom: 8px;
+}
+
+.error-window-details code {
+ font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier,
+ monospace;
+ background: rgba(201, 52, 52, 0.1);
+ padding: 2px 4px;
+ border-radius: 4px;
+ white-space: pre-wrap;
+ word-break: break-word;
+}
diff --git a/runtime/frontend/src/App.tsx b/runtime/frontend/src/App.tsx
index c87f5c59411..671e8ca1a15 100644
--- a/runtime/frontend/src/App.tsx
+++ b/runtime/frontend/src/App.tsx
@@ -10,7 +10,7 @@ import type { WindowData } from './components/Desktop';
import CommandInput from './components/CommandInput';
import GeneratedContent from './components/GeneratedContent';
import GeminiStatsWindow from './components/GeminiStatsWindow';
-import DrawingPadApp from './components/DrawingPadApp';
+import DrawingPadApp, { type DrawingCommand } from './components/DrawingPadApp';
import './App.css';
const API_BASE_URL = 'http://localhost:3001';
@@ -97,21 +97,32 @@ function App() {
return () => clearInterval(interval);
}, [refreshStatus]);
- const createCommandWindow = useCallback((request: string) => {
- const newWindow: WindowData = {
- id: `window-${Date.now()}`,
- title: `Request: ${truncate(request, 24)}`,
- content: (
-
- Preview
- Request: {request}
- Gemini will replace this window with a tailored experience.
-
- ),
- };
-
- setWindows((prev) => [...prev, newWindow]);
- }, []);
+ const createErrorWindow = useCallback(
+ (request: string, errorMessage: string) => {
+ const newWindow: WindowData = {
+ id: `error-window-${Date.now()}`,
+ title: `Error: ${truncate(request, 24)}`,
+ content: (
+
+ Agent Request Failed
+ Your request could not be completed. See details below.
+
+ Command:
+
+ {request}
+
+ Error:
+
+ {errorMessage}
+
+
+
+ ),
+ };
+ setWindows((prev) => [...prev, newWindow]);
+ },
+ [],
+ );
const handleCloseWindow = useCallback((id: string) => {
setWindows((prev) => prev.filter((window) => window.id !== id));
@@ -187,14 +198,17 @@ function App() {
});
}, [openStaticWindow]);
- const handleOpenDrawingPad = useCallback(() => {
- openStaticWindow({
- id: 'drawing-pad',
- title: 'Neon Sketch',
- content: ,
- position: { x: 520, y: 160 },
- });
- }, [openStaticWindow]);
+ const handleOpenDrawingPad = useCallback(
+ (initialCommands?: DrawingCommand[]) => {
+ openStaticWindow({
+ id: 'drawing-pad',
+ title: 'Neon Sketch',
+ content: ,
+ position: { x: 520, y: 160 },
+ });
+ },
+ [openStaticWindow],
+ );
const handleOpenUsageStats = useCallback(() => {
openStaticWindow({
@@ -236,6 +250,20 @@ function App() {
const mode =
(data.mode as AgentMode) ?? agentStatus.agentMode ?? 'UNKNOWN';
+ if (data.result?.output) {
+ try {
+ const agentJson = JSON.parse(data.result.output);
+ if (
+ agentJson.tool === 'drawing' &&
+ Array.isArray(agentJson.commands)
+ ) {
+ handleOpenDrawingPad(agentJson.commands);
+ }
+ } catch (e) {
+ console.warn('Could not parse agent JSON output', e);
+ }
+ }
+
setLastActivity({
command,
message: data.message || 'Command processed.',
@@ -259,14 +287,19 @@ function App() {
: null,
timestamp: Date.now(),
});
- createCommandWindow(command);
+ createErrorWindow(command, message);
} finally {
setIsProcessing(false);
setPendingCommand(null);
refreshStatus();
}
},
- [agentStatus.agentMode, createCommandWindow, refreshStatus],
+ [
+ agentStatus.agentMode,
+ createErrorWindow,
+ handleOpenDrawingPad,
+ refreshStatus,
+ ],
);
const commandHint = useMemo(() => {
diff --git a/runtime/frontend/src/components/CommandInput.css b/runtime/frontend/src/components/CommandInput.css
index f2d89937e1b..e50c9b9ea0e 100644
--- a/runtime/frontend/src/components/CommandInput.css
+++ b/runtime/frontend/src/components/CommandInput.css
@@ -247,6 +247,18 @@
margin: 0;
}
+.command-input-status.pending .loading-indicator {
+ display: inline-block;
+ width: 12px;
+ height: 12px;
+ border: 2px solid rgba(255, 255, 255, 0.4);
+ border-top-color: #fff;
+ border-radius: 50%;
+ animation: spin 1s linear infinite;
+ margin-right: 8px;
+ vertical-align: middle;
+}
+
@keyframes command-submit-spin {
to {
transform: rotate(360deg);
diff --git a/runtime/frontend/src/components/CommandInput.tsx b/runtime/frontend/src/components/CommandInput.tsx
index 2e3403e0ace..c441d17a124 100644
--- a/runtime/frontend/src/components/CommandInput.tsx
+++ b/runtime/frontend/src/components/CommandInput.tsx
@@ -80,6 +80,7 @@ export default function CommandInput({
+ {statusTone === 'pending' && }
{statusMessage}
{authError && (
{authError}
diff --git a/runtime/frontend/src/components/DrawingPadApp.css b/runtime/frontend/src/components/DrawingPadApp.css
index 5832001f3fd..8581d50c5e9 100644
--- a/runtime/frontend/src/components/DrawingPadApp.css
+++ b/runtime/frontend/src/components/DrawingPadApp.css
@@ -1,112 +1,72 @@
-.drawing-pad {
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+.drawing-pad-app {
display: flex;
flex-direction: column;
- gap: 16px;
- min-width: 420px;
- max-width: 680px;
-}
-
-.drawing-pad__header h3 {
- margin: 0 0 4px;
- font-size: 20px;
+ background-color: #222;
+ border-radius: 8px;
+ padding: 16px;
+ box-shadow: 0 4px 12px rgba(0, 0, 0, 0.5);
}
-.drawing-pad__header p {
- margin: 0;
- color: #4e4c63;
- font-size: 14px;
+.drawing-canvas {
+ background-color: #000;
+ border-radius: 4px;
+ cursor: crosshair;
+ touch-action: none;
}
-.drawing-pad__toolbar {
+.drawing-controls {
display: flex;
- flex-wrap: wrap;
+ justify-content: space-between;
align-items: center;
- gap: 14px;
- padding: 12px 16px;
- border: 1px solid #dad6ff;
- border-radius: 12px;
- background: linear-gradient(135deg, rgba(102, 126, 234, 0.1), rgba(118, 75, 162, 0.1));
- box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.5);
+ margin-top: 12px;
}
-.drawing-pad__tool {
+.color-palette {
display: flex;
- align-items: center;
gap: 8px;
- font-size: 12px;
- font-weight: 600;
- text-transform: uppercase;
- letter-spacing: 0.08em;
- color: #2d1b69;
-}
-
-.drawing-pad__tool input[type='range'] {
- accent-color: #704bff;
- width: 130px;
}
-.drawing-pad__tool input[type='color'] {
- width: 42px;
+.color-swatch {
+ width: 28px;
height: 28px;
- border: none;
- border-radius: 6px;
- padding: 0;
- background: transparent;
-}
-
-.drawing-pad__value {
- padding: 2px 8px;
- border-radius: 999px;
- background: rgba(45, 27, 105, 0.08);
- color: #2d1b69;
- font-size: 12px;
-}
-
-.drawing-pad__clear {
- margin-left: auto;
- padding: 8px 14px;
- border-radius: 999px;
- border: 1px solid rgba(45, 27, 105, 0.25);
- background: rgba(255, 255, 255, 0.9);
- color: #2d1b69;
- font-weight: 600;
- letter-spacing: 0.05em;
+ border-radius: 50%;
+ border: 2px solid #fff;
cursor: pointer;
- transition: transform 0.1s ease, box-shadow 0.1s ease;
+ transition:
+ transform 0.2s,
+ box-shadow 0.2s;
}
-.drawing-pad__clear:hover {
- transform: translateY(-1px);
- box-shadow: 0 4px 8px rgba(45, 27, 105, 0.1);
+.color-swatch:hover {
+ transform: scale(1.1);
}
-.drawing-pad__canvas {
- width: 100%;
- height: 420px;
- border-radius: 14px;
- border: 2px solid rgba(45, 27, 105, 0.25);
- background: repeating-linear-gradient(
- 0deg,
- rgba(126, 107, 197, 0.04),
- rgba(126, 107, 197, 0.04) 24px,
- rgba(255, 255, 255, 0.4) 24px,
- rgba(255, 255, 255, 0.4) 48px
- ),
- #ffffff;
- cursor: crosshair;
+.color-swatch.active {
+ box-shadow: 0 0 8px 2px var(--swatch-color);
+ transform: scale(1.15);
}
-@media (max-width: 600px) {
- .drawing-pad {
- min-width: 0;
- }
-
- .drawing-pad__toolbar {
- flex-direction: column;
- align-items: flex-start;
- }
+.clear-button {
+ background-color: #444;
+ color: #fff;
+ border: 1px solid #666;
+ border-radius: 4px;
+ padding: 8px 16px;
+ font-family: 'VT323', monospace;
+ font-size: 16px;
+ cursor: pointer;
+ transition:
+ background-color 0.2s,
+ box-shadow 0.2s;
+}
- .drawing-pad__clear {
- margin-left: 0;
- }
+.clear-button:hover {
+ background-color: #555;
+ box-shadow: 0 0 8px rgba(255, 255, 255, 0.3);
}
diff --git a/runtime/frontend/src/components/DrawingPadApp.tsx b/runtime/frontend/src/components/DrawingPadApp.tsx
index e5f52938518..9a81ea25fb9 100644
--- a/runtime/frontend/src/components/DrawingPadApp.tsx
+++ b/runtime/frontend/src/components/DrawingPadApp.tsx
@@ -4,155 +4,178 @@
* SPDX-License-Identifier: Apache-2.0
*/
-import { useEffect, useRef, useState } from 'react';
-import type { PointerEvent as ReactPointerEvent } from 'react';
-
+import { useRef, useEffect, useState } from 'react';
import './DrawingPadApp.css';
-const DEFAULT_COLOR = '#2d1b69';
-const DEFAULT_BRUSH_SIZE = 6;
+export interface DrawingCommand {
+ shape: 'line' | 'circle' | 'rectangle';
+ color: string;
+ positions: number[];
+}
-export default function DrawingPadApp() {
- const canvasRef = useRef(null);
- const contextRef = useRef(null);
- const colorRef = useRef(DEFAULT_COLOR);
- const brushSizeRef = useRef(DEFAULT_BRUSH_SIZE);
- const [isDrawing, setIsDrawing] = useState(false);
- const [color, setColor] = useState(DEFAULT_COLOR);
- const [brushSize, setBrushSize] = useState(DEFAULT_BRUSH_SIZE);
+interface DrawingPadAppProps {
+ initialCommands?: DrawingCommand[];
+}
- useEffect(() => {
- const canvas = canvasRef.current;
- if (!canvas) return;
-
- const resizeCanvas = () => {
- const dpr = window.devicePixelRatio || 1;
- const { offsetWidth, offsetHeight } = canvas;
- canvas.width = offsetWidth * dpr;
- canvas.height = offsetHeight * dpr;
-
- const context = canvas.getContext('2d');
- if (!context) return;
-
- context.scale(dpr, dpr);
- context.lineCap = 'round';
- context.lineJoin = 'round';
- context.strokeStyle = colorRef.current;
- context.lineWidth = brushSizeRef.current;
- contextRef.current = context;
- };
+const COLORS = ['#00f6ff', '#ff00ff', '#5eff00', '#ffff00', '#ff8000'];
- resizeCanvas();
+export default function DrawingPadApp({
+ initialCommands = [],
+}: DrawingPadAppProps) {
+ const canvasRef = useRef(null);
+ const [drawing, setDrawing] = useState(false);
+ const [color, setColor] = useState(COLORS[0]);
+ const [lastPos, setLastPos] = useState<{ x: number; y: number } | null>(null);
- window.addEventListener('resize', resizeCanvas);
- return () => {
- window.removeEventListener('resize', resizeCanvas);
- };
- }, []);
+ const getCanvasContext = () => {
+ const canvas = canvasRef.current;
+ if (!canvas) return null;
+ return canvas.getContext('2d');
+ };
useEffect(() => {
- colorRef.current = color;
- if (contextRef.current) {
- contextRef.current.strokeStyle = color;
- }
+ const ctx = getCanvasContext();
+ if (!ctx) return;
+
+ ctx.strokeStyle = color;
+ ctx.lineWidth = 5;
+ ctx.lineCap = 'round';
+ ctx.lineJoin = 'round';
+ ctx.shadowBlur = 10;
+ ctx.shadowColor = color;
}, [color]);
useEffect(() => {
- brushSizeRef.current = brushSize;
- if (contextRef.current) {
- contextRef.current.lineWidth = brushSize;
+ const ctx = getCanvasContext();
+ if (!ctx || !initialCommands.length) return;
+
+ for (const command of initialCommands) {
+ ctx.strokeStyle = command.color;
+ ctx.shadowColor = command.color;
+ ctx.lineWidth = 5;
+ ctx.lineCap = 'round';
+ ctx.lineJoin = 'round';
+ ctx.shadowBlur = 10;
+
+ ctx.beginPath();
+ if (command.shape === 'line' && command.positions.length >= 4) {
+ const [x1, y1, ...rest] = command.positions;
+ ctx.moveTo(x1, y1);
+ for (let i = 0; i < rest.length; i += 2) {
+ ctx.lineTo(rest[i], rest[i + 1]);
+ }
+ } else if (command.shape === 'circle' && command.positions.length >= 3) {
+ const [x, y, radius] = command.positions;
+ ctx.arc(x, y, radius, 0, 2 * Math.PI);
+ } else if (
+ command.shape === 'rectangle' &&
+ command.positions.length >= 4
+ ) {
+ const [x, y, width, height] = command.positions;
+ ctx.rect(x, y, width, height);
+ }
+ ctx.stroke();
}
- }, [brushSize]);
- const handlePointerDown = (event: ReactPointerEvent) => {
- const { offsetX, offsetY } = event.nativeEvent;
- const context = contextRef.current;
- if (!context) return;
+ ctx.strokeStyle = color;
+ ctx.shadowColor = color;
+ }, [initialCommands, color]);
- context.beginPath();
- context.moveTo(offsetX, offsetY);
- setIsDrawing(true);
+ const getPointerPosition = (
+ e: React.MouseEvent | React.TouchEvent,
+ ): { x: number; y: number } | null => {
+ const canvas = canvasRef.current;
+ if (!canvas) return null;
+
+ const rect = canvas.getBoundingClientRect();
+ const clientX = 'touches' in e ? e.touches[0].clientX : e.clientX;
+ const clientY = 'touches' in e ? e.touches[0].clientY : e.clientY;
+
+ return {
+ x: clientX - rect.left,
+ y: clientY - rect.top,
+ };
};
- const handlePointerUp = () => {
- if (!contextRef.current) return;
- contextRef.current.closePath();
- setIsDrawing(false);
+ const startDrawing = (e: React.MouseEvent | React.TouchEvent) => {
+ const pos = getPointerPosition(e);
+ if (!pos) return;
+
+ setDrawing(true);
+ setLastPos(pos);
};
- const handlePointerMove = (event: ReactPointerEvent) => {
- if (!isDrawing) return;
- const { offsetX, offsetY } = event.nativeEvent;
- const context = contextRef.current;
- if (!context) return;
+ const draw = (e: React.MouseEvent | React.TouchEvent) => {
+ if (!drawing) return;
+
+ const ctx = getCanvasContext();
+ const pos = getPointerPosition(e);
+ if (!ctx || !pos || !lastPos) return;
+
+ ctx.beginPath();
+ ctx.moveTo(lastPos.x, lastPos.y);
+ ctx.lineTo(pos.x, pos.y);
+ ctx.stroke();
- context.lineTo(offsetX, offsetY);
- context.stroke();
+ setLastPos(pos);
};
- const handlePointerLeave = () => {
- if (isDrawing) {
- handlePointerUp();
- }
+ const stopDrawing = () => {
+ setDrawing(false);
+ setLastPos(null);
};
const clearCanvas = () => {
const canvas = canvasRef.current;
- const context = contextRef.current;
- if (!canvas || !context) return;
+ const ctx = getCanvasContext();
+ if (!canvas || !ctx) return;
- context.clearRect(0, 0, canvas.width, canvas.height);
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
};
return (
-
-
- Neon Sketch
-
- Paint freely, remix colors, and clear the slate whenever inspiration
- strikes.
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+ {COLORS.map((swatch) => (
+ setColor(swatch)}
+ onKeyDown={(e) => {
+ if (e.key === 'Enter' || e.key === ' ') {
+ setColor(swatch);
+ }
+ }}
+ role="button"
+ tabIndex={0}
+ aria-label={`Set color to ${swatch}`}
+ />
+ ))}
+
+
+
);
}