From c5fb63367b7ed166f29f52e4b2fc28650471e737 Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Mon, 22 Dec 2025 00:04:42 +0200 Subject: [PATCH 01/15] Add initial project structure and configuration for RPA forms example --- .../rpa-forms-example/.env.example | 1 + python-examples/rpa-forms-example/.gitignore | 51 ++++ .../rpa-forms-example/Intuned.jsonc | 15 + python-examples/rpa-forms-example/README.md | 132 +++++++++ .../____testParameters/get_stock_details.json | 8 + .../rpa-forms-example/api/form_filler.py | 277 ++++++++++++++++++ .../rpa-forms-example/hooks/setup_context.py | 15 + .../rpa-forms-example/pyproject.toml | 28 ++ .../utils/types_and_schemas.py | 110 +++++++ 9 files changed, 637 insertions(+) create mode 100644 python-examples/rpa-forms-example/.env.example create mode 100644 python-examples/rpa-forms-example/.gitignore create mode 100644 python-examples/rpa-forms-example/Intuned.jsonc create mode 100644 python-examples/rpa-forms-example/README.md create mode 100644 python-examples/rpa-forms-example/____testParameters/get_stock_details.json create mode 100644 python-examples/rpa-forms-example/api/form_filler.py create mode 100644 python-examples/rpa-forms-example/hooks/setup_context.py create mode 100644 python-examples/rpa-forms-example/pyproject.toml create mode 100644 python-examples/rpa-forms-example/utils/types_and_schemas.py diff --git a/python-examples/rpa-forms-example/.env.example b/python-examples/rpa-forms-example/.env.example new file mode 100644 index 00000000..29b1e203 --- /dev/null +++ b/python-examples/rpa-forms-example/.env.example @@ -0,0 +1 @@ +INTUNED_API_KEY=your_api_key_here \ No newline at end of file diff --git a/python-examples/rpa-forms-example/.gitignore b/python-examples/rpa-forms-example/.gitignore new file mode 100644 index 00000000..ec933ac1 --- /dev/null +++ b/python-examples/rpa-forms-example/.gitignore @@ -0,0 +1,51 @@ +# Dependencies +node_modules/ +.pnp +.pnp.js + +# Production builds +/build +/dist +/.next/ +/out/ + +# Environment variables +.env +.env.local +.env.development.local +.env.test.local +.env.production.local + +# Logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime/temporary files +.cache +.parcel-cache +*.tsbuildinfo + +# Coverage +/coverage +.nyc_output + +# Python +__pycache__/ +*.py[cod] +*.pyc +.Python +build/ +*.egg-info/ +.venv/ +venv/ +.env + +# OS files +.DS_Store +Thumbs.db + +# IDE +.vscode/ +.idea/ diff --git a/python-examples/rpa-forms-example/Intuned.jsonc b/python-examples/rpa-forms-example/Intuned.jsonc new file mode 100644 index 00000000..429bec45 --- /dev/null +++ b/python-examples/rpa-forms-example/Intuned.jsonc @@ -0,0 +1,15 @@ +// For more information, see our Intuned settings reference +// https://docs.intunedhq.com/docs/05-references/intuned-json +{ + "projectName": "stagehand-template", + "apiAccess": { + "enabled": true + }, + "authSessions": { + "enabled": false + }, + "replication": { + "maxConcurrentRequests": 1, + "size": "standard" + } +} \ No newline at end of file diff --git a/python-examples/rpa-forms-example/README.md b/python-examples/rpa-forms-example/README.md new file mode 100644 index 00000000..047dafdd --- /dev/null +++ b/python-examples/rpa-forms-example/README.md @@ -0,0 +1,132 @@ +# stagehand-stock-details Intuned project + +Using Stagehand with Intuned + +## Getting Started + +To get started developing browser automation projects with Intuned, check out our [concepts and terminology](https://docs.intunedhq.com/docs/getting-started/conceptual-guides/core-concepts#runs%3A-executing-your-automations). + + +## Development + +> **_NOTE:_** All commands support `--help` flag to get more information about the command and its arguments and options. + +### Install dependencies +```bash +uv sync +``` + +After installing dependencies, `intuned` command should be available in your environment. + +### Run an API +```bash +uv run intuned run api +``` + +### Deploy project +```bash +uv run intuned deploy +``` + + + + +### `intuned-browser`: Intuned Browser SDK + +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). + + +### `intuned-runtime`: Intuned Runtime SDK + +All intuned projects use the Intuned runtime SDK. It also exposes some helpers for nested scheduling and auth sessions. This project uses some of these helpers. For more information, check out the documentation coming soon. + +This project uses the `setup_context` hook from the Intuned runtime SDK. This hook is used to set up the browser context and page for the project. For more information, check out the documentation coming soon. + + +## Project Structure +The project structure is as follows: +``` +/ +├── apis/ # Your API endpoints +│ └── ... +├── auth-sessions/ # Auth session related APIs +│ ├── check.py # API to check if the auth session is still valid +│ └── create.py # API to create/recreate the auth session programmatically +├── auth-sessions-instances/ # Auth session instances created and used by the CLI +│ └── ... +└── intuned.json # Intuned project configuration file +``` + + +## `Intuned.json` Reference +```jsonc +{ + // Your Intuned workspace ID. + // Optional - If not provided here, it must be supplied via the `--workspace-id` flag during deployment. + "workspaceId": "your_workspace_id", + + // The name of your Intuned project. + // Optional - If not provided here, it must be supplied via the command line when deploying. + "projectName": "your_project_name", + + // Replication settings + "replication": { + // The maximum number of concurrent executions allowed via Intuned API. This does not affect jobs. + // A number of machines equal to this will be allocated to handle API requests. + // Not applicable if api access is disabled. + "maxConcurrentRequests": 1, + + // The machine size to use for this project. This is applicable for both API requests and jobs. + // "standard": Standard machine size (6 shared vCPUs, 2GB RAM) + // "large": Large machine size (8 shared vCPUs, 4GB RAM) + // "xlarge": Extra large machine size (1 performance vCPU, 8GB RAM) + "size": "standard" + } + + // Auth session settings + "authSessions": { + // Whether auth sessions are enabled for this project. + // If enabled, "auth-sessions/check.ts" API must be implemented to validate the auth session. + "enabled": true, + + // Whether to save Playwright traces for auth session runs. + "saveTraces": false, + + // The type of auth session to use. + // "API" type requires implementing "auth-sessions/create.ts" API to create/recreate the auth session programmatically. + // "MANUAL" type uses a recorder to manually create the auth session. + "type": "API", + + + // Recorder start URL for the recorder to navigate to when creating the auth session. + // Required if "type" is "MANUAL". Not used if "type" is "API". + "startUrl": "https://example.com/login", + + // Recorder finish URL for the recorder. Once this URL is reached, the recorder stops and saves the auth session. + // Required if "type" is "MANUAL". Not used if "type" is "API". + "finishUrl": "https://example.com/dashboard", + + // Recorder browser mode + // "fullscreen": Launches the browser in fullscreen mode. + // "kiosk": Launches the browser in kiosk mode (no address bar, no navigation controls). + // Only applicable for "MANUAL" type. + "browserMode": "fullscreen" + } + + // API access settings + "apiAccess": { + // Whether to enable consumption through Intuned API. If this is false, the project can only be consumed through jobs. + // This is required for projects that use auth sessions. + "enabled": true + }, + + // Whether to run the deployed API in a headful browser. Running in headful can help with some anti-bot detections. However, it requires more resources and may work slower or crash if the machine size is "standard". + "headful": false, + + // The region where your Intuned project is hosted. + // For a list of available regions, contact support or refer to the documentation. + // Optional - Default: "us" + "region": "us" +} +``` + \ No newline at end of file diff --git a/python-examples/rpa-forms-example/____testParameters/get_stock_details.json b/python-examples/rpa-forms-example/____testParameters/get_stock_details.json new file mode 100644 index 00000000..6ca75210 --- /dev/null +++ b/python-examples/rpa-forms-example/____testParameters/get_stock_details.json @@ -0,0 +1,8 @@ +[ + { + "name": "Most advanced", + "value": "{\n \"criteria\": \"most advanced stock today\"\n}", + "lastUsed": true, + "id": "5db5d935-de48-4169-9de5-563930eec8d4" + } +] diff --git a/python-examples/rpa-forms-example/api/form_filler.py b/python-examples/rpa-forms-example/api/form_filler.py new file mode 100644 index 00000000..f17b0b34 --- /dev/null +++ b/python-examples/rpa-forms-example/api/form_filler.py @@ -0,0 +1,277 @@ +from typing import TypedDict, cast +from intuned_runtime import attempt_store +from pydantic import BaseModel +from stagehand import Stagehand, StagehandPage +from stagehand.types import ObserveResult +import os +from utils.types_and_schemas import ListParameters + + +class Params(TypedDict): + criteria: str + + +action_cache = {} + + +async def get_cached_action(page: StagehandPage, instruction: str): + if instruction in action_cache: + print(f"Using cached action for {instruction}") + return action_cache[instruction] + + results = await page.observe(instruction) + if results: + action = results[0] + action_cache[instruction] = action + return action + + return None + + +async def perform_action( + page: StagehandPage, action: ObserveResult | str, instruction: str | None = None +): + if action: + try: + await page.act(action) + await page._wait_for_settled_dom() + except Exception: + # If action failed and we have the instruction, clear cache and re-observe + if instruction and instruction in action_cache: + print( + f"Action failed for '{instruction}', clearing cache and re-observing..." + ) + del action_cache[instruction] + results = await page.observe(instruction) + if results: + new_action = results[0] + action_cache[instruction] = new_action + await page.act(new_action) + await page._wait_for_settled_dom() + else: + raise + else: + raise + + +async def get_and_perform_action(page: StagehandPage, instruction: str): + """Get cached or observe action, then perform it with self-healing on failure.""" + action = await get_cached_action(page, instruction) + if action: + await perform_action(page, action, instruction) + else: + raise ValueError(f"Could not find action for instruction: {instruction}") + + +# async def automation( +# page: StagehandPage, params: ListParameters, *args: ..., **kwargs: ... +# ): +# await page.set_viewport_size({"width": 1280, "height": 800}) +# params = ListParameters.model_validate(params) +# site = params.metadata.site +# insurance_object_type = params.metadata.insurance_object_type +# zip_code = params.address.zip_code +# # stagehand = cast(Stagehand, attempt_store.get("stagehand")) + +# await page.goto(site) + +# # agent = stagehand.agent( +# # provider="openai", +# # model="computer-use-preview", +# # options={"apiKey": os.getenv("OPENAI_API_KEY")}, +# # ) + +# # Agent runs on current Stagehand page +# await get_and_perform_action( +# page, f"Fill in the zip code {zip_code} in the zip code field" +# ) +# await page.act(f"click on the {insurance_object_type} option") +# await get_and_perform_action(page, "Click the Start My Quote button") +# await get_and_perform_action( +# page, f"Fill in the zip code {zip_code} in the zip code field again" +# ) +# await get_and_perform_action(page, "Click the Continue button") +# await get_and_perform_action( +# page, +# f"Fill in the date of birth {params.applicant.date_of_birth} in the date of birth field", +# ) +# await get_and_perform_action(page, "Click the Next button") +# await get_and_perform_action( +# page, +# f"Fill in the first name {params.applicant.first_name} in the first name field", +# ) +# await get_and_perform_action( +# page, +# f"Fill in the last name {params.applicant.last_name} in the last name field", +# ) +# await get_and_perform_action(page, "Click the Next button") +# # fill_address_action = await get_cached_action( +# # page, +# # f"""Type {params.address.street_line1} into the Address field. +# # Wait for the address suggestions dropdown to appear.""", +# # ) +# # await perform_action(page, cast(ObserveResult, fill_address_action)) +# # await page._wait_for_settled_dom() +# # fill_apt_action = await get_cached_action( +# # page, +# # f"""Fill {params.address.street_line2} into the Apt # field.""", +# # ) +# # await perform_action(page, cast(ObserveResult, fill_apt_action)) +# # await page._wait_for_settled_dom() +# # click_next_action = await get_cached_action(page, "Click the Next button") +# # await perform_action(page, cast(ObserveResult, click_next_action)) +# # await page._wait_for_settled_dom() +# # if params.vehicle.has_vin: +# # check_vin_action = await get_cached_action(page, "Click the Yes button") +# # await perform_action(page, cast(ObserveResult, check_vin_action)) +# # await page._wait_for_settled_dom() +# # fill_vin_action = await get_cached_action( +# # page, +# # f"""Fill {params.vehicle.vin} into the VIN field.""", +# # ) +# # await perform_action(page, cast(ObserveResult, fill_vin_action)) +# # await page._wait_for_settled_dom() +# # else: +# # check_vin_action = await get_cached_action(page, "Click the No button") +# # await perform_action(page, cast(ObserveResult, check_vin_action)) +# # await page._wait_for_settled_dom() +# # fill_year_action = await get_cached_action( +# # page, +# # f"""Fill {params.vehicle.year} into the Year field.""", +# # ) +# # await perform_action(page, cast(ObserveResult, fill_year_action)) +# # await page._wait_for_settled_dom() +# # fill_make_action = await get_cached_action( +# # page, +# # f"""Fill {params.vehicle.make} into the Make field.""", +# # ) +# # await perform_action(page, cast(ObserveResult, fill_make_action)) +# # await page._wait_for_settled_dom() +# # fill_model_action = await get_cached_action( +# # page, +# # f"""Fill {params.vehicle.model} into the Model field.""", +# # ) +# # await perform_action(page, cast(ObserveResult, fill_model_action)) +# # await page._wait_for_settled_dom() +# return await page.extract("Extract the title of the page") + + +async def automation( + page: StagehandPage, params: ListParameters, *args: ..., **kwargs: ... +): + await page.set_viewport_size({"width": 1280, "height": 800}) + + params = ListParameters.model_validate(params) + + site = params.metadata.site + insurance_object_type = params.metadata.insurance_object_type + zip_code = params.address.zip_code + + await page.goto(site) + + # --- ZIP entry --- + await get_and_perform_action( + page, f"Fill in the zip code {zip_code} in the zip code field" + ) + + # --- Object type selection --- + await get_and_perform_action(page, f"Click on the {insurance_object_type} option") + + await get_and_perform_action(page, "Click the Start My Quote button") + + # --- ZIP confirmation (GEICO repeats this) --- + await get_and_perform_action( + page, f"Fill in the zip code {zip_code} in the zip code field again" + ) + await get_and_perform_action(page, "Click the Continue button") + + # --- DOB --- + await get_and_perform_action( + page, + f"Fill in the date of birth {params.applicant.date_of_birth} in the date of birth field", + ) + await get_and_perform_action(page, "Click the Next button") + + # --- Name --- + await get_and_perform_action( + page, + f"Fill in the first name {params.applicant.first_name} in the first name field", + ) + await get_and_perform_action( + page, + f"Fill in the last name {params.applicant.last_name} in the last name field", + ) + await get_and_perform_action(page, "Click the Next button") + + # --- Address (autocomplete + apt split) --- + await get_and_perform_action( + page, + f""" + Type {params.address.street_line1} + into the Address field. + Wait for the address suggestions dropdown to appear. + Select the first suggested address. + """, + ) + + if params.address.street_line2: + await get_and_perform_action( + page, + f"Fill {params.address.street_line2} into the Apt # field", + ) + + await get_and_perform_action(page, "Click the Next button") + + # --- VIN decision --- + if params.vehicle.has_vin: + await get_and_perform_action(page, "Click the Yes button") + await get_and_perform_action( + page, f"Fill {params.vehicle.vin} into the VIN field" + ) + else: + await get_and_perform_action(page, "Click the No button") + await get_and_perform_action( + page, f"Fill {params.vehicle.year} into the Year field" + ) + await get_and_perform_action( + page, f"Fill {params.vehicle.make} into the Make field" + ) + await get_and_perform_action( + page, f"Fill {params.vehicle.model} into the Model field" + ) + + return await page.extract("Extract the title of the page") + + +""" +{ + "metadata": { + "site": "https://www.geico.com", + "insurance_object_type": "auto" + }, + "applicant": { + "first_name": "John", + "last_name": "Doe", + "date_of_birth": "04/12/1992", + "gender": "male", + "marital_status": "single" + }, + "address": { + "street_line1": "123 W 34th St, Savannah, GA 31401", + "street_line2": "5B", + "city": "Savannah", + "state": "GA", + "zip_code": "31401", + "residence_type": "apartment" + }, + "vehicle": { + "has_vin": false, + "vin": null, + "year": 2021, + "make": "Toyota", + "model": "Camry", + "ownership": "owned" + } +} + +""" diff --git a/python-examples/rpa-forms-example/hooks/setup_context.py b/python-examples/rpa-forms-example/hooks/setup_context.py new file mode 100644 index 00000000..cd1103d1 --- /dev/null +++ b/python-examples/rpa-forms-example/hooks/setup_context.py @@ -0,0 +1,15 @@ +from intuned_runtime import attempt_store +from stagehand import Stagehand + + +async def setup_context(*, api_name: str, api_parameters: str, cdp_url: str): + stagehand = Stagehand( + env="LOCAL", local_browser_launch_options=dict(cdp_url=cdp_url) + ) + await stagehand.init() + attempt_store.set("stagehand", stagehand) + + async def cleanup(): + await stagehand.close() + + return stagehand.context, stagehand.page, cleanup diff --git a/python-examples/rpa-forms-example/pyproject.toml b/python-examples/rpa-forms-example/pyproject.toml new file mode 100644 index 00000000..a3fd4467 --- /dev/null +++ b/python-examples/rpa-forms-example/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "stagehand-stock-details" +version = "0.0.1" +description = "Using Stagehand with Intuned" +authors = [{ name = "Intuned", email = "service@intunedhq.com" }] +requires-python = ">=3.12,<3.13" +readme = "README.md" +keywords = [ + "Python", + "AI", + "Stagehand", + "intuned-browser-sdk", + "intuned-runtime-sdk", + "intuned-runtime-sdk-setup-context-hook", +] +dependencies = [ + "playwright==1.56", + "intuned-runtime==1.3.12", + "stagehand==0.5.3", + "intuned-browser==0.1.10", +] + +[tool.uv] +package = false diff --git a/python-examples/rpa-forms-example/utils/types_and_schemas.py b/python-examples/rpa-forms-example/utils/types_and_schemas.py new file mode 100644 index 00000000..671a1922 --- /dev/null +++ b/python-examples/rpa-forms-example/utils/types_and_schemas.py @@ -0,0 +1,110 @@ +from typing import Optional, Literal +from pydantic import BaseModel, Field, model_validator + + +# ---------- Metadata ---------- + + +class Metadata(BaseModel): + site: str + run_mode: Literal["human", "fast"] = "human" + checkpointing: bool = True + insurance_object_type: Literal[ + "auto", "homeowners", "renters", "motorcycle", "boat", "commercial_auto" + ] + + +# ---------- Applicant ---------- + + +class Applicant(BaseModel): + first_name: str + last_name: str + date_of_birth: str + gender: Literal["male", "female", "other"] + marital_status: Literal["single", "married", "divorced", "widowed"] + + +# ---------- Address ---------- + + +class Address(BaseModel): + street_line1: str + street_line2: Optional[str] = None # Apt / Unit + city: str + state: str = Field(..., min_length=2, max_length=2) + zip_code: str = Field(..., pattern=r"^\d{5}$") + residence_type: Literal["apartment", "house", "condo", "townhouse"] + + +# ---------- Vehicle ---------- + + +class Vehicle(BaseModel): + has_vin: bool + vin: Optional[str] = None + + year: Optional[int] = None + make: Optional[str] = None + model: Optional[str] = None + + ownership: Optional[Literal["owned", "leased", "financed"]] = None + primary_use: Optional[Literal["commute", "pleasure", "business"]] = None + annual_mileage: Optional[int] = None + + @model_validator(mode="after") + def validate_vin_path(self): + if self.has_vin: + if not self.vin: + raise ValueError("vin is required when has_vin is true") + if any([self.year, self.make, self.model]): + raise ValueError("year/make/model must be omitted when vin is provided") + else: + if self.vin is not None: + raise ValueError("vin must be null when has_vin is false") + if not all([self.year, self.make, self.model]): + raise ValueError( + "year, make, and model are required when has_vin is false" + ) + return self + + +# ---------- Driving History ---------- + + +class DrivingHistory(BaseModel): + licensed_since: int + accidents_last_5_years: int = Field(ge=0) + violations_last_5_years: int = Field(ge=0) + claims_last_5_years: int = Field(ge=0) + + +# ---------- Coverage ---------- + + +class CoveragePreferences(BaseModel): + liability: Literal["state_minimum", "standard", "premium"] + collision: bool + comprehensive: bool + deductible: int = Field(ge=0) + + +# ---------- Constraints ---------- + + +class Constraints(BaseModel): + do_not_proceed_without_validation: bool = True + stop_after_step: Optional[str] = None + + +# ---------- Root Model ---------- + + +class ListParameters(BaseModel): + metadata: Metadata + applicant: Applicant + address: Address + vehicle: Vehicle + # driving_history: DrivingHistory + # coverage_preferences: CoveragePreferences + # constraints: Constraints From 4160755a57f0c732e3a14ddfe5f8cf3ede4c58ab Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Mon, 22 Dec 2025 00:05:04 +0200 Subject: [PATCH 02/15] Remove commented-out automation function in form_filler.py to clean up the codebase. --- .../rpa-forms-example/api/form_filler.py | 91 ------------------- 1 file changed, 91 deletions(-) diff --git a/python-examples/rpa-forms-example/api/form_filler.py b/python-examples/rpa-forms-example/api/form_filler.py index f17b0b34..577a4867 100644 --- a/python-examples/rpa-forms-example/api/form_filler.py +++ b/python-examples/rpa-forms-example/api/form_filler.py @@ -63,97 +63,6 @@ async def get_and_perform_action(page: StagehandPage, instruction: str): raise ValueError(f"Could not find action for instruction: {instruction}") -# async def automation( -# page: StagehandPage, params: ListParameters, *args: ..., **kwargs: ... -# ): -# await page.set_viewport_size({"width": 1280, "height": 800}) -# params = ListParameters.model_validate(params) -# site = params.metadata.site -# insurance_object_type = params.metadata.insurance_object_type -# zip_code = params.address.zip_code -# # stagehand = cast(Stagehand, attempt_store.get("stagehand")) - -# await page.goto(site) - -# # agent = stagehand.agent( -# # provider="openai", -# # model="computer-use-preview", -# # options={"apiKey": os.getenv("OPENAI_API_KEY")}, -# # ) - -# # Agent runs on current Stagehand page -# await get_and_perform_action( -# page, f"Fill in the zip code {zip_code} in the zip code field" -# ) -# await page.act(f"click on the {insurance_object_type} option") -# await get_and_perform_action(page, "Click the Start My Quote button") -# await get_and_perform_action( -# page, f"Fill in the zip code {zip_code} in the zip code field again" -# ) -# await get_and_perform_action(page, "Click the Continue button") -# await get_and_perform_action( -# page, -# f"Fill in the date of birth {params.applicant.date_of_birth} in the date of birth field", -# ) -# await get_and_perform_action(page, "Click the Next button") -# await get_and_perform_action( -# page, -# f"Fill in the first name {params.applicant.first_name} in the first name field", -# ) -# await get_and_perform_action( -# page, -# f"Fill in the last name {params.applicant.last_name} in the last name field", -# ) -# await get_and_perform_action(page, "Click the Next button") -# # fill_address_action = await get_cached_action( -# # page, -# # f"""Type {params.address.street_line1} into the Address field. -# # Wait for the address suggestions dropdown to appear.""", -# # ) -# # await perform_action(page, cast(ObserveResult, fill_address_action)) -# # await page._wait_for_settled_dom() -# # fill_apt_action = await get_cached_action( -# # page, -# # f"""Fill {params.address.street_line2} into the Apt # field.""", -# # ) -# # await perform_action(page, cast(ObserveResult, fill_apt_action)) -# # await page._wait_for_settled_dom() -# # click_next_action = await get_cached_action(page, "Click the Next button") -# # await perform_action(page, cast(ObserveResult, click_next_action)) -# # await page._wait_for_settled_dom() -# # if params.vehicle.has_vin: -# # check_vin_action = await get_cached_action(page, "Click the Yes button") -# # await perform_action(page, cast(ObserveResult, check_vin_action)) -# # await page._wait_for_settled_dom() -# # fill_vin_action = await get_cached_action( -# # page, -# # f"""Fill {params.vehicle.vin} into the VIN field.""", -# # ) -# # await perform_action(page, cast(ObserveResult, fill_vin_action)) -# # await page._wait_for_settled_dom() -# # else: -# # check_vin_action = await get_cached_action(page, "Click the No button") -# # await perform_action(page, cast(ObserveResult, check_vin_action)) -# # await page._wait_for_settled_dom() -# # fill_year_action = await get_cached_action( -# # page, -# # f"""Fill {params.vehicle.year} into the Year field.""", -# # ) -# # await perform_action(page, cast(ObserveResult, fill_year_action)) -# # await page._wait_for_settled_dom() -# # fill_make_action = await get_cached_action( -# # page, -# # f"""Fill {params.vehicle.make} into the Make field.""", -# # ) -# # await perform_action(page, cast(ObserveResult, fill_make_action)) -# # await page._wait_for_settled_dom() -# # fill_model_action = await get_cached_action( -# # page, -# # f"""Fill {params.vehicle.model} into the Model field.""", -# # ) -# # await perform_action(page, cast(ObserveResult, fill_model_action)) -# # await page._wait_for_settled_dom() -# return await page.extract("Extract the title of the page") async def automation( From b2f5966b0a53aa33ffe244fe7fdfb1a8d717e87a Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Tue, 23 Dec 2025 15:47:42 +0200 Subject: [PATCH 03/15] Add insurance form filler automation and project configuration updates --- .../api/insurance_form_filler/default.json | 39 ++++ .../{Intuned.jsonc => Intuned.json} | 4 +- python-examples/rpa-forms-example/README.md | 51 +++-- .../____testParameters/get_stock_details.json | 8 - .../rpa-forms-example/api/form_filler.py | 186 ------------------ .../api/insurance_form_filler.py | 183 +++++++++++++++++ .../rpa-forms-example/pyproject.toml | 4 +- .../utils/types_and_schemas.py | 117 ++++------- 8 files changed, 299 insertions(+), 293 deletions(-) create mode 100644 python-examples/rpa-forms-example/.parameters/api/insurance_form_filler/default.json rename python-examples/rpa-forms-example/{Intuned.jsonc => Intuned.json} (51%) delete mode 100644 python-examples/rpa-forms-example/____testParameters/get_stock_details.json delete mode 100644 python-examples/rpa-forms-example/api/form_filler.py create mode 100644 python-examples/rpa-forms-example/api/insurance_form_filler.py diff --git a/python-examples/rpa-forms-example/.parameters/api/insurance_form_filler/default.json b/python-examples/rpa-forms-example/.parameters/api/insurance_form_filler/default.json new file mode 100644 index 00000000..8445c3ec --- /dev/null +++ b/python-examples/rpa-forms-example/.parameters/api/insurance_form_filler/default.json @@ -0,0 +1,39 @@ +{ + "metadata": { + "site": "https://www.erieinsurance.com/", + "insurance_type": "auto" + }, + "applicant": { + "first_name": "John", + "last_name": "Doe", + "date_of_birth": "04/12/1992", + "gender": "male", + "marital_status": "single", + "accident_prevention_course": true, + "email": "john.doe@example.com", + "phone_number": "555-123-4567", + "is_cell_phone": true, + "can_text": true, + "preferred_name": "Johnny", + "home_multi_policy_discount": false, + "currently_has_auto_insurance": false, + "coverage_effective_date": "01/01/2026" + }, + "address": { + "street_line1": "350 W 42nd St", + "street_line2": "Apt 12B", + "city": "New York", + "state": "NY", + "zip_code": "10036" + }, + "vehicle": { + "vehicle_type": "Automobile", + "year": 2021, + "make": "Toyota", + "model": "Camry Le", + "primary_use": "Pleasure", + "annual_mileage": 12000, + "days_driven_per_week": 5, + "miles_driven_one_way": 15 + } + } \ No newline at end of file diff --git a/python-examples/rpa-forms-example/Intuned.jsonc b/python-examples/rpa-forms-example/Intuned.json similarity index 51% rename from python-examples/rpa-forms-example/Intuned.jsonc rename to python-examples/rpa-forms-example/Intuned.json index 429bec45..b82f7b2f 100644 --- a/python-examples/rpa-forms-example/Intuned.jsonc +++ b/python-examples/rpa-forms-example/Intuned.json @@ -1,7 +1,5 @@ -// For more information, see our Intuned settings reference -// https://docs.intunedhq.com/docs/05-references/intuned-json { - "projectName": "stagehand-template", + "projectName": "insurance-form-filler", "apiAccess": { "enabled": true }, diff --git a/python-examples/rpa-forms-example/README.md b/python-examples/rpa-forms-example/README.md index 047dafdd..eb07e1f5 100644 --- a/python-examples/rpa-forms-example/README.md +++ b/python-examples/rpa-forms-example/README.md @@ -1,6 +1,12 @@ -# stagehand-stock-details Intuned project +# rpa-forms-example Intuned project -Using Stagehand with Intuned +AI-powered form automation using Stagehand to automatically fill out insurance quote forms with applicant, address, and object information. + +## Run on Intuned + +Open this project in Intuned by clicking the button below. + +[![Run on Intuned](https://cdn1.intuned.io/button.svg)](https://app.intuned.io?repo=https://github.com/Intuned/cookbook/tree/main/python-examples/rpa-forms-example) ## Getting Started @@ -20,7 +26,7 @@ After installing dependencies, `intuned` command should be available in your env ### Run an API ```bash -uv run intuned run api +uv run intuned run api insurance_form_filler .parameters/api/insurance_form_filler/default.json ``` ### Deploy project @@ -36,29 +42,35 @@ uv run intuned deploy This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). -### `intuned-runtime`: Intuned Runtime SDK - -All intuned projects use the Intuned runtime SDK. It also exposes some helpers for nested scheduling and auth sessions. This project uses some of these helpers. For more information, check out the documentation coming soon. - -This project uses the `setup_context` hook from the Intuned runtime SDK. This hook is used to set up the browser context and page for the project. For more information, check out the documentation coming soon. ## Project Structure The project structure is as follows: ``` / -├── apis/ # Your API endpoints -│ └── ... -├── auth-sessions/ # Auth session related APIs -│ ├── check.py # API to check if the auth session is still valid -│ └── create.py # API to create/recreate the auth session programmatically -├── auth-sessions-instances/ # Auth session instances created and used by the CLI -│ └── ... -└── intuned.json # Intuned project configuration file +├── api/ # Your API endpoints +│ └── insurance_form_filler.py # Main automation API for filling insurance forms +├── hooks/ # Setup hooks +│ └── setup_context.py # Browser context setup hook +├── utils/ # Utility modules +│ └── types_and_schemas.py # Pydantic models for type validation +├── Intuned.json # Intuned project configuration file +└── pyproject.toml # Python project dependencies ``` +### How It Works -## `Intuned.json` Reference +1. **insurance_form_filler.py** - Uses Stagehand's AI-powered automation to navigate to the insurance website, select insurance type, and fill out multi-step forms including: + - Applicant information (name, date of birth, gender, marital status) + - Contact details (email, phone, text preferences) + - Address information (street, city, state, zip code) + - Vehicle details (type, year, make, model, usage) + - Additional preferences (multi-policy discount, current insurance status, coverage effective date) + + The automation uses natural language instructions to interact with form elements, making it resilient to UI changes. + + +## `Intuned.jsonc` Reference ```jsonc { // Your Intuned workspace ID. @@ -81,7 +93,7 @@ The project structure is as follows: // "large": Large machine size (8 shared vCPUs, 4GB RAM) // "xlarge": Extra large machine size (1 performance vCPU, 8GB RAM) "size": "standard" - } + }, // Auth session settings "authSessions": { @@ -111,7 +123,7 @@ The project structure is as follows: // "kiosk": Launches the browser in kiosk mode (no address bar, no navigation controls). // Only applicable for "MANUAL" type. "browserMode": "fullscreen" - } + }, // API access settings "apiAccess": { @@ -129,4 +141,3 @@ The project structure is as follows: "region": "us" } ``` - \ No newline at end of file diff --git a/python-examples/rpa-forms-example/____testParameters/get_stock_details.json b/python-examples/rpa-forms-example/____testParameters/get_stock_details.json deleted file mode 100644 index 6ca75210..00000000 --- a/python-examples/rpa-forms-example/____testParameters/get_stock_details.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "name": "Most advanced", - "value": "{\n \"criteria\": \"most advanced stock today\"\n}", - "lastUsed": true, - "id": "5db5d935-de48-4169-9de5-563930eec8d4" - } -] diff --git a/python-examples/rpa-forms-example/api/form_filler.py b/python-examples/rpa-forms-example/api/form_filler.py deleted file mode 100644 index 577a4867..00000000 --- a/python-examples/rpa-forms-example/api/form_filler.py +++ /dev/null @@ -1,186 +0,0 @@ -from typing import TypedDict, cast -from intuned_runtime import attempt_store -from pydantic import BaseModel -from stagehand import Stagehand, StagehandPage -from stagehand.types import ObserveResult -import os -from utils.types_and_schemas import ListParameters - - -class Params(TypedDict): - criteria: str - - -action_cache = {} - - -async def get_cached_action(page: StagehandPage, instruction: str): - if instruction in action_cache: - print(f"Using cached action for {instruction}") - return action_cache[instruction] - - results = await page.observe(instruction) - if results: - action = results[0] - action_cache[instruction] = action - return action - - return None - - -async def perform_action( - page: StagehandPage, action: ObserveResult | str, instruction: str | None = None -): - if action: - try: - await page.act(action) - await page._wait_for_settled_dom() - except Exception: - # If action failed and we have the instruction, clear cache and re-observe - if instruction and instruction in action_cache: - print( - f"Action failed for '{instruction}', clearing cache and re-observing..." - ) - del action_cache[instruction] - results = await page.observe(instruction) - if results: - new_action = results[0] - action_cache[instruction] = new_action - await page.act(new_action) - await page._wait_for_settled_dom() - else: - raise - else: - raise - - -async def get_and_perform_action(page: StagehandPage, instruction: str): - """Get cached or observe action, then perform it with self-healing on failure.""" - action = await get_cached_action(page, instruction) - if action: - await perform_action(page, action, instruction) - else: - raise ValueError(f"Could not find action for instruction: {instruction}") - - - - -async def automation( - page: StagehandPage, params: ListParameters, *args: ..., **kwargs: ... -): - await page.set_viewport_size({"width": 1280, "height": 800}) - - params = ListParameters.model_validate(params) - - site = params.metadata.site - insurance_object_type = params.metadata.insurance_object_type - zip_code = params.address.zip_code - - await page.goto(site) - - # --- ZIP entry --- - await get_and_perform_action( - page, f"Fill in the zip code {zip_code} in the zip code field" - ) - - # --- Object type selection --- - await get_and_perform_action(page, f"Click on the {insurance_object_type} option") - - await get_and_perform_action(page, "Click the Start My Quote button") - - # --- ZIP confirmation (GEICO repeats this) --- - await get_and_perform_action( - page, f"Fill in the zip code {zip_code} in the zip code field again" - ) - await get_and_perform_action(page, "Click the Continue button") - - # --- DOB --- - await get_and_perform_action( - page, - f"Fill in the date of birth {params.applicant.date_of_birth} in the date of birth field", - ) - await get_and_perform_action(page, "Click the Next button") - - # --- Name --- - await get_and_perform_action( - page, - f"Fill in the first name {params.applicant.first_name} in the first name field", - ) - await get_and_perform_action( - page, - f"Fill in the last name {params.applicant.last_name} in the last name field", - ) - await get_and_perform_action(page, "Click the Next button") - - # --- Address (autocomplete + apt split) --- - await get_and_perform_action( - page, - f""" - Type {params.address.street_line1} - into the Address field. - Wait for the address suggestions dropdown to appear. - Select the first suggested address. - """, - ) - - if params.address.street_line2: - await get_and_perform_action( - page, - f"Fill {params.address.street_line2} into the Apt # field", - ) - - await get_and_perform_action(page, "Click the Next button") - - # --- VIN decision --- - if params.vehicle.has_vin: - await get_and_perform_action(page, "Click the Yes button") - await get_and_perform_action( - page, f"Fill {params.vehicle.vin} into the VIN field" - ) - else: - await get_and_perform_action(page, "Click the No button") - await get_and_perform_action( - page, f"Fill {params.vehicle.year} into the Year field" - ) - await get_and_perform_action( - page, f"Fill {params.vehicle.make} into the Make field" - ) - await get_and_perform_action( - page, f"Fill {params.vehicle.model} into the Model field" - ) - - return await page.extract("Extract the title of the page") - - -""" -{ - "metadata": { - "site": "https://www.geico.com", - "insurance_object_type": "auto" - }, - "applicant": { - "first_name": "John", - "last_name": "Doe", - "date_of_birth": "04/12/1992", - "gender": "male", - "marital_status": "single" - }, - "address": { - "street_line1": "123 W 34th St, Savannah, GA 31401", - "street_line2": "5B", - "city": "Savannah", - "state": "GA", - "zip_code": "31401", - "residence_type": "apartment" - }, - "vehicle": { - "has_vin": false, - "vin": null, - "year": 2021, - "make": "Toyota", - "model": "Camry", - "ownership": "owned" - } -} - -""" diff --git a/python-examples/rpa-forms-example/api/insurance_form_filler.py b/python-examples/rpa-forms-example/api/insurance_form_filler.py new file mode 100644 index 00000000..79b82114 --- /dev/null +++ b/python-examples/rpa-forms-example/api/insurance_form_filler.py @@ -0,0 +1,183 @@ +from stagehand import StagehandPage +from utils.types_and_schemas import ListParameters + + +class InvalidActionError(Exception): + pass + + +async def perform_action(page: StagehandPage, instruction: str) -> None: # type: ignore + action = await page.observe(instruction) + if action: + await page.act(action[0]) + await page.wait_for_load_state("domcontentloaded") + await page.wait_for_timeout(2000) + else: + raise InvalidActionError( + f"Could not find action for instruction: {instruction}" + ) + + +async def automation( + page: StagehandPage, params: ListParameters, *args: ..., **kwargs: ... +): + await page.set_viewport_size({"width": 1280, "height": 800}) + + params = ListParameters.model_validate(params) + + site = params.metadata.site + await page.goto(site) + + # --- Object type selection --- + await perform_action( + page, + f"Choose the {params.metadata.insurance_type} option from the insurance type dropdown", + ) + + # --- ZIP entry --- + await perform_action( + page, f"Fill in the zip code {params.address.zip_code} in the zip code field" + ) + + await perform_action(page, "Click the Get a quote button") + await page.wait_for_selector("#mainContent") + # --- Name --- + await perform_action( + page, + f"Fill in the first name {params.applicant.first_name} in the first name field", + ) + await perform_action( + page, + f"Fill in the last name {params.applicant.last_name} in the last name field", + ) + # --- DOB --- + await perform_action( + page, + f"Fill in the date of birth {params.applicant.date_of_birth} in the date of birth field", + ) + + # --- Address --- + await perform_action( + page, + f"Fill in the address {params.address.street_line1} in the street address field", + ) + await perform_action( + page, + f"Fill in the city {params.address.city} in the city field", + ) + await perform_action( + page, + f"select the state {params.address.state} from the state dropdown", + ) + await perform_action( + page, + f"Fill in the zip code {params.address.zip_code} in the zip code field", + ) + await perform_action(page, "Click the Continue button") + # --- Vehicle --- + await perform_action( + page, + f"Choose the {params.vehicle.vehicle_type} option from the dropdown", + ) + await perform_action( + page, + f"Select the year {params.vehicle.year} from the year dropdown", + ) + await perform_action( + page, + f"Select the make {params.vehicle.make} from the make dropdown", + ) + await perform_action( + page, + f"Select the model {params.vehicle.model} from the model dropdown", + ) + await perform_action(page, "Click the Continue button") + + await perform_action(page, "Click the Continue button") + + # --- Fill driver information --- + await perform_action( + page, + f"Fill in the first name {params.applicant.first_name} in the first name field if the first name field is empty", + ) + await perform_action( + page, + f"Fill in the last name {params.applicant.last_name} in the last name field if the last name field is empty", + ) + await perform_action( + page, + f"click the {params.applicant.gender} radio button", + ) + await perform_action( + page, + f"choose the {params.applicant.marital_status} option from the marital status dropdown", + ) + if params.applicant.accident_prevention_course: + await perform_action(page, "Click the Yes radio button.") + else: + await perform_action(page, "Click the No radio button.") + await perform_action(page, "Click the Continue button") + await perform_action(page, "Click the Continue button") + + # --- Final details --- + await perform_action( + page, f"Fill in the email {params.applicant.email} in the email field" + ) + await perform_action( + page, + f"Fill in the phone number {params.applicant.phone_number} in the phone number field", + ) + if params.applicant.is_cell_phone: + await perform_action( + page, "Click the Yes radio button in the Is this a cell phone? field" + ) + else: + await perform_action( + page, "Click the No radio button in the Is this a cell phone? field" + ) + if params.applicant.can_text: + await perform_action( + page, + "Click the Yes radio button in the Can an ERIE Agent text you about this quote? field", + ) + else: + await perform_action( + page, + "Click the No radio button in the Can an ERIE Agent text you about this quote? field", + ) + if params.applicant.preferred_name: + await perform_action( + page, + f"Fill in the preferred name {params.applicant.preferred_name} in the preferred name field", + ) + if params.applicant.home_multi_policy_discount: + await perform_action( + page, + "Click the Yes radio button in the Would you like our Home Multi-Policy Discount applied to your quote? field", + ) + else: + await perform_action( + page, + "Click the No radio button in the Would you like our Home Multi-Policy Discount applied to your quote? field", + ) + if params.applicant.currently_has_auto_insurance: + await perform_action( + page, + "Click the Yes radio button in the Do you currently have auto insurance? field", + ) + else: + await perform_action( + page, + "Click the No radio button in the Do you currently have auto insurance? field", + ) + await perform_action( + page, + f"Fill in the coverage effective date {params.applicant.coverage_effective_date} in the coverage effective date field", + ) + await perform_action(page, "Click the Continue button") + await perform_action(page, "Click the Submit Quote to Agent button") + result = await page.extract("Extract the confirmation message") + if result: + return result + else: + raise InvalidActionError("Could not find confirmation message") diff --git a/python-examples/rpa-forms-example/pyproject.toml b/python-examples/rpa-forms-example/pyproject.toml index a3fd4467..e5f52555 100644 --- a/python-examples/rpa-forms-example/pyproject.toml +++ b/python-examples/rpa-forms-example/pyproject.toml @@ -3,9 +3,9 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "stagehand-stock-details" +name = "insurace-form-filler" version = "0.0.1" -description = "Using Stagehand with Intuned" +description = "" authors = [{ name = "Intuned", email = "service@intunedhq.com" }] requires-python = ">=3.12,<3.13" readme = "README.md" diff --git a/python-examples/rpa-forms-example/utils/types_and_schemas.py b/python-examples/rpa-forms-example/utils/types_and_schemas.py index 671a1922..5ecd5249 100644 --- a/python-examples/rpa-forms-example/utils/types_and_schemas.py +++ b/python-examples/rpa-forms-example/utils/types_and_schemas.py @@ -1,15 +1,13 @@ from typing import Optional, Literal -from pydantic import BaseModel, Field, model_validator +from pydantic import BaseModel, Field # ---------- Metadata ---------- class Metadata(BaseModel): - site: str - run_mode: Literal["human", "fast"] = "human" - checkpointing: bool = True - insurance_object_type: Literal[ + site: str = Field(..., format="uri") + insurance_type: Literal[ "auto", "homeowners", "renters", "motorcycle", "boat", "commercial_auto" ] @@ -18,84 +16,58 @@ class Metadata(BaseModel): class Applicant(BaseModel): - first_name: str - last_name: str - date_of_birth: str - gender: Literal["male", "female", "other"] - marital_status: Literal["single", "married", "divorced", "widowed"] + first_name: str = Field(..., min_length=1, description="The first name of the applicant") + last_name: str = Field(..., min_length=1, description="The last name of the applicant") + date_of_birth: str = Field(..., format="date", description="The date of birth of the applicant") + gender: Literal["male", "female", "other"] = Field(..., description="The gender of the applicant") + marital_status: Literal["single", "married", "divorced", "widowed"] = Field(..., description="The marital status of the applicant") + accident_prevention_course: Optional[bool] = Field(default=False, description="Whether the applicant has completed an accident prevention course") + email: str = Field(..., format="email", description="The email address of the applicant") + phone_number: str = Field(..., description="The phone number of the applicant") + is_cell_phone: bool = Field(..., description="Whether the phone number is a cell phone") + can_text: bool = Field(..., description="Can an ERIE Agent text you about this quote?") + preferred_name: Optional[str] = Field(default=None, description="The preferred name of the applicant") + home_multi_policy_discount: bool = Field(..., description="Would you like our Home Multi-Policy Discount applied to your quote?") + currently_has_auto_insurance: bool = Field(..., description="Do you currently have auto insurance?") + coverage_effective_date: str = Field(..., pattern=r"^\d{2}/\d{2}/\d{4}$", description="When does coverage need to be effective? (mm/dd/yyyy format)") # ---------- Address ---------- class Address(BaseModel): - street_line1: str + street_line1: str = Field(..., min_length=1, description="The street address of the applicant") street_line2: Optional[str] = None # Apt / Unit - city: str - state: str = Field(..., min_length=2, max_length=2) - zip_code: str = Field(..., pattern=r"^\d{5}$") - residence_type: Literal["apartment", "house", "condo", "townhouse"] - + city: str = Field(..., min_length=1, description="The city of the applicant") + state: str = Field(..., min_length=2, max_length=2, description="The state of the applicant") + zip_code: str = Field(..., pattern=r"^\d{5}$", description="The zip code of the applicant") # ---------- Vehicle ---------- class Vehicle(BaseModel): - has_vin: bool - vin: Optional[str] = None - - year: Optional[int] = None - make: Optional[str] = None - model: Optional[str] = None - - ownership: Optional[Literal["owned", "leased", "financed"]] = None - primary_use: Optional[Literal["commute", "pleasure", "business"]] = None - annual_mileage: Optional[int] = None - - @model_validator(mode="after") - def validate_vin_path(self): - if self.has_vin: - if not self.vin: - raise ValueError("vin is required when has_vin is true") - if any([self.year, self.make, self.model]): - raise ValueError("year/make/model must be omitted when vin is provided") - else: - if self.vin is not None: - raise ValueError("vin must be null when has_vin is false") - if not all([self.year, self.make, self.model]): - raise ValueError( - "year, make, and model are required when has_vin is false" - ) - return self - - -# ---------- Driving History ---------- - - -class DrivingHistory(BaseModel): - licensed_since: int - accidents_last_5_years: int = Field(ge=0) - violations_last_5_years: int = Field(ge=0) - claims_last_5_years: int = Field(ge=0) - - -# ---------- Coverage ---------- - - -class CoveragePreferences(BaseModel): - liability: Literal["state_minimum", "standard", "premium"] - collision: bool - comprehensive: bool - deductible: int = Field(ge=0) - - -# ---------- Constraints ---------- - - -class Constraints(BaseModel): - do_not_proceed_without_validation: bool = True - stop_after_step: Optional[str] = None - + vehicle_type: Literal[ + "Automobile", + "Travel Trailer", + "ATV", + "Utility Trailer", + "Snowmobile", + "Motor Home", + "Camper", + "Moped", + "Trail Bike", + "Dune Buggy", + "Mini Bike", + "Golf Cart", + "Recreational Trailer", + ] = Field(..., description="The type of vehicle") + year: int = Field(..., description="The year of the vehicle") + make: str = Field(..., min_length=1, description="The make of the vehicle") + model: str = Field(..., min_length=1, description="The model of the vehicle") + primary_use: Literal["Farm", "Business", "Pleasure", "Work/School"] | None = Field(default=None, description="The primary use of the vehicle") + annual_mileage: int | None = Field(default=None, description="The annual mileage of the vehicle") + days_driven_per_week: int | None = Field(default=None, description="The number of days driven per week") + miles_driven_one_way: int | None = Field(default=None, description="The number of miles driven one way") # ---------- Root Model ---------- @@ -105,6 +77,3 @@ class ListParameters(BaseModel): applicant: Applicant address: Address vehicle: Vehicle - # driving_history: DrivingHistory - # coverage_preferences: CoveragePreferences - # constraints: Constraints From 4276bab639e0eed49f1c046a0db40f3b9c0cffed Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Tue, 23 Dec 2025 15:50:03 +0200 Subject: [PATCH 04/15] Add Intuned.jsonc configuration for insurance form filler project and refactor field definitions in types_and_schemas.py for improved readability. --- .../{Intuned.json => Intuned.jsonc} | 0 .../api/insurance_form_filler.py | 2 +- .../utils/types_and_schemas.py | 86 ++++++++++++++----- 3 files changed, 67 insertions(+), 21 deletions(-) rename python-examples/rpa-forms-example/{Intuned.json => Intuned.jsonc} (100%) diff --git a/python-examples/rpa-forms-example/Intuned.json b/python-examples/rpa-forms-example/Intuned.jsonc similarity index 100% rename from python-examples/rpa-forms-example/Intuned.json rename to python-examples/rpa-forms-example/Intuned.jsonc diff --git a/python-examples/rpa-forms-example/api/insurance_form_filler.py b/python-examples/rpa-forms-example/api/insurance_form_filler.py index 79b82114..33217ec3 100644 --- a/python-examples/rpa-forms-example/api/insurance_form_filler.py +++ b/python-examples/rpa-forms-example/api/insurance_form_filler.py @@ -6,7 +6,7 @@ class InvalidActionError(Exception): pass -async def perform_action(page: StagehandPage, instruction: str) -> None: # type: ignore +async def perform_action(page: StagehandPage, instruction: str) -> None: action = await page.observe(instruction) if action: await page.act(action[0]) diff --git a/python-examples/rpa-forms-example/utils/types_and_schemas.py b/python-examples/rpa-forms-example/utils/types_and_schemas.py index 5ecd5249..f0d2bbaa 100644 --- a/python-examples/rpa-forms-example/utils/types_and_schemas.py +++ b/python-examples/rpa-forms-example/utils/types_and_schemas.py @@ -16,31 +16,68 @@ class Metadata(BaseModel): class Applicant(BaseModel): - first_name: str = Field(..., min_length=1, description="The first name of the applicant") - last_name: str = Field(..., min_length=1, description="The last name of the applicant") - date_of_birth: str = Field(..., format="date", description="The date of birth of the applicant") - gender: Literal["male", "female", "other"] = Field(..., description="The gender of the applicant") - marital_status: Literal["single", "married", "divorced", "widowed"] = Field(..., description="The marital status of the applicant") - accident_prevention_course: Optional[bool] = Field(default=False, description="Whether the applicant has completed an accident prevention course") - email: str = Field(..., format="email", description="The email address of the applicant") + first_name: str = Field( + ..., min_length=1, description="The first name of the applicant" + ) + last_name: str = Field( + ..., min_length=1, description="The last name of the applicant" + ) + date_of_birth: str = Field( + ..., format="date", description="The date of birth of the applicant" + ) + gender: Literal["male", "female", "other"] = Field( + ..., description="The gender of the applicant" + ) + marital_status: Literal["single", "married", "divorced", "widowed"] = Field( + ..., description="The marital status of the applicant" + ) + accident_prevention_course: Optional[bool] = Field( + default=False, + description="Whether the applicant has completed an accident prevention course", + ) + email: str = Field( + ..., format="email", description="The email address of the applicant" + ) phone_number: str = Field(..., description="The phone number of the applicant") - is_cell_phone: bool = Field(..., description="Whether the phone number is a cell phone") - can_text: bool = Field(..., description="Can an ERIE Agent text you about this quote?") - preferred_name: Optional[str] = Field(default=None, description="The preferred name of the applicant") - home_multi_policy_discount: bool = Field(..., description="Would you like our Home Multi-Policy Discount applied to your quote?") - currently_has_auto_insurance: bool = Field(..., description="Do you currently have auto insurance?") - coverage_effective_date: str = Field(..., pattern=r"^\d{2}/\d{2}/\d{4}$", description="When does coverage need to be effective? (mm/dd/yyyy format)") + is_cell_phone: bool = Field( + ..., description="Whether the phone number is a cell phone" + ) + can_text: bool = Field( + ..., description="Can an ERIE Agent text you about this quote?" + ) + preferred_name: Optional[str] = Field( + default=None, description="The preferred name of the applicant" + ) + home_multi_policy_discount: bool = Field( + ..., + description="Would you like our Home Multi-Policy Discount applied to your quote?", + ) + currently_has_auto_insurance: bool = Field( + ..., description="Do you currently have auto insurance?" + ) + coverage_effective_date: str = Field( + ..., + pattern=r"^\d{2}/\d{2}/\d{4}$", + description="When does coverage need to be effective? (mm/dd/yyyy format)", + ) # ---------- Address ---------- class Address(BaseModel): - street_line1: str = Field(..., min_length=1, description="The street address of the applicant") + street_line1: str = Field( + ..., min_length=1, description="The street address of the applicant" + ) street_line2: Optional[str] = None # Apt / Unit city: str = Field(..., min_length=1, description="The city of the applicant") - state: str = Field(..., min_length=2, max_length=2, description="The state of the applicant") - zip_code: str = Field(..., pattern=r"^\d{5}$", description="The zip code of the applicant") + state: str = Field( + ..., min_length=2, max_length=2, description="The state of the applicant" + ) + zip_code: str = Field( + ..., pattern=r"^\d{5}$", description="The zip code of the applicant" + ) + # ---------- Vehicle ---------- @@ -64,10 +101,19 @@ class Vehicle(BaseModel): year: int = Field(..., description="The year of the vehicle") make: str = Field(..., min_length=1, description="The make of the vehicle") model: str = Field(..., min_length=1, description="The model of the vehicle") - primary_use: Literal["Farm", "Business", "Pleasure", "Work/School"] | None = Field(default=None, description="The primary use of the vehicle") - annual_mileage: int | None = Field(default=None, description="The annual mileage of the vehicle") - days_driven_per_week: int | None = Field(default=None, description="The number of days driven per week") - miles_driven_one_way: int | None = Field(default=None, description="The number of miles driven one way") + primary_use: Literal["Farm", "Business", "Pleasure", "Work/School"] | None = Field( + default=None, description="The primary use of the vehicle" + ) + annual_mileage: int | None = Field( + default=None, description="The annual mileage of the vehicle" + ) + days_driven_per_week: int | None = Field( + default=None, description="The number of days driven per week" + ) + miles_driven_one_way: int | None = Field( + default=None, description="The number of miles driven one way" + ) + # ---------- Root Model ---------- From 6405774c40d16c3277907f5e90c490e0838aea38 Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Tue, 23 Dec 2025 15:52:51 +0200 Subject: [PATCH 05/15] Update README to include rpa-forms-example for AI-powered form automation --- README.md | 1 + python-examples/README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 121f58af..c2e556f0 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ A collection of examples for building browser automations with [Intuned](https:/ | [quick-recipes](./python-examples/quick-recipes/) | Quick browser automation recipes | | [rpa-example](./python-examples/rpa-example/) | Consultation booking automation | | [rpa-auth-example](./python-examples/rpa-auth-example/) | Authenticated consultation booking with Auth Sessions | +| [rpa-forms-example](./python-examples/rpa-forms-example/) | AI-powered form automation using Stagehand to fill insurance quote forms | | [auth-with-email-otp](./python-examples/auth-with-email-otp/) | Multi-step authentication with Email-based OTP using Resend API | | [auth-with-secret-otp](./python-examples/auth-with-secret-otp/) | Multi-step authentication with TOTP (Time-based OTP) verification | | [auth-with-email-otp](./python-examples/auth-with-email-otp/) | Multi-step authentication with email-based OTP verification | diff --git a/python-examples/README.md b/python-examples/README.md index 70a33f73..9886beb2 100644 --- a/python-examples/README.md +++ b/python-examples/README.md @@ -8,6 +8,7 @@ Intuned sample projects in Python. | [quick-recipes](./quick-recipes/) | Quick browser automation recipes | | [rpa-example](./rpa-example/) | Consultation booking automation | | [rpa-auth-example](./rpa-auth-example/) | Authenticated consultation booking with Auth Sessions | +| [rpa-forms-example](./rpa-forms-example/) | AI-powered form automation using Stagehand to fill insurance quote forms | | [auth-with-email-otp](./auth-with-email-otp/) | Multi-step authentication with Email-based OTP using Resend API | | [auth-with-secret-otp](./auth-with-secret-otp/) | Multi-step authentication with TOTP (Time-based OTP) verification | | [auth-with-email-otp](./auth-with-email-otp/) | Multi-step authentication with email-based OTP verification | From eb064b9242eea33d41563af7497fab414d449c95 Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Tue, 23 Dec 2025 15:57:36 +0200 Subject: [PATCH 06/15] Update rpa-forms-example configuration and add insurance form filler automation script --- python-examples/rpa-forms-example/Intuned.jsonc | 10 +++++++++- ...surance_form_filler.py => insurance-form-filler.py} | 0 2 files changed, 9 insertions(+), 1 deletion(-) rename python-examples/rpa-forms-example/api/{insurance_form_filler.py => insurance-form-filler.py} (100%) diff --git a/python-examples/rpa-forms-example/Intuned.jsonc b/python-examples/rpa-forms-example/Intuned.jsonc index b82f7b2f..ca3b09a7 100644 --- a/python-examples/rpa-forms-example/Intuned.jsonc +++ b/python-examples/rpa-forms-example/Intuned.jsonc @@ -1,5 +1,7 @@ +// For more information, see our Intuned settings reference +// https://docs.intunedhq.com/docs/05-references/intuned-json { - "projectName": "insurance-form-filler", + "projectName": "rpa-forms", "apiAccess": { "enabled": true }, @@ -9,5 +11,11 @@ "replication": { "maxConcurrentRequests": 1, "size": "standard" + }, + "metadata": { + "template": { + "name": "rpa-forms-example", + "description": "RPA example for filling insurance quote forms" + } } } \ No newline at end of file diff --git a/python-examples/rpa-forms-example/api/insurance_form_filler.py b/python-examples/rpa-forms-example/api/insurance-form-filler.py similarity index 100% rename from python-examples/rpa-forms-example/api/insurance_form_filler.py rename to python-examples/rpa-forms-example/api/insurance-form-filler.py From 5c143362905dd5e6c5ad4eff5f752a000cf31a19 Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Tue, 23 Dec 2025 15:58:55 +0200 Subject: [PATCH 07/15] Add default JSON parameters for insurance form filler in rpa-forms-example --- .../{insurance_form_filler => insurance-form-filler}/default.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename python-examples/rpa-forms-example/.parameters/api/{insurance_form_filler => insurance-form-filler}/default.json (100%) diff --git a/python-examples/rpa-forms-example/.parameters/api/insurance_form_filler/default.json b/python-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json similarity index 100% rename from python-examples/rpa-forms-example/.parameters/api/insurance_form_filler/default.json rename to python-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json From 42a47f811b57edfeabcc3d0b317629d42c28ce1d Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Tue, 23 Dec 2025 17:22:12 +0200 Subject: [PATCH 08/15] Enhance rpa-forms-example with AI-powered form automation, including project setup, API scripts, and updated README. Added tags to Intuned.jsonc and corrected API parameter paths. --- .../rpa-forms-example/Intuned.jsonc | 13 +- python-examples/rpa-forms-example/README.md | 2 +- typescript-examples/README.md | 1 + .../rpa-forms-example/.env.example | 1 + .../api/insurance-form-filler/default.json | 40 +++ .../rpa-forms-example/Intuned.jsonc | 30 ++ .../rpa-forms-example/README.md | 161 ++++++++++ .../api/insurance-form-filler.ts | 275 ++++++++++++++++++ .../rpa-forms-example/hooks/setupContext.ts | 11 + .../rpa-forms-example/package.json | 28 ++ .../rpa-forms-example/tsconfig.json | 21 ++ .../utils/typesAndSchemas.ts | 92 ++++++ 12 files changed, 672 insertions(+), 3 deletions(-) create mode 100644 typescript-examples/rpa-forms-example/.env.example create mode 100644 typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json create mode 100644 typescript-examples/rpa-forms-example/Intuned.jsonc create mode 100644 typescript-examples/rpa-forms-example/README.md create mode 100644 typescript-examples/rpa-forms-example/api/insurance-form-filler.ts create mode 100644 typescript-examples/rpa-forms-example/hooks/setupContext.ts create mode 100644 typescript-examples/rpa-forms-example/package.json create mode 100644 typescript-examples/rpa-forms-example/tsconfig.json create mode 100644 typescript-examples/rpa-forms-example/utils/typesAndSchemas.ts diff --git a/python-examples/rpa-forms-example/Intuned.jsonc b/python-examples/rpa-forms-example/Intuned.jsonc index ca3b09a7..1b4b6d69 100644 --- a/python-examples/rpa-forms-example/Intuned.jsonc +++ b/python-examples/rpa-forms-example/Intuned.jsonc @@ -15,7 +15,16 @@ "metadata": { "template": { "name": "rpa-forms-example", - "description": "RPA example for filling insurance quote forms" + "description": "RPA example for filling insurance quote forms", + "tags": [ + "Stagehand", + "AI", + "RPA", + "Forms", + "intuned-runtime-sdk", + "intuned-runtime-sdk-setup-context-hook" + ] } } -} \ No newline at end of file +} + diff --git a/python-examples/rpa-forms-example/README.md b/python-examples/rpa-forms-example/README.md index eb07e1f5..fe6250b8 100644 --- a/python-examples/rpa-forms-example/README.md +++ b/python-examples/rpa-forms-example/README.md @@ -26,7 +26,7 @@ After installing dependencies, `intuned` command should be available in your env ### Run an API ```bash -uv run intuned run api insurance_form_filler .parameters/api/insurance_form_filler/default.json +uv run intuned run api insurance_form_filler .parameters/api/insurance-form-filler/default.json ``` ### Deploy project diff --git a/typescript-examples/README.md b/typescript-examples/README.md index 29d912da..9b8aad6a 100644 --- a/typescript-examples/README.md +++ b/typescript-examples/README.md @@ -8,6 +8,7 @@ Intuned sample projects in TypeScript. | [quick-recipes](./quick-recipes/) | Quick browser automation recipes | | [rpa-example](./rpa-example/) | Consultation booking automation | | [rpa-auth-example](./rpa-auth-example/) | Authenticated consultation booking with Auth Sessions | +| [rpa-forms-example](./rpa-forms-example/) | AI-powered form automation using Stagehand to fill insurance quote forms | | [auth-with-secret-otp](./auth-with-secret-otp/) | Multi-step authentication with TOTP (Time-based OTP) verification | | [auth-with-email-otp](./auth-with-email-otp/) | Multi-step authentication with Email-based OTP using Resend API | | [e-commerce-scrapingcourse](./e-commerce-scrapingcourse/) | E-commerce product scraper with pagination | diff --git a/typescript-examples/rpa-forms-example/.env.example b/typescript-examples/rpa-forms-example/.env.example new file mode 100644 index 00000000..29b1e203 --- /dev/null +++ b/typescript-examples/rpa-forms-example/.env.example @@ -0,0 +1 @@ +INTUNED_API_KEY=your_api_key_here \ No newline at end of file diff --git a/typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json b/typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json new file mode 100644 index 00000000..558b715e --- /dev/null +++ b/typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json @@ -0,0 +1,40 @@ +{ + "metadata": { + "site": "https://www.erieinsurance.com/", + "insurance_type": "auto" + }, + "applicant": { + "first_name": "John", + "last_name": "Doe", + "date_of_birth": "04/12/1992", + "gender": "male", + "marital_status": "single", + "accident_prevention_course": true, + "email": "john.doe@example.com", + "phone_number": "555-123-4567", + "is_cell_phone": true, + "can_text": true, + "preferred_name": "Johnny", + "home_multi_policy_discount": false, + "currently_has_auto_insurance": false, + "coverage_effective_date": "01/01/2026" + }, + "address": { + "street_line1": "350 W 42nd St", + "street_line2": "Apt 12B", + "city": "New York", + "state": "NY", + "zip_code": "10036" + }, + "vehicle": { + "vehicle_type": "Automobile", + "year": 2021, + "make": "Toyota", + "model": "Camry Le", + "primary_use": "Pleasure", + "annual_mileage": 12000, + "days_driven_per_week": 5, + "miles_driven_one_way": 15 + } +} + diff --git a/typescript-examples/rpa-forms-example/Intuned.jsonc b/typescript-examples/rpa-forms-example/Intuned.jsonc new file mode 100644 index 00000000..1b4b6d69 --- /dev/null +++ b/typescript-examples/rpa-forms-example/Intuned.jsonc @@ -0,0 +1,30 @@ +// For more information, see our Intuned settings reference +// https://docs.intunedhq.com/docs/05-references/intuned-json +{ + "projectName": "rpa-forms", + "apiAccess": { + "enabled": true + }, + "authSessions": { + "enabled": false + }, + "replication": { + "maxConcurrentRequests": 1, + "size": "standard" + }, + "metadata": { + "template": { + "name": "rpa-forms-example", + "description": "RPA example for filling insurance quote forms", + "tags": [ + "Stagehand", + "AI", + "RPA", + "Forms", + "intuned-runtime-sdk", + "intuned-runtime-sdk-setup-context-hook" + ] + } + } +} + diff --git a/typescript-examples/rpa-forms-example/README.md b/typescript-examples/rpa-forms-example/README.md new file mode 100644 index 00000000..630f3b9d --- /dev/null +++ b/typescript-examples/rpa-forms-example/README.md @@ -0,0 +1,161 @@ +# rpa-forms-example Intuned project + +AI-powered form automation using Stagehand to automatically fill out insurance quote forms with applicant, address, and object information. + +## Run on Intuned + +Open this project in Intuned by clicking the button below. + +[![Run on Intuned](https://cdn1.intuned.io/button.svg)](https://app.intuned.io?repo=https://github.com/Intuned/cookbook/tree/main/typescript-examples/rpa-forms-example) + +## Getting Started + +To get started developing browser automation projects with Intuned, check out our [concepts and terminology](https://docs.intunedhq.com/docs/getting-started/conceptual-guides/core-concepts#runs%3A-executing-your-automations). + +### Prerequisites + +This project uses Stagehand for AI-powered form automation. You'll need to set the `OPENAI_API_KEY` environment variable: + +```bash +export OPENAI_API_KEY=your_openai_api_key_here +``` + +## Development + +> **_NOTE:_** All commands support `--help` flag to get more information about the command and its arguments and options. + +### Install Dependencies +```bash +# npm +npm install + +# yarn +yarn +``` + +### Run the API Locally +```bash +# npm +npm run intuned run api insurance-form-filler .parameters/api/insurance-form-filler/default.json + +# yarn +yarn intuned run api insurance-form-filler .parameters/api/insurance-form-filler/default.json +``` + +### Deploy to Intuned +```bash +# npm +npm run intuned deploy + +# yarn +yarn intuned deploy +``` + +### `intuned-browser`: Intuned Browser SDK + +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). + +### `intuned-runtime`: Intuned Runtime SDK + +All intuned projects use the Intuned runtime SDK. It also exposes some helpers for nested scheduling and auth sessions. This project uses some of these helpers. For more information, check out the documentation coming soon. + +This project uses the `setupContext` hook from the Intuned runtime SDK. This hook is used to set up the browser context and page for the project. For more information, check out the documentation coming soon. + +## Project Structure +The project structure is as follows: +``` +/ +├── api/ # Your API endpoints +│ └── insurance-form-filler.ts # Main automation API for filling insurance forms +├── hooks/ # Setup hooks +│ └── setupContext.ts # Browser context setup hook +├── utils/ # Utility modules +│ └── typesAndSchemas.ts # Zod schemas for type validation +├── Intuned.jsonc # Intuned project configuration file +└── package.json # Node.js project dependencies +``` + +### How It Works + +1. **insurance-form-filler.ts** - Uses Stagehand's AI-powered automation to navigate to the insurance website, select insurance type, and fill out multi-step forms including: + - Applicant information (name, date of birth, gender, marital status) + - Contact details (email, phone, text preferences) + - Address information (street, city, state, zip code) + - Vehicle details (type, year, make, model, usage) + - Additional preferences (multi-policy discount, current insurance status, coverage effective date) + + The automation uses natural language instructions to interact with form elements, making it resilient to UI changes. + + +## `Intuned.jsonc` Reference +```jsonc +{ + // Your Intuned workspace ID. + // Optional - If not provided here, it must be supplied via the `--workspace-id` flag during deployment. + "workspaceId": "your_workspace_id", + + // The name of your Intuned project. + // Optional - If not provided here, it must be supplied via the command line when deploying. + "projectName": "your_project_name", + + // Replication settings + "replication": { + // The maximum number of concurrent executions allowed via Intuned API. This does not affect jobs. + // A number of machines equal to this will be allocated to handle API requests. + // Not applicable if api access is disabled. + "maxConcurrentRequests": 1, + + // The machine size to use for this project. This is applicable for both API requests and jobs. + // "standard": Standard machine size (6 shared vCPUs, 2GB RAM) + // "large": Large machine size (8 shared vCPUs, 4GB RAM) + // "xlarge": Extra large machine size (1 performance vCPU, 8GB RAM) + "size": "standard" + }, + + // Auth session settings + "authSessions": { + // Whether auth sessions are enabled for this project. + // If enabled, "auth-sessions/check.ts" API must be implemented to validate the auth session. + "enabled": true, + + // Whether to save Playwright traces for auth session runs. + "saveTraces": false, + + // The type of auth session to use. + // "API" type requires implementing "auth-sessions/create.ts" API to create/recreate the auth session programmatically. + // "MANUAL" type uses a recorder to manually create the auth session. + "type": "API", + + + // Recorder start URL for the recorder to navigate to when creating the auth session. + // Required if "type" is "MANUAL". Not used if "type" is "API". + "startUrl": "https://example.com/login", + + // Recorder finish URL for the recorder. Once this URL is reached, the recorder stops and saves the auth session. + // Required if "type" is "MANUAL". Not used if "type" is "API". + "finishUrl": "https://example.com/dashboard", + + // Recorder browser mode + // "fullscreen": Launches the browser in fullscreen mode. + // "kiosk": Launches the browser in kiosk mode (no address bar, no navigation controls). + // Only applicable for "MANUAL" type. + "browserMode": "fullscreen" + }, + + // API access settings + "apiAccess": { + // Whether to enable consumption through Intuned API. If this is false, the project can only be consumed through jobs. + // This is required for projects that use auth sessions. + "enabled": true + }, + + // Whether to run the deployed API in a headful browser. Running in headful can help with some anti-bot detections. However, it requires more resources and may work slower or crash if the machine size is "standard". + "headful": false, + + // The region where your Intuned project is hosted. + // For a list of available regions, contact support or refer to the documentation. + // Optional - Default: "us" + "region": "us" +} +``` + diff --git a/typescript-examples/rpa-forms-example/api/insurance-form-filler.ts b/typescript-examples/rpa-forms-example/api/insurance-form-filler.ts new file mode 100644 index 00000000..33dacc43 --- /dev/null +++ b/typescript-examples/rpa-forms-example/api/insurance-form-filler.ts @@ -0,0 +1,275 @@ +import { Stagehand, AISdkClient } from "@browserbasehq/stagehand"; +import { createOpenAI } from "@ai-sdk/openai"; +import type { Page, BrowserContext } from "playwright"; +import { attemptStore, getAiGatewayConfig } from "@intuned/runtime"; +import { listParametersSchema, ListParameters } from "../utils/typesAndSchemas"; + +class InvalidActionError extends Error { + constructor(message: string) { + super(message); + this.name = "InvalidActionError"; + } +} + +async function getWebSocketUrl(cdpUrl: string): Promise { + const versionUrl = cdpUrl.endsWith("/") + ? `${cdpUrl}json/version` + : `${cdpUrl}/json/version`; + const response = await fetch(versionUrl); + const data = await response.json(); + return data.webSocketDebuggerUrl; +} + +async function performAction( + stagehand: Stagehand, + instruction: string +): Promise { + const action = await stagehand.observe(instruction); + if (action && action.length > 0) { + await stagehand.act(action[0]); + await stagehand.context.pages()[0]?.waitForLoadState("domcontentloaded"); + await new Promise((resolve) => setTimeout(resolve, 2000)); + } else { + throw new InvalidActionError( + `Could not find action for instruction: ${instruction}` + ); + } +} + +export default async function handler( + params: ListParameters, + page: Page, + _context: BrowserContext +) { + // Get AI gateway config for Stagehand + const { baseUrl, apiKey } = await getAiGatewayConfig(); + const cdpUrl = attemptStore.get("cdpUrl") as string; + const webSocketUrl = await getWebSocketUrl(cdpUrl); + + // Create AI SDK provider with Intuned's AI gateway + const openai = createOpenAI({ + apiKey, + baseURL: baseUrl, + }); + + const llmClient = new AISdkClient({ + model: openai("gpt-5-mini"), + }); + + // Initialize Stagehand with act/extract/observe capabilities + const stagehand = new Stagehand({ + env: "LOCAL", + localBrowserLaunchOptions: { + cdpUrl: webSocketUrl, + viewport: { width: 1280, height: 800 }, + }, + // llmClient, + logger: console.log, + }); + await stagehand.init(); + console.log("\nInitialized 🤘 Stagehand"); + // Validate parameters + const validatedParams = listParametersSchema.parse(params); + + const { metadata, applicant, address, vehicle } = validatedParams; + // Navigate to site + await page.goto(metadata.site); + + // --- Object type selection --- + await performAction( + stagehand, + `Choose the ${metadata.insurance_type} option from the insurance type dropdown` + ); + + // --- ZIP entry --- + await performAction( + stagehand, + `Fill in the zip code ${address.zip_code} in the zip code field` + ); + + await performAction(stagehand, "Click the Get a quote button"); + await page.waitForSelector("#mainContent"); + + // --- Name --- + await performAction( + stagehand, + `Fill in the first name ${applicant.first_name} in the first name field` + ); + await performAction( + stagehand, + + `Fill in the last name ${applicant.last_name} in the last name field` + ); + + // --- DOB --- + await performAction( + stagehand, + + `Fill in the date of birth ${applicant.date_of_birth} in the date of birth field` + ); + + // --- Address --- + await performAction( + stagehand, + + `Fill in the address ${address.street_line1} in the street address field` + ); + await performAction( + stagehand, + + `Fill in the city ${address.city} in the city field` + ); + await performAction( + stagehand, + + `select the state ${address.state} from the state dropdown` + ); + await performAction( + stagehand, + + `Fill in the zip code ${address.zip_code} in the zip code field` + ); + await performAction(stagehand, "Click the Continue button"); + + // --- Vehicle --- + await performAction( + stagehand, + + `Choose the ${vehicle.vehicle_type} option from the dropdown` + ); + await performAction( + stagehand, + + `Select the year ${vehicle.year} from the year dropdown` + ); + await performAction( + stagehand, + + `Select the make ${vehicle.make} from the make dropdown` + ); + await performAction( + stagehand, + + `Select the model ${vehicle.model} from the model dropdown` + ); + await performAction(stagehand, "Click the Continue button"); + await performAction(stagehand, "Click the Continue button"); + + // --- Fill driver information --- + await performAction( + stagehand, + + `Fill in the first name ${applicant.first_name} in the first name field if the first name field is empty` + ); + await performAction( + stagehand, + + `Fill in the last name ${applicant.last_name} in the last name field if the last name field is empty` + ); + await performAction( + stagehand, + + `click the ${applicant.gender} radio button` + ); + await performAction( + stagehand, + + `choose the ${applicant.marital_status} option from the marital status dropdown` + ); + if (applicant.accident_prevention_course) { + await performAction(stagehand, "Click the Yes radio button."); + } else { + await performAction(stagehand, "Click the No radio button."); + } + await performAction(stagehand, "Click the Continue button"); + await performAction(stagehand, "Click the Continue button"); + + // --- Final details --- + await performAction( + stagehand, + + `Fill in the email ${applicant.email} in the email field` + ); + await performAction( + stagehand, + + `Fill in the phone number ${applicant.phone_number} in the phone number field` + ); + if (applicant.is_cell_phone) { + await performAction( + stagehand, + + "Click the Yes radio button in the Is this a cell phone? field" + ); + } else { + await performAction( + stagehand, + + "Click the No radio button in the Is this a cell phone? field" + ); + } + if (applicant.can_text) { + await performAction( + stagehand, + + "Click the Yes radio button in the Can an ERIE Agent text you about this quote? field" + ); + } else { + await performAction( + stagehand, + + "Click the No radio button in the Can an ERIE Agent text you about this quote? field" + ); + } + if (applicant.preferred_name) { + await performAction( + stagehand, + + `Fill in the preferred name ${applicant.preferred_name} in the preferred name field` + ); + } + if (applicant.home_multi_policy_discount) { + await performAction( + stagehand, + + "Click the Yes radio button in the Would you like our Home Multi-Policy Discount applied to your quote? field" + ); + } else { + await performAction( + stagehand, + + "Click the No radio button in the Would you like our Home Multi-Policy Discount applied to your quote? field" + ); + } + if (applicant.currently_has_auto_insurance) { + await performAction( + stagehand, + + "Click the Yes radio button in the Do you currently have auto insurance? field" + ); + } else { + await performAction( + stagehand, + + "Click the No radio button in the Do you currently have auto insurance? field" + ); + } + await performAction( + stagehand, + + `Fill in the coverage effective date ${applicant.coverage_effective_date} in the coverage effective date field` + ); + await performAction(stagehand, "Click the Continue button"); + await performAction( + stagehand, + + "Click the Submit Quote to Agent button" + ); + + const result = await stagehand.extract("Extract the confirmation message"); + if (result) { + return result; + } else { + throw new InvalidActionError("Could not find confirmation message"); + } +} diff --git a/typescript-examples/rpa-forms-example/hooks/setupContext.ts b/typescript-examples/rpa-forms-example/hooks/setupContext.ts new file mode 100644 index 00000000..2b3bf817 --- /dev/null +++ b/typescript-examples/rpa-forms-example/hooks/setupContext.ts @@ -0,0 +1,11 @@ +import { attemptStore } from "@intuned/runtime"; + +export default async function setupContext({ + cdpUrl, +}: { + cdpUrl: string; + apiName: string; + apiParameters: any; +}) { + attemptStore.set("cdpUrl", cdpUrl); +} diff --git a/typescript-examples/rpa-forms-example/package.json b/typescript-examples/rpa-forms-example/package.json new file mode 100644 index 00000000..b3649721 --- /dev/null +++ b/typescript-examples/rpa-forms-example/package.json @@ -0,0 +1,28 @@ +{ + "name": "insurance-form-filler", + "version": "1.0.0", + "description": "Insurance form filler using Stagehand in Intuned", + "tags": [ + "Stagehand", + "AI", + "RPA", + "Forms", + "intuned-runtime-sdk", + "intuned-runtime-sdk-setup-context-hook", + "intuned-browser-sdk" + ], + "main": "index.js", + "scripts": { + "intuned": "intuned" + }, + "author": "", + "license": "ISC", + "dependencies": { + "@browserbasehq/stagehand": "^3.0.6", + "@intuned/browser": "0.1.8", + "@intuned/runtime": "^1.3.15", + "@types/node": "^20.10.3", + "playwright": "~1.56.0", + "zod": "3.25.67" + } +} diff --git a/typescript-examples/rpa-forms-example/tsconfig.json b/typescript-examples/rpa-forms-example/tsconfig.json new file mode 100644 index 00000000..f0046e60 --- /dev/null +++ b/typescript-examples/rpa-forms-example/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "moduleResolution": "node", + "module": "ESNext", + "target": "ES2021", + "outDir": "./dist", + "sourceMap": false, + "declaration": true, + "esModuleInterop": true, + "skipLibCheck": true, + "allowSyntheticDefaultImports": true + }, + "include": [ + "**/*.ts" + ], + "exclude": [ + "node_modules", + "dist" + ] +} + diff --git a/typescript-examples/rpa-forms-example/utils/typesAndSchemas.ts b/typescript-examples/rpa-forms-example/utils/typesAndSchemas.ts new file mode 100644 index 00000000..52ec17ed --- /dev/null +++ b/typescript-examples/rpa-forms-example/utils/typesAndSchemas.ts @@ -0,0 +1,92 @@ +import { z } from "zod"; + +// ---------- Metadata ---------- + +const metadataSchema = z.object({ + site: z.string().url(), + insurance_type: z.enum([ + "auto", + "homeowners", + "renters", + "motorcycle", + "boat", + "commercial_auto", + ]), +}); + +// ---------- Applicant ---------- + +const applicantSchema = z.object({ + first_name: z.string().min(1, "First name is required"), + last_name: z.string().min(1, "Last name is required"), + date_of_birth: z + .string() + .regex(/^\d{2}\/\d{2}\/\d{4}$/, "Date must be in MM/DD/YYYY format"), + gender: z.enum(["male", "female", "other"]), + marital_status: z.enum(["single", "married", "divorced", "widowed"]), + accident_prevention_course: z.boolean().default(false), + email: z.string().email("Invalid email address"), + phone_number: z.string(), + is_cell_phone: z.boolean(), + can_text: z.boolean(), + preferred_name: z.string().optional(), + home_multi_policy_discount: z.boolean(), + currently_has_auto_insurance: z.boolean(), + coverage_effective_date: z + .string() + .regex(/^\d{2}\/\d{2}\/\d{4}$/, "Date must be in MM/DD/YYYY format"), +}); + +// ---------- Address ---------- + +const addressSchema = z.object({ + street_line1: z.string().min(1, "Street address is required"), + street_line2: z.string().optional(), + city: z.string().min(1, "City is required"), + state: z.string().length(2, "State must be 2 characters"), + zip_code: z.string().regex(/^\d{5}$/, "Zip code must be 5 digits"), +}); + +// ---------- Vehicle ---------- + +const vehicleSchema = z.object({ + vehicle_type: z.enum([ + "Automobile", + "Travel Trailer", + "ATV", + "Utility Trailer", + "Snowmobile", + "Motor Home", + "Camper", + "Moped", + "Trail Bike", + "Dune Buggy", + "Mini Bike", + "Golf Cart", + "Recreational Trailer", + ]), + year: z.number().int().positive(), + make: z.string().min(1, "Make is required"), + model: z.string().min(1, "Model is required"), + primary_use: z + .enum(["Farm", "Business", "Pleasure", "Work/School"]) + .optional(), + annual_mileage: z.number().int().positive().optional(), + days_driven_per_week: z.number().int().positive().optional(), + miles_driven_one_way: z.number().int().positive().optional(), +}); + +// ---------- Root Schema ---------- + +export const listParametersSchema = z.object({ + metadata: metadataSchema, + applicant: applicantSchema, + address: addressSchema, + vehicle: vehicleSchema, +}); + +export type ListParameters = z.infer; +export type Metadata = z.infer; +export type Applicant = z.infer; +export type Address = z.infer; +export type Vehicle = z.infer; From 84074933ea8e276b3935ca316839970fddd14471 Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Tue, 23 Dec 2025 17:24:53 +0200 Subject: [PATCH 09/15] Update main README to include rpa-forms-example for AI-powered form automation --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c3889d88..64fc3407 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ A collection of examples for building browser automations with [Intuned](https:/ | [quick-recipes](./typescript-examples/quick-recipes/) | Quick browser automation recipes | | [rpa-example](./typescript-examples/rpa-example/) | Consultation booking automation | | [rpa-auth-example](./typescript-examples/rpa-auth-example/) | Authenticated consultation booking with Auth Sessions | +| [rpa-forms-example](./python-examples/rpa-forms-example/) | AI-powered form automation using Stagehand to fill insurance quote forms | | [auth-with-secret-otp](./typescript-examples/auth-with-secret-otp/) | Multi-step authentication with TOTP (Time-based OTP) using secret keys | | [auth-with-email-otp](./typescript-examples/auth-with-email-otp/) | Multi-step authentication with Email-based OTP using Resend API | | [e-commerce-scrapingcourse](./typescript-examples/e-commerce-scrapingcourse/) | E-commerce product scraper with pagination | From 8ead4e90fb953eb71318b3d068d31a62603fc2e3 Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Tue, 23 Dec 2025 17:29:14 +0200 Subject: [PATCH 10/15] Fix link for rpa-forms-example in main README to point to TypeScript examples directory --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 64fc3407..2595e1f9 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ A collection of examples for building browser automations with [Intuned](https:/ | [quick-recipes](./typescript-examples/quick-recipes/) | Quick browser automation recipes | | [rpa-example](./typescript-examples/rpa-example/) | Consultation booking automation | | [rpa-auth-example](./typescript-examples/rpa-auth-example/) | Authenticated consultation booking with Auth Sessions | -| [rpa-forms-example](./python-examples/rpa-forms-example/) | AI-powered form automation using Stagehand to fill insurance quote forms | +| [rpa-forms-example](./typescript-examples/rpa-forms-example/) | AI-powered form automation using Stagehand to fill insurance quote forms | | [auth-with-secret-otp](./typescript-examples/auth-with-secret-otp/) | Multi-step authentication with TOTP (Time-based OTP) using secret keys | | [auth-with-email-otp](./typescript-examples/auth-with-email-otp/) | Multi-step authentication with Email-based OTP using Resend API | | [e-commerce-scrapingcourse](./typescript-examples/e-commerce-scrapingcourse/) | E-commerce product scraper with pagination | From 068b0138932e79d0f6cd8f34d2e89f56c7e8cf16 Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Tue, 23 Dec 2025 17:57:11 +0200 Subject: [PATCH 11/15] Update dependencies and refactor insurance form filler automation in rpa-forms-example. Upgraded intuned-runtime to version 1.3.14, corrected README command syntax, and modified the automation script to utilize the Stagehand class for improved page interactions. --- python-examples/rpa-forms-example/README.md | 2 +- .../api/insurance-form-filler.py | 111 ++++++++++-------- .../rpa-forms-example/hooks/setup_context.py | 12 +- .../rpa-forms-example/pyproject.toml | 2 +- .../stagehand/hooks/setup_context.py | 4 +- 5 files changed, 69 insertions(+), 62 deletions(-) diff --git a/python-examples/rpa-forms-example/README.md b/python-examples/rpa-forms-example/README.md index fe6250b8..881e417c 100644 --- a/python-examples/rpa-forms-example/README.md +++ b/python-examples/rpa-forms-example/README.md @@ -26,7 +26,7 @@ After installing dependencies, `intuned` command should be available in your env ### Run an API ```bash -uv run intuned run api insurance_form_filler .parameters/api/insurance-form-filler/default.json +uv run intuned run api insurance-form-filler .parameters/api/insurance-form-filler/default.json ``` ### Deploy project diff --git a/python-examples/rpa-forms-example/api/insurance-form-filler.py b/python-examples/rpa-forms-example/api/insurance-form-filler.py index 33217ec3..e7f63184 100644 --- a/python-examples/rpa-forms-example/api/insurance-form-filler.py +++ b/python-examples/rpa-forms-example/api/insurance-form-filler.py @@ -1,4 +1,6 @@ -from stagehand import StagehandPage +from stagehand import Stagehand +from playwright.async_api import Page +from intuned_runtime import attempt_store, get_ai_gateway_config from utils.types_and_schemas import ListParameters @@ -6,7 +8,7 @@ class InvalidActionError(Exception): pass -async def perform_action(page: StagehandPage, instruction: str) -> None: +async def perform_action(page: Page, instruction: str) -> None: action = await page.observe(instruction) if action: await page.act(action[0]) @@ -18,165 +20,182 @@ async def perform_action(page: StagehandPage, instruction: str) -> None: ) -async def automation( - page: StagehandPage, params: ListParameters, *args: ..., **kwargs: ... -): +async def automation(page: Page, params: ListParameters, *args: ..., **kwargs: ...): + base_url, api_key = get_ai_gateway_config() + cdp_url = attempt_store.get("cdp_url") + + # Initialize Stagehand with act/extract/observe capabilities + stagehand = Stagehand( + env="LOCAL", + local_browser_launch_options=dict( + cdp_url=cdp_url, viewport=dict(width=1280, height=800) + ), + model_api_key=api_key, + model_client_options={ + "baseURL": base_url, + }, + ) + await stagehand.init() + print("\nInitialized 🤘 Stagehand") await page.set_viewport_size({"width": 1280, "height": 800}) params = ListParameters.model_validate(params) site = params.metadata.site - await page.goto(site) + await stagehand.page.goto(site) # --- Object type selection --- await perform_action( - page, + stagehand.page, f"Choose the {params.metadata.insurance_type} option from the insurance type dropdown", ) # --- ZIP entry --- await perform_action( - page, f"Fill in the zip code {params.address.zip_code} in the zip code field" + stagehand.page, + f"Fill in the zip code {params.address.zip_code} in the zip code field", ) - await perform_action(page, "Click the Get a quote button") + await perform_action(stagehand.page, "Click the Get a quote button") await page.wait_for_selector("#mainContent") # --- Name --- await perform_action( - page, + stagehand.page, f"Fill in the first name {params.applicant.first_name} in the first name field", ) await perform_action( - page, + stagehand.page, f"Fill in the last name {params.applicant.last_name} in the last name field", ) # --- DOB --- await perform_action( - page, + stagehand.page, f"Fill in the date of birth {params.applicant.date_of_birth} in the date of birth field", ) # --- Address --- await perform_action( - page, + stagehand.page, f"Fill in the address {params.address.street_line1} in the street address field", ) await perform_action( - page, + stagehand.page, f"Fill in the city {params.address.city} in the city field", ) await perform_action( - page, + stagehand.page, f"select the state {params.address.state} from the state dropdown", ) await perform_action( - page, + stagehand.page, f"Fill in the zip code {params.address.zip_code} in the zip code field", ) - await perform_action(page, "Click the Continue button") + await perform_action(stagehand.page, "Click the Continue button") # --- Vehicle --- await perform_action( - page, + stagehand.page, f"Choose the {params.vehicle.vehicle_type} option from the dropdown", ) await perform_action( - page, + stagehand.page, f"Select the year {params.vehicle.year} from the year dropdown", ) await perform_action( - page, + stagehand.page, f"Select the make {params.vehicle.make} from the make dropdown", ) await perform_action( - page, + stagehand.page, f"Select the model {params.vehicle.model} from the model dropdown", ) - await perform_action(page, "Click the Continue button") + await perform_action(stagehand.page, "Click the Continue button") - await perform_action(page, "Click the Continue button") + await perform_action(stagehand.page, "Click the Continue button") # --- Fill driver information --- await perform_action( - page, + stagehand.page, f"Fill in the first name {params.applicant.first_name} in the first name field if the first name field is empty", ) await perform_action( - page, + stagehand.page, f"Fill in the last name {params.applicant.last_name} in the last name field if the last name field is empty", ) await perform_action( - page, + stagehand.page, f"click the {params.applicant.gender} radio button", ) await perform_action( - page, + stagehand.page, f"choose the {params.applicant.marital_status} option from the marital status dropdown", ) if params.applicant.accident_prevention_course: - await perform_action(page, "Click the Yes radio button.") + await perform_action(stagehand.page, "Click the Yes radio button.") else: - await perform_action(page, "Click the No radio button.") - await perform_action(page, "Click the Continue button") - await perform_action(page, "Click the Continue button") + await perform_action(stagehand.page, "Click the No radio button.") + await perform_action(stagehand.page, "Click the Continue button") + await perform_action(stagehand.page, "Click the Continue button") # --- Final details --- await perform_action( - page, f"Fill in the email {params.applicant.email} in the email field" + stagehand.page, f"Fill in the email {params.applicant.email} in the email field" ) await perform_action( - page, + stagehand.page, f"Fill in the phone number {params.applicant.phone_number} in the phone number field", ) if params.applicant.is_cell_phone: await perform_action( - page, "Click the Yes radio button in the Is this a cell phone? field" + stagehand.page, + "Click the Yes radio button in the Is this a cell phone? field", ) else: await perform_action( - page, "Click the No radio button in the Is this a cell phone? field" + stagehand.page, + "Click the No radio button in the Is this a cell phone? field", ) if params.applicant.can_text: await perform_action( - page, + stagehand.page, "Click the Yes radio button in the Can an ERIE Agent text you about this quote? field", ) else: await perform_action( - page, + stagehand.page, "Click the No radio button in the Can an ERIE Agent text you about this quote? field", ) if params.applicant.preferred_name: await perform_action( - page, + stagehand.page, f"Fill in the preferred name {params.applicant.preferred_name} in the preferred name field", ) if params.applicant.home_multi_policy_discount: await perform_action( - page, + stagehand.page, "Click the Yes radio button in the Would you like our Home Multi-Policy Discount applied to your quote? field", ) else: await perform_action( - page, + stagehand.page, "Click the No radio button in the Would you like our Home Multi-Policy Discount applied to your quote? field", ) if params.applicant.currently_has_auto_insurance: await perform_action( - page, + stagehand.page, "Click the Yes radio button in the Do you currently have auto insurance? field", ) else: await perform_action( - page, + stagehand.page, "Click the No radio button in the Do you currently have auto insurance? field", ) await perform_action( - page, + stagehand.page, f"Fill in the coverage effective date {params.applicant.coverage_effective_date} in the coverage effective date field", ) - await perform_action(page, "Click the Continue button") - await perform_action(page, "Click the Submit Quote to Agent button") - result = await page.extract("Extract the confirmation message") + await perform_action(stagehand.page, "Click the Continue button") + await perform_action(stagehand.page, "Click the Submit Quote to Agent button") + result = await stagehand.page.extract("Extract the confirmation message") if result: return result else: diff --git a/python-examples/rpa-forms-example/hooks/setup_context.py b/python-examples/rpa-forms-example/hooks/setup_context.py index cd1103d1..de57852f 100644 --- a/python-examples/rpa-forms-example/hooks/setup_context.py +++ b/python-examples/rpa-forms-example/hooks/setup_context.py @@ -1,15 +1,5 @@ from intuned_runtime import attempt_store -from stagehand import Stagehand async def setup_context(*, api_name: str, api_parameters: str, cdp_url: str): - stagehand = Stagehand( - env="LOCAL", local_browser_launch_options=dict(cdp_url=cdp_url) - ) - await stagehand.init() - attempt_store.set("stagehand", stagehand) - - async def cleanup(): - await stagehand.close() - - return stagehand.context, stagehand.page, cleanup + attempt_store.set("cdp_url", cdp_url) diff --git a/python-examples/rpa-forms-example/pyproject.toml b/python-examples/rpa-forms-example/pyproject.toml index e5f52555..3fdfa2b3 100644 --- a/python-examples/rpa-forms-example/pyproject.toml +++ b/python-examples/rpa-forms-example/pyproject.toml @@ -19,7 +19,7 @@ keywords = [ ] dependencies = [ "playwright==1.56", - "intuned-runtime==1.3.12", + "intuned-runtime==1.3.14", "stagehand==0.5.3", "intuned-browser==0.1.10", ] diff --git a/python-examples/stagehand/hooks/setup_context.py b/python-examples/stagehand/hooks/setup_context.py index 3a3a003f..23ff2368 100644 --- a/python-examples/stagehand/hooks/setup_context.py +++ b/python-examples/stagehand/hooks/setup_context.py @@ -3,6 +3,4 @@ async def setup_context(*, api_name: str, api_parameters: str, cdp_url: str): attempt_store.set("cdp_url", cdp_url) - - - + \ No newline at end of file From e367ce446281956aff9b28544689cc831857a3ca Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Tue, 23 Dec 2025 23:33:12 +0200 Subject: [PATCH 12/15] Refactor insurance form filler automation in rpa-forms-example. Updated project name and metadata in Intuned.jsonc files, and enhanced the performAction function to include page parameter for improved interaction and error handling. --- .../rpa-forms-example/Intuned.jsonc | 6 +- .../rpa-forms-example/Intuned.jsonc | 6 +- .../api/insurance-form-filler.ts | 107 ++++++++++-------- 3 files changed, 64 insertions(+), 55 deletions(-) diff --git a/python-examples/rpa-forms-example/Intuned.jsonc b/python-examples/rpa-forms-example/Intuned.jsonc index 1b4b6d69..fce0f013 100644 --- a/python-examples/rpa-forms-example/Intuned.jsonc +++ b/python-examples/rpa-forms-example/Intuned.jsonc @@ -1,7 +1,7 @@ // For more information, see our Intuned settings reference // https://docs.intunedhq.com/docs/05-references/intuned-json { - "projectName": "rpa-forms", + "projectName": "insurance-form-filler", "apiAccess": { "enabled": true }, @@ -14,8 +14,8 @@ }, "metadata": { "template": { - "name": "rpa-forms-example", - "description": "RPA example for filling insurance quote forms", + "name": "insurance-form-filler", + "description": "Insurance form filler using Stagehand in Intuned", "tags": [ "Stagehand", "AI", diff --git a/typescript-examples/rpa-forms-example/Intuned.jsonc b/typescript-examples/rpa-forms-example/Intuned.jsonc index 1b4b6d69..fce0f013 100644 --- a/typescript-examples/rpa-forms-example/Intuned.jsonc +++ b/typescript-examples/rpa-forms-example/Intuned.jsonc @@ -1,7 +1,7 @@ // For more information, see our Intuned settings reference // https://docs.intunedhq.com/docs/05-references/intuned-json { - "projectName": "rpa-forms", + "projectName": "insurance-form-filler", "apiAccess": { "enabled": true }, @@ -14,8 +14,8 @@ }, "metadata": { "template": { - "name": "rpa-forms-example", - "description": "RPA example for filling insurance quote forms", + "name": "insurance-form-filler", + "description": "Insurance form filler using Stagehand in Intuned", "tags": [ "Stagehand", "AI", diff --git a/typescript-examples/rpa-forms-example/api/insurance-form-filler.ts b/typescript-examples/rpa-forms-example/api/insurance-form-filler.ts index 33dacc43..2d848ece 100644 --- a/typescript-examples/rpa-forms-example/api/insurance-form-filler.ts +++ b/typescript-examples/rpa-forms-example/api/insurance-form-filler.ts @@ -22,18 +22,23 @@ async function getWebSocketUrl(cdpUrl: string): Promise { async function performAction( stagehand: Stagehand, + page: Page, instruction: string ): Promise { - const action = await stagehand.observe(instruction); - if (action && action.length > 0) { - await stagehand.act(action[0]); - await stagehand.context.pages()[0]?.waitForLoadState("domcontentloaded"); - await new Promise((resolve) => setTimeout(resolve, 2000)); - } else { - throw new InvalidActionError( - `Could not find action for instruction: ${instruction}` - ); + for (let i = 0; i < 3; i++) { + const action = await stagehand.observe(instruction); + if (action && action.length > 0) { + await stagehand.act(action[0]); + await page.waitForLoadState("domcontentloaded"); + await page.waitForTimeout(2000); + return; + } else { + await page.waitForTimeout(2000); + } } + throw new InvalidActionError( + `Could not find action for instruction: ${instruction} after 3 attempts` + ); } export default async function handler( @@ -63,12 +68,13 @@ export default async function handler( cdpUrl: webSocketUrl, viewport: { width: 1280, height: 800 }, }, - // llmClient, + llmClient, logger: console.log, }); await stagehand.init(); console.log("\nInitialized 🤘 Stagehand"); // Validate parameters + await page.setViewportSize({ width: 1280, height: 800 }); const validatedParams = listParametersSchema.parse(params); const { metadata, applicant, address, vehicle } = validatedParams; @@ -78,191 +84,194 @@ export default async function handler( // --- Object type selection --- await performAction( stagehand, - `Choose the ${metadata.insurance_type} option from the insurance type dropdown` + page, + `Choose the ${metadata.insurance_type} option from the insurance type dropdown if not chosen` ); // --- ZIP entry --- await performAction( stagehand, + page, `Fill in the zip code ${address.zip_code} in the zip code field` ); - await performAction(stagehand, "Click the Get a quote button"); + await performAction(stagehand, page, "Click the Get a quote button"); await page.waitForSelector("#mainContent"); // --- Name --- await performAction( stagehand, + page, `Fill in the first name ${applicant.first_name} in the first name field` ); await performAction( stagehand, - + page, `Fill in the last name ${applicant.last_name} in the last name field` ); // --- DOB --- await performAction( stagehand, - + page, `Fill in the date of birth ${applicant.date_of_birth} in the date of birth field` ); // --- Address --- await performAction( stagehand, - + page, `Fill in the address ${address.street_line1} in the street address field` ); await performAction( stagehand, - + page, `Fill in the city ${address.city} in the city field` ); await performAction( stagehand, - + page, `select the state ${address.state} from the state dropdown` ); await performAction( stagehand, - + page, `Fill in the zip code ${address.zip_code} in the zip code field` ); - await performAction(stagehand, "Click the Continue button"); + await performAction(stagehand, page, "Click the Continue button"); // --- Vehicle --- await performAction( stagehand, - - `Choose the ${vehicle.vehicle_type} option from the dropdown` + page, + `Choose the ${vehicle.vehicle_type} option from the dropdown if not chosen` ); await performAction( stagehand, - + page, `Select the year ${vehicle.year} from the year dropdown` ); await performAction( stagehand, - + page, `Select the make ${vehicle.make} from the make dropdown` ); await performAction( stagehand, - + page, `Select the model ${vehicle.model} from the model dropdown` ); - await performAction(stagehand, "Click the Continue button"); - await performAction(stagehand, "Click the Continue button"); + await performAction(stagehand, page, "Click the Continue button"); + await performAction(stagehand, page, "Click the Continue button"); // --- Fill driver information --- await performAction( stagehand, - + page, `Fill in the first name ${applicant.first_name} in the first name field if the first name field is empty` ); await performAction( stagehand, - + page, `Fill in the last name ${applicant.last_name} in the last name field if the last name field is empty` ); await performAction( stagehand, - + page, `click the ${applicant.gender} radio button` ); await performAction( stagehand, - + page, `choose the ${applicant.marital_status} option from the marital status dropdown` ); if (applicant.accident_prevention_course) { - await performAction(stagehand, "Click the Yes radio button."); + await performAction(stagehand, page, "Click the Yes radio button."); } else { - await performAction(stagehand, "Click the No radio button."); + await performAction(stagehand, page, "Click the No radio button."); } - await performAction(stagehand, "Click the Continue button"); - await performAction(stagehand, "Click the Continue button"); + await performAction(stagehand, page, "Click the Continue button"); + await performAction(stagehand, page, "Click the Continue button"); // --- Final details --- await performAction( stagehand, - + page, `Fill in the email ${applicant.email} in the email field` ); await performAction( stagehand, - + page, `Fill in the phone number ${applicant.phone_number} in the phone number field` ); if (applicant.is_cell_phone) { await performAction( stagehand, - + page, "Click the Yes radio button in the Is this a cell phone? field" ); } else { await performAction( stagehand, - + page, "Click the No radio button in the Is this a cell phone? field" ); } if (applicant.can_text) { await performAction( stagehand, - + page, "Click the Yes radio button in the Can an ERIE Agent text you about this quote? field" ); } else { await performAction( stagehand, - + page, "Click the No radio button in the Can an ERIE Agent text you about this quote? field" ); } if (applicant.preferred_name) { await performAction( stagehand, - + page, `Fill in the preferred name ${applicant.preferred_name} in the preferred name field` ); } if (applicant.home_multi_policy_discount) { await performAction( stagehand, - + page, "Click the Yes radio button in the Would you like our Home Multi-Policy Discount applied to your quote? field" ); } else { await performAction( stagehand, - + page, "Click the No radio button in the Would you like our Home Multi-Policy Discount applied to your quote? field" ); } if (applicant.currently_has_auto_insurance) { await performAction( stagehand, - + page, "Click the Yes radio button in the Do you currently have auto insurance? field" ); } else { await performAction( stagehand, - + page, "Click the No radio button in the Do you currently have auto insurance? field" ); } await performAction( stagehand, - + page, `Fill in the coverage effective date ${applicant.coverage_effective_date} in the coverage effective date field` ); - await performAction(stagehand, "Click the Continue button"); + await performAction(stagehand, page, "Click the Continue button"); await performAction( stagehand, - + page, "Click the Submit Quote to Agent button" ); @@ -272,4 +281,4 @@ export default async function handler( } else { throw new InvalidActionError("Could not find confirmation message"); } -} +} \ No newline at end of file From 180b4830faad060fe61e6ee6e9f2d956b6be19ce Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Tue, 23 Dec 2025 23:39:49 +0200 Subject: [PATCH 13/15] Update main README to correct the link for rpa-forms-example and ensure accurate descriptions for Python examples. --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 2595e1f9..927cedac 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,7 @@ A collection of examples for building browser automations with [Intuned](https:/ | [quick-recipes](./python-examples/quick-recipes/) | Quick browser automation recipes | | [rpa-example](./python-examples/rpa-example/) | Consultation booking automation | | [rpa-auth-example](./python-examples/rpa-auth-example/) | Authenticated consultation booking with Auth Sessions | -| [rpa-forms-example](./python-examples/rpa-forms-example/) | AI-powered form automation using Stagehand to fill insurance quote forms | -| [auth-with-email-otp](./python-examples/auth-with-email-otp/) | Multi-step authentication with Email-based OTP using Resend API | +| [rpa-forms-example](./python-examples/rpa-forms-example/) | AI-powered form automation using Stagehand to fill insurance quote forms |ß | [auth-with-secret-otp](./python-examples/auth-with-secret-otp/) | Multi-step authentication with TOTP (Time-based OTP) verification | | [auth-with-email-otp](./python-examples/auth-with-email-otp/) | Multi-step authentication with Email-based OTP using Resend API | | [e-commerce-scrapingcourse](./python-examples/e-commerce-scrapingcourse/) | E-commerce product scraper with pagination | From 3ade30f65ecd170afc4adc877ac522798e5cf3f3 Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Wed, 24 Dec 2025 11:40:45 +0200 Subject: [PATCH 14/15] Enhance insurance form filler by adding a new Honda vehicle JSON configuration, updating the default JSON to include model details, and improving the perform_action function for better error handling and retries. --- .../api/insurance-form-filler/default.json | 4 +- .../api/insurance-form-filler/honda.json | 40 +++++++++++++++++++ .../api/insurance-form-filler.py | 23 ++++++----- .../utils/types_and_schemas.py | 3 ++ 4 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 python-examples/rpa-forms-example/.parameters/api/insurance-form-filler/honda.json diff --git a/python-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json b/python-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json index 8445c3ec..69e9441f 100644 --- a/python-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json +++ b/python-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json @@ -21,7 +21,6 @@ }, "address": { "street_line1": "350 W 42nd St", - "street_line2": "Apt 12B", "city": "New York", "state": "NY", "zip_code": "10036" @@ -34,6 +33,7 @@ "primary_use": "Pleasure", "annual_mileage": 12000, "days_driven_per_week": 5, - "miles_driven_one_way": 15 + "miles_driven_one_way": 15, + "model_details": "Car04 4D,2.5,4" } } \ No newline at end of file diff --git a/python-examples/rpa-forms-example/.parameters/api/insurance-form-filler/honda.json b/python-examples/rpa-forms-example/.parameters/api/insurance-form-filler/honda.json new file mode 100644 index 00000000..1c4c22f6 --- /dev/null +++ b/python-examples/rpa-forms-example/.parameters/api/insurance-form-filler/honda.json @@ -0,0 +1,40 @@ +{ + "metadata": { + "site": "https://www.erieinsurance.com/", + "insurance_type": "auto" + }, + "applicant": { + "first_name": "Michael", + "last_name": "Smith", + "date_of_birth": "09/18/1988", + "gender": "male", + "marital_status": "married", + "accident_prevention_course": false, + "email": "michael.smith@example.com", + "phone_number": "555-987-6543", + "is_cell_phone": true, + "can_text": true, + "preferred_name": "Mike", + "home_multi_policy_discount": true, + "currently_has_auto_insurance": true, + "coverage_effective_date": "02/15/2026" + }, + "address": { + "street_line1": "245 Market St", + "city": "Philadelphia", + "state": "PA", + "zip_code": "19106" + }, + "vehicle": { + "vehicle_type": "Automobile", + "year": 2019, + "make": "Honda", + "model": "Civic EX", + "primary_use": "Farm", + "annual_mileage": 14000, + "days_driven_per_week": 5, + "miles_driven_one_way": 12, + "model_details": "Car02 2D,1.5,4" + } + } + \ No newline at end of file diff --git a/python-examples/rpa-forms-example/api/insurance-form-filler.py b/python-examples/rpa-forms-example/api/insurance-form-filler.py index e7f63184..cdc11ad4 100644 --- a/python-examples/rpa-forms-example/api/insurance-form-filler.py +++ b/python-examples/rpa-forms-example/api/insurance-form-filler.py @@ -9,15 +9,16 @@ class InvalidActionError(Exception): async def perform_action(page: Page, instruction: str) -> None: - action = await page.observe(instruction) - if action: - await page.act(action[0]) - await page.wait_for_load_state("domcontentloaded") - await page.wait_for_timeout(2000) - else: - raise InvalidActionError( - f"Could not find action for instruction: {instruction}" - ) + for _ in range(3): + action = await page.observe(instruction) + if action: + await page.act(action[0]) + await page.wait_for_load_state("domcontentloaded") + await page.wait_for_timeout(2000) + return + else: + await page.wait_for_timeout(2000) + raise InvalidActionError(f"Could not find action for instruction: {instruction}") async def automation(page: Page, params: ListParameters, *args: ..., **kwargs: ...): @@ -108,6 +109,10 @@ async def automation(page: Page, params: ListParameters, *args: ..., **kwargs: . stagehand.page, f"Select the model {params.vehicle.model} from the model dropdown", ) + await perform_action( + stagehand.page, + f"Fill in the model details {params.vehicle.model_details} in the model details field if not already filled", + ) await perform_action(stagehand.page, "Click the Continue button") await perform_action(stagehand.page, "Click the Continue button") diff --git a/python-examples/rpa-forms-example/utils/types_and_schemas.py b/python-examples/rpa-forms-example/utils/types_and_schemas.py index f0d2bbaa..12c4c772 100644 --- a/python-examples/rpa-forms-example/utils/types_and_schemas.py +++ b/python-examples/rpa-forms-example/utils/types_and_schemas.py @@ -113,6 +113,9 @@ class Vehicle(BaseModel): miles_driven_one_way: int | None = Field( default=None, description="The number of miles driven one way" ) + model_details: str | None = Field( + ..., description="The details of the model of the vehicle" + ) # ---------- Root Model ---------- From e45c2b490a1b2f8aa65d0561ca66205ae8ed8c19 Mon Sep 17 00:00:00 2001 From: mohamedmamdouh22 Date: Wed, 24 Dec 2025 11:51:07 +0200 Subject: [PATCH 15/15] Update insurance form filler by adding a new Honda vehicle configuration, modifying the default JSON to include model details, and enhancing the vehicle schema to accommodate model details in validation. --- .../api/insurance-form-filler/default.json | 7 ++-- .../api/insurance-form-filler/honda.json | 40 +++++++++++++++++++ .../api/insurance-form-filler.ts | 7 +++- .../utils/typesAndSchemas.ts | 1 + 4 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/honda.json diff --git a/typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json b/typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json index 558b715e..958797a5 100644 --- a/typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json +++ b/typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/default.json @@ -21,7 +21,6 @@ }, "address": { "street_line1": "350 W 42nd St", - "street_line2": "Apt 12B", "city": "New York", "state": "NY", "zip_code": "10036" @@ -34,7 +33,7 @@ "primary_use": "Pleasure", "annual_mileage": 12000, "days_driven_per_week": 5, - "miles_driven_one_way": 15 + "miles_driven_one_way": 15, + "model_details": "Car04 4D,2.5,4" } -} - +} \ No newline at end of file diff --git a/typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/honda.json b/typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/honda.json new file mode 100644 index 00000000..1c4c22f6 --- /dev/null +++ b/typescript-examples/rpa-forms-example/.parameters/api/insurance-form-filler/honda.json @@ -0,0 +1,40 @@ +{ + "metadata": { + "site": "https://www.erieinsurance.com/", + "insurance_type": "auto" + }, + "applicant": { + "first_name": "Michael", + "last_name": "Smith", + "date_of_birth": "09/18/1988", + "gender": "male", + "marital_status": "married", + "accident_prevention_course": false, + "email": "michael.smith@example.com", + "phone_number": "555-987-6543", + "is_cell_phone": true, + "can_text": true, + "preferred_name": "Mike", + "home_multi_policy_discount": true, + "currently_has_auto_insurance": true, + "coverage_effective_date": "02/15/2026" + }, + "address": { + "street_line1": "245 Market St", + "city": "Philadelphia", + "state": "PA", + "zip_code": "19106" + }, + "vehicle": { + "vehicle_type": "Automobile", + "year": 2019, + "make": "Honda", + "model": "Civic EX", + "primary_use": "Farm", + "annual_mileage": 14000, + "days_driven_per_week": 5, + "miles_driven_one_way": 12, + "model_details": "Car02 2D,1.5,4" + } + } + \ No newline at end of file diff --git a/typescript-examples/rpa-forms-example/api/insurance-form-filler.ts b/typescript-examples/rpa-forms-example/api/insurance-form-filler.ts index 2d848ece..cac03980 100644 --- a/typescript-examples/rpa-forms-example/api/insurance-form-filler.ts +++ b/typescript-examples/rpa-forms-example/api/insurance-form-filler.ts @@ -161,6 +161,11 @@ export default async function handler( page, `Select the model ${vehicle.model} from the model dropdown` ); + await performAction( + stagehand, + page, + `Fill in the model details ${vehicle.model_details} in the model details field if not already filled` + ); await performAction(stagehand, page, "Click the Continue button"); await performAction(stagehand, page, "Click the Continue button"); @@ -281,4 +286,4 @@ export default async function handler( } else { throw new InvalidActionError("Could not find confirmation message"); } -} \ No newline at end of file +} diff --git a/typescript-examples/rpa-forms-example/utils/typesAndSchemas.ts b/typescript-examples/rpa-forms-example/utils/typesAndSchemas.ts index 52ec17ed..db335631 100644 --- a/typescript-examples/rpa-forms-example/utils/typesAndSchemas.ts +++ b/typescript-examples/rpa-forms-example/utils/typesAndSchemas.ts @@ -74,6 +74,7 @@ const vehicleSchema = z.object({ annual_mileage: z.number().int().positive().optional(), days_driven_per_week: z.number().int().positive().optional(), miles_driven_one_way: z.number().int().positive().optional(), + model_details: z.string(), }); // ---------- Root Schema ----------