From 0b7cd3fc7ce49d3f197fa93192f2d2170e28917a Mon Sep 17 00:00:00 2001 From: Bdair2002 <107470382+Bdair2002@users.noreply.github.com> Date: Wed, 24 Dec 2025 15:04:44 +0200 Subject: [PATCH] Multiple fixes --- .claude/CLAUDE.md | 2 +- .cursorrules | 2 +- .../browser-sdk-showcase/README.md | 2 +- .../e-commerce-auth-scrapingcourse/README.md | 2 +- .../.parameters/api/list/default.json | 4 +- .../e-commerce-scrapingcourse/Intuned.jsonc | 7 +- .../e-commerce-scrapingcourse/README.md | 154 +++++------------ .../e-commerce-scrapingcourse/api/details.py | 23 +-- .../e-commerce-scrapingcourse/api/list.py | 22 +-- python-examples/empty-auth/README.md | 2 +- python-examples/empty/README.md | 2 +- python-examples/hyprid-automation/README.md | 4 +- python-examples/playwright-python/README.md | 2 +- .../project_getting_started_template.md | 2 +- python-examples/quick-recipes/README.md | 4 +- python-examples/rpa-example/README.md | 4 +- .../auth-with-email-otp/README.md | 2 +- .../auth-with-secret-otp/README.md | 2 +- .../browser-sdk-showcase/README.md | 2 +- .../captcha-solving-auth-example/README.md | 2 +- .../captcha-solving-basic-example/README.md | 2 +- typescript-examples/computer-use/README.md | 2 +- .../e-commerce-auth-scrapingcourse/README.md | 2 +- .../.parameters/api/list/default.json | 4 +- .../e-commerce-scrapingcourse/Intuned.jsonc | 13 +- .../e-commerce-scrapingcourse/README.md | 160 +++++------------- .../ehr-integration-ts/README.md | 2 +- typescript-examples/empty-auth/README.md | 2 +- typescript-examples/empty/README.md | 2 +- .../hybrid-automation/README.md | 4 +- .../playwright-basics-ts/README.md | 2 +- .../project_getting_started_template.md | 2 +- typescript-examples/quick-recipes/README.md | 4 +- .../rpa-auth-example/README.md | 2 +- typescript-examples/rpa-example/README.md | 4 +- typescript-examples/stagehand/README.md | 2 +- 36 files changed, 164 insertions(+), 291 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 25ae4333..bfe4cd1c 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -379,7 +379,7 @@ Each project must have a `.parameters/` folder containing test parameters for ru ## Documentation Links - AuthSessions: https://docs.intunedhq.com/docs/02-features/auth-sessions -- Browser SDK: https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview +- Browser SDK: https://docs.intunedhq.com/automation-sdks/overview - Intuned in depth: https://docs.intunedhq.com/docs/01-learn/deep-dives/intuned-indepth - Introduction / Quickstarts: https://docs.intunedhq.com/docs/00-getting-started/introduction - Recipe docs: https://docs.intunedhq.com/docs/01-learn/recipes/ diff --git a/.cursorrules b/.cursorrules index 17f4a298..4310d76c 100644 --- a/.cursorrules +++ b/.cursorrules @@ -22,7 +22,7 @@ ## Doc Links - AuthSessions: https://docs.intunedhq.com/docs/02-features/auth-sessions -- Browser SDK: https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview +- Browser SDK: https://docs.intunedhq.com/automation-sdks/overview - Intuned in depth: https://docs.intunedhq.com/docs/01-learn/deep-dives/intuned-indepth - Introduction / Quickstarts: https://docs.intunedhq.com/docs/00-getting-started/introduction - Recipe docs: download-file, pagination, upload-files, capture-screenshots under https://docs.intunedhq.com/docs/01-learn/recipes/ diff --git a/python-examples/browser-sdk-showcase/README.md b/python-examples/browser-sdk-showcase/README.md index 8d59c6c6..4b794fde 100644 --- a/python-examples/browser-sdk-showcase/README.md +++ b/python-examples/browser-sdk-showcase/README.md @@ -155,4 +155,4 @@ See [ai/README.md](./api/ai/README.md) for AI helpers that require API keys and For detailed documentation on each helper function, visit: - [Intuned Browser SDK - Python](https://docs.intunedhq.com/automation-sdks/intuned-sdk/python/helpers/functions/) -- [Browser SDK Overview](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview) +- [Browser SDK Overview](https://docs.intunedhq.com/automation-sdks/overview) diff --git a/python-examples/e-commerce-auth-scrapingcourse/README.md b/python-examples/e-commerce-auth-scrapingcourse/README.md index 419f975d..e164521f 100644 --- a/python-examples/e-commerce-auth-scrapingcourse/README.md +++ b/python-examples/e-commerce-auth-scrapingcourse/README.md @@ -193,4 +193,4 @@ This project uses the Intuned browser SDK for enhanced reliability: - **`save_file_to_s3`**: Automatically upload images and files to S3 storage - **`extend_payload`**: Trigger additional API calls dynamically (used to trigger `details` API for each product) -For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/python-examples/e-commerce-scrapingcourse/.parameters/api/list/default.json b/python-examples/e-commerce-scrapingcourse/.parameters/api/list/default.json index 9e26dfee..dc0135a2 100644 --- a/python-examples/e-commerce-scrapingcourse/.parameters/api/list/default.json +++ b/python-examples/e-commerce-scrapingcourse/.parameters/api/list/default.json @@ -1 +1,3 @@ -{} \ No newline at end of file +{ + "limit": 10 +} \ No newline at end of file diff --git a/python-examples/e-commerce-scrapingcourse/Intuned.jsonc b/python-examples/e-commerce-scrapingcourse/Intuned.jsonc index a2b12a12..5c8b8e66 100644 --- a/python-examples/e-commerce-scrapingcourse/Intuned.jsonc +++ b/python-examples/e-commerce-scrapingcourse/Intuned.jsonc @@ -12,7 +12,12 @@ "metadata": { "template": { "name": "e-commerce-scrapingcourse", - "description": "Basic e-commerce scraper using scrapingcourse.com" + "description": "Basic e-commerce scraper using scrapingcourse.com", + "tags": ["web-scraping", "e-commerce", "pagination", "jobs"] + }, + "defaultRunPlaygroundInput": { + "apiName": "list", + "parameters": {} }, "defaultJobInput": { "configuration": { diff --git a/python-examples/e-commerce-scrapingcourse/README.md b/python-examples/e-commerce-scrapingcourse/README.md index f4d35d06..8bd42d2a 100644 --- a/python-examples/e-commerce-scrapingcourse/README.md +++ b/python-examples/e-commerce-scrapingcourse/README.md @@ -1,7 +1,15 @@ -# E-Commerce Product Scraper +# e-commerce-scrapingcourse Intuned project E-commerce scraping automation that extracts product information from an online store with pagination support. +## Key Features + +- **Automatic Pagination**: The `list` API automatically handles pagination to scrape multiple pages +- **Dynamic API Chaining**: Uses `extend_payload` to automatically trigger the `details` API for each product found +- **S3 File Upload**: Product images are automatically uploaded to S3 using `save_file_to_s3` +- **Job Configuration**: Configured as a job template with retry logic and concurrent request handling + + ## Run on Intuned Open this project in Intuned by clicking the button below. @@ -26,141 +34,61 @@ After installing dependencies, `intuned` command should be available in your env ### Run an API - ```bash uv run intuned run api list .parameters/api/list/default.json uv run intuned run api details .parameters/api/details/default.json ``` -#### Example: List Products - +### Save project ```bash -# List products with default page limit -uv run intuned run api list .parameters/api/list/default.json +uv run intuned run save ``` -#### Example: Get Product Details - -```bash -# Get details for a specific product -uv run intuned run api details .parameters/api/details/default.json -``` +Reference for saving project [here](https://docs.intunedhq.com/docs/02-features/local-development-cli#use-runtime-sdk-and-browser-sdk-helpers) ### Deploy project ```bash uv run intuned deploy ``` +### `intuned-browser`: Intuned Browser SDK + +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). + + + ## Project Structure -The project structure is as follows: ``` / -├── api/ # Your API endpoints -│ ├── list.py # API to scrape product list with pagination -│ └── details.py # API to scrape detailed product information -├── utils/ # Utility files -│ └── types_and_schemas.py # Python types and Pydantic models -└── Intuned.jsonc # Intuned project configuration file +├── .parameters/ # Test parameters for APIs +│ └── api/ +│ ├── list/ +│ │ └── default.json +│ └── details/ +│ └── default.json +├── api/ # API endpoints +│ ├── list.py # Scrape product list with pagination +│ └── details.py # Extract detailed product information +├── utils/ # Utility modules +│ └── types_and_schemas.py # Type definitions and Pydantic models +├── Intuned.jsonc # Intuned project configuration +└── pyproject.toml # Python project dependencies ``` ## APIs -### `list` - Product List Scraper - -Scrapes products from the e-commerce store with pagination support. - -**Parameters:** -- `limit` (optional): Maximum number of pages to scrape (default: 50) - -**Returns:** -List of products with: -- `name`: Product name -- `detailsUrl`: URL to product details page - -**Features:** -- Automatic pagination handling -- Triggers `details` API for each product using `extend_payload` -- Configurable page limit - -### `details` - Product Details Scraper - -Scrapes detailed information for a specific product. - -**Parameters:** -- `name`: Product name -- `detailsUrl`: URL to the product details page - -**Returns:** -Product details object with: -- `name`: Product name -- `price`: Product price -- `sku`: Stock Keeping Unit -- `category`: Product category -- `shortDescription`: Brief product description -- `fullDescription`: Complete product description -- `imageAttachments`: List of product images (uploaded to S3) -- `availableSizes`: List of available sizes -- `availableColors`: List of available colors -- `variants`: List of product variants with stock information - - -## `Intuned.jsonc` Reference -```jsonc -{ - // API access settings - "apiAccess": { - // Whether to enable consumption through Intuned API - "enabled": false - }, - - // Auth session settings - "authSessions": { - // Auth sessions are not used in this project - "enabled": false - }, - - // Replication settings - "replication": { - // The maximum number of concurrent executions allowed via Intuned API - "maxConcurrentRequests": 1, - - // The machine size to use for this project - // "standard": Standard machine size (6 shared vCPUs, 2GB RAM) - // "large": Large machine size (8 shared vCPUs, 4GB RAM) - // "xlarge": Extra large machine size (1 performance vCPU, 8GB RAM) - "size": "standard" - }, - - // Default job configuration - "metadata": { - "defaultJobInput": { - "configuration": { - // Number of concurrent API calls within the job - "maxConcurrentRequests": 2, - // Retry configuration - "retry": { - "maximumAttempts": 3 - } - }, - "payload": [ - { - "apiName": "list", - "parameters": {} - } - ] - } - } -} -``` - -## Using `intuned_browser` SDK +| API | Description | +|-----|-------------| +| `list` | Scrapes products from the e-commerce store with pagination support. Automatically triggers `details` API for each product using `extend_payload` | +| `details` | Extracts detailed information for a specific product including price, SKU, category, descriptions, images (uploaded to S3), sizes, colors, and variants | -This project uses the Intuned browser SDK for enhanced reliability: -- **`go_to_url`**: Navigate to URLs with automatic retries and intelligent timeout detection -- **`save_file_to_s3`**: Automatically upload images and files to S3 storage -- **`extend_payload`**: Trigger additional API calls dynamically (used to trigger `details` API for each product) +## Learn More -For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +- [Intuned Documentation](https://docs.intunedhq.com) +- [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/overview) +- [Web Scraping Recipe](https://docs.intunedhq.com/docs/01-learn/recipes/) +- [extend_payload Helper](https://docs.intunedhq.com/docs/05-references/runtime-sdk-python/extend-payload) +- [save_file_to_s3 Helper](https://docs.intunedhq.com/automation-sdks/intuned-sdk/python/helpers/functions/save_file_to_s3) diff --git a/python-examples/e-commerce-scrapingcourse/api/details.py b/python-examples/e-commerce-scrapingcourse/api/details.py index ae2d8312..31d3144d 100644 --- a/python-examples/e-commerce-scrapingcourse/api/details.py +++ b/python-examples/e-commerce-scrapingcourse/api/details.py @@ -1,9 +1,10 @@ +# Extract detailed product information from e-commerce product page from playwright.async_api import Page -from typing import List, Optional +from typing import TypedDict, List from intuned_browser import go_to_url, save_file_to_s3, Attachment import json import re - +from pydantic import HttpUrl from utils.types_and_schemas import ( ProductDetails, ProductVariant, @@ -11,6 +12,11 @@ ) +class Params(TypedDict): + name: str + detailsUrl: HttpUrl + + async def get_product_images(page: Page) -> List[Attachment]: # Extract all product images from the gallery image_elements = await page.locator(".woocommerce-product-gallery__image img").all() @@ -109,7 +115,7 @@ async def extract_product_details(page: Page, params: DetailsSchema) -> ProductD price = await price_element.text_content() # Extract id - id_element = page.locator(".sku_wrapper .id") + id_element = page.locator(".sku_wrapper .sku") id = await id_element.text_content() or "" # Extract category @@ -148,14 +154,7 @@ async def extract_product_details(page: Page, params: DetailsSchema) -> ProductD ) -async def handler( - page: Page, - params: Optional[dict] = None, - **_kwargs, -) -> ProductDetails: - if params is None: - raise ValueError("Params are required for this handler") - +async def automation(page: Page, params: Params, **_kwargs) -> ProductDetails: # Validate params using pydantic model validated_params = DetailsSchema(**params) @@ -168,5 +167,7 @@ async def handler( # Extract all detailed product information product_details = await extract_product_details(page, validated_params) + print(f"Successfully extracted details for product: {product_details.name}") + # Return the complete product details return product_details diff --git a/python-examples/e-commerce-scrapingcourse/api/list.py b/python-examples/e-commerce-scrapingcourse/api/list.py index 4b6a3222..dc8e35b2 100644 --- a/python-examples/e-commerce-scrapingcourse/api/list.py +++ b/python-examples/e-commerce-scrapingcourse/api/list.py @@ -1,11 +1,16 @@ +# List products from e-commerce site with pagination from playwright.async_api import Page -from typing import TypedDict, List, Optional +from typing import TypedDict, List from runtime_helpers import extend_payload from intuned_browser import go_to_url from utils.types_and_schemas import ListSchema +class Params(TypedDict): + limit: int + + class Product(TypedDict): name: str detailsUrl: str @@ -83,16 +88,9 @@ async def navigate_to_next_page(page: Page) -> None: await page.locator("#product-list").wait_for(state="visible") -async def handler( - page: Page, - params: Optional[dict] = None, - **_kwargs, -) -> List[Product]: +async def automation(page: Page, params: Params, **_kwargs) -> List[Product]: # Get the page limit from params, default to 50 if not provided - if params is None: - params = {} - - validated_params = ListSchema(**params) + validated_params = ListSchema(**(params or {})) page_limit = validated_params.limit or 50 # Navigate to the e-commerce website @@ -133,5 +131,9 @@ async def handler( current_page += 1 + print( + f"Successfully scraped {len(all_products)} products from {current_page} page(s)" + ) + # Return the scraped products return all_products diff --git a/python-examples/empty-auth/README.md b/python-examples/empty-auth/README.md index 66ae01f8..763c6df3 100644 --- a/python-examples/empty-auth/README.md +++ b/python-examples/empty-auth/README.md @@ -64,7 +64,7 @@ uv run intuned run authsession validate ### `intuned-browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/python-examples/empty/README.md b/python-examples/empty/README.md index 7f419777..2f46522c 100644 --- a/python-examples/empty/README.md +++ b/python-examples/empty/README.md @@ -46,7 +46,7 @@ uv run intuned deploy ### `intuned-browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/python-examples/hyprid-automation/README.md b/python-examples/hyprid-automation/README.md index c70a14a2..0744ac38 100644 --- a/python-examples/hyprid-automation/README.md +++ b/python-examples/hyprid-automation/README.md @@ -1,6 +1,6 @@ # Hybrid Automation -This example demonstrates **hybrid automation** - a flexible approach that combines the [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview) with AI-powered tools like [Stagehand](https://docs.stagehand.dev/) and `extract_structured_data`. This gives you the speed and reliability of traditional automation with the adaptability of AI when needed. +This example demonstrates **hybrid automation** - a flexible approach that combines the [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/overview) with AI-powered tools like [Stagehand](https://docs.stagehand.dev/) and `extract_structured_data`. This gives you the speed and reliability of traditional automation with the adaptability of AI when needed. ## Run on Intuned @@ -85,6 +85,6 @@ utils/crawler/ # Crawler utilities ## Learn More - [Flexible Automations](https://docs.intunedhq.com/docs/02-features/flexible-automation) -- [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview) +- [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/overview) - [Extract Structured Data](https://docs.intunedhq.com/automation-sdks/intuned-sdk/python/helpers/functions/extract_structured_data) - [Stagehand act/extract/observe](https://docs.stagehand.dev/v2/basics/act) diff --git a/python-examples/playwright-python/README.md b/python-examples/playwright-python/README.md index 971e514a..be4b5887 100644 --- a/python-examples/playwright-python/README.md +++ b/python-examples/playwright-python/README.md @@ -112,4 +112,4 @@ uv run intuned deploy ## Related - [Playwright deep dive](https://docs.intunedhq.com/docs/01-learn/deep-dives/playwright) -- [Intuned SDK](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview) +- [Intuned SDK](https://docs.intunedhq.com/automation-sdks/overview) diff --git a/python-examples/project_getting_started_template.md b/python-examples/project_getting_started_template.md index b2cb4175..86a7280c 100644 --- a/python-examples/project_getting_started_template.md +++ b/python-examples/project_getting_started_template.md @@ -71,7 +71,7 @@ uv run intuned deploy ### `intuned-browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/python-examples/quick-recipes/README.md b/python-examples/quick-recipes/README.md index c61c0b24..6b8574b2 100644 --- a/python-examples/quick-recipes/README.md +++ b/python-examples/quick-recipes/README.md @@ -52,7 +52,7 @@ uv run intuned deploy ### `intuned-browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). @@ -94,4 +94,4 @@ This project uses Intuned browser SDK. For more information, check out the [Intu - [Pagination Recipe](https://docs.intunedhq.com/docs/01-learn/recipes/pagination) - [Upload Files Recipe](https://docs.intunedhq.com/docs/01-learn/recipes/upload-files) - [Capture Screenshots Recipe](https://docs.intunedhq.com/docs/01-learn/recipes/capture-screenshots) -- [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview) +- [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/overview) diff --git a/python-examples/rpa-example/README.md b/python-examples/rpa-example/README.md index ee444e02..7488459c 100644 --- a/python-examples/rpa-example/README.md +++ b/python-examples/rpa-example/README.md @@ -55,7 +55,7 @@ uv run intuned deploy ## `intuned-browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). @@ -86,5 +86,5 @@ This project uses Intuned browser SDK. For more information, check out the [Intu ## Learn More - **Intuned Concepts**: https://docs.intunedhq.com/docs/00-getting-started/introduction -- **Intuned Browser SDK**: https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview +- **Intuned Browser SDK**: https://docs.intunedhq.com/automation-sdks/overview - **CLI Documentation**: https://docs.intunedhq.com/docs/05-references/cli diff --git a/typescript-examples/auth-with-email-otp/README.md b/typescript-examples/auth-with-email-otp/README.md index 1a161ec3..1e6b18ef 100644 --- a/typescript-examples/auth-with-email-otp/README.md +++ b/typescript-examples/auth-with-email-otp/README.md @@ -76,7 +76,7 @@ yarn intuned run authsession validate ### `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/typescript-examples/auth-with-secret-otp/README.md b/typescript-examples/auth-with-secret-otp/README.md index 482c40c7..9932a56b 100644 --- a/typescript-examples/auth-with-secret-otp/README.md +++ b/typescript-examples/auth-with-secret-otp/README.md @@ -85,7 +85,7 @@ yarn intuned run authsession validate ### `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/typescript-examples/browser-sdk-showcase/README.md b/typescript-examples/browser-sdk-showcase/README.md index c128c9b3..3c834934 100644 --- a/typescript-examples/browser-sdk-showcase/README.md +++ b/typescript-examples/browser-sdk-showcase/README.md @@ -185,4 +185,4 @@ See [ai/README.md](./api/ai/README.md) for AI helpers that require API keys and For detailed documentation on each helper function, visit: - [Intuned Browser SDK - TypeScript](https://docs.intunedhq.com/automation-sdks/intuned-sdk/typescript/helpers/functions/) -- [Browser SDK Overview](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview) +- [Browser SDK Overview](https://docs.intunedhq.com/automation-sdks/overview) diff --git a/typescript-examples/captcha-solving-auth-example/README.md b/typescript-examples/captcha-solving-auth-example/README.md index 72ce1e45..ba3a1fcc 100644 --- a/typescript-examples/captcha-solving-auth-example/README.md +++ b/typescript-examples/captcha-solving-auth-example/README.md @@ -62,7 +62,7 @@ yarn intuned deploy ### `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/typescript-examples/captcha-solving-basic-example/README.md b/typescript-examples/captcha-solving-basic-example/README.md index b8ae2e36..d8f4d494 100644 --- a/typescript-examples/captcha-solving-basic-example/README.md +++ b/typescript-examples/captcha-solving-basic-example/README.md @@ -55,7 +55,7 @@ yarn intuned deploy ### `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/typescript-examples/computer-use/README.md b/typescript-examples/computer-use/README.md index 2002f17b..f8604525 100644 --- a/typescript-examples/computer-use/README.md +++ b/typescript-examples/computer-use/README.md @@ -196,7 +196,7 @@ yarn intuned deploy ### `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). ## Learn More diff --git a/typescript-examples/e-commerce-auth-scrapingcourse/README.md b/typescript-examples/e-commerce-auth-scrapingcourse/README.md index 2cbd332c..f3ed2b06 100644 --- a/typescript-examples/e-commerce-auth-scrapingcourse/README.md +++ b/typescript-examples/e-commerce-auth-scrapingcourse/README.md @@ -226,4 +226,4 @@ This project uses the Intuned browser SDK for enhanced reliability: - **`saveFileToS3`**: Automatically upload images and files to S3 storage - **`extendPayload`**: Trigger additional API calls dynamically (used to trigger `details` API for each product) -For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/typescript-examples/e-commerce-scrapingcourse/.parameters/api/list/default.json b/typescript-examples/e-commerce-scrapingcourse/.parameters/api/list/default.json index 9e26dfee..dc0135a2 100644 --- a/typescript-examples/e-commerce-scrapingcourse/.parameters/api/list/default.json +++ b/typescript-examples/e-commerce-scrapingcourse/.parameters/api/list/default.json @@ -1 +1,3 @@ -{} \ No newline at end of file +{ + "limit": 10 +} \ No newline at end of file diff --git a/typescript-examples/e-commerce-scrapingcourse/Intuned.jsonc b/typescript-examples/e-commerce-scrapingcourse/Intuned.jsonc index a2b12a12..8c71329d 100644 --- a/typescript-examples/e-commerce-scrapingcourse/Intuned.jsonc +++ b/typescript-examples/e-commerce-scrapingcourse/Intuned.jsonc @@ -12,7 +12,14 @@ "metadata": { "template": { "name": "e-commerce-scrapingcourse", - "description": "Basic e-commerce scraper using scrapingcourse.com" + "description": "Basic e-commerce scraper using scrapingcourse.com", + "tags": ["web-scraping", "e-commerce", "pagination", "jobs"] + }, + "defaultRunPlaygroundInput": { + "apiName": "list", + "parameters": { + "limit": 10 + } }, "defaultJobInput": { "configuration": { @@ -24,7 +31,9 @@ "payload": [ { "apiName": "list", - "parameters": {} + "parameters": { + "limit": 10 + } } ] } diff --git a/typescript-examples/e-commerce-scrapingcourse/README.md b/typescript-examples/e-commerce-scrapingcourse/README.md index 9ac0dbe4..0fd27be8 100644 --- a/typescript-examples/e-commerce-scrapingcourse/README.md +++ b/typescript-examples/e-commerce-scrapingcourse/README.md @@ -1,7 +1,15 @@ -# E-Commerce Product Scraper +# e-commerce-scrapingcourse Intuned project E-commerce scraping automation that extracts product information from an online store with pagination support. +## Key Features + +- **Automatic Pagination**: The `list` API automatically handles pagination to scrape multiple pages +- **Dynamic API Chaining**: Uses `extendPayload` to automatically trigger the `details` API for each product found +- **S3 File Upload**: Product images are automatically uploaded to S3 using `saveFileToS3` +- **Job Configuration**: Configured as a job template with retry logic and concurrent request handling + + ## Run on Intuned Open this project in Intuned by clicking the button below. @@ -30,7 +38,6 @@ yarn ### Run an API - ```bash # npm npm run intuned run api list .parameters/api/list/default.json @@ -41,27 +48,16 @@ yarn intuned run api list .parameters/api/list/default.json yarn intuned run api details .parameters/api/details/default.json ``` -#### Example: List Products - +### Save project ```bash -# List products with default page limit # npm -npm run intuned run api list .parameters/api/list/default.json +npm run intuned run save # yarn -yarn intuned run api list .parameters/api/list/default.json +yarn intuned run save ``` -#### Example: Get Product Details - -```bash -# Get details for a specific product -# npm -npm run intuned run api details .parameters/api/details/default.json - -# yarn -yarn intuned run api details .parameters/api/details/default.json -``` +Reference for saving project [here](https://docs.intunedhq.com/docs/02-features/local-development-cli#use-runtime-sdk-and-browser-sdk-helpers) ### Deploy project ```bash @@ -72,116 +68,44 @@ npm run intuned deploy yarn intuned deploy ``` +### `@intuned/browser`: Intuned Browser SDK + +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). + + + ## Project Structure -The project structure is as follows: ``` / -├── api/ # Your API endpoints -│ ├── list.ts # API to scrape product list with pagination -│ └── details.ts # API to scrape detailed product information -├── utils/ # Utility files -│ └── typesAndSchemas.ts # TypeScript types and Zod schemas -└── Intuned.jsonc # Intuned project configuration file +├── .parameters/ # Test parameters for APIs +│ └── api/ +│ ├── list/ +│ │ └── default.json +│ └── details/ +│ └── default.json +├── api/ # API endpoints +│ ├── list.ts # Scrape product list with pagination +│ └── details.ts # Extract detailed product information +├── utils/ # Utility modules +│ └── typesAndSchemas.ts # Type definitions and Zod schemas +├── Intuned.jsonc # Intuned project configuration +└── package.json # Node.js project dependencies ``` ## APIs -### `list` - Product List Scraper - -Scrapes products from the e-commerce store with pagination support. - -**Parameters:** -- `limit` (optional): Maximum number of pages to scrape (default: 50) - -**Returns:** -Array of products with: -- `name`: Product name -- `detailsUrl`: URL to product details page - -**Features:** -- Automatic pagination handling -- Triggers `details` API for each product using `extendPayload` -- Configurable page limit - -### `details` - Product Details Scraper - -Scrapes detailed information for a specific product. - -**Parameters:** -- `name`: Product name -- `detailsUrl`: URL to the product details page - -**Returns:** -Product details object with: -- `name`: Product name -- `price`: Product price -- `sku`: Stock Keeping Unit -- `category`: Product category -- `shortDescription`: Brief product description -- `fullDescription`: Complete product description -- `imageAttachments`: Array of product images (uploaded to S3) -- `availableSizes`: Array of available sizes -- `availableColors`: Array of available colors -- `variants`: Array of product variants with stock information - - -## `Intuned.jsonc` Reference -```jsonc -{ - // API access settings - "apiAccess": { - // Whether to enable consumption through Intuned API - "enabled": false - }, - - // Auth session settings - "authSessions": { - // Auth sessions are not used in this project - "enabled": false - }, - - // Replication settings - "replication": { - // The maximum number of concurrent executions allowed via Intuned API - "maxConcurrentRequests": 1, - - // The machine size to use for this project - // "standard": Standard machine size (6 shared vCPUs, 2GB RAM) - // "large": Large machine size (8 shared vCPUs, 4GB RAM) - // "xlarge": Extra large machine size (1 performance vCPU, 8GB RAM) - "size": "standard" - }, - - // Default job configuration - "metadata": { - "defaultJobInput": { - "configuration": { - // Number of concurrent API calls within the job - "maxConcurrentRequests": 2, - // Retry configuration - "retry": { - "maximumAttempts": 3 - } - }, - "payload": [ - { - "apiName": "list", - "parameters": {} - } - ] - } - } -} -``` - -## Using `@intuned/browser` SDK +| API | Description | +|-----|-------------| +| `list` | Scrapes products from the e-commerce store with pagination support. Automatically triggers `details` API for each product using `extendPayload` | +| `details` | Extracts detailed information for a specific product including price, SKU, category, descriptions, images (uploaded to S3), sizes, colors, and variants | -This project uses the Intuned browser SDK for enhanced reliability: -- **`goToUrl`**: Navigate to URLs with automatic retries and intelligent timeout detection -- **`saveFileToS3`**: Automatically upload images and files to S3 storage -- **`extendPayload`**: Trigger additional API calls dynamically (used to trigger `details` API for each product) +## Learn More -For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +- [Intuned Documentation](https://docs.intunedhq.com) +- [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/overview) +- [Web Scraping Recipe](https://docs.intunedhq.com/docs/01-learn/recipes/) +- [extendPayload Helper](https://docs.intunedhq.com/docs/05-references/runtime-sdk-typescript/extend-timeout) +- [saveFileToS3 Helper](https://docs.intunedhq.com/automation-sdks/intuned-sdk/typescript/helpers/functions/saveFileToS3) diff --git a/typescript-examples/ehr-integration-ts/README.md b/typescript-examples/ehr-integration-ts/README.md index 3af9f14e..8e427ce5 100644 --- a/typescript-examples/ehr-integration-ts/README.md +++ b/typescript-examples/ehr-integration-ts/README.md @@ -69,7 +69,7 @@ yarn intuned deploy ### `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/typescript-examples/empty-auth/README.md b/typescript-examples/empty-auth/README.md index b6fd44d1..57ea5eee 100644 --- a/typescript-examples/empty-auth/README.md +++ b/typescript-examples/empty-auth/README.md @@ -89,7 +89,7 @@ yarn intuned run authsession validate ### `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/typescript-examples/empty/README.md b/typescript-examples/empty/README.md index ab6b6763..9a372602 100644 --- a/typescript-examples/empty/README.md +++ b/typescript-examples/empty/README.md @@ -65,7 +65,7 @@ yarn intuned deploy ### `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/typescript-examples/hybrid-automation/README.md b/typescript-examples/hybrid-automation/README.md index 76a03f7d..ad1e79f2 100644 --- a/typescript-examples/hybrid-automation/README.md +++ b/typescript-examples/hybrid-automation/README.md @@ -1,6 +1,6 @@ # Hybrid Automation -This example demonstrates **hybrid automation** - a flexible approach that combines the [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview) (`@intuned/browser`) with AI-powered tools like [Stagehand](https://docs.stagehand.dev/) and `extractStructuredData`. This gives you the speed and reliability of traditional automation with the adaptability of AI when needed. +This example demonstrates **hybrid automation** - a flexible approach that combines the [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/overview) (`@intuned/browser`) with AI-powered tools like [Stagehand](https://docs.stagehand.dev/) and `extractStructuredData`. This gives you the speed and reliability of traditional automation with the adaptability of AI when needed. ## Run on Intuned @@ -85,6 +85,6 @@ utils/crawler/ # Crawler utilities ## Learn More - [Flexible Automations](https://docs.intunedhq.com/docs/02-features/flexible-automation) -- [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview) +- [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/overview) - [Extract Structured Data](https://docs.intunedhq.com/automation-sdks/intuned-sdk/typescript/helpers/functions/extractStructuredData) - [Stagehand act/extract/observe](https://docs.stagehand.dev/v2/basics/act) diff --git a/typescript-examples/playwright-basics-ts/README.md b/typescript-examples/playwright-basics-ts/README.md index ec7821f0..d0e47e32 100644 --- a/typescript-examples/playwright-basics-ts/README.md +++ b/typescript-examples/playwright-basics-ts/README.md @@ -113,4 +113,4 @@ yarn intuned deploy ## Related - [Playwright deep dive](https://docs.intunedhq.com/docs/01-learn/deep-dives/playwright) -- [Intuned SDK](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview) +- [Intuned SDK](https://docs.intunedhq.com/automation-sdks/overview) diff --git a/typescript-examples/project_getting_started_template.md b/typescript-examples/project_getting_started_template.md index 7334852c..e51b5dd8 100644 --- a/typescript-examples/project_getting_started_template.md +++ b/typescript-examples/project_getting_started_template.md @@ -104,7 +104,7 @@ yarn intuned deploy ### `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/typescript-examples/quick-recipes/README.md b/typescript-examples/quick-recipes/README.md index 2841143b..506ef905 100644 --- a/typescript-examples/quick-recipes/README.md +++ b/typescript-examples/quick-recipes/README.md @@ -73,7 +73,7 @@ yarn intuned deploy ### `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). @@ -116,4 +116,4 @@ This project uses Intuned browser SDK. For more information, check out the [Intu - [Pagination Recipe](https://docs.intunedhq.com/docs/01-learn/recipes/pagination) - [Upload Files Recipe](https://docs.intunedhq.com/docs/01-learn/recipes/upload-files) - [Capture Screenshots Recipe](https://docs.intunedhq.com/docs/01-learn/recipes/capture-screenshots) -- [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview) +- [Intuned Browser SDK](https://docs.intunedhq.com/automation-sdks/overview) diff --git a/typescript-examples/rpa-auth-example/README.md b/typescript-examples/rpa-auth-example/README.md index 6bd4b5f0..3a8d5183 100644 --- a/typescript-examples/rpa-auth-example/README.md +++ b/typescript-examples/rpa-auth-example/README.md @@ -92,7 +92,7 @@ yarn intuned run authsession validate ### `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). diff --git a/typescript-examples/rpa-example/README.md b/typescript-examples/rpa-example/README.md index ab442e88..6af3cd15 100644 --- a/typescript-examples/rpa-example/README.md +++ b/typescript-examples/rpa-example/README.md @@ -78,7 +78,7 @@ yarn intuned deploy ## `@intuned/browser`: Intuned Browser SDK -This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview). +This project uses Intuned browser SDK. For more information, check out the [Intuned Browser SDK documentation](https://docs.intunedhq.com/automation-sdks/overview). ## Project Structure @@ -107,5 +107,5 @@ This project uses Intuned browser SDK. For more information, check out the [Intu ## Learn More - **Intuned Concepts**: https://docs.intunedhq.com/docs/00-getting-started/introduction -- **Intuned Browser SDK**: https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview +- **Intuned Browser SDK**: https://docs.intunedhq.com/automation-sdks/overview - **CLI Documentation**: https://docs.intunedhq.com/docs/05-references/cli diff --git a/typescript-examples/stagehand/README.md b/typescript-examples/stagehand/README.md index 403c58c2..8df8459d 100644 --- a/typescript-examples/stagehand/README.md +++ b/typescript-examples/stagehand/README.md @@ -108,5 +108,5 @@ yarn intuned deploy - **Intuned Concepts**: https://docs.intunedhq.com/docs/00-getting-started/introduction - **Stagehand Documentation**: https://docs.stagehand.dev/ -- **Intuned Runtime SDK**: https://docs.intunedhq.com/automation-sdks/intuned-sdk/overview +- **Intuned Runtime SDK**: https://docs.intunedhq.com/automation-sdks/overview - **Setup Hooks**: https://docs.intunedhq.com/docs/01-learn/recipes/setup-hooks