From 0d70b807f7d5b0adc8ba2a2046bf5a2c39b44b6d Mon Sep 17 00:00:00 2001 From: aeluce Date: Mon, 15 Dec 2025 12:47:13 -0600 Subject: [PATCH 1/4] separate suiteql connector docs --- .../Connectors/capture-connectors/README.md | 7 +- .../netsuite-suiteanalytics.md | 93 +++++--- .../capture-connectors/netsuite-suiteql.md | 208 ++++++++++++++++++ .../capture-connectors/netsuite-suitetalk.md | 188 ---------------- 4 files changed, 278 insertions(+), 218 deletions(-) create mode 100644 site/docs/reference/Connectors/capture-connectors/netsuite-suiteql.md delete mode 100644 site/docs/reference/Connectors/capture-connectors/netsuite-suitetalk.md diff --git a/site/docs/reference/Connectors/capture-connectors/README.md b/site/docs/reference/Connectors/capture-connectors/README.md index 8e00a086854..e4d18798fee 100644 --- a/site/docs/reference/Connectors/capture-connectors/README.md +++ b/site/docs/reference/Connectors/capture-connectors/README.md @@ -194,9 +194,12 @@ All Estuary connectors capture data in real time, as it appears in the source sy - Navan - [Configuration](./navan.md) - Package - ghcr.io/estuary/source-navan:dev -- NetSuite +- NetSuite SuiteAnalytics - [Configuration](./netsuite-suiteanalytics.md) - - Package - ghcr.io/estuary/source-netsuite:dev + - Package - ghcr.io/estuary/source-netsuite-v2:dev +- NetSuite SuiteQL + - [Configuration](./netsuite-suiteql.md) + - Package - ghcr.io/estuary/source-netsuite-suiteql:dev - OneDrive - [Configuration](./onedrive.md) - Package - ghcr.io/estuary/source-onedrive:dev diff --git a/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md b/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md index cbc4e0a8416..b87c2c52849 100644 --- a/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md +++ b/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md @@ -2,18 +2,19 @@ import ReactPlayer from "react-player"; # NetSuite SuiteAnalytics Connect -This connector captures data from Oracle NetSuite into Flow collections. It uses the SuiteAnalytics Connect feature in order to both load large amounts of data quickly, as well as introspect the available tables, their schemas, keys, and cursor fields. +This connector captures data from Oracle NetSuite into Estuary collections. It uses the SuiteAnalytics Connect feature in order to both load large amounts of data quickly, as well as introspect the available tables, their schemas, keys, and cursor fields. -[`ghcr.io/estuary/source-netsuite:dev`](https://ghcr.io/estuary/source-netsuite:dev) provides the +[`ghcr.io/estuary/source-netsuite:dev`](https://ghcr.io/estuary/source-netsuite-v2:dev) provides the latest connector image. You can also follow the link in your browser to see past image versions. -If you don't have SuiteAnalytics Connect, the connector can be used in SuiteQL mode. +In general, SuiteAnalytics Connect is the preferred method to retrieve data from NetSuite. +However, if you don't have SuiteAnalytics, see the [SuiteQL connector](./netsuite-suiteql.md) instead. ## Supported data resources -Flow discovers all of the tables to which you grant access during [setup](#setup), including `Transactions`, `Reports`, `Lists`, and `Setup`. +Estuary discovers all of the tables to which you grant access during [setup](#setup), including `Transactions`, `Reports`, `Lists`, and `Setup`. ## Prerequisites @@ -26,15 +27,29 @@ Flow discovers all of the tables to which you grant access during [setup](#setup ## Setup -**Create a NetSuite account** +#### 1. Create a NetSuite account 1. Create an account on the [Oracle NetSuite](https://www.netsuite.com/portal/home.shtml) portal. 2. Confirm your email address. -**Set up your NetSuite account** +#### 2. Enable SuiteAnalytics Connect -1. Find your _Realm_, or Account ID. You'll use this to connect with Flow. +1. Navigate to **Setup** > **Company** > **Enable Features**. + +2. Click the **SuiteCloud** tab. + +3. In the **Manage Authentication** section, check the checkbox labeled **TOKEN-BASED AUTHENTICATION**. + +4. Save your changes. + +5. Next, navigate to **Setup** > **Company** > **Analytics** > **Connectivity** and check the checkbox labeled **SuiteAnalytics Connect**. + +6. Save your changes. + +#### 3. Find Your Account ID + +Find your _Realm_, or Account ID. You'll use this to connect with Estuary. 1. In your NetSuite portal, go to **Setup** > **Company** > **Company Information**. @@ -42,7 +57,9 @@ Flow discovers all of the tables to which you grant access during [setup](#setup If you have a production account, it will look like `2345678`. If you're using a sandbox, it'll look like `2345678_SB2`. -2. Create a NetSuite _integration_ to obtain a Consumer Key and Consumer Secret. +#### 4. Generate Consumer Tokens + +Create a NetSuite _integration_ to obtain a Consumer Key and Consumer Secret. 1. Navigate to **Setup** > **Integration** > **Manage Integrations** > **New**. @@ -56,7 +73,7 @@ Flow discovers all of the tables to which you grant access during [setup](#setup Your Consumer Key and Consumer Secret will be shown once. Copy them to a safe place. -3. Set up a role for use with Flow. +#### 5. Set Up a Custom Role 1. Go to **Setup** > **Users/Roles** > **Manage Roles** > **New**. @@ -72,21 +89,23 @@ Flow discovers all of the tables to which you grant access during [setup](#setup 7. (IMPORTANT) Click **Setup** an add all the dropdown entities with either **full** or **view** access level. + 8. (IMPORTANT) If you have multiple subsidiaries, make sure to select all of the subsidiaries you want the connector to have access to under the **Role** > **Subsidiary Restrictions** configuration. + To allow your custom role to reflect future changes, be sure to edit these parameters again when you rename or customize any NetSuite object. -4. Set up user for use with Flow. +#### 6. Assign the Role to a User 1. Go to **Setup** > **Users/Roles** > **Manage Users**. - 2. Find the user you want to give access to use with Flow. In the **Name** column, click the user's name. Then, click the **Edit** button. + 2. Find the user you want to give access to use with Estuary. In the **Name** column, click the user's name. Then, click the **Edit** button. 3. Find the **Access** tab. - 4. From the dropdown list, select role you created previously; for example, `estuary-integration-role`. + 4. From the dropdown list, select the role you created previously; for example, `estuary-integration-role`. 5. Save your changes. -5. Generate an access token. +#### 7. Generate User Access Tokens 1. Go to **Setup** > **Users/Roles** > **Access Tokens** > **New**. @@ -102,7 +121,7 @@ Flow discovers all of the tables to which you grant access during [setup](#setup Your Token ID and Token Secret will be shown once. Copy them to a safe place. -You now have a properly configured account with the correct permissions and all the information you need to connect with Flow: +You now have a properly configured account with the correct permissions and all the information you need to connect with Estuary: - Realm (Account ID) - Consumer Key @@ -116,7 +135,7 @@ You can also authenticate with a username and password, but a consumer/token is ## Configuration -You configure connectors either in the Flow web app, or by directly editing the catalog specification file. +You configure connectors either in the Estuary web app, or by directly editing the catalog specification file. See [connectors](../../../concepts/connectors.md#using-connectors) to learn more about using connectors. The values and specification sample below provide configuration details specific to the NetSuite source connector. ### Properties @@ -126,34 +145,36 @@ See [connectors](../../../concepts/connectors.md#using-connectors) to learn more | Property | Title | Description | Type | Required/Default | | ----------------------------- | ---------------------- | ------------------------------------------------------------------------------------------------ | ------ | ---------------- | | `/account` | Netsuite Account ID | Netsuite realm/Account ID e.g. 2344535, as for `production` or 2344535_SB1, as for `sandbox` | string | Required | -| `/role_id` | Role ID | The ID of the role you created. Defaults to 3, which is the ID of the administrator role. | int | 3 | -| `/connection_type` | Connection Type | Type of connection to use. `suiteanalytics` is preferred over `suiteql`. | string | Required | | `/suiteanalytics_data_source` | Data Source | Which NetSuite data source to use. Options are `NetSuite.com`, or `NetSuite2.com` | string | Required | | `/authentication` | Authentication Details | Credentials to access your NetSuite account | object | Required | +| `/authentication/auth_type` | Authentication Type | Type of authentication used, either `token` or `user_pass`. | string | `token` | -#### Token/Consumer Authentication +##### Token/Consumer Authentication | Property | Title | Description | Type | Required/Default | | --------------------------------- | --------------- | ------------------------------------------------- | ------ | ---------------- | | `/authentication/consumer_key` | Consumer Key | Consumer key associated with your integration. | string | Required | | `/authentication/consumer_secret` | Consumer Secret | Consumer secret associated with your integration. | string | Required | -| `/authentication/token_key` | Token Key | Access token key | string | Required | +| `/authentication/token_id` | Token ID | Access token key | string | Required | | `/authentication/token_secret` | Token Secret | Access token secret | string | Required | -#### Username/Password Authentication +##### Username/Password Authentication | Property | Title | Description | Type | Required/Default | | -------------------------- | -------- | -------------------------------------- | ------ | ---------------- | | `/authentication/username` | Username | Your NetSuite account's email/username | string | Required | | `/authentication/password` | Password | Your NetSuite account's password. | string | Required | +| `/authentication/role_id` | Role ID | The ID of the role you created. Defaults to 3, which is the ID of the administrator role. | int | `3` | -#### Advanced Config options +##### Advanced Config options | Property | Title | Description | Type | Required/Default | | ---------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | ---------------- | | `/advanced/connection_limit` | Connection Limit | The maximum number of concurrent data streams to attempt at once. | int | 10 Connections | | `/advanced/task_limit` | Task Limit | The maximum number of concurrent tasks to run at once. A task is either a backfill or incremental load. Backfills can load multiple chunks in parallel, so this must be strictly <= `/advanced/connection_limit` | int | 5 Tasks | | `/advanced/start_date` | Start Date | The date that we should attempt to start backfilling from. If not provided, backfill from the beginning. | date | Not Required | +| `/advanced/cursor_fields` | Cursor Fields | Columns to use as cursor for incremental replication, in order of preference. Case insensitive. | string array | Not Required. `last_modified_date`, `lastmodifieddate`, and more are used as the default list. | +| `/advanced/query_idle_timeout_seconds` | Query Idle Timeout | Maximum time to wait for the next row during query execution. Query will timeout if no rows are received within this duration. | [`ISO8601` Duration](https://www.digi.com/resources/documentation/digidocs/90001488-13/reference/r_iso_8601_duration_format.htm) | `PT30M` | #### Bindings @@ -161,11 +182,18 @@ See [connectors](../../../concepts/connectors.md#using-connectors) to learn more | ------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | | `/name` | Name | The name of the table this binding refers to | string | Required | | `/interval` | Interval | How frequently to check for incremental changes | [`ISO8601` Duration](https://www.digi.com/resources/documentation/digidocs/90001488-13/reference/r_iso_8601_duration_format.htm) | `PT1H` (1 Hour) | +| `/schedule` | Schedule | Schedule to automatically rebackfill this binding. Accepts a cron expression. | string | | | `/log_cursor` | Log Cursor | A date-time column to use for incremental capture of modifications. | String | Required (Automatically Discovered) | | `/page_cursor` | Page Cursor | An indexed, non-NULL integer column to use for ordered table backfills. Does not need to be unique, but should have high cardinality. | String | Required (Automatically Discovered) | | `/concurrency` | Concurrency | Maximum number of concurrent connections to use for backfilling. | int | 1 Connection | | `/query_limit` | Query Limit | Maximum number of rows to fetch in a query. Will be divided between all connections if `/concurrency` > 1 | int | 100,000 Rows | -| `/query_timeout` | Query Timeout | Timeout for queries. Typically left as the default as some tables just take a very long time to respond. | [`ISO8601` Duration](https://www.digi.com/resources/documentation/digidocs/90001488-13/reference/r_iso_8601_duration_format.htm) | `PT10M` (10 Minutes) | +| `/select_columns` | Manually Selected Columns | Override the columns to load from the table. If empty, all columns will be loaded. Ideally this should only be set when loading specific columns is neccesary, as it won't automatically update when new columns are added or removed. | string array | `[]` | +| `/snapshot_backfill` | Single-shot Backfill | Attempt to backfill using a single-shot query to load all rows. Useful when no good page cursor exists, and the table is of reasonable size. Incremental updates are still possible if a log cursor is defined. | boolean | `false` | + +##### Table Associations + +| Property | Title | Description | Type | Required/Default | +| --- | --- | --- | --- | --- | | `/associations` | Associations | List of associated tables for which related data should be loaded. | Array[TableAssociation] | [] | | `/associations/[n]/child_table_name` | Foreign Table Name | The name of the "foreign" table that should be associated with the "parent" binding containing this association | String | Required | | `/associations/[n]/parent_join_column_name` | Parent Join Column | The name of the column on the "parent" table to be used as the join key | String | Required | @@ -173,10 +201,21 @@ See [connectors](../../../concepts/connectors.md#using-connectors) to learn more | `/associations/[n]/load_during_backfill` | Load During Backfill | Whether or not to load associated documents during backfill | Boolean | False | | `/associations/[n]/load_during_incremental` | Load During Incremental | Whether or not to load associated documents during incremental loads | Boolean | True | +##### Advanced Options + +| Property | Title | Description | Type | Required/Default | +| --- | --- | --- | --- | --- | +| `/advanced` | Advanced | Advanced options to customize your binding. | object | | +| `/advanced/initial_backfill_cursor` | Initial Backfill Cursor | Manually set the starting cursor value for backfill operations. If specified, the backfill will start from this cursor value instead of the table's minimum value. Useful for partial backfills or resuming from a specific point. Only applies to the initial backfill. | int | | +| `/advanced/exclude_calculated` | Exclude Calculated Columns | Exclude calculated columns from queries. Keys and cursors are never excluded. | boolean | `false` | +| `/advanced/exclude_custom` | Exclude Custom Columns | Exclude custom columns from queries. Keys and cursors are never excluded. | boolean | `false`| +| `/advanced/exclude_hidden` | Exclude Hidden Columns | Exclude hidden columns from queries. Keys and cursors are never excluded. | boolean | `false` | +| `/advanced/exclude_non_display` | Exclude Non-Display Columns | Exclude non-display columns from queries. Keys and cursors are never excluded. | boolean | `false` | +| `/advanced/extra_columns` | Additional Columns | Columns to include even if they match exclusion criteria. Useful for selectively re-including specific columns that would otherwise be filtered out. Cannot be used with 'Manually Selected Columns'. | string array | `[]` | + ### Sample ```yaml - captures: ${PREFIX}/${CAPTURE_NAME}: endpoint: @@ -190,16 +229,13 @@ captures: consumer_secret_sops: xxx token_id: xxx token_secret_sops: xxx - connection_type: suiteanalytics - role_id: 3 suiteanalytics_data_source: NetSuite2.com advanced: connection_limit: 20 cursor_fields: [] - enable_auto_cursor: false - resource_tracing: false start_date: null task_limit: 10 + query_idle_timeout_seconds: PT30M bindings: - resource: associations: @@ -213,8 +249,9 @@ captures: page_cursor: id query_limit: 100000 concurrency: 1 - query_timeout: PT10M log_cursor: lastmodifieddate + select_columns: [] + snapshot_backfill: false target: ${PREFIX}/${CAPTURE_NAME}/transaction {...} ``` diff --git a/site/docs/reference/Connectors/capture-connectors/netsuite-suiteql.md b/site/docs/reference/Connectors/capture-connectors/netsuite-suiteql.md new file mode 100644 index 00000000000..51e32f35d1d --- /dev/null +++ b/site/docs/reference/Connectors/capture-connectors/netsuite-suiteql.md @@ -0,0 +1,208 @@ + +# Netsuite SuiteQL (Beta) + +This connector captures data from Oracle NetSuite into Estuary collections using SuiteQL. + +To use SuiteAnalytics Connect to sync your data instead, see the [Netsuite SuiteAnalytics connector](./netsuite-suiteanalytics.md). + +## Supported data resources + +Currently, this connector supports a subset of NetSuite tables, such as: + +* Customer +* Item +* Term +* Transaction +* TransactionHistory +* TransactionLine +* TransactionShippingAddress +* TransactionStatus + +If you need to capture a table that is not yet supported, [contact support](mailto:support@estuary.dev) with the table name(s). +Estuary support will be able to confirm availability and, if needed, add the table(s) to the connector. + +## Prerequisites + +- Oracle NetSuite [account](https://system.netsuite.com/pages/customerlogin.jsp?country=US) +- Allowed access to all Account permissions options +- A new integration with token-based authentication +- A custom role with access to objects you want to capture. See [setup](#setup). +- A new user assigned to the custom role +- Access token generated for the custom role + +## Setup + +#### 1. Create a NetSuite account + +1. Create an account on the [Oracle NetSuite](https://www.netsuite.com/portal/home.shtml) portal. + +2. Confirm your email address. + +#### 2. Enable SuiteQL + +1. Navigate to **Setup** > **Company** > **Enable Features**. + +2. Click the **SuiteCloud** tab. + +3. In the **Manage Authentication** section, check the checkbox labeled **TOKEN-BASED AUTHENTICATION**. + +4. Save your changes. + +5. Next, in the **SuiteTalk (Web Services)** section, check the checkbox labeled **REST WEB SERVICES**. + +6. Save your changes. + +#### 3. Find Your Account ID + +Find your _Realm_, or Account ID. You'll use this to connect with Estuary. + + 1. In your NetSuite portal, go to **Setup** > **Company** > **Company Information**. + + 2. Copy your Account ID. + + If you have a production account, it will look like `2345678`. If you're using a sandbox, it'll look like `2345678_SB2`. + +#### 4. Generate Consumer Tokens + +Create a NetSuite _integration_ to obtain a Consumer Key and Consumer Secret. + + 1. Navigate to **Setup** > **Integration** > **Manage Integrations** > **New**. + + 2. Give the integration a name, for example, `estuary-rest-integration`. + + 3. Make sure the **State** option is enabled. + + 4. In the **Authentication** section, check the **Token-Based Authentication** checkbox. + + 5. Save your changes. + + Your Consumer Key and Consumer Secret will be shown once. Copy them to a safe place. + +#### 5. Set Up a Custom Role + + 1. Go to **Setup** > **Users/Roles** > **Manage Roles** > **New**. + + 2. Give the role a name, for example, `estuary-integration-role`. + + 3. Scroll to the **Permissions** section. + + 4. (IMPORTANT) Click **Transactions** and add all the dropdown entities with either **full** or **view** access level. + + 5. (IMPORTANT) Click **Reports** and add all the dropdown entities with either **full** or **view** access level. + + 6. (IMPORTANT) Click **Lists** and add all the dropdown entities with either **full** or **view** access level. + + 7. (IMPORTANT) Click **Setup** an add all the dropdown entities with either **full** or **view** access level. + + 8. (IMPORTANT) If you have multiple subsidiaries, make sure to select all of the subsidiaries you want the connector to have access to under the **Role** > **Subsidiary Restrictions** configuration. + + To allow your custom role to reflect future changes, be sure to edit these parameters again when you rename or customize any NetSuite object. + +#### 6. Assign the Role to a User + + 1. Go to **Setup** > **Users/Roles** > **Manage Users**. + + 2. Find the user you want to give access to use with Estuary. In the **Name** column, click the user's name. Then, click the **Edit** button. + + 3. Find the **Access** tab. + + 4. From the dropdown list, select the role you created previously; for example, `estuary-integration-role`. + + 5. Save your changes. + +#### 7. Generate User Access Tokens + + 1. Go to **Setup** > **Users/Roles** > **Access Tokens** > **New**. + + 2. Select an **Application Name**. + + 3. Under **User**, select the user you assigned the role previously. + + 4. Under **Role**, select the role you assigned to the user previously. + + 5. Under **Token Name**, give a descriptive name to the token you are creating, for example `estuary-rest-integration-token`. + + 6. Save your changes. + + Your Token ID and Token Secret will be shown once. Copy them to a safe place. + +You now have a properly configured account with the correct permissions and all the information you need to connect with Estuary: + +- Realm (Account ID) +- Consumer Key +- Consumer Secret +- Token ID +- Token Secret + +## Configuration + +You configure connectors either in Estuary's web app, or by directly editing the catalog specification file. +See [connectors](/concepts/connectors.md#using-connectors) to learn more about using connectors. The values and specification sample below provide configuration details specific to the NetSuite source connector. + +### Properties + +#### Endpoint + +| Property | Title | Description | Type | Required/Default | +| --- | --- | --- | --- | --- | +| `/account` | Netsuite Account ID | Netsuite realm/Account ID e.g. 2344535, as for `production` or 2344535_SB1, as for `sandbox` | string | Required | +| `/authentication` | Authentication Details | Credentials to access your NetSuite account | object | Required | +| `/tables` | Tables | List of tables to capture with their keys | array of objects, each containing a string name and array of keys | Defaults to all supported data resources | +| `/advanced` | Advanced | Any advanced options to use for the connector | object | | + +#### Authentication Config + +| Property | Title | Description | Type | Required/Default | +| --- | --- | --- | --- | --- | +| `/authentication/consumer_key` | Consumer Key | Consumer key associated with your integration. | string | Required | +| `/authentication/consumer_secret` | Consumer Secret | Consumer secret associated with your integration. | string | Required | +| `/authentication/token_id` | Token ID | Access token ID | string | Required | +| `/authentication/token_secret` | Token Secret | Access token secret | string | Required | + +#### Advanced Config options + +| Property | Title | Description | Type | Required/Default | +| --- | --- | --- | --- | --- | +| `/advanced/connection_limit` | Connection Limit | The maximum number of concurrent data streams to NetSuite. | int | `2` | + +#### Bindings + +| Property | Title | Description | Type | Required/Default | +| --- | --- | --- | --- | --- | +| `/name` | Name | The name of the table this binding refers to | string | Required | +| `/interval` | Interval | How frequently to check for incremental changes | [`ISO8601` Duration](https://www.digi.com/resources/documentation/digidocs/90001488-13/reference/r_iso_8601_duration_format.htm) | `PT1H` (1 Hour) | +| `/schedule` | Schedule | Schedule to automatically rebackfill this binding. Accepts a cron expression. | string | | +| `/page_cursor` | Page Cursor | Cursor field for paginated backfills | string | Defaults to first key field | +| `/columns` | Columns | List of columns to select. Empty means `SELECT *` (fails silently if >100 columns). | string array | `[]` | +| `/query_limit` | Query Limit | Number of rows to fetch per query page | int | `10000` | + +### Sample + +```yaml +captures: + ${PREFIX}/${CAPTURE_NAME}: + endpoint: + connector: + image: ghcr.io/estuary/source-netsuite-suiteql:dev + config: + account: "12345678" + authentication: + consumer_key: xxx + consumer_secret_sops: xxx + token_id: xxx + token_secret_sops: xxx + tables: + - name: transaction + keys: [id] + advanced: + connection_limit: 2 + bindings: + - resource: + name: transaction + interval: PT1H + page_cursor: id + columns: [] + query_limit: 10000 + target: ${PREFIX}/${CAPTURE_NAME}/transaction + {...} +``` diff --git a/site/docs/reference/Connectors/capture-connectors/netsuite-suitetalk.md b/site/docs/reference/Connectors/capture-connectors/netsuite-suitetalk.md deleted file mode 100644 index f26601fcd2b..00000000000 --- a/site/docs/reference/Connectors/capture-connectors/netsuite-suitetalk.md +++ /dev/null @@ -1,188 +0,0 @@ -# NetSuite SuiteTalk REST (Deprecated) - -:::warning -This connector is deprecated. For new pipelines, we recommend you use our actively maintained [NetSuite connector](./netsuite-suiteanalytics.md) instead. -::: - -This connector captures data from Oracle NetSuite into Flow collections. It connects to the NetSuite Analytics Data Warehouse using the SuiteQL REST endpoint and a custom role. - -It is available for use in the Flow web application. - -## SuiteAnalytics vs SuiteQL via REST API - -These two different connection modes have some key differences: - -### [SuiteAnalytics Connect](../netsuite-suiteanalytics) - -- Requires the SuiteAnalytics Connect feature to be purchased on your NetSuite account -- Can inspect which tables (standard & custom) exist in your account -- Can inspect the exact data types specified on these table columns -- This means you can connect to any table in your account and all fields (booleans, date, and datetimes) are properly formatted in Estuary - -### SuiteQL via REST API - -- Custom tables are not supported without manual work -- Some standard tables may not yet be supported and will require additional work from the Estuary team -- Datetime values are represented as dates without the time specification (this is a limitation of the REST API) -- Data types on custom columns may not be properly represented -- You are repsonsible for determining the right set of permissions to grant the connector, which can often be complicated and unintuitive - -## Prerequisites - -- Oracle NetSuite [account](https://system.netsuite.com/pages/customerlogin.jsp?country=US) -- Allowed access to all Account permissions options -- A new integration with token-based authentication -- A custom role with access to objects you want to capture _or_ a purchased SuiteAnalytics Module. See [setup](#general-setup). -- A new user assigned to the custom role -- Access token generated for the custom role - -## General Setup - -**Set up required features on your NetSuite account** - -1. Find your Account ID (also know as the "Realm"). You'll use this to connect with Flow. - - 1. In your NetSuite portal, go to **Setup** > **Company** > **Company Information**. - - 2. Copy your Account ID. - - If you have a production account, it will look like `2345678`. If you're using a sandbox, it'll look like `2345678_SB2`. - -2. Enable the required features. - - 1. Navigate to **Setup** > **Company** > **Enable Features**. - - 2. Click the **SuiteCloud** tab. - - 3. In the **Manage Authentication** section, check the checkbox labeled **TOKEN-BASED AUTHENTICATION**. - - 4. If you are using the SuiteQL connection, in the **SuiteTalk (Web Services)** section, check the checkbox labeled **REST WEB SERVICES**. - - 5. Save your changes. - - 6. If you are using SuiteAnalytics Connect, navigate to **Setup** > **Company** > **Analytics** > **Connectivity** and check the checkbox labeled **SuiteAnalytics Connect**. - - 7. Save your changes. - -3. Create a NetSuite _integration_ to obtain a Consumer Key and Consumer Secret. - - 1. Navigate to **Setup** > **Integration** > **Manage Integrations** > **New**. - - 2. Give the integration a name, for example, `estuary-netsuite-integration`. - - 3. Make sure the **State** option is enabled. - - 4. In the **Authentication** section, check the **Token-Based Authentication** checkbox. - - 5. Save your changes. - - Your Consumer Key and Consumer Secret will be shown once. Copy them to a safe place. They will never show up again - and will be key to the integration working properly. - -4. If you are using the **SuiteQL** over REST API connection, Set up a role for use with Flow. - - 1. Go to **Setup** > **Users/Roles** > **Manage Roles** > **New**. - - 2. Give the role a name, for example, `estuary-integration-role`. - - 3. The easiest thing to do here is to click "Core Administrative Permissions". If you want to scope down the permissions given to the connector (which you should) you'll have to determine which permissions are necessary. This is challenging because many different settings and configurations can expand the required permissions. [Check out this repository](https://github.com/iloveitaly/netsuite-permissions) for help with determining exactly which permissions are required in your case. - - 4. Scroll to the **Permissions** section. - - 5. (IMPORTANT) Click **Transactions** and add all the dropdown entities with either **full** or **view** access level. - - - Find Transaction - - 6. (IMPORTANT) Click **Setup** an add the following entities with either **full** or **view** access level. - - - Log in using Access Tokens - - REST Web Services - - User Access Tokens - - To allow your custom role to reflect future changes, be sure to edit these parameters again when you rename or customize any NetSuite object. - -5. If you are using **SuiteAnalytics Connect** you don't need a custom role. Instead, you can use the bundled "Data Warehouse Integrator" - -6. Set up user for use with the connector. - - 1. Go to **Setup** > **Users/Roles** > **Manage Users**. - - 2. Find the user you want to give access to use with Flow. In the **Name** column, click the user's name. Then, click the **Edit** button. - - 3. Find the **Access** tab. - - 4. From the dropdown list, select either role you created previously (e.g. `estuary-integration-role`) or the **Data Warehouse Integrator** role if you are using SuiteAnalytics Connect. - - 5. Save your changes. - -7. Generate an access token. - - 1. Go to **Setup** > **Users/Roles** > **Access Tokens** > **New**. - - 2. Select the **Application Name** you created earlier. - - 3. Under **User**, select the user you assigned the role previously. - - 4. Under **Role**, select the role you assigned to the user previously. - - 5. Under **Token Name**, give a descriptive name to the token you are creating, for example `estuary-rest-integration-token`. - - 6. Save your changes. - - Your Token ID and Token Secret will be shown once. Copy them to a safe place. - -You now have a properly configured account with the correct permissions and all the information you need to connect with Flow: - -- Account ID (Realm) -- Consumer Key -- Consumer Secret -- Token ID -- Token Secret - -## Configuration - -You configure connectors either in the Flow web app, or by directly editing the catalog specification file. -See [connectors](../../../concepts/connectors.md#using-connectors) to learn more about using connectors. The values and specification sample below provide configuration details specific to the NetSuite source connector. - -### Properties - -#### Endpoint - -| Property | Title | Description | Type | Required/Default | -| ------------------ | --------------- | ------------------------------------------------------------------------------------- | ------ | ---------------- | -| `/account_id` | Realm | Netsuite realm e.g. 2344535, as for `production` or 2344535_SB1, as for the `sandbox` | string | Required | -| `/start_date` | Token Secret | The date to start collecting data from | date | Required | -| `/consumer_key` | Consumer Key | Consumer key associated with your integration. | string | Required | -| `/consumer_secret` | Consumer Secret | Consumer secret associated with your integration. | string | Required | -| `/token_key` | Token Key | Access token key | string | Required | -| `/token_secret` | Token Secret | Access token secret | string | Required | - -#### Bindings - -| Property | Title | Description | Type | Required/Default | -| --------------- | --------- | ---------------------------------------------------------------------- | ------ | ---------------- | -| **`/stream`** | Stream | Resource of your NetSuite project from which collections are captured. | string | Required | -| **`/syncMode`** | Sync Mode | Connection method. | string | Required | - -### Sample - -```yaml - -captures: - ${PREFIX}/${CAPTURE_NAME}: - endpoint: - connector: - image: ghcr.io/estuary/source-netsuite:dev - config: - account_id: - consumer_key: - consumer_secret: - token_key: - token_secret: - start_date: "2023-11-01T00:00:00Z" - bindings: - - resource: - stream: Transaction - target: ${PREFIX}/Transaction - {...} -``` From 59ee62a142ef45efe39216dca876ffe3bd2b8fb2 Mon Sep 17 00:00:00 2001 From: aeluce Date: Wed, 7 Jan 2026 14:19:00 -0600 Subject: [PATCH 2/4] add netsuite special column info --- .../netsuite-suiteanalytics.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md b/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md index b87c2c52849..a681631fa5c 100644 --- a/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md +++ b/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md @@ -255,3 +255,24 @@ captures: target: ${PREFIX}/${CAPTURE_NAME}/transaction {...} ``` + +## Special Columns + +NetSuite tables can include several special column types: + +* Calculated Columns +* Custom Columns +* Hidden Columns +* Non-Display Columns + +Ingesting these types of column can slow down queries (calculated columns, for example, require computation on every row) or cause other blockages in the data flow. +Newly discovered bindings will therefore default to excluding calculated, custom, and hidden columns from your collections. + +If your bindings allow special column types, newly discovered columns may impact your capture in the future, even if everything currently works as expected. +It can therefore be prudent to select only the subset of NetSuite special columns you need to capture. + +To set exclusions for particular special column types, configure the resource's [Advanced Options](#advanced-options). + +If you find these exclusions too broad, you can add back individual filtered-out fields using the resource's **Additional Columns** Advanced Option. + +You can find out whether a specific column falls under one of these special types in NetSuite's column metadata under the `userdata` field. From d616e4c6b1c119cc7036ef44b66bebebd47ac36e Mon Sep 17 00:00:00 2001 From: aeluce Date: Wed, 7 Jan 2026 14:24:01 -0600 Subject: [PATCH 3/4] uncomment netsuite redirect --- site/docusaurus.config.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/site/docusaurus.config.js b/site/docusaurus.config.js index dd465bf36a0..477e779f01a 100644 --- a/site/docusaurus.config.js +++ b/site/docusaurus.config.js @@ -153,12 +153,10 @@ const config = { from: '/reference/Connectors/materialization-connectors/Parquet/', to: '/reference/Connectors/materialization-connectors/amazon-s3-parquet/' }, - - // Waiting on https://github.com/estuary/flow/pull/2559/ - // { - // from : '/reference/Connectors/capture-connectors/netsuite-suitetalk/', - // to: '/reference/Connectors/capture-connectors/netsuite-suiteql/', - // }, + { + from: '/reference/Connectors/capture-connectors/netsuite-suitetalk/', + to: '/reference/Connectors/capture-connectors/netsuite-suiteql/', + }, ], }, ], From b740333a69396ad3cfc8c339010804aeb3633833 Mon Sep 17 00:00:00 2001 From: aeluce Date: Thu, 8 Jan 2026 09:21:14 -0600 Subject: [PATCH 4/4] remove cursor fields --- .../Connectors/capture-connectors/netsuite-suiteanalytics.md | 1 - 1 file changed, 1 deletion(-) diff --git a/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md b/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md index a681631fa5c..bb2b25f36ad 100644 --- a/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md +++ b/site/docs/reference/Connectors/capture-connectors/netsuite-suiteanalytics.md @@ -173,7 +173,6 @@ See [connectors](../../../concepts/connectors.md#using-connectors) to learn more | `/advanced/connection_limit` | Connection Limit | The maximum number of concurrent data streams to attempt at once. | int | 10 Connections | | `/advanced/task_limit` | Task Limit | The maximum number of concurrent tasks to run at once. A task is either a backfill or incremental load. Backfills can load multiple chunks in parallel, so this must be strictly <= `/advanced/connection_limit` | int | 5 Tasks | | `/advanced/start_date` | Start Date | The date that we should attempt to start backfilling from. If not provided, backfill from the beginning. | date | Not Required | -| `/advanced/cursor_fields` | Cursor Fields | Columns to use as cursor for incremental replication, in order of preference. Case insensitive. | string array | Not Required. `last_modified_date`, `lastmodifieddate`, and more are used as the default list. | | `/advanced/query_idle_timeout_seconds` | Query Idle Timeout | Maximum time to wait for the next row during query execution. Query will timeout if no rows are received within this duration. | [`ISO8601` Duration](https://www.digi.com/resources/documentation/digidocs/90001488-13/reference/r_iso_8601_duration_format.htm) | `PT30M` | #### Bindings