From f3347ff54fd0282edee10a455c4d47517a1036ea Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 2 Feb 2026 08:58:31 -0800 Subject: [PATCH 01/15] bring index, get started, edit --- modules/ROOT/nav.adoc | 2 +- modules/ROOT/pages/faqs.adoc | 2 +- modules/ROOT/pages/index.adoc | 2 +- modules/ROOT/partials/sinks/edit.adoc | 26 + modules/ROOT/partials/sinks/get-started.adoc | 40 ++ modules/ROOT/partials/sources/edit.adoc | 26 + .../ROOT/partials/sources/get-started.adoc | 40 ++ modules/connectors/pages/index.adoc | 624 ++++++++++++++++++ 8 files changed, 759 insertions(+), 3 deletions(-) create mode 100644 modules/ROOT/partials/sinks/edit.adoc create mode 100644 modules/ROOT/partials/sinks/get-started.adoc create mode 100644 modules/ROOT/partials/sources/edit.adoc create mode 100644 modules/ROOT/partials/sources/get-started.adoc create mode 100644 modules/connectors/pages/index.adoc diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 59f861dc..3fb44a88 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -23,6 +23,6 @@ * xref:operations:auth.adoc[] * xref:operations:functions.adoc[] * xref:operations:functions-transform.adoc[] -* xref:operations:io-connectors.adoc[] +* xref:connectors:index.adoc[] * xref:operations:scale-cluster.adoc[] * xref:operations:troubleshooting.adoc[] \ No newline at end of file diff --git a/modules/ROOT/pages/faqs.adoc b/modules/ROOT/pages/faqs.adoc index d37965ff..ba115f8d 100644 --- a/modules/ROOT/pages/faqs.adoc +++ b/modules/ROOT/pages/faqs.adoc @@ -81,7 +81,7 @@ https://github.com/datastax/pulsar-heartbeat[{pulsar-short} Heartbeat] monitors == What are the features provided by {company} {pulsar} Connector (`pulsar-sink`) that are not supported in `kafka-sink`? -The https://pulsar.apache.org/docs/en/io-overview/[{pulsar-short} IO framework] provides many features that are not possible in Kafka, and has different compression formats and auth/security features. The features are handled by {pulsar-short}. For more, see xref:operations:io-connectors.adoc[Luna Streaming IO Connectors]. +The https://pulsar.apache.org/docs/en/io-overview/[{pulsar-short} IO framework] provides many features that are not possible in Kafka, and has different compression formats and auth/security features. The features are handled by {pulsar-short}. For more, see xref:connectors:index.adoc[Luna Streaming IO Connectors]. The {company} {pulsar} Connector allows single-record acknowledgement and negative acknowledgements. diff --git a/modules/ROOT/pages/index.adoc b/modules/ROOT/pages/index.adoc index 13244e46..7364bed9 100644 --- a/modules/ROOT/pages/index.adoc +++ b/modules/ROOT/pages/index.adoc @@ -17,7 +17,7 @@ In addition to the distribution of https://pulsar.apache.org/en/versions/[{pulsa * A xref:install-upgrade:quickstart-helm-installs.adoc[Helm chart] that deploys and manages {pulsar-short} on your current CNCF-conformant Kubernetes infrastructure -* Cassandra, Elastic, Kinesis, Kafka, and JDBC xref:operations:io-connectors.adoc[connectors] +* Cassandra, Elastic, Kinesis, Kafka, and JDBC xref:connectors:index.adoc[connectors] * The xref:components:starlight.adoc[Starlight suite of {pulsar-short} protocol handlers for Kafka, RabbitMQ, and JMS] diff --git a/modules/ROOT/partials/sinks/edit.adoc b/modules/ROOT/partials/sinks/edit.adoc new file mode 100644 index 00000000..bddd1dd4 --- /dev/null +++ b/modules/ROOT/partials/sinks/edit.adoc @@ -0,0 +1,26 @@ +To update a connector, pass the new configuration definition to the connector. +For example, if you created the connector with a configuration file, you can pass an updated configuration file. + +You can include the entire configuration or only the properties that you want to change. +Additionally, some properties can be modified with specific arguments, such as `--parallelism`. + +To get the current configuration, see xref:connectors:index.adoc#get-sink-connector-configuration-data[Get sink connector configuration data]. + +.pulsar-admin CLI +[source,shell,subs="+attributes"] +---- +./bin/pulsar-admin sinks update \ + --sink-type {connectorType} \ + --name "$SINK_NAME" \ + --inputs "persistent://$TENANT/$NAMESPACE/$TOPIC" \ + --tenant "$TENANT" \ + --parallelism 2 +---- + +.{pulsar-short} Admin API +[source,shell] +---- +curl -sS --fail -L -X PUT "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME" \ + --header "Authorization: Bearer $PULSAR_TOKEN" \ + --form "sinkConfig=@configs.json;type=application/json" +---- \ No newline at end of file diff --git a/modules/ROOT/partials/sinks/get-started.adoc b/modules/ROOT/partials/sinks/get-started.adoc new file mode 100644 index 00000000..a1fbc4bb --- /dev/null +++ b/modules/ROOT/partials/sinks/get-started.adoc @@ -0,0 +1,40 @@ +. Optional: If you are using the `pulsar-admin` CLI or {pulsar-short} Admin API, set the following commonly-used environment variables: ++ +[source,shell,subs="+quotes"] +---- +export TENANT="**TENANT_NAME**" +export TOPIC="**INPUT_TOPIC_NAME**" +export NAMESPACE="**NAMESPACE_NAME**" # or default +export SINK_NAME="**SINK_CONNECTOR_UNIQUE_NAME**" +export PULSAR_TOKEN="**TENANT_PULSAR_TOKEN**" # API only +export WEB_SERVICE_URL="**TENANT_PULSAR_WEB_SERVICE_URL**" # API only +---- ++ +`**SINK_NAME**` is the name for your new sink connector. +{company} recommends a memorable, human-readable name that summarizes the connector's purpose. +For example: `{connectorType}-sink-prod-us-east-1`. + +. Create the connector using JSON-formatted connector configuration data. +You can pass the configuration directly or with a configuration file. ++ +.pulsar-admin CLI +[source,shell,subs="+attributes"] +---- +./bin/pulsar-admin sinks create \ + --sink-type {connectorType} \ + --name "$SINK_NAME" \ + --inputs "persistent://$TENANT/$NAMESPACE/$TOPIC" \ + --tenant "$TENANT" \ + --sink-config-file configs.json +---- ++ +.{pulsar-short} Admin API +[source,shell] +---- +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME" \ + --header "Authorization: Bearer $PULSAR_TOKEN" \ + --form "sinkConfig=@configs.json;type=application/json" +---- ++ +.Example configuration data structure +include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.adoc[] \ No newline at end of file diff --git a/modules/ROOT/partials/sources/edit.adoc b/modules/ROOT/partials/sources/edit.adoc new file mode 100644 index 00000000..99660e6a --- /dev/null +++ b/modules/ROOT/partials/sources/edit.adoc @@ -0,0 +1,26 @@ +To update a connector, pass the new configuration definition to the connector. +For example, if you created the connector with a configuration file, you can pass an updated configuration file. + +You can include the entire configuration or only the properties that you want to change. +Additionally, some properties can be modified with specific arguments, such as `--parallelism`. + +To get the current configuration, see xref:connectors:index.adoc#get-source-connector-configuration-data[Get source connector configuration data]. + +.pulsar-admin CLI +[source,shell,subs="+attributes"] +---- +./bin/pulsar-admin sources update \ + --source-type {connectorType} \ + --name "$SOURCE_NAME" \ + --destination-topic-name "persistent://$TENANT/$NAMESPACE/$TOPIC" \ + --tenant "$TENANT" \ + --parallelism 2 +---- + +.{pulsar-short} Admin API +[source,shell] +---- +curl -sS --fail -L -X PUT "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME" \ + --header "Authorization: Bearer $PULSAR_TOKEN" \ + --form "sourceConfig=@mynetty-source-config.json;type=application/json" +---- \ No newline at end of file diff --git a/modules/ROOT/partials/sources/get-started.adoc b/modules/ROOT/partials/sources/get-started.adoc new file mode 100644 index 00000000..b521827d --- /dev/null +++ b/modules/ROOT/partials/sources/get-started.adoc @@ -0,0 +1,40 @@ +. Optional: If you are using the `pulsar-admin` CLI or {pulsar-short} Admin API, set the following commonly-used environment variables: ++ +[source,shell,subs="+quotes"] +---- +export TENANT="**TENANT_NAME**" +export TOPIC="**OUTPUT_TOPIC_NAME**" +export NAMESPACE="**NAMESPACE_NAME**" # or default +export SOURCE_NAME="**SOURCE_CONNECTOR_UNIQUE_NAME**" +export PULSAR_TOKEN="**TENANT_PULSAR_TOKEN**" # API only +export WEB_SERVICE_URL="**TENANT_PULSAR_WEB_SERVICE_URL**" # API only +---- ++ +`**SOURCE_NAME**` is the name for your new source connector. +{company} recommends a memorable, human-readable name that summarizes the connector's purpose. +For example: `{connectorType}-source-prod-us-east-1`. + +. Create the connector using JSON-formatted connector configuration data. +You can pass the configuration directly or with a configuration file. ++ +.pulsar-admin CLI +[source,shell,subs="+attributes"] +---- +./bin/pulsar-admin sources create \ + --source-type {connectorType} \ + --name "$SOURCE_NAME" \ + --destination-topic-name "persistent://$TENANT/$NAMESPACE/$TOPIC" \ + --tenant "$TENANT" \ + --source-config-file configs.json +---- ++ +.{pulsar-short} Admin API +[source,shell] +---- +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME" \ + --header "Authorization: Bearer $PULSAR_TOKEN" \ + --form "sourceConfig=@mynetty-source-config.json;type=application/json" +---- ++ +.Example configuration data structure +include::common:streaming:example$connectors/sources/{connectorType}/sample-data.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/index.adoc b/modules/connectors/pages/index.adoc new file mode 100644 index 00000000..d70504f2 --- /dev/null +++ b/modules/connectors/pages/index.adoc @@ -0,0 +1,624 @@ += {pulsar-short} I/O connectors in {product} +:navtitle: {pulsar-short} I/O connectors reference +:page-aliases: operations:io-connectors.adoc + +{product} offers fully-managed versions of https://pulsar.apache.org/docs/en/io-overview/[{pulsar-reg} sink and source connectors]. + +[IMPORTANT] +==== +{product} doesn't support custom sink or source connectors. +==== + +//Reconcile w/ io-connectors in each version & delete io-connectors page. + +//Luna Streaming: Cloud Storage sink, Google BigQuery sink are experimental +//Elasticsearch is lowercase +//Luna Streaming extra Experimental source: Netty +//Luna Streaming -2.x and -1.x have fewer connectors + +[#sink-connectors] +== Sink connectors + +Sink connectors export messages from {pulsar-short} topics to external services. + +Supported sink connectors:: +The following sink connectors are included in {product}: ++ +* xref:connectors/sinks/astra-db.adoc[{astra-db} and {cass} sink] +* xref:connectors/sinks/kafka.adoc[Apache Kafka sink] +* xref:connectors/sinks/kinesis.adoc[AWS Kinesis sink] +* xref:connectors/sinks/cloud-storage.adoc[Cloud Storage sink] +* xref:connectors/sinks/elastic-search.adoc[Elasticsearch sink] +* xref:connectors/sinks/google-bigquery.adoc[Google BigQuery sink] +* xref:connectors/sinks/jdbc-clickhouse.adoc[JDBC ClickHouse sink] +* xref:connectors/sinks/jdbc-mariadb.adoc[JDBC MariaDB sink] +* xref:connectors/sinks/jdbc-postgres.adoc[JDBC PostgreSQL sink] +* xref:connectors/sinks/jdbc-sqllite.adoc[JDBC SQLite sink] +* xref:connectors/sinks/snowflake.adoc[Snowflake sink] + +Experimental sink connectors:: +The following sink connectors are experimental. +They are in development, and they aren't fully supported or tested with {product}. +To get access to experimental connectors, contact {support-url}[{company} Support]. ++ +* Aerospike +* Apache Geode +* Apache HBase +* Apache Kudu +* Apache Phoenix +* Apache Solr +* Batch Data Generator +* CoAP +* Couchbase +* DataDog +* Diffusion +* Flume +* Hazelcast +* HDFS 2 +* HDFS 3 +* Humio +* InfluxDB +* JMS +* Kinetica +* MarkLogic +* Microsoft Azure Data Explorer (Kusto) +* Microsoft Azure DocumentDB +* MongoDB +* MQTT +* Neo4J +* New Relic +* OrientDB +* PLC4X +* RabbitMQ +* Redis +* SAP HANA +* SingleStore +* Splunk +* XTDB +* Zeebe + +[#source-connectors] +== Source connectors + +Source connectors ingest messages from external services into {pulsar-short} topics. + +Supported source connectors:: +The following source connectors are included in {product}: ++ +* xref:connectors/sources/kafka.adoc[Apache Kafka source] +* xref:connectors/sources/kinesis.adoc[AWS Kinesis source] +* xref:connectors/sources/data-generator.adoc[Data Generator source] +* xref:connectors/sources/debezium-mongodb.adoc[Debezium MongoDB source] +* xref:connectors/sources/debezium-mysql.adoc[Debezium MySQL source] +* xref:connectors/sources/debezium-oracle.adoc[Debezium Oracle source] +* xref:connectors/sources/debezium-postgres.adoc[Debezium PostgreSQL source] +* xref:connectors/sources/debezium-sqlserver.adoc[Debezium SQL Server source] + +Experimental source connectors:: +The following source connectors are experimental. +They are in development, and they aren't fully supported or tested with {product}. +To get access to experimental connectors, contact {support-url}[{company} Support]. ++ +* {cass} +* Apache Geode +* Apache Kudu +* Apache Phoenix +* Batch Data Generator +* Big Query +* Canal +* CoAP +* Couchbase +* DataDog +* Diffusion +* DynamoDB +* File +* Flume +* Hazelcast +* Humio +* JMS +* Kinetica +* MarkLogic +* Microsoft Azure DocumentDB +* MongoDB +* MQTT +* Neo4J +* New Relic +* NSQ +* OrientDB +* PLC4X +* RabbitMQ +* Redis +* SAP HANA +* SingleStore +* Splunk +* Twitter +* XTDB +* Zeebe + +== Create and manage connectors + +You can use the {astra-ui}, the `xref:developing:produce-consume-pulsar-client.adoc[pulsar-admin]` CLI, or the xref:apis:api-operations.adoc[{pulsar-short} Admin API] to create, monitor, and manage sink and source connectors. + +//Combine below sections w/ "use the Pulsar CLI" page (separate pulsar-client and pulsar-admin) and Admin API ref pages; then replace below w/ links to specific sections + +=== Get available sink connectors + +Get a list of sink connectors that are available in your {product} {pulsar-short} tenant: + +.pulsar-admin CLI +[source,shell,subs="attributes+"] +---- +./bin/pulsar-admin sinks available-sinks +---- + +.{pulsar-short} Admin API +[source,shell,subs="attributes+"] +---- +curl "$WEB_SERVICE_URL/admin/v3/sinks/builtinsinks" -H "Authorization: $PULSAR_TOKEN" +---- + +=== Get available source connectors + +Get a list of source connectors that are available in your {product} {pulsar-short} tenant: + +.pulsar-admin CLI +[source,shell,subs="attributes+"] +---- +./bin/pulsar-admin sources available-sources +---- + +.{pulsar-short} Admin API +[source,shell,subs="attributes+"] +---- +curl "$WEB_SERVICE_URL/admin/v3/sources/builtinsources" -H "Authorization: $PULSAR_TOKEN" +---- + +=== Create or update a connector + +Although you use the same base commands to create and update all {product} {pulsar-short} connectors, each connector has different configuration options. +For example commands and configuration details, see the documentation for your preferred <> and <>. + +[#get-sink-connector-configuration-data] +=== Get sink connector configuration data + +Get the configuration for an existing sink connector: + +.pulsar-admin CLI +[source,shell] +---- +# Get information about a connector +./bin/pulsar-admin sinks get \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" +---- + +.{pulsar-short} Admin API +[source,shell] +---- +# Get information about a connector +curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +[#get-source-connector-configuration-data] +=== Get source connector configuration data + +Get the configuration for an existing source connector: + +.pulsar-admin CLI +[source,shell] +---- +# Get information about connector +./bin/pulsar-admin sources get \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" +---- + +.{pulsar-short} Admin API +[source,shell] +---- +# Get a connector's information +curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +[#start-a-sink-connector] +=== Start a sink connector + +.pulsar-admin CLI +[source,shell] +---- +# Start all instances of a connector +./bin/pulsar-admin sinks start \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only start an individual instance +---- + +.{pulsar-short} Admin API +[source,shell,subs="attributes+"] +---- +# Start all instances of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/start" \ + --header "Authorization: Bearer $PULSAR_TOKEN" + +# Start an individual instance of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/$SINK_INSTANCE_ID/start" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +[#start-a-source-connector] +=== Start a source connector + +.pulsar-admin CLI +[source,shell] +---- +# Start all instances of a connector +./bin/pulsar-admin sources start \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only start an individual instance +---- + +.{pulsar-short} Admin API +[source,shell] +---- +# Start all instances of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/start" \ + --header "Authorization: Bearer $PULSAR_TOKEN" + +# Start an individual instance of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/$SOURCE_INSTANCE_ID/start" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +[#stop-a-sink-connector] +=== Stop a sink connector + +.pulsar-admin CLI +[source,shell] +---- +# Stop all instances of a connector +./bin/pulsar-admin sinks stop \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only stop an individual instance +---- + +.{pulsar-short} Admin API +[source,shell] +---- +# Stop all instances of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/stop" \ + --header "Authorization: Bearer $PULSAR_TOKEN" + +# Stop an individual instance of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/$SINK_INSTANCE_ID/stop" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +[#stop-a-source-connector] +=== Stop a source connector + +.pulsar-admin CLI +[source,shell] +---- +# Stop all instances of a connector +./bin/pulsar-admin sources stop \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only stop an individual instance +---- + +.{pulsar-short} Admin API +[source,shell] +---- +# Stop all instances of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/stop" \ + --header "Authorization: Bearer $PULSAR_TOKEN" + +# Stop an individual instance of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/$SOURCE_INSTANCE_ID/stop" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +[#restart-a-sink-connector] +=== Restart a sink connector + +.pulsar-admin CLI +[source,shell] +---- +# Restart all instances of a connector +./bin/pulsar-admin sinks restart \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only restart an individual instance +---- + +.{pulsar-short} Admin API +[source,shell] +---- +# Restart all instances of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/restart" \ + --header "Authorization: Bearer $PULSAR_TOKEN" + +# Restart an individual instance of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/$SINK_INSTANCE_ID/restart" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +[#restart-a-source-connector] +=== Restart a source connector + +.pulsar-admin CLI +[source,shell] +---- +# Restart all instances of a connector +./bin/pulsar-admin sources restart \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only restart an individual instance +---- + +.{pulsar-short} Admin API +[source,shell] +---- +# Restart all instances of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/restart" \ + --header "Authorization: Bearer $PULSAR_TOKEN" + +# Restart an individual instance of a connector +curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/$SOURCE_INSTANCE_ID/restart" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +[#get-sink-connector-status] +=== Get sink connector status + +.pulsar-admin CLI +[source,shell] +---- +# Check connector status +./bin/pulsar-admin sinks status \ + --instance-id "$SINK_INSTANCE_ID" \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" +---- + +.{pulsar-short} Admin API +[source,shell] +---- +# Get the status of all connector instances +curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/status" \ + --header "Authorization: Bearer $PULSAR_TOKEN" + +# Get the status of an individual connector instance +curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/$SINK_INSTANCE_ID/status" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +Status response for all connector instances: + +.Result +[source,json] +---- +{ + "numInstances": 0, + "numRunning": 0, + "instances": [ + { + "instanceId": 0, + "status": { + "running": true, + "error": "string", + "numRestarts": 0, + "numReadFromPulsar": 0, + "numSystemExceptions": 0, + "latestSystemExceptions": [ + { + "exceptionString": "string", + "timestampMs": 0 + } + ], + "numSinkExceptions": 0, + "latestSinkExceptions": [ + { + "exceptionString": "string", + "timestampMs": 0 + } + ], + "numWrittenToSink": 0, + "lastReceivedTime": 0, + "workerId": "string" + } + } + ] +} +---- + +Status response for individual connector instance: + +.Result +[source,json] +---- +{ + "running": true, + "error": "string", + "numRestarts": 0, + "numReadFromPulsar": 0, + "numSystemExceptions": 0, + "latestSystemExceptions": [ + { + "exceptionString": "string", + "timestampMs": 0 + } + ], + "numSinkExceptions": 0, + "latestSinkExceptions": [ + { + "exceptionString": "string", + "timestampMs": 0 + } + ], + "numWrittenToSink": 0, + "lastReceivedTime": 0, + "workerId": "string" +} +---- + +[#get-source-connector-status] +=== Get source connector status + +.pulsar-admin CLI +[source,shell] +---- +# Check connector status +./bin/pulsar-admin sources status \ + --instance-id "$SOURCE_INSTANCE_ID" \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" +---- + +.{pulsar-short} Admin API +[source,shell] +---- +# Get the status of all connector instances +curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/status" \ + --header "Authorization: Bearer $PULSAR_TOKEN" + +# Get the status of an individual connector instance +curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/$SOURCE_INSTANCE_ID/status" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +Status response for all connector instances: + +.Result +[source,json] +---- +{ + "numInstances": 0, + "numRunning": 0, + "instances": [ + { + "instanceId": 0, + "status": { + "running": true, + "error": "string", + "numRestarts": 0, + "numReceivedFromSource": 0, + "numSystemExceptions": 0, + "latestSystemExceptions": [ + { + "exceptionString": "string", + "timestampMs": 0 + } + ], + "numSourceExceptions": 0, + "latestSourceExceptions": [ + { + "exceptionString": "string", + "timestampMs": 0 + } + ], + "numWritten": 0, + "lastReceivedTime": 0, + "workerId": "string" + } + } + ] +} +---- + +Status response for individual connector instance: + +.Result +[source,json] +---- +{ + "running": true, + "error": "string", + "numRestarts": 0, + "numReceivedFromSource": 0, + "numSystemExceptions": 0, + "latestSystemExceptions": [ + { + "exceptionString": "string", + "timestampMs": 0 + } + ], + "numSourceExceptions": 0, + "latestSourceExceptions": [ + { + "exceptionString": "string", + "timestampMs": 0 + } + ], + "numWritten": 0, + "lastReceivedTime": 0, + "workerId": "string" +} +---- + +[#delete-a-sink-connector] +=== Delete a sink connector + +.pulsar-admin CLI +[source,shell] +---- +# Delete all instances of a connector +./bin/pulsar-admin sinks delete \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" +---- + +.{pulsar-short} Admin API +[source,shell] +---- +# Delete all instances of a connector +curl -sS --fail -L -X DELETE "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +[#delete-a-source-connector] +=== Delete a source connector + +.pulsar-admin CLI +[source,shell] +---- +# Delete all instances of a connector +./bin/pulsar-admin sources delete \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" +---- + +.{pulsar-short} Admin API +[source,shell] +---- +# Delete all instances of a connector +curl -sS --fail -L -X DELETE "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME" \ + --header "Authorization: Bearer $PULSAR_TOKEN" +---- + +== Connector metrics + +{product} exposes Prometheus-formatted metrics for every connector. +For more information, see xref:operations:astream-scrape-metrics.adoc[]. \ No newline at end of file From 579db291b7e131ba2df592e79913696b4215e38d Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 2 Feb 2026 09:28:55 -0800 Subject: [PATCH 02/15] add scb attributes --- antora.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/antora.yml b/antora.yml index 3702ce40..0c1c1dcb 100644 --- a/antora.yml +++ b/antora.yml @@ -17,4 +17,7 @@ asciidoc: starlight-rabbitmq: 'Starlight for RabbitMQ' pulsar-reg: 'Apache Pulsar(TM)' pulsar: 'Apache Pulsar' - pulsar-short: 'Pulsar' \ No newline at end of file + pulsar-short: 'Pulsar' + scb: 'Secure Connect Bundle (SCB)' + scb-short: 'SCB' + scb-brief: 'Secure Connect Bundle' \ No newline at end of file From 00e51e16c8335e2663c564b13dae6ec0214f6704 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 2 Feb 2026 09:40:04 -0800 Subject: [PATCH 03/15] add sink source pages and nav --- antora.yml | 3 ++- modules/ROOT/nav.adoc | 24 ++++++++++++++++++- modules/connectors/pages/index.adoc | 2 +- modules/connectors/pages/sinks/astra-db.adoc | 1 + .../connectors/pages/sinks/cloud-storage.adoc | 1 + .../pages/sinks/elastic-search.adoc | 1 + .../pages/sinks/google-bigquery.adoc | 1 + .../pages/sinks/jdbc-clickhouse.adoc | 1 + .../connectors/pages/sinks/jdbc-mariadb.adoc | 1 + .../connectors/pages/sinks/jdbc-postgres.adoc | 1 + .../connectors/pages/sinks/jdbc-sqllite.adoc | 1 + modules/connectors/pages/sinks/kafka.adoc | 1 + modules/connectors/pages/sinks/kinesis.adoc | 1 + modules/connectors/pages/sinks/snowflake.adoc | 1 + .../pages/sources/data-generator.adoc | 1 + .../pages/sources/debezium-mongodb.adoc | 1 + .../pages/sources/debezium-mysql.adoc | 1 + .../pages/sources/debezium-oracle.adoc | 1 + .../pages/sources/debezium-postgres.adoc | 1 + .../pages/sources/debezium-sqlserver.adoc | 1 + modules/connectors/pages/sources/kafka.adoc | 1 + modules/connectors/pages/sources/kinesis.adoc | 1 + 22 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 modules/connectors/pages/sinks/astra-db.adoc create mode 100644 modules/connectors/pages/sinks/cloud-storage.adoc create mode 100644 modules/connectors/pages/sinks/elastic-search.adoc create mode 100644 modules/connectors/pages/sinks/google-bigquery.adoc create mode 100644 modules/connectors/pages/sinks/jdbc-clickhouse.adoc create mode 100644 modules/connectors/pages/sinks/jdbc-mariadb.adoc create mode 100644 modules/connectors/pages/sinks/jdbc-postgres.adoc create mode 100644 modules/connectors/pages/sinks/jdbc-sqllite.adoc create mode 100644 modules/connectors/pages/sinks/kafka.adoc create mode 100644 modules/connectors/pages/sinks/kinesis.adoc create mode 100644 modules/connectors/pages/sinks/snowflake.adoc create mode 100644 modules/connectors/pages/sources/data-generator.adoc create mode 100644 modules/connectors/pages/sources/debezium-mongodb.adoc create mode 100644 modules/connectors/pages/sources/debezium-mysql.adoc create mode 100644 modules/connectors/pages/sources/debezium-oracle.adoc create mode 100644 modules/connectors/pages/sources/debezium-postgres.adoc create mode 100644 modules/connectors/pages/sources/debezium-sqlserver.adoc create mode 100644 modules/connectors/pages/sources/kafka.adoc create mode 100644 modules/connectors/pages/sources/kinesis.adoc diff --git a/antora.yml b/antora.yml index 0c1c1dcb..bb966a8e 100644 --- a/antora.yml +++ b/antora.yml @@ -20,4 +20,5 @@ asciidoc: pulsar-short: 'Pulsar' scb: 'Secure Connect Bundle (SCB)' scb-short: 'SCB' - scb-brief: 'Secure Connect Bundle' \ No newline at end of file + scb-brief: 'Secure Connect Bundle' + web-ui: 'Admin Console' \ No newline at end of file diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 3fb44a88..adf74d05 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -23,6 +23,28 @@ * xref:operations:auth.adoc[] * xref:operations:functions.adoc[] * xref:operations:functions-transform.adoc[] -* xref:connectors:index.adoc[] +* I/O connectors +** xref:connectors:index.adoc[] +** Sinks +*** xref:connectors:sinks/astra-db.adoc[] +*** xref:connectors:sinks/cloud-storage.adoc[] +*** xref:connectors:sinks/elastic-search.adoc[] +*** xref:connectors:sinks/google-bigquery.adoc[] +*** xref:connectors:sinks/jdbc-clickhouse.adoc[] +*** xref:connectors:sinks/jdbc-mariadb.adoc[] +*** xref:connectors:sinks/jdbc-postgres.adoc[] +*** xref:connectors:sinks/jsbc-sqllite.adoc[] +*** xref:connectors:sinks/kafka.adoc[] +*** xref:connectors:sinks/kinesis.adoc[] +*** xref:connectors:sinks/snowflake.adoc[] +** Sources +*** xref:connectors:sources/data-generator.adoc[] +*** xref:connectors:sources/debezium-mongodb.adoc[] +*** xref:connectors:sources/debezium-mysql.adoc[] +*** xref:connectors:sources/debezium-oracle.adoc[] +*** xref:connectors:sources/debezium-postgres.adoc[] +*** xref:connectors:sources/debezium-sqlserver.adoc[] +*** xref:connectors:sources/kafka.adoc[] +*** xref:connectors:sources/kinesis.adoc[] * xref:operations:scale-cluster.adoc[] * xref:operations:troubleshooting.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/index.adoc b/modules/connectors/pages/index.adoc index d70504f2..2a221f71 100644 --- a/modules/connectors/pages/index.adoc +++ b/modules/connectors/pages/index.adoc @@ -137,7 +137,7 @@ To get access to experimental connectors, contact {support-url}[{company} Suppor == Create and manage connectors -You can use the {astra-ui}, the `xref:developing:produce-consume-pulsar-client.adoc[pulsar-admin]` CLI, or the xref:apis:api-operations.adoc[{pulsar-short} Admin API] to create, monitor, and manage sink and source connectors. +You can use the {web-ui}, the `xref:developing:produce-consume-pulsar-client.adoc[pulsar-admin]` CLI, or the xref:apis:api-operations.adoc[{pulsar-short} Admin API] to create, monitor, and manage sink and source connectors. //Combine below sections w/ "use the Pulsar CLI" page (separate pulsar-client and pulsar-admin) and Admin API ref pages; then replace below w/ links to specific sections diff --git a/modules/connectors/pages/sinks/astra-db.adoc b/modules/connectors/pages/sinks/astra-db.adoc new file mode 100644 index 00000000..bba17bd0 --- /dev/null +++ b/modules/connectors/pages/sinks/astra-db.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/astra-db.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/cloud-storage.adoc b/modules/connectors/pages/sinks/cloud-storage.adoc new file mode 100644 index 00000000..d817171f --- /dev/null +++ b/modules/connectors/pages/sinks/cloud-storage.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/cloud-storage.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/elastic-search.adoc b/modules/connectors/pages/sinks/elastic-search.adoc new file mode 100644 index 00000000..255f956c --- /dev/null +++ b/modules/connectors/pages/sinks/elastic-search.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/elastic-search.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/google-bigquery.adoc b/modules/connectors/pages/sinks/google-bigquery.adoc new file mode 100644 index 00000000..2f5ea794 --- /dev/null +++ b/modules/connectors/pages/sinks/google-bigquery.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/google-bigquery.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-clickhouse.adoc b/modules/connectors/pages/sinks/jdbc-clickhouse.adoc new file mode 100644 index 00000000..52bc0de9 --- /dev/null +++ b/modules/connectors/pages/sinks/jdbc-clickhouse.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/jdbc-clickhouse.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-mariadb.adoc b/modules/connectors/pages/sinks/jdbc-mariadb.adoc new file mode 100644 index 00000000..b1ab9377 --- /dev/null +++ b/modules/connectors/pages/sinks/jdbc-mariadb.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/jdbc-mariadb.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-postgres.adoc b/modules/connectors/pages/sinks/jdbc-postgres.adoc new file mode 100644 index 00000000..a94fa742 --- /dev/null +++ b/modules/connectors/pages/sinks/jdbc-postgres.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/jdbc-postgres.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-sqllite.adoc b/modules/connectors/pages/sinks/jdbc-sqllite.adoc new file mode 100644 index 00000000..79ba3d50 --- /dev/null +++ b/modules/connectors/pages/sinks/jdbc-sqllite.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/jdbc-sqllite.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/kafka.adoc b/modules/connectors/pages/sinks/kafka.adoc new file mode 100644 index 00000000..eb30fdd9 --- /dev/null +++ b/modules/connectors/pages/sinks/kafka.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/kafka.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/kinesis.adoc b/modules/connectors/pages/sinks/kinesis.adoc new file mode 100644 index 00000000..ea1ceee2 --- /dev/null +++ b/modules/connectors/pages/sinks/kinesis.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/kinesis.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/snowflake.adoc b/modules/connectors/pages/sinks/snowflake.adoc new file mode 100644 index 00000000..a31e2cdc --- /dev/null +++ b/modules/connectors/pages/sinks/snowflake.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/snowflake.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/data-generator.adoc b/modules/connectors/pages/sources/data-generator.adoc new file mode 100644 index 00000000..7b91b223 --- /dev/null +++ b/modules/connectors/pages/sources/data-generator.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/data-generator.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-mongodb.adoc b/modules/connectors/pages/sources/debezium-mongodb.adoc new file mode 100644 index 00000000..ef7baf1d --- /dev/null +++ b/modules/connectors/pages/sources/debezium-mongodb.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/debezium-mongodb.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-mysql.adoc b/modules/connectors/pages/sources/debezium-mysql.adoc new file mode 100644 index 00000000..72a9bca4 --- /dev/null +++ b/modules/connectors/pages/sources/debezium-mysql.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/debezium-mysql.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-oracle.adoc b/modules/connectors/pages/sources/debezium-oracle.adoc new file mode 100644 index 00000000..e47ba969 --- /dev/null +++ b/modules/connectors/pages/sources/debezium-oracle.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/debezium-oracle.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-postgres.adoc b/modules/connectors/pages/sources/debezium-postgres.adoc new file mode 100644 index 00000000..396f9dfc --- /dev/null +++ b/modules/connectors/pages/sources/debezium-postgres.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/debezium-postgres.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-sqlserver.adoc b/modules/connectors/pages/sources/debezium-sqlserver.adoc new file mode 100644 index 00000000..9b6243f6 --- /dev/null +++ b/modules/connectors/pages/sources/debezium-sqlserver.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/debezium-sqlserver.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/kafka.adoc b/modules/connectors/pages/sources/kafka.adoc new file mode 100644 index 00000000..1f5f4232 --- /dev/null +++ b/modules/connectors/pages/sources/kafka.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/kafka.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/kinesis.adoc b/modules/connectors/pages/sources/kinesis.adoc new file mode 100644 index 00000000..bbc36c5a --- /dev/null +++ b/modules/connectors/pages/sources/kinesis.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/kinesis.adoc[] \ No newline at end of file From 950ff7696c1ec5c6bc8460cc8796270d6a99d5a6 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 2 Feb 2026 11:13:21 -0800 Subject: [PATCH 04/15] fix links --- modules/connectors/pages/index.adoc | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/modules/connectors/pages/index.adoc b/modules/connectors/pages/index.adoc index 2a221f71..fdb0938d 100644 --- a/modules/connectors/pages/index.adoc +++ b/modules/connectors/pages/index.adoc @@ -24,17 +24,17 @@ Sink connectors export messages from {pulsar-short} topics to external services. Supported sink connectors:: The following sink connectors are included in {product}: + -* xref:connectors/sinks/astra-db.adoc[{astra-db} and {cass} sink] -* xref:connectors/sinks/kafka.adoc[Apache Kafka sink] -* xref:connectors/sinks/kinesis.adoc[AWS Kinesis sink] -* xref:connectors/sinks/cloud-storage.adoc[Cloud Storage sink] -* xref:connectors/sinks/elastic-search.adoc[Elasticsearch sink] -* xref:connectors/sinks/google-bigquery.adoc[Google BigQuery sink] -* xref:connectors/sinks/jdbc-clickhouse.adoc[JDBC ClickHouse sink] -* xref:connectors/sinks/jdbc-mariadb.adoc[JDBC MariaDB sink] -* xref:connectors/sinks/jdbc-postgres.adoc[JDBC PostgreSQL sink] -* xref:connectors/sinks/jdbc-sqllite.adoc[JDBC SQLite sink] -* xref:connectors/sinks/snowflake.adoc[Snowflake sink] +* xref:connectors:sinks/astra-db.adoc[{astra-db} and {cass} sink] +* xref:connectors:sinks/kafka.adoc[Apache Kafka sink] +* xref:connectors:sinks/kinesis.adoc[AWS Kinesis sink] +* xref:connectors:sinks/cloud-storage.adoc[Cloud Storage sink] +* xref:connectors:sinks/elastic-search.adoc[Elasticsearch sink] +* xref:connectors:sinks/google-bigquery.adoc[Google BigQuery sink] +* xref:connectors:sinks/jdbc-clickhouse.adoc[JDBC ClickHouse sink] +* xref:connectors:sinks/jdbc-mariadb.adoc[JDBC MariaDB sink] +* xref:connectors:sinks/jdbc-postgres.adoc[JDBC PostgreSQL sink] +* xref:connectors:sinks/jdbc-sqllite.adoc[JDBC SQLite sink] +* xref:connectors:sinks/snowflake.adoc[Snowflake sink] Experimental sink connectors:: The following sink connectors are experimental. @@ -85,14 +85,14 @@ Source connectors ingest messages from external services into {pulsar-short} top Supported source connectors:: The following source connectors are included in {product}: + -* xref:connectors/sources/kafka.adoc[Apache Kafka source] -* xref:connectors/sources/kinesis.adoc[AWS Kinesis source] -* xref:connectors/sources/data-generator.adoc[Data Generator source] -* xref:connectors/sources/debezium-mongodb.adoc[Debezium MongoDB source] -* xref:connectors/sources/debezium-mysql.adoc[Debezium MySQL source] -* xref:connectors/sources/debezium-oracle.adoc[Debezium Oracle source] -* xref:connectors/sources/debezium-postgres.adoc[Debezium PostgreSQL source] -* xref:connectors/sources/debezium-sqlserver.adoc[Debezium SQL Server source] +* xref:connectors:sources/kafka.adoc[Apache Kafka source] +* xref:connectors:sources/kinesis.adoc[AWS Kinesis source] +* xref:connectors:sources/data-generator.adoc[Data Generator source] +* xref:connectors:sources/debezium-mongodb.adoc[Debezium MongoDB source] +* xref:connectors:sources/debezium-mysql.adoc[Debezium MySQL source] +* xref:connectors:sources/debezium-oracle.adoc[Debezium Oracle source] +* xref:connectors:sources/debezium-postgres.adoc[Debezium PostgreSQL source] +* xref:connectors:sources/debezium-sqlserver.adoc[Debezium SQL Server source] Experimental source connectors:: The following source connectors are experimental. From 275b939ecef48134172ca202cfddeedf66d86a13 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 2 Feb 2026 13:27:11 -0800 Subject: [PATCH 05/15] unify connector index, remove curl examples --- modules/ROOT/nav.adoc | 4 +- modules/ROOT/partials/sinks/edit.adoc | 9 - modules/ROOT/partials/sinks/get-started.adoc | 15 +- modules/ROOT/partials/sources/edit.adoc | 9 - .../ROOT/partials/sources/get-started.adoc | 15 +- modules/connectors/pages/index.adoc | 473 ++++-------------- modules/operations/pages/io-connectors.adoc | 225 --------- 7 files changed, 92 insertions(+), 658 deletions(-) delete mode 100644 modules/operations/pages/io-connectors.adoc diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index adf74d05..b79ec6ca 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -27,9 +27,9 @@ ** xref:connectors:index.adoc[] ** Sinks *** xref:connectors:sinks/astra-db.adoc[] -*** xref:connectors:sinks/cloud-storage.adoc[] +*** xref:connectors:sinks/cloud-storage.adoc[Cloud Storage (experimental)] *** xref:connectors:sinks/elastic-search.adoc[] -*** xref:connectors:sinks/google-bigquery.adoc[] +*** xref:connectors:sinks/google-bigquery.adoc[Google BigQuery (experimental)] *** xref:connectors:sinks/jdbc-clickhouse.adoc[] *** xref:connectors:sinks/jdbc-mariadb.adoc[] *** xref:connectors:sinks/jdbc-postgres.adoc[] diff --git a/modules/ROOT/partials/sinks/edit.adoc b/modules/ROOT/partials/sinks/edit.adoc index bddd1dd4..90999bb1 100644 --- a/modules/ROOT/partials/sinks/edit.adoc +++ b/modules/ROOT/partials/sinks/edit.adoc @@ -6,7 +6,6 @@ Additionally, some properties can be modified with specific arguments, such as ` To get the current configuration, see xref:connectors:index.adoc#get-sink-connector-configuration-data[Get sink connector configuration data]. -.pulsar-admin CLI [source,shell,subs="+attributes"] ---- ./bin/pulsar-admin sinks update \ @@ -15,12 +14,4 @@ To get the current configuration, see xref:connectors:index.adoc#get-sink-connec --inputs "persistent://$TENANT/$NAMESPACE/$TOPIC" \ --tenant "$TENANT" \ --parallelism 2 ----- - -.{pulsar-short} Admin API -[source,shell] ----- -curl -sS --fail -L -X PUT "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME" \ - --header "Authorization: Bearer $PULSAR_TOKEN" \ - --form "sinkConfig=@configs.json;type=application/json" ---- \ No newline at end of file diff --git a/modules/ROOT/partials/sinks/get-started.adoc b/modules/ROOT/partials/sinks/get-started.adoc index a1fbc4bb..c00eff4f 100644 --- a/modules/ROOT/partials/sinks/get-started.adoc +++ b/modules/ROOT/partials/sinks/get-started.adoc @@ -1,13 +1,11 @@ -. Optional: If you are using the `pulsar-admin` CLI or {pulsar-short} Admin API, set the following commonly-used environment variables: +. Optional: If you are using the `pulsar-admin` CLI, set the following commonly-used environment variables: + [source,shell,subs="+quotes"] ---- export TENANT="**TENANT_NAME**" export TOPIC="**INPUT_TOPIC_NAME**" -export NAMESPACE="**NAMESPACE_NAME**" # or default +export NAMESPACE="**NAMESPACE_NAME**" export SINK_NAME="**SINK_CONNECTOR_UNIQUE_NAME**" -export PULSAR_TOKEN="**TENANT_PULSAR_TOKEN**" # API only -export WEB_SERVICE_URL="**TENANT_PULSAR_WEB_SERVICE_URL**" # API only ---- + `**SINK_NAME**` is the name for your new sink connector. @@ -17,7 +15,6 @@ For example: `{connectorType}-sink-prod-us-east-1`. . Create the connector using JSON-formatted connector configuration data. You can pass the configuration directly or with a configuration file. + -.pulsar-admin CLI [source,shell,subs="+attributes"] ---- ./bin/pulsar-admin sinks create \ @@ -28,13 +25,5 @@ You can pass the configuration directly or with a configuration file. --sink-config-file configs.json ---- + -.{pulsar-short} Admin API -[source,shell] ----- -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME" \ - --header "Authorization: Bearer $PULSAR_TOKEN" \ - --form "sinkConfig=@configs.json;type=application/json" ----- -+ .Example configuration data structure include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.adoc[] \ No newline at end of file diff --git a/modules/ROOT/partials/sources/edit.adoc b/modules/ROOT/partials/sources/edit.adoc index 99660e6a..b51ea3c2 100644 --- a/modules/ROOT/partials/sources/edit.adoc +++ b/modules/ROOT/partials/sources/edit.adoc @@ -6,7 +6,6 @@ Additionally, some properties can be modified with specific arguments, such as ` To get the current configuration, see xref:connectors:index.adoc#get-source-connector-configuration-data[Get source connector configuration data]. -.pulsar-admin CLI [source,shell,subs="+attributes"] ---- ./bin/pulsar-admin sources update \ @@ -15,12 +14,4 @@ To get the current configuration, see xref:connectors:index.adoc#get-source-conn --destination-topic-name "persistent://$TENANT/$NAMESPACE/$TOPIC" \ --tenant "$TENANT" \ --parallelism 2 ----- - -.{pulsar-short} Admin API -[source,shell] ----- -curl -sS --fail -L -X PUT "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME" \ - --header "Authorization: Bearer $PULSAR_TOKEN" \ - --form "sourceConfig=@mynetty-source-config.json;type=application/json" ---- \ No newline at end of file diff --git a/modules/ROOT/partials/sources/get-started.adoc b/modules/ROOT/partials/sources/get-started.adoc index b521827d..9fdda519 100644 --- a/modules/ROOT/partials/sources/get-started.adoc +++ b/modules/ROOT/partials/sources/get-started.adoc @@ -1,13 +1,11 @@ -. Optional: If you are using the `pulsar-admin` CLI or {pulsar-short} Admin API, set the following commonly-used environment variables: +. Optional: If you are using the `pulsar-admin` CLI, set the following commonly-used environment variables: + [source,shell,subs="+quotes"] ---- export TENANT="**TENANT_NAME**" export TOPIC="**OUTPUT_TOPIC_NAME**" -export NAMESPACE="**NAMESPACE_NAME**" # or default +export NAMESPACE="**NAMESPACE_NAME**" export SOURCE_NAME="**SOURCE_CONNECTOR_UNIQUE_NAME**" -export PULSAR_TOKEN="**TENANT_PULSAR_TOKEN**" # API only -export WEB_SERVICE_URL="**TENANT_PULSAR_WEB_SERVICE_URL**" # API only ---- + `**SOURCE_NAME**` is the name for your new source connector. @@ -17,7 +15,6 @@ For example: `{connectorType}-source-prod-us-east-1`. . Create the connector using JSON-formatted connector configuration data. You can pass the configuration directly or with a configuration file. + -.pulsar-admin CLI [source,shell,subs="+attributes"] ---- ./bin/pulsar-admin sources create \ @@ -28,13 +25,5 @@ You can pass the configuration directly or with a configuration file. --source-config-file configs.json ---- + -.{pulsar-short} Admin API -[source,shell] ----- -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME" \ - --header "Authorization: Bearer $PULSAR_TOKEN" \ - --form "sourceConfig=@mynetty-source-config.json;type=application/json" ----- -+ .Example configuration data structure include::common:streaming:example$connectors/sources/{connectorType}/sample-data.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/index.adoc b/modules/connectors/pages/index.adoc index fdb0938d..e82f0e30 100644 --- a/modules/connectors/pages/index.adoc +++ b/modules/connectors/pages/index.adoc @@ -4,17 +4,20 @@ {product} offers fully-managed versions of https://pulsar.apache.org/docs/en/io-overview/[{pulsar-reg} sink and source connectors]. -[IMPORTANT] +[TIP] ==== -{product} doesn't support custom sink or source connectors. +There are three versions of the {company} Luna Streaming distribution. +The `lunastreaming-all` version includes all connectors. ==== //Reconcile w/ io-connectors in each version & delete io-connectors page. - //Luna Streaming: Cloud Storage sink, Google BigQuery sink are experimental //Elasticsearch is lowercase -//Luna Streaming extra Experimental source: Netty -//Luna Streaming -2.x and -1.x have fewer connectors +//3.x are the same + +//Luna Streaming -2.x and -1.x have fewer connectors: +//No experimental +//No Cloud storage, BigQuery, JDBC SQLlite, snowflake, data generator, debezium oracle, debezium sqlserver [#sink-connectors] == Sink connectors @@ -27,9 +30,7 @@ The following sink connectors are included in {product}: * xref:connectors:sinks/astra-db.adoc[{astra-db} and {cass} sink] * xref:connectors:sinks/kafka.adoc[Apache Kafka sink] * xref:connectors:sinks/kinesis.adoc[AWS Kinesis sink] -* xref:connectors:sinks/cloud-storage.adoc[Cloud Storage sink] * xref:connectors:sinks/elastic-search.adoc[Elasticsearch sink] -* xref:connectors:sinks/google-bigquery.adoc[Google BigQuery sink] * xref:connectors:sinks/jdbc-clickhouse.adoc[JDBC ClickHouse sink] * xref:connectors:sinks/jdbc-mariadb.adoc[JDBC MariaDB sink] * xref:connectors:sinks/jdbc-postgres.adoc[JDBC PostgreSQL sink] @@ -38,8 +39,8 @@ The following sink connectors are included in {product}: Experimental sink connectors:: The following sink connectors are experimental. -They are in development, and they aren't fully supported or tested with {product}. -To get access to experimental connectors, contact {support-url}[{company} Support]. +They are available in the `luna-streaming-all` version of {product}, and you can use them in your Luna Streaming cluster. +They are considered experimental because they aren't fully supported or tested with Astra Streaming. + * Aerospike * Apache Geode @@ -48,11 +49,13 @@ To get access to experimental connectors, contact {support-url}[{company} Suppor * Apache Phoenix * Apache Solr * Batch Data Generator +* xref:connectors:sinks/cloud-storage.adoc[Cloud Storage] * CoAP * Couchbase * DataDog * Diffusion * Flume +* xref:connectors:sinks/google-bigquery.adoc[Google BigQuery] * Hazelcast * HDFS 2 * HDFS 3 @@ -96,8 +99,8 @@ The following source connectors are included in {product}: Experimental source connectors:: The following source connectors are experimental. -They are in development, and they aren't fully supported or tested with {product}. -To get access to experimental connectors, contact {support-url}[{company} Support]. +They are available in the `luna-streaming-all` version of {product}, and you can use them in your Luna Streaming cluster. +They are considered experimental because they aren't fully supported or tested with Astra Streaming. + * {cass} * Apache Geode @@ -122,6 +125,7 @@ To get access to experimental connectors, contact {support-url}[{company} Suppor * MongoDB * MQTT * Neo4J +* Netty * New Relic * NSQ * OrientDB @@ -135,55 +139,28 @@ To get access to experimental connectors, contact {support-url}[{company} Suppor * XTDB * Zeebe +[#create-and-manage-connectors] == Create and manage connectors -You can use the {web-ui}, the `xref:developing:produce-consume-pulsar-client.adoc[pulsar-admin]` CLI, or the xref:apis:api-operations.adoc[{pulsar-short} Admin API] to create, monitor, and manage sink and source connectors. +You can use the {web-ui} and the `xref:components:admin-console-tutorial.adoc[pulsar-admin]` CLI to create, monitor, and manage sink and source connectors. -//Combine below sections w/ "use the Pulsar CLI" page (separate pulsar-client and pulsar-admin) and Admin API ref pages; then replace below w/ links to specific sections +Although you use the same base commands to create and update all {product} {pulsar-short} connectors, each connector has different configuration options. +For example commands and configuration details, see the documentation for your preferred <> and <>. -=== Get available sink connectors +=== `pulsar-admin` CLI sink operations +Get available sink connectors:: Get a list of sink connectors that are available in your {product} {pulsar-short} tenant: - -.pulsar-admin CLI -[source,shell,subs="attributes+"] ++ +[source,shell] ---- ./bin/pulsar-admin sinks available-sinks ---- -.{pulsar-short} Admin API -[source,shell,subs="attributes+"] ----- -curl "$WEB_SERVICE_URL/admin/v3/sinks/builtinsinks" -H "Authorization: $PULSAR_TOKEN" ----- - -=== Get available source connectors - -Get a list of source connectors that are available in your {product} {pulsar-short} tenant: - -.pulsar-admin CLI -[source,shell,subs="attributes+"] ----- -./bin/pulsar-admin sources available-sources ----- - -.{pulsar-short} Admin API -[source,shell,subs="attributes+"] ----- -curl "$WEB_SERVICE_URL/admin/v3/sources/builtinsources" -H "Authorization: $PULSAR_TOKEN" ----- - -=== Create or update a connector - -Although you use the same base commands to create and update all {product} {pulsar-short} connectors, each connector has different configuration options. -For example commands and configuration details, see the documentation for your preferred <> and <>. - [#get-sink-connector-configuration-data] -=== Get sink connector configuration data - +Get sink connector configuration data:: Get the configuration for an existing sink connector: - -.pulsar-admin CLI ++ [source,shell] ---- # Get information about a connector @@ -193,41 +170,8 @@ Get the configuration for an existing sink connector: --tenant "$TENANT" ---- -.{pulsar-short} Admin API -[source,shell] ----- -# Get information about a connector -curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME" \ - --header "Authorization: Bearer $PULSAR_TOKEN" ----- - -[#get-source-connector-configuration-data] -=== Get source connector configuration data - -Get the configuration for an existing source connector: - -.pulsar-admin CLI -[source,shell] ----- -# Get information about connector -./bin/pulsar-admin sources get \ - --namespace "$NAMESPACE" \ - --name "$SOURCE_NAME" \ - --tenant "$TENANT" ----- - -.{pulsar-short} Admin API -[source,shell] ----- -# Get a connector's information -curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME" \ - --header "Authorization: Bearer $PULSAR_TOKEN" ----- - -[#start-a-sink-connector] -=== Start a sink connector - -.pulsar-admin CLI +Start a sink connector:: ++ [source,shell] ---- # Start all instances of a connector @@ -239,49 +183,8 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/ # optionally add --instance-id to only start an individual instance ---- -.{pulsar-short} Admin API -[source,shell,subs="attributes+"] ----- -# Start all instances of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/start" \ - --header "Authorization: Bearer $PULSAR_TOKEN" - -# Start an individual instance of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/$SINK_INSTANCE_ID/start" \ - --header "Authorization: Bearer $PULSAR_TOKEN" ----- - -[#start-a-source-connector] -=== Start a source connector - -.pulsar-admin CLI -[source,shell] ----- -# Start all instances of a connector -./bin/pulsar-admin sources start \ - --namespace "$NAMESPACE" \ - --name "$SOURCE_NAME" \ - --tenant "$TENANT" - -# optionally add --instance-id to only start an individual instance ----- - -.{pulsar-short} Admin API -[source,shell] ----- -# Start all instances of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/start" \ - --header "Authorization: Bearer $PULSAR_TOKEN" - -# Start an individual instance of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/$SOURCE_INSTANCE_ID/start" \ - --header "Authorization: Bearer $PULSAR_TOKEN" ----- - -[#stop-a-sink-connector] -=== Stop a sink connector - -.pulsar-admin CLI +Stop a sink connector:: ++ [source,shell] ---- # Stop all instances of a connector @@ -293,49 +196,8 @@ curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE # optionally add --instance-id to only stop an individual instance ---- -.{pulsar-short} Admin API -[source,shell] ----- -# Stop all instances of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/stop" \ - --header "Authorization: Bearer $PULSAR_TOKEN" - -# Stop an individual instance of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/$SINK_INSTANCE_ID/stop" \ - --header "Authorization: Bearer $PULSAR_TOKEN" ----- - -[#stop-a-source-connector] -=== Stop a source connector - -.pulsar-admin CLI -[source,shell] ----- -# Stop all instances of a connector -./bin/pulsar-admin sources stop \ - --namespace "$NAMESPACE" \ - --name "$SOURCE_NAME" \ - --tenant "$TENANT" - -# optionally add --instance-id to only stop an individual instance ----- - -.{pulsar-short} Admin API -[source,shell] ----- -# Stop all instances of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/stop" \ - --header "Authorization: Bearer $PULSAR_TOKEN" - -# Stop an individual instance of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/$SOURCE_INSTANCE_ID/stop" \ - --header "Authorization: Bearer $PULSAR_TOKEN" ----- - -[#restart-a-sink-connector] -=== Restart a sink connector - -.pulsar-admin CLI +Restart a sink connector:: ++ [source,shell] ---- # Restart all instances of a connector @@ -347,49 +209,8 @@ curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE # optionally add --instance-id to only restart an individual instance ---- -.{pulsar-short} Admin API -[source,shell] ----- -# Restart all instances of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/restart" \ - --header "Authorization: Bearer $PULSAR_TOKEN" - -# Restart an individual instance of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/$SINK_INSTANCE_ID/restart" \ - --header "Authorization: Bearer $PULSAR_TOKEN" ----- - -[#restart-a-source-connector] -=== Restart a source connector - -.pulsar-admin CLI -[source,shell] ----- -# Restart all instances of a connector -./bin/pulsar-admin sources restart \ - --namespace "$NAMESPACE" \ - --name "$SOURCE_NAME" \ - --tenant "$TENANT" - -# optionally add --instance-id to only restart an individual instance ----- - -.{pulsar-short} Admin API -[source,shell] ----- -# Restart all instances of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/restart" \ - --header "Authorization: Bearer $PULSAR_TOKEN" - -# Restart an individual instance of a connector -curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/$SOURCE_INSTANCE_ID/restart" \ - --header "Authorization: Bearer $PULSAR_TOKEN" ----- - -[#get-sink-connector-status] -=== Get sink connector status - -.pulsar-admin CLI +Get sink connector status:: ++ [source,shell] ---- # Check connector status @@ -400,222 +221,100 @@ curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE --tenant "$TENANT" ---- -.{pulsar-short} Admin API +Delete a sink connector:: ++ [source,shell] ---- -# Get the status of all connector instances -curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/status" \ - --header "Authorization: Bearer $PULSAR_TOKEN" - -# Get the status of an individual connector instance -curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME/$SINK_INSTANCE_ID/status" \ - --header "Authorization: Bearer $PULSAR_TOKEN" ----- - -Status response for all connector instances: - -.Result -[source,json] ----- -{ - "numInstances": 0, - "numRunning": 0, - "instances": [ - { - "instanceId": 0, - "status": { - "running": true, - "error": "string", - "numRestarts": 0, - "numReadFromPulsar": 0, - "numSystemExceptions": 0, - "latestSystemExceptions": [ - { - "exceptionString": "string", - "timestampMs": 0 - } - ], - "numSinkExceptions": 0, - "latestSinkExceptions": [ - { - "exceptionString": "string", - "timestampMs": 0 - } - ], - "numWrittenToSink": 0, - "lastReceivedTime": 0, - "workerId": "string" - } - } - ] -} +# Delete all instances of a connector +./bin/pulsar-admin sinks delete \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" ---- -Status response for individual connector instance: +=== `pulsar-admin` CLI source operations -.Result -[source,json] +Get available source connectors:: +Get a list of source connectors that are available in your {product} {pulsar-short} tenant: ++ +[source,shell] ---- -{ - "running": true, - "error": "string", - "numRestarts": 0, - "numReadFromPulsar": 0, - "numSystemExceptions": 0, - "latestSystemExceptions": [ - { - "exceptionString": "string", - "timestampMs": 0 - } - ], - "numSinkExceptions": 0, - "latestSinkExceptions": [ - { - "exceptionString": "string", - "timestampMs": 0 - } - ], - "numWrittenToSink": 0, - "lastReceivedTime": 0, - "workerId": "string" -} +./bin/pulsar-admin sources available-sources ---- -[#get-source-connector-status] -=== Get source connector status - -.pulsar-admin CLI +[#get-source-connector-configuration-data] +Get source connector configuration data:: +Get the configuration for an existing source connector: ++ [source,shell] ---- -# Check connector status -./bin/pulsar-admin sources status \ - --instance-id "$SOURCE_INSTANCE_ID" \ +# Get information about connector +./bin/pulsar-admin sources get \ --namespace "$NAMESPACE" \ --name "$SOURCE_NAME" \ --tenant "$TENANT" ---- -.{pulsar-short} Admin API +Start a source connector:: ++ [source,shell] ---- -# Get the status of all connector instances -curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/status" \ - --header "Authorization: Bearer $PULSAR_TOKEN" +# Start all instances of a connector +./bin/pulsar-admin sources start \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" -# Get the status of an individual connector instance -curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME/$SOURCE_INSTANCE_ID/status" \ - --header "Authorization: Bearer $PULSAR_TOKEN" +# optionally add --instance-id to only start an individual instance ---- -Status response for all connector instances: - -.Result -[source,json] ----- -{ - "numInstances": 0, - "numRunning": 0, - "instances": [ - { - "instanceId": 0, - "status": { - "running": true, - "error": "string", - "numRestarts": 0, - "numReceivedFromSource": 0, - "numSystemExceptions": 0, - "latestSystemExceptions": [ - { - "exceptionString": "string", - "timestampMs": 0 - } - ], - "numSourceExceptions": 0, - "latestSourceExceptions": [ - { - "exceptionString": "string", - "timestampMs": 0 - } - ], - "numWritten": 0, - "lastReceivedTime": 0, - "workerId": "string" - } - } - ] -} +Stop a source connector:: ++ +[source,shell] ---- +# Stop all instances of a connector +./bin/pulsar-admin sources stop \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" -Status response for individual connector instance: - -.Result -[source,json] ----- -{ - "running": true, - "error": "string", - "numRestarts": 0, - "numReceivedFromSource": 0, - "numSystemExceptions": 0, - "latestSystemExceptions": [ - { - "exceptionString": "string", - "timestampMs": 0 - } - ], - "numSourceExceptions": 0, - "latestSourceExceptions": [ - { - "exceptionString": "string", - "timestampMs": 0 - } - ], - "numWritten": 0, - "lastReceivedTime": 0, - "workerId": "string" -} +# optionally add --instance-id to only stop an individual instance ---- -[#delete-a-sink-connector] -=== Delete a sink connector - -.pulsar-admin CLI +Restart a source connector:: ++ [source,shell] ---- -# Delete all instances of a connector -./bin/pulsar-admin sinks delete \ +# Restart all instances of a connector +./bin/pulsar-admin sources restart \ --namespace "$NAMESPACE" \ - --name "$SINK_NAME" \ + --name "$SOURCE_NAME" \ --tenant "$TENANT" ----- -.{pulsar-short} Admin API -[source,shell] ----- -# Delete all instances of a connector -curl -sS --fail -L -X DELETE "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$SINK_NAME" \ - --header "Authorization: Bearer $PULSAR_TOKEN" +# optionally add --instance-id to only restart an individual instance ---- -[#delete-a-source-connector] -=== Delete a source connector - -.pulsar-admin CLI +Get source connector status:: ++ [source,shell] ---- -# Delete all instances of a connector -./bin/pulsar-admin sources delete \ +# Check connector status +./bin/pulsar-admin sources status \ + --instance-id "$SOURCE_INSTANCE_ID" \ --namespace "$NAMESPACE" \ --name "$SOURCE_NAME" \ --tenant "$TENANT" ---- -.{pulsar-short} Admin API +Delete a source connector:: ++ [source,shell] ---- # Delete all instances of a connector -curl -sS --fail -L -X DELETE "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/$SOURCE_NAME" \ - --header "Authorization: Bearer $PULSAR_TOKEN" +./bin/pulsar-admin sources delete \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" ---- == Connector metrics diff --git a/modules/operations/pages/io-connectors.adoc b/modules/operations/pages/io-connectors.adoc deleted file mode 100644 index 31800cd1..00000000 --- a/modules/operations/pages/io-connectors.adoc +++ /dev/null @@ -1,225 +0,0 @@ -= Luna Streaming IO connectors - -When you have Luna Streaming xref:install-upgrade:quickstart-server-installs.adoc[installed] and running, add IO connectors to connect your deployment to external systems like https://cassandra.apache.org/_/index.html[Apache Cassandra], https://www.elastic.co/[ElasticSearch], and more. - -[NOTE] -==== -There are three versions of the {company} Luna Streaming distribution. -The lunastreaming-all version contains all connectors. -==== - -[#sink-connectors] -== Sink Connectors - -[#astradb-sink] -=== AstraDB sink - -The AstraDB sink connector reads messages from {pulsar} topics and writes them to AstraDB systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/astra-db.adoc[AstraDB sink documentation] - -[#elasticsearch-sink] -=== ElasticSearch sink - -The Elasticsearch sink connector reads messages from {pulsar} topics and writes them to Elasticsearch systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/elastic-search.adoc[Elasticsearch sink documentation] - -[#jdbc-clickhouse-sink] -=== JDBC-Clickhouse sink - -The JDBC-ClickHouse sink connector reads messages from {pulsar} topics and writes them to JDBC-ClickHouse systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/jdbc-clickhouse.adoc[JDBC ClickHouse sink documentation] - -[#jdbc-mariadb-sink] -=== JDBC-MariaDB sink - -The JDBC-MariaDB sink connector reads messages from {pulsar} topics and writes them to JDBC-MariaDB systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/jdbc-mariadb.adoc[JDBC MariaDB sink documentation] - -[#jdbc-postgres-sink] -=== JDBC-PostgreSQL sink - -The JDBC-PostgreSQL sink connector reads messages from {pulsar} topics and writes them to JDBC-PostgreSQL systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/jdbc-postgres.adoc[JDBC PostgreSQL sink documentation] - -[#jdbc-sqlite-sink] -=== JDBC-SQLite - -The JDBC-SQLite sink connector reads messages from {pulsar} topics and writes them to JDBC-SQLite systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/jdbc-sqllite.adoc[JDBC SQLite sink documentation] - -[#kafka-sink] -=== Kafka - -The Kafka sink connector reads messages from {pulsar} topics and writes them to Kafka systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/kafka.adoc[Kafka sink documentation] - -[#kinesis-sink] -=== Kinesis - -The Kinesis sink connector reads messages from {pulsar} topics and writes them to Kinesis systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/kinesis.adoc[Kinesis sink documentation] - -[#snowflake-sink] -=== Snowflake - -The Snowflake sink connector reads messages from {pulsar} topics and writes them to Snowflake systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/snowflake.adoc[Snowflake sink documentation] - -[#source-connectors] -== Source Connectors - -[#datagenerator-source] -=== Data Generator source - -The Data generator source connector produces messages for testing and persists the messages to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/data-generator.adoc[Data Generator source documentation] - -[#debezium-mongodb-source] -=== Debezium MongoDB source - -The Debezium MongoDB source connector reads data from Debezium MongoDB systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/debezium-mongodb.adoc[Debezium MongoDB source documentation] - -[#debezium-mysql-source] -=== Debezium MySQL source - -The Debezium MySQL source connector reads data from Debezium MySQL systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/debezium-mysql.adoc[Debezium MySQL source documentation] - -[#debezium-oracle-source] -=== Debezium Oracle source - -The Debezium Oracle source connector reads data from Debezium Oracle systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/debezium-oracle.adoc[Debezium Oracle source documentation] - -[#debezium-postgres-source] -=== Debezium Postgres source - -The Debezium PostgreSQL source connector reads data from Debezium PostgreSQL systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/debezium-postgres.adoc[Debezium PostgreSQL source documentation] - -[#debezium-sql-server-source] -=== Debezium SQL Server source - -The Debezium SQL Server source connector reads data from Debezium SQL Server systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/debezium-sqlserver.adoc[Debezium SQL Server source documentation] - -[#kafka-source] -=== Kafka source - -The Kafka source connector reads data from Kafka systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/kafka.adoc[Kafka source connector documentation] - -[#kinesis-source] -=== AWS Kinesis source - -The AWS Kinesis source connector reads data from Kinesis systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/kinesis.adoc[Kinesis source connector documentation] - -== Experimental Connectors - -{company} is always experimenting with connectors. -Below are the connectors available in the luna-streaming-all version of *Luna Streaming*. -We call these *experimental connectors* because they have not yet been promoted to official support in our *Astra Streaming* clusters, but they will work in your Luna Streaming cluster. - -[#sink-experimental] -=== Sink Connectors (experimental) - -* Cloud Storage -* Kinetica -* Aerospike -* Azure DocumentDB -* Azure Data Explorer (Kusto) -* Batch Data Generator -* Big Query -* CoAP -* Couchbase -* DataDog -* Diffusion -* Flume -* Apache Geode -* Hazelcast -* Apache HBase -* HDFS 2 -* HDFS 3 -* Humio -* InfluxDB -* JMS -* Apache Kudu -* MarkLogic -* MongoDB -* MQTT -* Neo4J -* New Relic -* OrientDB -* Apache Phoenix -* PLC4X -* RabbitMQ -* Redis -* SAP HANA -* SingleStore -* Apache Solr -* Splunk -* XTDB -* Zeebe - -[#source-experimental] -=== Source Connectors (experimental) - -* Cassandra Source -* Kinetica -* Azure DocumentDB -* Batch Data Generator -* Big Query -* canal -* CoAP -* Couchbase -* datadog -* diffusion -* DynamoDB -* file -* flume -* Apache Geode -* Hazelcast -* Humio -* JMS -* Apache Kudu -* MarkLogic -* MongoDB -* MQTT -* Neo4J -* Netty -* New Relic -* NSQ -* OrientDB -* Apache Phoenix -* PLC4X -* RabbitMQ -* Redis -* SAP HANA -* SingleStore -* Splunk -* Twitter -* XTDB -* Zeebe - -== Next steps - -For more on connectors and {pulsar-short}, see xref:streaming-learning:pulsar-io:connectors/index.adoc[] and the https://pulsar.apache.org/docs/2.11.x/io-overview/[{pulsar-short} documentation]. \ No newline at end of file From 32f91fd4c82de52a89032d0dca4b9d4b866f2cc4 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 2 Feb 2026 13:50:39 -0800 Subject: [PATCH 06/15] remove comment --- modules/connectors/pages/index.adoc | 9 --------- 1 file changed, 9 deletions(-) diff --git a/modules/connectors/pages/index.adoc b/modules/connectors/pages/index.adoc index e82f0e30..3fdd5d47 100644 --- a/modules/connectors/pages/index.adoc +++ b/modules/connectors/pages/index.adoc @@ -10,15 +10,6 @@ There are three versions of the {company} Luna Streaming distribution. The `lunastreaming-all` version includes all connectors. ==== -//Reconcile w/ io-connectors in each version & delete io-connectors page. -//Luna Streaming: Cloud Storage sink, Google BigQuery sink are experimental -//Elasticsearch is lowercase -//3.x are the same - -//Luna Streaming -2.x and -1.x have fewer connectors: -//No experimental -//No Cloud storage, BigQuery, JDBC SQLlite, snowflake, data generator, debezium oracle, debezium sqlserver - [#sink-connectors] == Sink connectors From e0a0f9da7888e8ec1a98120f5438471fb075d4b7 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 05:15:33 -0800 Subject: [PATCH 07/15] missing attributes --- antora.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/antora.yml b/antora.yml index bb966a8e..59d2e021 100644 --- a/antora.yml +++ b/antora.yml @@ -13,6 +13,7 @@ asciidoc: pulsar-version: '3.1' admin-console-version: '2.0.4' heartbeat-version: '1.0.12' + debezium-version: '1.7' starlight-kafka: 'Starlight for Kafka' starlight-rabbitmq: 'Starlight for RabbitMQ' pulsar-reg: 'Apache Pulsar(TM)' @@ -21,4 +22,9 @@ asciidoc: scb: 'Secure Connect Bundle (SCB)' scb-short: 'SCB' scb-brief: 'Secure Connect Bundle' - web-ui: 'Admin Console' \ No newline at end of file + web-ui: 'Admin Console' + astra: 'Astra' + astra-db: 'Astra DB' + cass-reg: 'Apache Cassandra(R)' + cass: 'Apache Cassandra' + cass-short: 'Cassandra' \ No newline at end of file From caf1cefcd4f33e81c518dfd3c47da95867aee833 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 05:35:47 -0800 Subject: [PATCH 08/15] attributes hacks, missing attributes --- antora.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/antora.yml b/antora.yml index 59d2e021..45ade280 100644 --- a/antora.yml +++ b/antora.yml @@ -13,18 +13,24 @@ asciidoc: pulsar-version: '3.1' admin-console-version: '2.0.4' heartbeat-version: '1.0.12' - debezium-version: '1.7' starlight-kafka: 'Starlight for Kafka' starlight-rabbitmq: 'Starlight for RabbitMQ' pulsar-reg: 'Apache Pulsar(TM)' pulsar: 'Apache Pulsar' pulsar-short: 'Pulsar' - scb: 'Secure Connect Bundle (SCB)' - scb-short: 'SCB' - scb-brief: 'Secure Connect Bundle' + + # Required for include::common partials that are shared with Astra Streaming web-ui: 'Admin Console' astra: 'Astra' astra-db: 'Astra DB' + scb: 'Secure Connect Bundle (SCB)' + scb-short: 'SCB' + scb-brief: 'Secure Connect Bundle' cass-reg: 'Apache Cassandra(R)' cass: 'Apache Cassandra' - cass-short: 'Cassandra' \ No newline at end of file + cass-short: 'Cassandra' + debezium-version: '1.7' + get-started-sink: 'include::ROOT:partial$sinks/get-started.adoc[]' + edit-sink: 'include::ROOT:partial$sinks/edit.adoc[]' + get-started-source: 'include::ROOT:partial$sources/get-started.adoc[]' + edit-source: 'include::ROOT:partial$sources/edit.adoc[]' \ No newline at end of file From 602bec30dbbb17914fc162ae3aa40452ed9b048e Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 05:37:13 -0800 Subject: [PATCH 09/15] fix nav --- modules/ROOT/nav.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index b79ec6ca..d695ef03 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -33,7 +33,7 @@ *** xref:connectors:sinks/jdbc-clickhouse.adoc[] *** xref:connectors:sinks/jdbc-mariadb.adoc[] *** xref:connectors:sinks/jdbc-postgres.adoc[] -*** xref:connectors:sinks/jsbc-sqllite.adoc[] +*** xref:connectors:sinks/jdbc-sqllite.adoc[] *** xref:connectors:sinks/kafka.adoc[] *** xref:connectors:sinks/kinesis.adoc[] *** xref:connectors:sinks/snowflake.adoc[] From 22228c076cced43ca814d60834caa299fab22380 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 05:44:56 -0800 Subject: [PATCH 10/15] fix link --- modules/connectors/pages/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/connectors/pages/index.adoc b/modules/connectors/pages/index.adoc index 3fdd5d47..f6f300f3 100644 --- a/modules/connectors/pages/index.adoc +++ b/modules/connectors/pages/index.adoc @@ -311,4 +311,4 @@ Delete a source connector:: == Connector metrics {product} exposes Prometheus-formatted metrics for every connector. -For more information, see xref:operations:astream-scrape-metrics.adoc[]. \ No newline at end of file +For more information, see xref:components:pulsar-monitor.adoc[]. \ No newline at end of file From 18b630239dfd5f08593ee6b069e020adcf24c347 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 07:02:54 -0800 Subject: [PATCH 11/15] bring main topic content --- modules/ROOT/nav.adoc | 4 +- modules/connectors/pages/sinks/astra-db.adoc | 48 ++++++++++++++- .../connectors/pages/sinks/cloud-storage.adoc | 36 +++++++++++- .../pages/sinks/elastic-search.adoc | 38 +++++++++++- .../pages/sinks/google-bigquery.adoc | 36 +++++++++++- .../pages/sinks/jdbc-clickhouse.adoc | 35 ++++++++++- .../connectors/pages/sinks/jdbc-mariadb.adoc | 35 ++++++++++- .../connectors/pages/sinks/jdbc-postgres.adoc | 35 ++++++++++- .../connectors/pages/sinks/jdbc-sqllite.adoc | 35 ++++++++++- modules/connectors/pages/sinks/kafka.adoc | 38 +++++++++++- modules/connectors/pages/sinks/kinesis.adoc | 40 ++++++++++++- modules/connectors/pages/sinks/snowflake.adoc | 35 ++++++++++- .../pages/sources/data-generator.adoc | 35 ++++++++++- .../pages/sources/debezium-mongodb.adoc | 43 +++++++++++++- .../pages/sources/debezium-mysql.adoc | 38 +++++++++++- .../pages/sources/debezium-oracle.adoc | 41 ++++++++++++- .../pages/sources/debezium-postgres.adoc | 38 +++++++++++- .../pages/sources/debezium-sqlserver.adoc | 58 ++++++++++++++++++- modules/connectors/pages/sources/kafka.adoc | 33 ++++++++++- modules/connectors/pages/sources/kinesis.adoc | 33 ++++++++++- 20 files changed, 713 insertions(+), 21 deletions(-) diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index d695ef03..e2ad1353 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -27,9 +27,9 @@ ** xref:connectors:index.adoc[] ** Sinks *** xref:connectors:sinks/astra-db.adoc[] -*** xref:connectors:sinks/cloud-storage.adoc[Cloud Storage (experimental)] +*** xref:connectors:sinks/cloud-storage.adoc[] *** xref:connectors:sinks/elastic-search.adoc[] -*** xref:connectors:sinks/google-bigquery.adoc[Google BigQuery (experimental)] +*** xref:connectors:sinks/google-bigquery.adoc[] *** xref:connectors:sinks/jdbc-clickhouse.adoc[] *** xref:connectors:sinks/jdbc-mariadb.adoc[] *** xref:connectors:sinks/jdbc-postgres.adoc[] diff --git a/modules/connectors/pages/sinks/astra-db.adoc b/modules/connectors/pages/sinks/astra-db.adoc index bba17bd0..b98105ed 100644 --- a/modules/connectors/pages/sinks/astra-db.adoc +++ b/modules/connectors/pages/sinks/astra-db.adoc @@ -1 +1,47 @@ -include::common:streaming:partial$connectors/sinks/astra-db.adoc[] \ No newline at end of file += {astra-db} and {cass} +:connectorType: cassandra-enhanced + +include::common:streaming:partial$connectors/sinks/astra-db-intro.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +[TIP] +==== +The preceding example connects to a self-managed {cass-short} cluster. + +For an {astra-db} example, see <>. + +For all connection properties, see <>. +==== + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== {company} {pulsar} sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/astra-db-properties.adoc[] + +[#topic] +=== Topic-to-table mapping properties (`topic`) + +include::common:streaming:partial$connectors/sinks/astra-db-topic.adoc[] + +== See also + +* https://github.com/datastax/pulsar-sink[{company} {pulsar} connector GitHub repository] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/cloud-storage.adoc b/modules/connectors/pages/sinks/cloud-storage.adoc index d817171f..bdf936e3 100644 --- a/modules/connectors/pages/sinks/cloud-storage.adoc +++ b/modules/connectors/pages/sinks/cloud-storage.adoc @@ -1 +1,35 @@ -include::common:streaming:partial$connectors/sinks/cloud-storage.adoc[] \ No newline at end of file += Cloud Storage (experimental) +:connectorType: cloud-storage + +include::common:streaming:partial$connectors/sinks/cloud-storage-intro.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +[#pulsar-sink-connector-properties] +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +== Cloud Storage sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/cloud-storage-properties.adoc[] + +[#data-format-types] +== Data format types + +include::common:streaming:partial$connectors/sinks/cloud-storage-format-types.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/elastic-search.adoc b/modules/connectors/pages/sinks/elastic-search.adoc index 255f956c..310fb2de 100644 --- a/modules/connectors/pages/sinks/elastic-search.adoc +++ b/modules/connectors/pages/sinks/elastic-search.adoc @@ -1 +1,37 @@ -include::common:streaming:partial$connectors/sinks/elastic-search.adoc[] \ No newline at end of file += Elasticsearch +:connectorType: elastic_search + +The Elasticsearch sink connector reads messages from {pulsar-short} topics and writes them to https://www.elastic.co/elasticsearch/[Elasticsearch]. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sinks/elastic-search-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== Elasticsearch sink connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://pulsar.apache.org/docs/io-elasticsearch-sink[OSS {pulsar} Elasticsearch sink connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that are only present in <>. \ No newline at end of file diff --git a/modules/connectors/pages/sinks/google-bigquery.adoc b/modules/connectors/pages/sinks/google-bigquery.adoc index 2f5ea794..df3db75c 100644 --- a/modules/connectors/pages/sinks/google-bigquery.adoc +++ b/modules/connectors/pages/sinks/google-bigquery.adoc @@ -1 +1,35 @@ -include::common:streaming:partial$connectors/sinks/google-bigquery.adoc[] \ No newline at end of file += Google BigQuery (experimental) +:connectorType: bigquery + +include::common:streaming:partial$connectors/sinks/google-bigquery-intro.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +[#pulsar-sink-connector-properties] +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== Kafka Connect adaptor properties (`configs`) + +include::common:streaming:partial$connectors/sinks/google-bigquery-properties.adoc[] + +[#kafkaConnectorConfigProperties] +=== Kafka Connect BigQuery Sink properties (`kafkaConnectorConfigProperties`) + +include::common:streaming:partial$connectors/sinks/google-bigquery-sink-properties.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-clickhouse.adoc b/modules/connectors/pages/sinks/jdbc-clickhouse.adoc index 52bc0de9..c7935fff 100644 --- a/modules/connectors/pages/sinks/jdbc-clickhouse.adoc +++ b/modules/connectors/pages/sinks/jdbc-clickhouse.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sinks/jdbc-clickhouse.adoc[] \ No newline at end of file += JDBC ClickHouse +:connectorType: jdbc-clickhouse + +You can use the JDBC ClickHouse sink connector to stream data from {pulsar-short} topics into https://clickhouse.com/[ClickHouse] tables. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sinks/jdbc-clickhouse-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== JDBC ClickHouse sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/jdbc-config-params.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-mariadb.adoc b/modules/connectors/pages/sinks/jdbc-mariadb.adoc index b1ab9377..6530017d 100644 --- a/modules/connectors/pages/sinks/jdbc-mariadb.adoc +++ b/modules/connectors/pages/sinks/jdbc-mariadb.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sinks/jdbc-mariadb.adoc[] \ No newline at end of file += JDBC MariaDB +:connectorType: jdbc-mariadb + +You can use the JDBC MariaDB sink connector to stream data from {pulsar-short} topics into https://mariadb.org/[MariaDB] tables. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sinks/jdbc-mariadb-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== JDBC MariaDB sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/jdbc-config-params.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-postgres.adoc b/modules/connectors/pages/sinks/jdbc-postgres.adoc index a94fa742..447ce52b 100644 --- a/modules/connectors/pages/sinks/jdbc-postgres.adoc +++ b/modules/connectors/pages/sinks/jdbc-postgres.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sinks/jdbc-postgres.adoc[] \ No newline at end of file += JDBC PostgreSQL +:connectorType: jdbc-postgres + +You can use the JDBC PostgreSQL sink connector to stream data from {pulsar-short} topics into https://www.postgresql.org/[PostgreSQL] tables. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sinks/jdbc-postgres-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== JDBC PostgreSQL sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/jdbc-config-params.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-sqllite.adoc b/modules/connectors/pages/sinks/jdbc-sqllite.adoc index 79ba3d50..40cd6a85 100644 --- a/modules/connectors/pages/sinks/jdbc-sqllite.adoc +++ b/modules/connectors/pages/sinks/jdbc-sqllite.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sinks/jdbc-sqllite.adoc[] \ No newline at end of file += JDBC SQLite +:connectorType: jdbc-sqlite + +You can use the JDBC SQLite sink connector to stream data from {pulsar-short} topics into https://www.sqlite.org/index.html[SQLite] tables. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sinks/jdbc-sqllite-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== JDBC SQLite sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/jdbc-config-params.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/kafka.adoc b/modules/connectors/pages/sinks/kafka.adoc index eb30fdd9..1586b92c 100644 --- a/modules/connectors/pages/sinks/kafka.adoc +++ b/modules/connectors/pages/sinks/kafka.adoc @@ -1 +1,37 @@ -include::common:streaming:partial$connectors/sinks/kafka.adoc[] \ No newline at end of file += Kafka +:connectorType: kafka + +The Kafka sink connector reads messages from {pulsar-short} topics and writes them to https://kafka.apache.org/[Kafka] topics. + +[#compatibility] +== Compatibility + +{product} supports {pulsar-reg} {pulsar-version}, which uses the https://github.com/apache/kafka/tree/2.7[Kafka 2.7.2 library]. + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== Kafka sink connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://pulsar.apache.org/docs/io-kafka-sink[OSS {pulsar} Kafka sink connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that aren't present in {pulsar} {pulsar-version}. \ No newline at end of file diff --git a/modules/connectors/pages/sinks/kinesis.adoc b/modules/connectors/pages/sinks/kinesis.adoc index ea1ceee2..0d8def28 100644 --- a/modules/connectors/pages/sinks/kinesis.adoc +++ b/modules/connectors/pages/sinks/kinesis.adoc @@ -1 +1,39 @@ -include::common:streaming:partial$connectors/sinks/kinesis.adoc[] \ No newline at end of file += Kinesis +:connectorType: kinesis + +The Kinesis sink connector reads messages from {pulsar-short} topics and writes them to https://aws.amazon.com/kinesis/[Amazon Kinesis]. + +[#compatibility] +== Compatibility + +The Amazon Kinesis Client Library for Java (Amazon KCL) is used to consume and process data from Amazon Kinesis in Java applications. + +{product} supports {pulsar-reg} {pulsar-version}, which uses the https://github.com/awslabs/amazon-kinesis-client[Amazon Kinesis Client 2.2.8 library] and the https://github.com/aws/aws-sdk-java[AWS Java SDK 0.14.0 library]. + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== Kinesis sink connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://pulsar.apache.org/docs/io-kinesis-sink[OSS {pulsar} Kinesis sink connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that aren't present in {pulsar} {pulsar-version}. \ No newline at end of file diff --git a/modules/connectors/pages/sinks/snowflake.adoc b/modules/connectors/pages/sinks/snowflake.adoc index a31e2cdc..43f7e0df 100644 --- a/modules/connectors/pages/sinks/snowflake.adoc +++ b/modules/connectors/pages/sinks/snowflake.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sinks/snowflake.adoc[] \ No newline at end of file += Snowflake +:connectorType: snowflake + +A https://www.snowflake.com/en/[Snowflake] database ingests structured and semi-structured datasets for processing and analysis. +Snowflake automatically manages all parts of the data storage process, including organization, structure, metadata, file size, compression, and statistics. + +The Snowflake sink connector reads messages from {pulsar-short} topics and writes them to Snowflake databases. + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== Snowflake sink connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +For all supported properties, examples, and exceptions, see the https://github.com/datastax/snowflake-connector[{company} Snowflake connector GitHub repository]. \ No newline at end of file diff --git a/modules/connectors/pages/sources/data-generator.adoc b/modules/connectors/pages/sources/data-generator.adoc index 7b91b223..ae31b23e 100644 --- a/modules/connectors/pages/sources/data-generator.adoc +++ b/modules/connectors/pages/sources/data-generator.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sources/data-generator.adoc[] \ No newline at end of file += Data Generator +:connectorType: data-generator + +include::common:streaming:partial$connectors/sources/data-generator-intro.adoc[] + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/data-generator-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Data Generator source connector properties (`configs`) + +include::common:streaming:partial$connectors/sources/data-generator-properties.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-mongodb.adoc b/modules/connectors/pages/sources/debezium-mongodb.adoc index ef7baf1d..9463b576 100644 --- a/modules/connectors/pages/sources/debezium-mongodb.adoc +++ b/modules/connectors/pages/sources/debezium-mongodb.adoc @@ -1 +1,42 @@ -include::common:streaming:partial$connectors/sources/debezium-mongodb.adoc[] \ No newline at end of file += Debezium MongoDB +:connectorType: debezium-mongodb + +The Debezium MongoDB source connector tracks either a MongoDB replica set or a MongoDB sharded cluster for document changes in databases and collections. +It streams those changes as messages to a {pulsar-short} topic. + +The connector automatically handles the following: + +* Addition and removal of shards in a sharded cluster. +* Changes in membership for each replica set. +* Elections within each replica set. +* Resolution of communications issues with replica set members. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/debezium-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Debezium MongoDB source connector properties (`configs`) + +include::common:streaming:partial$connectors/sources/debezium-mongodb-propeties.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-mysql.adoc b/modules/connectors/pages/sources/debezium-mysql.adoc index 72a9bca4..5c979ab0 100644 --- a/modules/connectors/pages/sources/debezium-mysql.adoc +++ b/modules/connectors/pages/sources/debezium-mysql.adoc @@ -1 +1,37 @@ -include::common:streaming:partial$connectors/sources/debezium-mysql.adoc[] \ No newline at end of file += Debezium MySQL +:connectorType: debezium-mysql + +The Debezium MySQL source connector reads the `binlog` from MySQL database servers, produces change events for row-level `INSERT`, `UPDATE`, and `DELETE` operations, and then sends change event messages to {pulsar-short} topics. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/debezium-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Debezium MySQL source connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-connector-properties[Debezium connector for MySQL] and the https://pulsar.apache.org/docs/io-debezium-source[OSS {pulsar} Debezium source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that are only present in <>. \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-oracle.adoc b/modules/connectors/pages/sources/debezium-oracle.adoc index e47ba969..2f609f36 100644 --- a/modules/connectors/pages/sources/debezium-oracle.adoc +++ b/modules/connectors/pages/sources/debezium-oracle.adoc @@ -1 +1,40 @@ -include::common:streaming:partial$connectors/sources/debezium-oracle.adoc[] \ No newline at end of file += Debezium Oracle +:connectorType: debezium-oracle + +The Debezium Oracle source connector captures and records row-level changes that occur in databases on Oracle servers, including tables that are added while the connector is running. + +You can configure the connector according to the events and data that you want to stream. +For example, you can configure it to emit change events for a subset of schemas and tables, or configure it to ignore, mask, or truncate values in specific columns. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/debezium-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Debezium Oracle source connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://debezium.io/documentation/reference/stable/connectors/oracle.html#oracle-connector-properties[Debezium connector for Oracle] and the https://pulsar.apache.org/docs/io-debezium-source[OSS {pulsar} Debezium source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that are only present in <>. \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-postgres.adoc b/modules/connectors/pages/sources/debezium-postgres.adoc index 396f9dfc..2e398b10 100644 --- a/modules/connectors/pages/sources/debezium-postgres.adoc +++ b/modules/connectors/pages/sources/debezium-postgres.adoc @@ -1 +1,37 @@ -include::common:streaming:partial$connectors/sources/debezium-postgres.adoc[] \ No newline at end of file += Debezium PostgreSQL +:connectorType: debezium-postgres + +The Debezium PostgreSQL source connector produces a change event for every row-level `INSERT`, `UPDATE`, and `DELETE` operation that it captures, and then it sends change event records for each table to separate {pulsar-short} topics. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/debezium-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Debezium PostgreSQL source connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://debezium.io/documentation/reference/stable/connectors/postgresql.html#postgresql-connector-properties[Debezium connector for PostgreSQL] and the https://pulsar.apache.org/docs/io-debezium-source[OSS {pulsar} Debezium source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that are only present in <>. \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-sqlserver.adoc b/modules/connectors/pages/sources/debezium-sqlserver.adoc index 9b6243f6..5c7d1d9e 100644 --- a/modules/connectors/pages/sources/debezium-sqlserver.adoc +++ b/modules/connectors/pages/sources/debezium-sqlserver.adoc @@ -1 +1,57 @@ -include::common:streaming:partial$connectors/sources/debezium-sqlserver.adoc[] \ No newline at end of file += Debezium SQL Server +:connectorType: debezium-sqlserver + +The Debezium SQL Server source connector reads data from Debezium SQL Server databases and tables, and then emits change data event messages to {pulsar-short} topics. + +It is based on the change data capture (CDC) feature available in SQL Server 2016 Service Pack 1 (SP1) and later editions. + +The SQL Server capture process monitors designated databases and tables, and then stores changes into dedicated change tables with stored procedure facades. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/debezium-compatibility.adoc[] + +[#create-the-connector] +== Create the connector + +include::common:streaming:partial$connectors/sources/debezium-sqlserver-setup.adoc[] + +include::ROOT:partial$sources/get-started.adoc[] + +. Set your consumers to subscribe to the `events` topic in your {pulsar-short} namespace to receive CDC events. +This topic contains the CDC events emitted by Debezium for the configured tables. + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +[WARNING] +==== +The `name` property must match the connector name you used in the topic names as explained in <>. +==== + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Debezium SQL Server source connector properties (`configs`) + +[WARNING] +==== +Make sure you set the `topic.prefix`, `database.server.name`, and `task.id` properties as explained in <>. +==== + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://debezium.io/documentation/reference/stable/connectors/sqlserver.html#sqlserver-connector-properties[Debezium connector for SQL Server] and the https://pulsar.apache.org/docs/io-debezium-source[OSS {pulsar} Debezium source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that are only present in <>. \ No newline at end of file diff --git a/modules/connectors/pages/sources/kafka.adoc b/modules/connectors/pages/sources/kafka.adoc index 1f5f4232..5ac91104 100644 --- a/modules/connectors/pages/sources/kafka.adoc +++ b/modules/connectors/pages/sources/kafka.adoc @@ -1 +1,32 @@ -include::common:streaming:partial$connectors/sources/kafka.adoc[] \ No newline at end of file += Kafka +:connectorType: kafka + +The Kafka source connector pulls data from https://kafka.apache.org/[Kafka] topics and persists it to {pulsar-short} topics. + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Kafka source connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://pulsar.apache.org/docs/io-kafka-source[OSS {pulsar} Kafka source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that aren't present in {pulsar} {pulsar-version}. \ No newline at end of file diff --git a/modules/connectors/pages/sources/kinesis.adoc b/modules/connectors/pages/sources/kinesis.adoc index bbc36c5a..5edd895a 100644 --- a/modules/connectors/pages/sources/kinesis.adoc +++ b/modules/connectors/pages/sources/kinesis.adoc @@ -1 +1,32 @@ -include::common:streaming:partial$connectors/sources/kinesis.adoc[] \ No newline at end of file += Kinesis +:connectorType: kinesis + +The Kinesis source connector pulls data from https://aws.amazon.com/kinesis/[Amazon Kinesis] and persists it to {pulsar-short} topics. + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Kinesis source connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://pulsar.apache.org/docs/io-kinesis-source[OSS {pulsar} Kinesis source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that aren't present in {pulsar} {pulsar-version}. \ No newline at end of file From 1f4488c04416a3997c35bd82f39367ad1d6c64b3 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 07:03:29 -0800 Subject: [PATCH 12/15] unused attributes --- antora.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/antora.yml b/antora.yml index 45ade280..422d00c5 100644 --- a/antora.yml +++ b/antora.yml @@ -29,8 +29,4 @@ asciidoc: cass-reg: 'Apache Cassandra(R)' cass: 'Apache Cassandra' cass-short: 'Cassandra' - debezium-version: '1.7' - get-started-sink: 'include::ROOT:partial$sinks/get-started.adoc[]' - edit-sink: 'include::ROOT:partial$sinks/edit.adoc[]' - get-started-source: 'include::ROOT:partial$sources/get-started.adoc[]' - edit-source: 'include::ROOT:partial$sources/edit.adoc[]' \ No newline at end of file + debezium-version: '1.7' \ No newline at end of file From eb751213b5461bd0f6cc48cbd6c1140184fb017b Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 07:46:43 -0800 Subject: [PATCH 13/15] change example syntax --- modules/ROOT/partials/sinks/get-started.adoc | 5 ++++- modules/ROOT/partials/sources/get-started.adoc | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/ROOT/partials/sinks/get-started.adoc b/modules/ROOT/partials/sinks/get-started.adoc index c00eff4f..ca68578d 100644 --- a/modules/ROOT/partials/sinks/get-started.adoc +++ b/modules/ROOT/partials/sinks/get-started.adoc @@ -26,4 +26,7 @@ You can pass the configuration directly or with a configuration file. ---- + .Example configuration data structure -include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.adoc[] \ No newline at end of file +[source,json,subs="+attributes"] +---- +include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.adoc[] +---- \ No newline at end of file diff --git a/modules/ROOT/partials/sources/get-started.adoc b/modules/ROOT/partials/sources/get-started.adoc index 9fdda519..4aa49700 100644 --- a/modules/ROOT/partials/sources/get-started.adoc +++ b/modules/ROOT/partials/sources/get-started.adoc @@ -26,4 +26,7 @@ You can pass the configuration directly or with a configuration file. ---- + .Example configuration data structure -include::common:streaming:example$connectors/sources/{connectorType}/sample-data.adoc[] \ No newline at end of file +[source,json,subs="+attributes"] +---- +include::common:streaming:example$connectors/sources/{connectorType}/sample-data.adoc[] +---- \ No newline at end of file From 148380dced71f2e3e4cda4f7de367151dffd92fd Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 07:52:29 -0800 Subject: [PATCH 14/15] fix file type --- modules/ROOT/partials/sinks/get-started.adoc | 2 +- modules/ROOT/partials/sources/get-started.adoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ROOT/partials/sinks/get-started.adoc b/modules/ROOT/partials/sinks/get-started.adoc index ca68578d..534eff36 100644 --- a/modules/ROOT/partials/sinks/get-started.adoc +++ b/modules/ROOT/partials/sinks/get-started.adoc @@ -28,5 +28,5 @@ You can pass the configuration directly or with a configuration file. .Example configuration data structure [source,json,subs="+attributes"] ---- -include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.adoc[] +include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.json[] ---- \ No newline at end of file diff --git a/modules/ROOT/partials/sources/get-started.adoc b/modules/ROOT/partials/sources/get-started.adoc index 4aa49700..63cde397 100644 --- a/modules/ROOT/partials/sources/get-started.adoc +++ b/modules/ROOT/partials/sources/get-started.adoc @@ -28,5 +28,5 @@ You can pass the configuration directly or with a configuration file. .Example configuration data structure [source,json,subs="+attributes"] ---- -include::common:streaming:example$connectors/sources/{connectorType}/sample-data.adoc[] +include::common:streaming:example$connectors/sources/{connectorType}/sample-data.json[] ---- \ No newline at end of file From 80dbc9c79b25a9ddc882b1a251b90b87e1aa3108 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 07:58:23 -0800 Subject: [PATCH 15/15] once more --- modules/ROOT/partials/sinks/get-started.adoc | 2 +- modules/ROOT/partials/sources/get-started.adoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ROOT/partials/sinks/get-started.adoc b/modules/ROOT/partials/sinks/get-started.adoc index 534eff36..e4acdf41 100644 --- a/modules/ROOT/partials/sinks/get-started.adoc +++ b/modules/ROOT/partials/sinks/get-started.adoc @@ -26,7 +26,7 @@ You can pass the configuration directly or with a configuration file. ---- + .Example configuration data structure -[source,json,subs="+attributes"] +[source,json] ---- include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.json[] ---- \ No newline at end of file diff --git a/modules/ROOT/partials/sources/get-started.adoc b/modules/ROOT/partials/sources/get-started.adoc index 63cde397..fb4cd25c 100644 --- a/modules/ROOT/partials/sources/get-started.adoc +++ b/modules/ROOT/partials/sources/get-started.adoc @@ -26,7 +26,7 @@ You can pass the configuration directly or with a configuration file. ---- + .Example configuration data structure -[source,json,subs="+attributes"] +[source,json] ---- include::common:streaming:example$connectors/sources/{connectorType}/sample-data.json[] ---- \ No newline at end of file