From 98774d1e7132b5612a1b7e7f29af6f76a96ac312 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 2 Feb 2026 13:49:26 -0800 Subject: [PATCH 01/11] bring new files --- modules/ROOT/partials/sinks/edit.adoc | 17 + modules/ROOT/partials/sinks/get-started.adoc | 29 ++ modules/ROOT/partials/sources/edit.adoc | 17 + .../ROOT/partials/sources/get-started.adoc | 29 ++ modules/connectors/pages/index.adoc | 323 ++++++++++++++++++ modules/connectors/pages/sinks/astra-db.adoc | 1 + .../connectors/pages/sinks/cloud-storage.adoc | 1 + .../pages/sinks/elastic-search.adoc | 1 + .../pages/sinks/google-bigquery.adoc | 1 + .../pages/sinks/jdbc-clickhouse.adoc | 1 + .../connectors/pages/sinks/jdbc-mariadb.adoc | 1 + .../connectors/pages/sinks/jdbc-postgres.adoc | 1 + .../connectors/pages/sinks/jdbc-sqllite.adoc | 1 + modules/connectors/pages/sinks/kafka.adoc | 1 + modules/connectors/pages/sinks/kinesis.adoc | 1 + modules/connectors/pages/sinks/snowflake.adoc | 1 + .../pages/sources/data-generator.adoc | 1 + .../pages/sources/debezium-mongodb.adoc | 1 + .../pages/sources/debezium-mysql.adoc | 1 + .../pages/sources/debezium-oracle.adoc | 1 + .../pages/sources/debezium-postgres.adoc | 1 + .../pages/sources/debezium-sqlserver.adoc | 1 + modules/connectors/pages/sources/kafka.adoc | 1 + modules/connectors/pages/sources/kinesis.adoc | 1 + 24 files changed, 434 insertions(+) create mode 100644 modules/ROOT/partials/sinks/edit.adoc create mode 100644 modules/ROOT/partials/sinks/get-started.adoc create mode 100644 modules/ROOT/partials/sources/edit.adoc create mode 100644 modules/ROOT/partials/sources/get-started.adoc create mode 100644 modules/connectors/pages/index.adoc create mode 100644 modules/connectors/pages/sinks/astra-db.adoc create mode 100644 modules/connectors/pages/sinks/cloud-storage.adoc create mode 100644 modules/connectors/pages/sinks/elastic-search.adoc create mode 100644 modules/connectors/pages/sinks/google-bigquery.adoc create mode 100644 modules/connectors/pages/sinks/jdbc-clickhouse.adoc create mode 100644 modules/connectors/pages/sinks/jdbc-mariadb.adoc create mode 100644 modules/connectors/pages/sinks/jdbc-postgres.adoc create mode 100644 modules/connectors/pages/sinks/jdbc-sqllite.adoc create mode 100644 modules/connectors/pages/sinks/kafka.adoc create mode 100644 modules/connectors/pages/sinks/kinesis.adoc create mode 100644 modules/connectors/pages/sinks/snowflake.adoc create mode 100644 modules/connectors/pages/sources/data-generator.adoc create mode 100644 modules/connectors/pages/sources/debezium-mongodb.adoc create mode 100644 modules/connectors/pages/sources/debezium-mysql.adoc create mode 100644 modules/connectors/pages/sources/debezium-oracle.adoc create mode 100644 modules/connectors/pages/sources/debezium-postgres.adoc create mode 100644 modules/connectors/pages/sources/debezium-sqlserver.adoc create mode 100644 modules/connectors/pages/sources/kafka.adoc create mode 100644 modules/connectors/pages/sources/kinesis.adoc diff --git a/modules/ROOT/partials/sinks/edit.adoc b/modules/ROOT/partials/sinks/edit.adoc new file mode 100644 index 0000000..90999bb --- /dev/null +++ b/modules/ROOT/partials/sinks/edit.adoc @@ -0,0 +1,17 @@ +To update a connector, pass the new configuration definition to the connector. +For example, if you created the connector with a configuration file, you can pass an updated configuration file. + +You can include the entire configuration or only the properties that you want to change. +Additionally, some properties can be modified with specific arguments, such as `--parallelism`. + +To get the current configuration, see xref:connectors:index.adoc#get-sink-connector-configuration-data[Get sink connector configuration data]. + +[source,shell,subs="+attributes"] +---- +./bin/pulsar-admin sinks update \ + --sink-type {connectorType} \ + --name "$SINK_NAME" \ + --inputs "persistent://$TENANT/$NAMESPACE/$TOPIC" \ + --tenant "$TENANT" \ + --parallelism 2 +---- \ No newline at end of file diff --git a/modules/ROOT/partials/sinks/get-started.adoc b/modules/ROOT/partials/sinks/get-started.adoc new file mode 100644 index 0000000..c00eff4 --- /dev/null +++ b/modules/ROOT/partials/sinks/get-started.adoc @@ -0,0 +1,29 @@ +. Optional: If you are using the `pulsar-admin` CLI, set the following commonly-used environment variables: ++ +[source,shell,subs="+quotes"] +---- +export TENANT="**TENANT_NAME**" +export TOPIC="**INPUT_TOPIC_NAME**" +export NAMESPACE="**NAMESPACE_NAME**" +export SINK_NAME="**SINK_CONNECTOR_UNIQUE_NAME**" +---- ++ +`**SINK_NAME**` is the name for your new sink connector. +{company} recommends a memorable, human-readable name that summarizes the connector's purpose. +For example: `{connectorType}-sink-prod-us-east-1`. + +. Create the connector using JSON-formatted connector configuration data. +You can pass the configuration directly or with a configuration file. ++ +[source,shell,subs="+attributes"] +---- +./bin/pulsar-admin sinks create \ + --sink-type {connectorType} \ + --name "$SINK_NAME" \ + --inputs "persistent://$TENANT/$NAMESPACE/$TOPIC" \ + --tenant "$TENANT" \ + --sink-config-file configs.json +---- ++ +.Example configuration data structure +include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.adoc[] \ No newline at end of file diff --git a/modules/ROOT/partials/sources/edit.adoc b/modules/ROOT/partials/sources/edit.adoc new file mode 100644 index 0000000..b51ea3c --- /dev/null +++ b/modules/ROOT/partials/sources/edit.adoc @@ -0,0 +1,17 @@ +To update a connector, pass the new configuration definition to the connector. +For example, if you created the connector with a configuration file, you can pass an updated configuration file. + +You can include the entire configuration or only the properties that you want to change. +Additionally, some properties can be modified with specific arguments, such as `--parallelism`. + +To get the current configuration, see xref:connectors:index.adoc#get-source-connector-configuration-data[Get source connector configuration data]. + +[source,shell,subs="+attributes"] +---- +./bin/pulsar-admin sources update \ + --source-type {connectorType} \ + --name "$SOURCE_NAME" \ + --destination-topic-name "persistent://$TENANT/$NAMESPACE/$TOPIC" \ + --tenant "$TENANT" \ + --parallelism 2 +---- \ No newline at end of file diff --git a/modules/ROOT/partials/sources/get-started.adoc b/modules/ROOT/partials/sources/get-started.adoc new file mode 100644 index 0000000..9fdda51 --- /dev/null +++ b/modules/ROOT/partials/sources/get-started.adoc @@ -0,0 +1,29 @@ +. Optional: If you are using the `pulsar-admin` CLI, set the following commonly-used environment variables: ++ +[source,shell,subs="+quotes"] +---- +export TENANT="**TENANT_NAME**" +export TOPIC="**OUTPUT_TOPIC_NAME**" +export NAMESPACE="**NAMESPACE_NAME**" +export SOURCE_NAME="**SOURCE_CONNECTOR_UNIQUE_NAME**" +---- ++ +`**SOURCE_NAME**` is the name for your new source connector. +{company} recommends a memorable, human-readable name that summarizes the connector's purpose. +For example: `{connectorType}-source-prod-us-east-1`. + +. Create the connector using JSON-formatted connector configuration data. +You can pass the configuration directly or with a configuration file. ++ +[source,shell,subs="+attributes"] +---- +./bin/pulsar-admin sources create \ + --source-type {connectorType} \ + --name "$SOURCE_NAME" \ + --destination-topic-name "persistent://$TENANT/$NAMESPACE/$TOPIC" \ + --tenant "$TENANT" \ + --source-config-file configs.json +---- ++ +.Example configuration data structure +include::common:streaming:example$connectors/sources/{connectorType}/sample-data.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/index.adoc b/modules/connectors/pages/index.adoc new file mode 100644 index 0000000..e82f0e3 --- /dev/null +++ b/modules/connectors/pages/index.adoc @@ -0,0 +1,323 @@ += {pulsar-short} I/O connectors in {product} +:navtitle: {pulsar-short} I/O connectors reference +:page-aliases: operations:io-connectors.adoc + +{product} offers fully-managed versions of https://pulsar.apache.org/docs/en/io-overview/[{pulsar-reg} sink and source connectors]. + +[TIP] +==== +There are three versions of the {company} Luna Streaming distribution. +The `lunastreaming-all` version includes all connectors. +==== + +//Reconcile w/ io-connectors in each version & delete io-connectors page. +//Luna Streaming: Cloud Storage sink, Google BigQuery sink are experimental +//Elasticsearch is lowercase +//3.x are the same + +//Luna Streaming -2.x and -1.x have fewer connectors: +//No experimental +//No Cloud storage, BigQuery, JDBC SQLlite, snowflake, data generator, debezium oracle, debezium sqlserver + +[#sink-connectors] +== Sink connectors + +Sink connectors export messages from {pulsar-short} topics to external services. + +Supported sink connectors:: +The following sink connectors are included in {product}: ++ +* xref:connectors:sinks/astra-db.adoc[{astra-db} and {cass} sink] +* xref:connectors:sinks/kafka.adoc[Apache Kafka sink] +* xref:connectors:sinks/kinesis.adoc[AWS Kinesis sink] +* xref:connectors:sinks/elastic-search.adoc[Elasticsearch sink] +* xref:connectors:sinks/jdbc-clickhouse.adoc[JDBC ClickHouse sink] +* xref:connectors:sinks/jdbc-mariadb.adoc[JDBC MariaDB sink] +* xref:connectors:sinks/jdbc-postgres.adoc[JDBC PostgreSQL sink] +* xref:connectors:sinks/jdbc-sqllite.adoc[JDBC SQLite sink] +* xref:connectors:sinks/snowflake.adoc[Snowflake sink] + +Experimental sink connectors:: +The following sink connectors are experimental. +They are available in the `luna-streaming-all` version of {product}, and you can use them in your Luna Streaming cluster. +They are considered experimental because they aren't fully supported or tested with Astra Streaming. ++ +* Aerospike +* Apache Geode +* Apache HBase +* Apache Kudu +* Apache Phoenix +* Apache Solr +* Batch Data Generator +* xref:connectors:sinks/cloud-storage.adoc[Cloud Storage] +* CoAP +* Couchbase +* DataDog +* Diffusion +* Flume +* xref:connectors:sinks/google-bigquery.adoc[Google BigQuery] +* Hazelcast +* HDFS 2 +* HDFS 3 +* Humio +* InfluxDB +* JMS +* Kinetica +* MarkLogic +* Microsoft Azure Data Explorer (Kusto) +* Microsoft Azure DocumentDB +* MongoDB +* MQTT +* Neo4J +* New Relic +* OrientDB +* PLC4X +* RabbitMQ +* Redis +* SAP HANA +* SingleStore +* Splunk +* XTDB +* Zeebe + +[#source-connectors] +== Source connectors + +Source connectors ingest messages from external services into {pulsar-short} topics. + +Supported source connectors:: +The following source connectors are included in {product}: ++ +* xref:connectors:sources/kafka.adoc[Apache Kafka source] +* xref:connectors:sources/kinesis.adoc[AWS Kinesis source] +* xref:connectors:sources/data-generator.adoc[Data Generator source] +* xref:connectors:sources/debezium-mongodb.adoc[Debezium MongoDB source] +* xref:connectors:sources/debezium-mysql.adoc[Debezium MySQL source] +* xref:connectors:sources/debezium-oracle.adoc[Debezium Oracle source] +* xref:connectors:sources/debezium-postgres.adoc[Debezium PostgreSQL source] +* xref:connectors:sources/debezium-sqlserver.adoc[Debezium SQL Server source] + +Experimental source connectors:: +The following source connectors are experimental. +They are available in the `luna-streaming-all` version of {product}, and you can use them in your Luna Streaming cluster. +They are considered experimental because they aren't fully supported or tested with Astra Streaming. ++ +* {cass} +* Apache Geode +* Apache Kudu +* Apache Phoenix +* Batch Data Generator +* Big Query +* Canal +* CoAP +* Couchbase +* DataDog +* Diffusion +* DynamoDB +* File +* Flume +* Hazelcast +* Humio +* JMS +* Kinetica +* MarkLogic +* Microsoft Azure DocumentDB +* MongoDB +* MQTT +* Neo4J +* Netty +* New Relic +* NSQ +* OrientDB +* PLC4X +* RabbitMQ +* Redis +* SAP HANA +* SingleStore +* Splunk +* Twitter +* XTDB +* Zeebe + +[#create-and-manage-connectors] +== Create and manage connectors + +You can use the {web-ui} and the `xref:components:admin-console-tutorial.adoc[pulsar-admin]` CLI to create, monitor, and manage sink and source connectors. + +Although you use the same base commands to create and update all {product} {pulsar-short} connectors, each connector has different configuration options. +For example commands and configuration details, see the documentation for your preferred <> and <>. + +=== `pulsar-admin` CLI sink operations + +Get available sink connectors:: +Get a list of sink connectors that are available in your {product} {pulsar-short} tenant: ++ +[source,shell] +---- +./bin/pulsar-admin sinks available-sinks +---- + +[#get-sink-connector-configuration-data] +Get sink connector configuration data:: +Get the configuration for an existing sink connector: ++ +[source,shell] +---- +# Get information about a connector +./bin/pulsar-admin sinks get \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" +---- + +Start a sink connector:: ++ +[source,shell] +---- +# Start all instances of a connector +./bin/pulsar-admin sinks start \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only start an individual instance +---- + +Stop a sink connector:: ++ +[source,shell] +---- +# Stop all instances of a connector +./bin/pulsar-admin sinks stop \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only stop an individual instance +---- + +Restart a sink connector:: ++ +[source,shell] +---- +# Restart all instances of a connector +./bin/pulsar-admin sinks restart \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only restart an individual instance +---- + +Get sink connector status:: ++ +[source,shell] +---- +# Check connector status +./bin/pulsar-admin sinks status \ + --instance-id "$SINK_INSTANCE_ID" \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" +---- + +Delete a sink connector:: ++ +[source,shell] +---- +# Delete all instances of a connector +./bin/pulsar-admin sinks delete \ + --namespace "$NAMESPACE" \ + --name "$SINK_NAME" \ + --tenant "$TENANT" +---- + +=== `pulsar-admin` CLI source operations + +Get available source connectors:: +Get a list of source connectors that are available in your {product} {pulsar-short} tenant: ++ +[source,shell] +---- +./bin/pulsar-admin sources available-sources +---- + +[#get-source-connector-configuration-data] +Get source connector configuration data:: +Get the configuration for an existing source connector: ++ +[source,shell] +---- +# Get information about connector +./bin/pulsar-admin sources get \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" +---- + +Start a source connector:: ++ +[source,shell] +---- +# Start all instances of a connector +./bin/pulsar-admin sources start \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only start an individual instance +---- + +Stop a source connector:: ++ +[source,shell] +---- +# Stop all instances of a connector +./bin/pulsar-admin sources stop \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only stop an individual instance +---- + +Restart a source connector:: ++ +[source,shell] +---- +# Restart all instances of a connector +./bin/pulsar-admin sources restart \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" + +# optionally add --instance-id to only restart an individual instance +---- + +Get source connector status:: ++ +[source,shell] +---- +# Check connector status +./bin/pulsar-admin sources status \ + --instance-id "$SOURCE_INSTANCE_ID" \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" +---- + +Delete a source connector:: ++ +[source,shell] +---- +# Delete all instances of a connector +./bin/pulsar-admin sources delete \ + --namespace "$NAMESPACE" \ + --name "$SOURCE_NAME" \ + --tenant "$TENANT" +---- + +== Connector metrics + +{product} exposes Prometheus-formatted metrics for every connector. +For more information, see xref:operations:astream-scrape-metrics.adoc[]. \ No newline at end of file diff --git a/modules/connectors/pages/sinks/astra-db.adoc b/modules/connectors/pages/sinks/astra-db.adoc new file mode 100644 index 0000000..bba17bd --- /dev/null +++ b/modules/connectors/pages/sinks/astra-db.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/astra-db.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/cloud-storage.adoc b/modules/connectors/pages/sinks/cloud-storage.adoc new file mode 100644 index 0000000..d817171 --- /dev/null +++ b/modules/connectors/pages/sinks/cloud-storage.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/cloud-storage.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/elastic-search.adoc b/modules/connectors/pages/sinks/elastic-search.adoc new file mode 100644 index 0000000..255f956 --- /dev/null +++ b/modules/connectors/pages/sinks/elastic-search.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/elastic-search.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/google-bigquery.adoc b/modules/connectors/pages/sinks/google-bigquery.adoc new file mode 100644 index 0000000..2f5ea79 --- /dev/null +++ b/modules/connectors/pages/sinks/google-bigquery.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/google-bigquery.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-clickhouse.adoc b/modules/connectors/pages/sinks/jdbc-clickhouse.adoc new file mode 100644 index 0000000..52bc0de --- /dev/null +++ b/modules/connectors/pages/sinks/jdbc-clickhouse.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/jdbc-clickhouse.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-mariadb.adoc b/modules/connectors/pages/sinks/jdbc-mariadb.adoc new file mode 100644 index 0000000..b1ab937 --- /dev/null +++ b/modules/connectors/pages/sinks/jdbc-mariadb.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/jdbc-mariadb.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-postgres.adoc b/modules/connectors/pages/sinks/jdbc-postgres.adoc new file mode 100644 index 0000000..a94fa74 --- /dev/null +++ b/modules/connectors/pages/sinks/jdbc-postgres.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/jdbc-postgres.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-sqllite.adoc b/modules/connectors/pages/sinks/jdbc-sqllite.adoc new file mode 100644 index 0000000..79ba3d5 --- /dev/null +++ b/modules/connectors/pages/sinks/jdbc-sqllite.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/jdbc-sqllite.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/kafka.adoc b/modules/connectors/pages/sinks/kafka.adoc new file mode 100644 index 0000000..eb30fdd --- /dev/null +++ b/modules/connectors/pages/sinks/kafka.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/kafka.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/kinesis.adoc b/modules/connectors/pages/sinks/kinesis.adoc new file mode 100644 index 0000000..ea1ceee --- /dev/null +++ b/modules/connectors/pages/sinks/kinesis.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/kinesis.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/snowflake.adoc b/modules/connectors/pages/sinks/snowflake.adoc new file mode 100644 index 0000000..a31e2cd --- /dev/null +++ b/modules/connectors/pages/sinks/snowflake.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sinks/snowflake.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/data-generator.adoc b/modules/connectors/pages/sources/data-generator.adoc new file mode 100644 index 0000000..7b91b22 --- /dev/null +++ b/modules/connectors/pages/sources/data-generator.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/data-generator.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-mongodb.adoc b/modules/connectors/pages/sources/debezium-mongodb.adoc new file mode 100644 index 0000000..ef7baf1 --- /dev/null +++ b/modules/connectors/pages/sources/debezium-mongodb.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/debezium-mongodb.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-mysql.adoc b/modules/connectors/pages/sources/debezium-mysql.adoc new file mode 100644 index 0000000..72a9bca --- /dev/null +++ b/modules/connectors/pages/sources/debezium-mysql.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/debezium-mysql.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-oracle.adoc b/modules/connectors/pages/sources/debezium-oracle.adoc new file mode 100644 index 0000000..e47ba96 --- /dev/null +++ b/modules/connectors/pages/sources/debezium-oracle.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/debezium-oracle.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-postgres.adoc b/modules/connectors/pages/sources/debezium-postgres.adoc new file mode 100644 index 0000000..396f9df --- /dev/null +++ b/modules/connectors/pages/sources/debezium-postgres.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/debezium-postgres.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-sqlserver.adoc b/modules/connectors/pages/sources/debezium-sqlserver.adoc new file mode 100644 index 0000000..9b6243f --- /dev/null +++ b/modules/connectors/pages/sources/debezium-sqlserver.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/debezium-sqlserver.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/kafka.adoc b/modules/connectors/pages/sources/kafka.adoc new file mode 100644 index 0000000..1f5f423 --- /dev/null +++ b/modules/connectors/pages/sources/kafka.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/kafka.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/kinesis.adoc b/modules/connectors/pages/sources/kinesis.adoc new file mode 100644 index 0000000..bbc36c5 --- /dev/null +++ b/modules/connectors/pages/sources/kinesis.adoc @@ -0,0 +1 @@ +include::common:streaming:partial$connectors/sources/kinesis.adoc[] \ No newline at end of file From fff94ff7f5b1c07822c11c085d141394019132b7 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 2 Feb 2026 13:58:45 -0800 Subject: [PATCH 02/11] other changes --- antora.yml | 6 +- modules/ROOT/nav.adoc | 24 ++- modules/ROOT/pages/faqs.adoc | 2 +- modules/ROOT/pages/index.adoc | 2 +- modules/connectors/pages/index.adoc | 9 - modules/operations/pages/io-connectors.adoc | 225 -------------------- 6 files changed, 30 insertions(+), 238 deletions(-) delete mode 100644 modules/operations/pages/io-connectors.adoc diff --git a/antora.yml b/antora.yml index afb5c0b..8cce6c2 100644 --- a/antora.yml +++ b/antora.yml @@ -17,4 +17,8 @@ asciidoc: starlight-rabbitmq: 'Starlight for RabbitMQ' pulsar-reg: 'Apache Pulsar(TM)' pulsar: 'Apache Pulsar' - pulsar-short: 'Pulsar' \ No newline at end of file + pulsar-short: 'Pulsar' + scb: 'Secure Connect Bundle (SCB)' + scb-short: 'SCB' + scb-brief: 'Secure Connect Bundle' + web-ui: 'Admin Console' \ No newline at end of file diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 28b9a78..13fc7cd 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -22,6 +22,28 @@ * xref:operations:auth.adoc[] * xref:operations:functions.adoc[] * xref:operations:functions-transform.adoc[] -* xref:operations:io-connectors.adoc[] +* I/O connectors +** xref:connectors:index.adoc[] +** Sinks +*** xref:connectors:sinks/astra-db.adoc[] +*** xref:connectors:sinks/cloud-storage.adoc[Cloud Storage (experimental)] +*** xref:connectors:sinks/elastic-search.adoc[] +*** xref:connectors:sinks/google-bigquery.adoc[Google BigQuery (experimental)] +*** xref:connectors:sinks/jdbc-clickhouse.adoc[] +*** xref:connectors:sinks/jdbc-mariadb.adoc[] +*** xref:connectors:sinks/jdbc-postgres.adoc[] +*** xref:connectors:sinks/jsbc-sqllite.adoc[] +*** xref:connectors:sinks/kafka.adoc[] +*** xref:connectors:sinks/kinesis.adoc[] +*** xref:connectors:sinks/snowflake.adoc[] +** Sources +*** xref:connectors:sources/data-generator.adoc[] +*** xref:connectors:sources/debezium-mongodb.adoc[] +*** xref:connectors:sources/debezium-mysql.adoc[] +*** xref:connectors:sources/debezium-oracle.adoc[] +*** xref:connectors:sources/debezium-postgres.adoc[] +*** xref:connectors:sources/debezium-sqlserver.adoc[] +*** xref:connectors:sources/kafka.adoc[] +*** xref:connectors:sources/kinesis.adoc[] * xref:operations:scale-cluster.adoc[] * xref:operations:troubleshooting.adoc[] \ No newline at end of file diff --git a/modules/ROOT/pages/faqs.adoc b/modules/ROOT/pages/faqs.adoc index 2706b79..4333713 100644 --- a/modules/ROOT/pages/faqs.adoc +++ b/modules/ROOT/pages/faqs.adoc @@ -80,7 +80,7 @@ https://github.com/datastax/pulsar-heartbeat[{pulsar-short} Heartbeat] monitors == What are the features provided by {company} {pulsar} Connector (`pulsar-sink`) that are not supported in `kafka-sink`? -The https://pulsar.apache.org/docs/en/io-overview/[{pulsar-short} IO framework] provides many features that are not possible in Kafka, and has different compression formats and auth/security features. The features are handled by {pulsar-short}. For more, see xref:operations:io-connectors.adoc[Luna Streaming IO Connectors]. +The https://pulsar.apache.org/docs/en/io-overview/[{pulsar-short} IO framework] provides many features that are not possible in Kafka, and has different compression formats and auth/security features. The features are handled by {pulsar-short}. For more, see xref:connectors:index.adoc[Luna Streaming IO Connectors]. The {company} {pulsar} Connector allows single-record acknowledgement and negative acknowledgements. diff --git a/modules/ROOT/pages/index.adoc b/modules/ROOT/pages/index.adoc index adbf15d..50a901d 100644 --- a/modules/ROOT/pages/index.adoc +++ b/modules/ROOT/pages/index.adoc @@ -15,7 +15,7 @@ In addition to the distribution of https://pulsar.apache.org/en/versions/[{pulsa * A xref:install-upgrade:quickstart-helm-installs.adoc[Helm chart] that deploys and manages {pulsar-short} on your current CNCF-conformant Kubernetes infrastructure -* Cassandra, Elastic, Kinesis, Kafka, and JDBC xref:operations:io-connectors.adoc[connectors] +* Cassandra, Elastic, Kinesis, Kafka, and JDBC xref:connectors:index.adoc[connectors] * The xref:components:starlight.adoc[Starlight suite of {pulsar-short} protocol handlers for Kafka, RabbitMQ, and JMS] diff --git a/modules/connectors/pages/index.adoc b/modules/connectors/pages/index.adoc index e82f0e3..3fdd5d4 100644 --- a/modules/connectors/pages/index.adoc +++ b/modules/connectors/pages/index.adoc @@ -10,15 +10,6 @@ There are three versions of the {company} Luna Streaming distribution. The `lunastreaming-all` version includes all connectors. ==== -//Reconcile w/ io-connectors in each version & delete io-connectors page. -//Luna Streaming: Cloud Storage sink, Google BigQuery sink are experimental -//Elasticsearch is lowercase -//3.x are the same - -//Luna Streaming -2.x and -1.x have fewer connectors: -//No experimental -//No Cloud storage, BigQuery, JDBC SQLlite, snowflake, data generator, debezium oracle, debezium sqlserver - [#sink-connectors] == Sink connectors diff --git a/modules/operations/pages/io-connectors.adoc b/modules/operations/pages/io-connectors.adoc deleted file mode 100644 index 958a96c..0000000 --- a/modules/operations/pages/io-connectors.adoc +++ /dev/null @@ -1,225 +0,0 @@ -= Luna Streaming IO connectors - -When you have Luna Streaming xref:install-upgrade:quickstart-server-installs.adoc[installed] and running, add IO connectors to connect your deployment to external systems like https://cassandra.apache.org/_/index.html[Apache Cassandra], https://www.elastic.co/[ElasticSearch], and more. - -[NOTE] -==== -There are three versions of the {company} Luna Streaming distribution. -The lunastreaming-all version contains all connectors. -==== - -[#sink-connectors] -== Sink Connectors - -[#astradb-sink] -=== AstraDB sink - -The AstraDB sink connector reads messages from {pulsar} topics and writes them to AstraDB systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/astra-db.adoc[AstraDB sink documentation] - -[#elasticsearch-sink] -=== ElasticSearch sink - -The Elasticsearch sink connector reads messages from {pulsar} topics and writes them to Elasticsearch systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/elastic-search.adoc[Elasticsearch sink documentation] - -[#jdbc-clickhouse-sink] -=== JDBC-Clickhouse sink - -The JDBC-ClickHouse sink connector reads messages from {pulsar} topics and writes them to JDBC-ClickHouse systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/jdbc-clickhouse.adoc[JDBC ClickHouse sink documentation] - -[#jdbc-mariadb-sink] -=== JDBC-MariaDB sink - -The JDBC-MariaDB sink connector reads messages from {pulsar} topics and writes them to JDBC-MariaDB systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/jdbc-mariadb.adoc[JDBC MariaDB sink documentation] - -[#jdbc-postgres-sink] -=== JDBC-PostgreSQL sink - -The JDBC-PostgreSQL sink connector reads messages from {pulsar} topics and writes them to JDBC-PostgreSQL systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/jdbc-postgres.adoc[JDBC PostgreSQL sink documentation] - -[#jdbc-sqlite-sink] -=== JDBC-SQLite - -The JDBC-SQLite sink connector reads messages from {pulsar} topics and writes them to JDBC-SQLite systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/jdbc-sqllite.adoc[JDBC SQLite sink documentation] - -[#kafka-sink] -=== Kafka - -The Kafka sink connector reads messages from {pulsar} topics and writes them to Kafka systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/kafka.adoc[Kafka sink documentation] - -[#kinesis-sink] -=== Kinesis - -The Kinesis sink connector reads messages from {pulsar} topics and writes them to Kinesis systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/kinesis.adoc[Kinesis sink documentation] - -[#snowflake-sink] -=== Snowflake - -The Snowflake sink connector reads messages from {pulsar} topics and writes them to Snowflake systems. - -xref:streaming-learning:pulsar-io:connectors/sinks/snowflake.adoc[Snowflake sink documentation] - -[#source-connectors] -== Source Connectors - -[#datagenerator-source] -=== Data Generator source - -The Data generator source connector produces messages for testing and persists the messages to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/data-generator.adoc[Data Generator source documentation] - -[#debezium-mongodb-source] -=== Debezium MongoDB source - -The Debezium MongoDB source connector reads data from Debezium MongoDB systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/debezium-mongodb.adoc[Debezium MongoDB source documentation] - -[#debezium-mysql-source] -=== Debezium MySQL source - -The Debezium MySQL source connector reads data from Debezium MySQL systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/debezium-mysql.adoc[Debezium MySQL source documentation] - -[#debezium-oracle-source] -=== Debezium Oracle source - -The Debezium Oracle source connector reads data from Debezium Oracle systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/debezium-oracle.adoc[Debezium Oracle source documentation] - -[#debezium-postgres-source] -=== Debezium Postgres source - -The Debezium PostgreSQL source connector reads data from Debezium PostgreSQL systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/debezium-postgres.adoc[Debezium PostgreSQL source documentation] - -[#debezium-sql-server-source] -=== Debezium SQL Server source - -The Debezium SQL Server source connector reads data from Debezium SQL Server systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/debezium-sqlserver.adoc[Debezium SQL Server source documentation] - -[#kafka-source] -=== Kafka source - -The Kafka source connector reads data from Kafka systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/kafka.adoc[Kafka source connector documentation] - -[#kinesis-source] -=== AWS Kinesis source - -The AWS Kinesis source connector reads data from Kinesis systems and produces data to {pulsar-short} topics. - -xref:streaming-learning:pulsar-io:connectors/sources/kinesis.adoc[Kinesis source connector documentation] - -== Experimental Connectors - -{company} is always experimenting with connectors. -Below are the connectors available in the luna-streaming-all version of *Luna Streaming*. -We call these *experimental connectors* because they have not yet been promoted to official support in our *Astra Streaming* clusters, but they will work in your Luna Streaming cluster. - -[#sink-experimental] -=== Sink Connectors (experimental) - -* Cloud Storage -* Kinetica -* Aerospike -* Azure DocumentDB -* Azure Data Explorer (Kusto) -* Batch Data Generator -* Big Query -* CoAP -* Couchbase -* DataDog -* Diffusion -* Flume -* Apache Geode -* Hazelcast -* Apache HBase -* HDFS 2 -* HDFS 3 -* Humio -* InfluxDB -* JMS -* Apache Kudu -* MarkLogic -* MongoDB -* MQTT -* Neo4J -* New Relic -* OrientDB -* Apache Phoenix -* PLC4X -* RabbitMQ -* Redis -* SAP HANA -* SingleStore -* Apache Solr -* Splunk -* XTDB -* Zeebe - -[#source-experimental] -=== Source Connectors (experimental) - -* Cassandra Source -* Kinetica -* Azure DocumentDB -* Batch Data Generator -* Big Query -* canal -* CoAP -* Couchbase -* DataDog -* Diffusion -* DynamoDB -* file -* flume -* Apache Geode -* Hazelcast -* Humio -* JMS -* Apache Kudu -* MarkLogic -* MongoDB -* MQTT -* Neo4J -* Netty -* New Relic -* NSQ -* OrientDB -* Apache Phoenix -* PLC4X -* RabbitMQ -* Redis -* SAP HANA -* SingleStore -* Splunk -* Twitter -* XTDB -* Zeebe - -== Next steps - -For more on connectors and {pulsar-short}, see xref:streaming-learning:pulsar-io:connectors/index.adoc[] and the https://pulsar.apache.org/docs/2.11.x/io-overview/[{pulsar-short} documentation]. \ No newline at end of file From 3d8c6afc5d247633cad76f9c7025b867836375fb Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 05:15:33 -0800 Subject: [PATCH 03/11] missing attributes --- antora.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/antora.yml b/antora.yml index 8cce6c2..a409219 100644 --- a/antora.yml +++ b/antora.yml @@ -13,6 +13,7 @@ asciidoc: pulsar-version: '2.10' admin-console-version: '2.0.4' heartbeat-version: '1.0.12' + debezium-version: '1.7' starlight-kafka: 'Starlight for Kafka' starlight-rabbitmq: 'Starlight for RabbitMQ' pulsar-reg: 'Apache Pulsar(TM)' @@ -21,4 +22,9 @@ asciidoc: scb: 'Secure Connect Bundle (SCB)' scb-short: 'SCB' scb-brief: 'Secure Connect Bundle' - web-ui: 'Admin Console' \ No newline at end of file + web-ui: 'Admin Console' + astra: 'Astra' + astra-db: 'Astra DB' + cass-reg: 'Apache Cassandra(R)' + cass: 'Apache Cassandra' + cass-short: 'Cassandra' \ No newline at end of file From 3c27b3e078e13edf7faeba35914b923c56530755 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 05:35:47 -0800 Subject: [PATCH 04/11] attributes hacks, missing attributes --- antora.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/antora.yml b/antora.yml index a409219..5c53832 100644 --- a/antora.yml +++ b/antora.yml @@ -13,18 +13,24 @@ asciidoc: pulsar-version: '2.10' admin-console-version: '2.0.4' heartbeat-version: '1.0.12' - debezium-version: '1.7' starlight-kafka: 'Starlight for Kafka' starlight-rabbitmq: 'Starlight for RabbitMQ' pulsar-reg: 'Apache Pulsar(TM)' pulsar: 'Apache Pulsar' pulsar-short: 'Pulsar' - scb: 'Secure Connect Bundle (SCB)' - scb-short: 'SCB' - scb-brief: 'Secure Connect Bundle' + + # Required for include::common partials that are shared with Astra Streaming web-ui: 'Admin Console' astra: 'Astra' astra-db: 'Astra DB' + scb: 'Secure Connect Bundle (SCB)' + scb-short: 'SCB' + scb-brief: 'Secure Connect Bundle' cass-reg: 'Apache Cassandra(R)' cass: 'Apache Cassandra' - cass-short: 'Cassandra' \ No newline at end of file + cass-short: 'Cassandra' + debezium-version: '1.7' + get-started-sink: 'include::ROOT:partial$sinks/get-started.adoc[]' + edit-sink: 'include::ROOT:partial$sinks/edit.adoc[]' + get-started-source: 'include::ROOT:partial$sources/get-started.adoc[]' + edit-source: 'include::ROOT:partial$sources/edit.adoc[]' \ No newline at end of file From 4e2e773e30283b5fb2ade4c3e0340dce7e33e125 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 05:38:30 -0800 Subject: [PATCH 05/11] fix nav --- modules/ROOT/nav.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 13fc7cd..935274d 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -32,7 +32,7 @@ *** xref:connectors:sinks/jdbc-clickhouse.adoc[] *** xref:connectors:sinks/jdbc-mariadb.adoc[] *** xref:connectors:sinks/jdbc-postgres.adoc[] -*** xref:connectors:sinks/jsbc-sqllite.adoc[] +*** xref:connectors:sinks/jdbc-sqllite.adoc[] *** xref:connectors:sinks/kafka.adoc[] *** xref:connectors:sinks/kinesis.adoc[] *** xref:connectors:sinks/snowflake.adoc[] From c85f91aa7812bed6943493e01da0800767e69101 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 05:44:56 -0800 Subject: [PATCH 06/11] fix link --- modules/connectors/pages/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/connectors/pages/index.adoc b/modules/connectors/pages/index.adoc index 3fdd5d4..f6f300f 100644 --- a/modules/connectors/pages/index.adoc +++ b/modules/connectors/pages/index.adoc @@ -311,4 +311,4 @@ Delete a source connector:: == Connector metrics {product} exposes Prometheus-formatted metrics for every connector. -For more information, see xref:operations:astream-scrape-metrics.adoc[]. \ No newline at end of file +For more information, see xref:components:pulsar-monitor.adoc[]. \ No newline at end of file From fadce54ab0f17f129b2af0681dc3cf787748ce8b Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 07:02:54 -0800 Subject: [PATCH 07/11] bring main topic content --- modules/ROOT/nav.adoc | 4 +- modules/connectors/pages/sinks/astra-db.adoc | 48 ++++++++++++++- .../connectors/pages/sinks/cloud-storage.adoc | 36 +++++++++++- .../pages/sinks/elastic-search.adoc | 38 +++++++++++- .../pages/sinks/google-bigquery.adoc | 36 +++++++++++- .../pages/sinks/jdbc-clickhouse.adoc | 35 ++++++++++- .../connectors/pages/sinks/jdbc-mariadb.adoc | 35 ++++++++++- .../connectors/pages/sinks/jdbc-postgres.adoc | 35 ++++++++++- .../connectors/pages/sinks/jdbc-sqllite.adoc | 35 ++++++++++- modules/connectors/pages/sinks/kafka.adoc | 38 +++++++++++- modules/connectors/pages/sinks/kinesis.adoc | 40 ++++++++++++- modules/connectors/pages/sinks/snowflake.adoc | 35 ++++++++++- .../pages/sources/data-generator.adoc | 35 ++++++++++- .../pages/sources/debezium-mongodb.adoc | 43 +++++++++++++- .../pages/sources/debezium-mysql.adoc | 38 +++++++++++- .../pages/sources/debezium-oracle.adoc | 41 ++++++++++++- .../pages/sources/debezium-postgres.adoc | 38 +++++++++++- .../pages/sources/debezium-sqlserver.adoc | 58 ++++++++++++++++++- modules/connectors/pages/sources/kafka.adoc | 33 ++++++++++- modules/connectors/pages/sources/kinesis.adoc | 33 ++++++++++- 20 files changed, 713 insertions(+), 21 deletions(-) diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 935274d..c84c5e7 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -26,9 +26,9 @@ ** xref:connectors:index.adoc[] ** Sinks *** xref:connectors:sinks/astra-db.adoc[] -*** xref:connectors:sinks/cloud-storage.adoc[Cloud Storage (experimental)] +*** xref:connectors:sinks/cloud-storage.adoc[] *** xref:connectors:sinks/elastic-search.adoc[] -*** xref:connectors:sinks/google-bigquery.adoc[Google BigQuery (experimental)] +*** xref:connectors:sinks/google-bigquery.adoc[] *** xref:connectors:sinks/jdbc-clickhouse.adoc[] *** xref:connectors:sinks/jdbc-mariadb.adoc[] *** xref:connectors:sinks/jdbc-postgres.adoc[] diff --git a/modules/connectors/pages/sinks/astra-db.adoc b/modules/connectors/pages/sinks/astra-db.adoc index bba17bd..b98105e 100644 --- a/modules/connectors/pages/sinks/astra-db.adoc +++ b/modules/connectors/pages/sinks/astra-db.adoc @@ -1 +1,47 @@ -include::common:streaming:partial$connectors/sinks/astra-db.adoc[] \ No newline at end of file += {astra-db} and {cass} +:connectorType: cassandra-enhanced + +include::common:streaming:partial$connectors/sinks/astra-db-intro.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +[TIP] +==== +The preceding example connects to a self-managed {cass-short} cluster. + +For an {astra-db} example, see <>. + +For all connection properties, see <>. +==== + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== {company} {pulsar} sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/astra-db-properties.adoc[] + +[#topic] +=== Topic-to-table mapping properties (`topic`) + +include::common:streaming:partial$connectors/sinks/astra-db-topic.adoc[] + +== See also + +* https://github.com/datastax/pulsar-sink[{company} {pulsar} connector GitHub repository] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/cloud-storage.adoc b/modules/connectors/pages/sinks/cloud-storage.adoc index d817171..bdf936e 100644 --- a/modules/connectors/pages/sinks/cloud-storage.adoc +++ b/modules/connectors/pages/sinks/cloud-storage.adoc @@ -1 +1,35 @@ -include::common:streaming:partial$connectors/sinks/cloud-storage.adoc[] \ No newline at end of file += Cloud Storage (experimental) +:connectorType: cloud-storage + +include::common:streaming:partial$connectors/sinks/cloud-storage-intro.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +[#pulsar-sink-connector-properties] +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +== Cloud Storage sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/cloud-storage-properties.adoc[] + +[#data-format-types] +== Data format types + +include::common:streaming:partial$connectors/sinks/cloud-storage-format-types.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/elastic-search.adoc b/modules/connectors/pages/sinks/elastic-search.adoc index 255f956..310fb2d 100644 --- a/modules/connectors/pages/sinks/elastic-search.adoc +++ b/modules/connectors/pages/sinks/elastic-search.adoc @@ -1 +1,37 @@ -include::common:streaming:partial$connectors/sinks/elastic-search.adoc[] \ No newline at end of file += Elasticsearch +:connectorType: elastic_search + +The Elasticsearch sink connector reads messages from {pulsar-short} topics and writes them to https://www.elastic.co/elasticsearch/[Elasticsearch]. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sinks/elastic-search-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== Elasticsearch sink connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://pulsar.apache.org/docs/io-elasticsearch-sink[OSS {pulsar} Elasticsearch sink connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that are only present in <>. \ No newline at end of file diff --git a/modules/connectors/pages/sinks/google-bigquery.adoc b/modules/connectors/pages/sinks/google-bigquery.adoc index 2f5ea79..df3db75 100644 --- a/modules/connectors/pages/sinks/google-bigquery.adoc +++ b/modules/connectors/pages/sinks/google-bigquery.adoc @@ -1 +1,35 @@ -include::common:streaming:partial$connectors/sinks/google-bigquery.adoc[] \ No newline at end of file += Google BigQuery (experimental) +:connectorType: bigquery + +include::common:streaming:partial$connectors/sinks/google-bigquery-intro.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +[#pulsar-sink-connector-properties] +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== Kafka Connect adaptor properties (`configs`) + +include::common:streaming:partial$connectors/sinks/google-bigquery-properties.adoc[] + +[#kafkaConnectorConfigProperties] +=== Kafka Connect BigQuery Sink properties (`kafkaConnectorConfigProperties`) + +include::common:streaming:partial$connectors/sinks/google-bigquery-sink-properties.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-clickhouse.adoc b/modules/connectors/pages/sinks/jdbc-clickhouse.adoc index 52bc0de..c7935ff 100644 --- a/modules/connectors/pages/sinks/jdbc-clickhouse.adoc +++ b/modules/connectors/pages/sinks/jdbc-clickhouse.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sinks/jdbc-clickhouse.adoc[] \ No newline at end of file += JDBC ClickHouse +:connectorType: jdbc-clickhouse + +You can use the JDBC ClickHouse sink connector to stream data from {pulsar-short} topics into https://clickhouse.com/[ClickHouse] tables. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sinks/jdbc-clickhouse-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== JDBC ClickHouse sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/jdbc-config-params.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-mariadb.adoc b/modules/connectors/pages/sinks/jdbc-mariadb.adoc index b1ab937..6530017 100644 --- a/modules/connectors/pages/sinks/jdbc-mariadb.adoc +++ b/modules/connectors/pages/sinks/jdbc-mariadb.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sinks/jdbc-mariadb.adoc[] \ No newline at end of file += JDBC MariaDB +:connectorType: jdbc-mariadb + +You can use the JDBC MariaDB sink connector to stream data from {pulsar-short} topics into https://mariadb.org/[MariaDB] tables. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sinks/jdbc-mariadb-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== JDBC MariaDB sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/jdbc-config-params.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-postgres.adoc b/modules/connectors/pages/sinks/jdbc-postgres.adoc index a94fa74..447ce52 100644 --- a/modules/connectors/pages/sinks/jdbc-postgres.adoc +++ b/modules/connectors/pages/sinks/jdbc-postgres.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sinks/jdbc-postgres.adoc[] \ No newline at end of file += JDBC PostgreSQL +:connectorType: jdbc-postgres + +You can use the JDBC PostgreSQL sink connector to stream data from {pulsar-short} topics into https://www.postgresql.org/[PostgreSQL] tables. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sinks/jdbc-postgres-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== JDBC PostgreSQL sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/jdbc-config-params.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/jdbc-sqllite.adoc b/modules/connectors/pages/sinks/jdbc-sqllite.adoc index 79ba3d5..40cd6a8 100644 --- a/modules/connectors/pages/sinks/jdbc-sqllite.adoc +++ b/modules/connectors/pages/sinks/jdbc-sqllite.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sinks/jdbc-sqllite.adoc[] \ No newline at end of file += JDBC SQLite +:connectorType: jdbc-sqlite + +You can use the JDBC SQLite sink connector to stream data from {pulsar-short} topics into https://www.sqlite.org/index.html[SQLite] tables. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sinks/jdbc-sqllite-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== JDBC SQLite sink connector properties (`configs`) + +include::common:streaming:partial$connectors/sinks/jdbc-config-params.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sinks/kafka.adoc b/modules/connectors/pages/sinks/kafka.adoc index eb30fdd..1586b92 100644 --- a/modules/connectors/pages/sinks/kafka.adoc +++ b/modules/connectors/pages/sinks/kafka.adoc @@ -1 +1,37 @@ -include::common:streaming:partial$connectors/sinks/kafka.adoc[] \ No newline at end of file += Kafka +:connectorType: kafka + +The Kafka sink connector reads messages from {pulsar-short} topics and writes them to https://kafka.apache.org/[Kafka] topics. + +[#compatibility] +== Compatibility + +{product} supports {pulsar-reg} {pulsar-version}, which uses the https://github.com/apache/kafka/tree/2.7[Kafka 2.7.2 library]. + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== Kafka sink connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://pulsar.apache.org/docs/io-kafka-sink[OSS {pulsar} Kafka sink connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that aren't present in {pulsar} {pulsar-version}. \ No newline at end of file diff --git a/modules/connectors/pages/sinks/kinesis.adoc b/modules/connectors/pages/sinks/kinesis.adoc index ea1ceee..0d8def2 100644 --- a/modules/connectors/pages/sinks/kinesis.adoc +++ b/modules/connectors/pages/sinks/kinesis.adoc @@ -1 +1,39 @@ -include::common:streaming:partial$connectors/sinks/kinesis.adoc[] \ No newline at end of file += Kinesis +:connectorType: kinesis + +The Kinesis sink connector reads messages from {pulsar-short} topics and writes them to https://aws.amazon.com/kinesis/[Amazon Kinesis]. + +[#compatibility] +== Compatibility + +The Amazon Kinesis Client Library for Java (Amazon KCL) is used to consume and process data from Amazon Kinesis in Java applications. + +{product} supports {pulsar-reg} {pulsar-version}, which uses the https://github.com/awslabs/amazon-kinesis-client[Amazon Kinesis Client 2.2.8 library] and the https://github.com/aws/aws-sdk-java[AWS Java SDK 0.14.0 library]. + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== Kinesis sink connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://pulsar.apache.org/docs/io-kinesis-sink[OSS {pulsar} Kinesis sink connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that aren't present in {pulsar} {pulsar-version}. \ No newline at end of file diff --git a/modules/connectors/pages/sinks/snowflake.adoc b/modules/connectors/pages/sinks/snowflake.adoc index a31e2cd..43f7e0d 100644 --- a/modules/connectors/pages/sinks/snowflake.adoc +++ b/modules/connectors/pages/sinks/snowflake.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sinks/snowflake.adoc[] \ No newline at end of file += Snowflake +:connectorType: snowflake + +A https://www.snowflake.com/en/[Snowflake] database ingests structured and semi-structured datasets for processing and analysis. +Snowflake automatically manages all parts of the data storage process, including organization, structure, metadata, file size, compression, and statistics. + +The Snowflake sink connector reads messages from {pulsar-short} topics and writes them to Snowflake databases. + +== Create the connector + +include::ROOT:partial$sinks/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sinks/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} sink connector properties + +include::common:streaming:partial$connectors/sinks/pulsar-config-params.adoc[] + +[#configs] +=== Snowflake sink connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +For all supported properties, examples, and exceptions, see the https://github.com/datastax/snowflake-connector[{company} Snowflake connector GitHub repository]. \ No newline at end of file diff --git a/modules/connectors/pages/sources/data-generator.adoc b/modules/connectors/pages/sources/data-generator.adoc index 7b91b22..ae31b23 100644 --- a/modules/connectors/pages/sources/data-generator.adoc +++ b/modules/connectors/pages/sources/data-generator.adoc @@ -1 +1,34 @@ -include::common:streaming:partial$connectors/sources/data-generator.adoc[] \ No newline at end of file += Data Generator +:connectorType: data-generator + +include::common:streaming:partial$connectors/sources/data-generator-intro.adoc[] + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/data-generator-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Data Generator source connector properties (`configs`) + +include::common:streaming:partial$connectors/sources/data-generator-properties.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-mongodb.adoc b/modules/connectors/pages/sources/debezium-mongodb.adoc index ef7baf1..9463b57 100644 --- a/modules/connectors/pages/sources/debezium-mongodb.adoc +++ b/modules/connectors/pages/sources/debezium-mongodb.adoc @@ -1 +1,42 @@ -include::common:streaming:partial$connectors/sources/debezium-mongodb.adoc[] \ No newline at end of file += Debezium MongoDB +:connectorType: debezium-mongodb + +The Debezium MongoDB source connector tracks either a MongoDB replica set or a MongoDB sharded cluster for document changes in databases and collections. +It streams those changes as messages to a {pulsar-short} topic. + +The connector automatically handles the following: + +* Addition and removal of shards in a sharded cluster. +* Changes in membership for each replica set. +* Elections within each replica set. +* Resolution of communications issues with replica set members. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/debezium-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Debezium MongoDB source connector properties (`configs`) + +include::common:streaming:partial$connectors/sources/debezium-mongodb-propeties.adoc[] \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-mysql.adoc b/modules/connectors/pages/sources/debezium-mysql.adoc index 72a9bca..5c979ab 100644 --- a/modules/connectors/pages/sources/debezium-mysql.adoc +++ b/modules/connectors/pages/sources/debezium-mysql.adoc @@ -1 +1,37 @@ -include::common:streaming:partial$connectors/sources/debezium-mysql.adoc[] \ No newline at end of file += Debezium MySQL +:connectorType: debezium-mysql + +The Debezium MySQL source connector reads the `binlog` from MySQL database servers, produces change events for row-level `INSERT`, `UPDATE`, and `DELETE` operations, and then sends change event messages to {pulsar-short} topics. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/debezium-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Debezium MySQL source connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-connector-properties[Debezium connector for MySQL] and the https://pulsar.apache.org/docs/io-debezium-source[OSS {pulsar} Debezium source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that are only present in <>. \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-oracle.adoc b/modules/connectors/pages/sources/debezium-oracle.adoc index e47ba96..2f609f3 100644 --- a/modules/connectors/pages/sources/debezium-oracle.adoc +++ b/modules/connectors/pages/sources/debezium-oracle.adoc @@ -1 +1,40 @@ -include::common:streaming:partial$connectors/sources/debezium-oracle.adoc[] \ No newline at end of file += Debezium Oracle +:connectorType: debezium-oracle + +The Debezium Oracle source connector captures and records row-level changes that occur in databases on Oracle servers, including tables that are added while the connector is running. + +You can configure the connector according to the events and data that you want to stream. +For example, you can configure it to emit change events for a subset of schemas and tables, or configure it to ignore, mask, or truncate values in specific columns. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/debezium-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Debezium Oracle source connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://debezium.io/documentation/reference/stable/connectors/oracle.html#oracle-connector-properties[Debezium connector for Oracle] and the https://pulsar.apache.org/docs/io-debezium-source[OSS {pulsar} Debezium source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that are only present in <>. \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-postgres.adoc b/modules/connectors/pages/sources/debezium-postgres.adoc index 396f9df..2e398b1 100644 --- a/modules/connectors/pages/sources/debezium-postgres.adoc +++ b/modules/connectors/pages/sources/debezium-postgres.adoc @@ -1 +1,37 @@ -include::common:streaming:partial$connectors/sources/debezium-postgres.adoc[] \ No newline at end of file += Debezium PostgreSQL +:connectorType: debezium-postgres + +The Debezium PostgreSQL source connector produces a change event for every row-level `INSERT`, `UPDATE`, and `DELETE` operation that it captures, and then it sends change event records for each table to separate {pulsar-short} topics. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/debezium-compatibility.adoc[] + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Debezium PostgreSQL source connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://debezium.io/documentation/reference/stable/connectors/postgresql.html#postgresql-connector-properties[Debezium connector for PostgreSQL] and the https://pulsar.apache.org/docs/io-debezium-source[OSS {pulsar} Debezium source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that are only present in <>. \ No newline at end of file diff --git a/modules/connectors/pages/sources/debezium-sqlserver.adoc b/modules/connectors/pages/sources/debezium-sqlserver.adoc index 9b6243f..5c7d1d9 100644 --- a/modules/connectors/pages/sources/debezium-sqlserver.adoc +++ b/modules/connectors/pages/sources/debezium-sqlserver.adoc @@ -1 +1,57 @@ -include::common:streaming:partial$connectors/sources/debezium-sqlserver.adoc[] \ No newline at end of file += Debezium SQL Server +:connectorType: debezium-sqlserver + +The Debezium SQL Server source connector reads data from Debezium SQL Server databases and tables, and then emits change data event messages to {pulsar-short} topics. + +It is based on the change data capture (CDC) feature available in SQL Server 2016 Service Pack 1 (SP1) and later editions. + +The SQL Server capture process monitors designated databases and tables, and then stores changes into dedicated change tables with stored procedure facades. + +[#compatibility] +== Compatibility + +include::common:streaming:partial$connectors/sources/debezium-compatibility.adoc[] + +[#create-the-connector] +== Create the connector + +include::common:streaming:partial$connectors/sources/debezium-sqlserver-setup.adoc[] + +include::ROOT:partial$sources/get-started.adoc[] + +. Set your consumers to subscribe to the `events` topic in your {pulsar-short} namespace to receive CDC events. +This topic contains the CDC events emitted by Debezium for the configured tables. + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +[WARNING] +==== +The `name` property must match the connector name you used in the topic names as explained in <>. +==== + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Debezium SQL Server source connector properties (`configs`) + +[WARNING] +==== +Make sure you set the `topic.prefix`, `database.server.name`, and `task.id` properties as explained in <>. +==== + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://debezium.io/documentation/reference/stable/connectors/sqlserver.html#sqlserver-connector-properties[Debezium connector for SQL Server] and the https://pulsar.apache.org/docs/io-debezium-source[OSS {pulsar} Debezium source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that are only present in <>. \ No newline at end of file diff --git a/modules/connectors/pages/sources/kafka.adoc b/modules/connectors/pages/sources/kafka.adoc index 1f5f423..5ac9110 100644 --- a/modules/connectors/pages/sources/kafka.adoc +++ b/modules/connectors/pages/sources/kafka.adoc @@ -1 +1,32 @@ -include::common:streaming:partial$connectors/sources/kafka.adoc[] \ No newline at end of file += Kafka +:connectorType: kafka + +The Kafka source connector pulls data from https://kafka.apache.org/[Kafka] topics and persists it to {pulsar-short} topics. + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Kafka source connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://pulsar.apache.org/docs/io-kafka-source[OSS {pulsar} Kafka source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that aren't present in {pulsar} {pulsar-version}. \ No newline at end of file diff --git a/modules/connectors/pages/sources/kinesis.adoc b/modules/connectors/pages/sources/kinesis.adoc index bbc36c5..5edd895 100644 --- a/modules/connectors/pages/sources/kinesis.adoc +++ b/modules/connectors/pages/sources/kinesis.adoc @@ -1 +1,32 @@ -include::common:streaming:partial$connectors/sources/kinesis.adoc[] \ No newline at end of file += Kinesis +:connectorType: kinesis + +The Kinesis source connector pulls data from https://aws.amazon.com/kinesis/[Amazon Kinesis] and persists it to {pulsar-short} topics. + +== Create the connector + +include::ROOT:partial$sources/get-started.adoc[] + +== Edit the connector + +include::ROOT:partial$sources/edit.adoc[] + +== Manage the connector + +include::common:streaming:partial$connectors/manage.adoc[] + +== Connector configuration reference + +include::common:streaming:partial$connectors/connector-params-intro.adoc[] + +=== {pulsar-short} source connector properties + +include::common:streaming:partial$connectors/sources/pulsar-config-params.adoc[] + +[#configs] +=== Kinesis source connector properties (`configs`) + +Set these properties in the `configs` section of the connector configuration. + +Generally, all properties provided in the https://pulsar.apache.org/docs/io-kinesis-source[OSS {pulsar} Kinesis source connector] are supported. +Exceptions include properties that aren't relevant to {product} and properties that aren't present in {pulsar} {pulsar-version}. \ No newline at end of file From 67797b336a19c6752be33d6939ee70f780778f56 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 07:03:29 -0800 Subject: [PATCH 08/11] unused attributes --- antora.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/antora.yml b/antora.yml index 5c53832..e895ac8 100644 --- a/antora.yml +++ b/antora.yml @@ -29,8 +29,4 @@ asciidoc: cass-reg: 'Apache Cassandra(R)' cass: 'Apache Cassandra' cass-short: 'Cassandra' - debezium-version: '1.7' - get-started-sink: 'include::ROOT:partial$sinks/get-started.adoc[]' - edit-sink: 'include::ROOT:partial$sinks/edit.adoc[]' - get-started-source: 'include::ROOT:partial$sources/get-started.adoc[]' - edit-source: 'include::ROOT:partial$sources/edit.adoc[]' \ No newline at end of file + debezium-version: '1.7' \ No newline at end of file From 8753ed72036a3e3a8ac67014daf4dc07a3f3f46b Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 07:46:43 -0800 Subject: [PATCH 09/11] change example syntax --- modules/ROOT/partials/sinks/get-started.adoc | 5 ++++- modules/ROOT/partials/sources/get-started.adoc | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/ROOT/partials/sinks/get-started.adoc b/modules/ROOT/partials/sinks/get-started.adoc index c00eff4..ca68578 100644 --- a/modules/ROOT/partials/sinks/get-started.adoc +++ b/modules/ROOT/partials/sinks/get-started.adoc @@ -26,4 +26,7 @@ You can pass the configuration directly or with a configuration file. ---- + .Example configuration data structure -include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.adoc[] \ No newline at end of file +[source,json,subs="+attributes"] +---- +include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.adoc[] +---- \ No newline at end of file diff --git a/modules/ROOT/partials/sources/get-started.adoc b/modules/ROOT/partials/sources/get-started.adoc index 9fdda51..4aa4970 100644 --- a/modules/ROOT/partials/sources/get-started.adoc +++ b/modules/ROOT/partials/sources/get-started.adoc @@ -26,4 +26,7 @@ You can pass the configuration directly or with a configuration file. ---- + .Example configuration data structure -include::common:streaming:example$connectors/sources/{connectorType}/sample-data.adoc[] \ No newline at end of file +[source,json,subs="+attributes"] +---- +include::common:streaming:example$connectors/sources/{connectorType}/sample-data.adoc[] +---- \ No newline at end of file From e645e33b933c3ae252cd2bc98b95b5fb993dc0b6 Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 07:52:29 -0800 Subject: [PATCH 10/11] fix file type --- modules/ROOT/partials/sinks/get-started.adoc | 2 +- modules/ROOT/partials/sources/get-started.adoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ROOT/partials/sinks/get-started.adoc b/modules/ROOT/partials/sinks/get-started.adoc index ca68578..534eff3 100644 --- a/modules/ROOT/partials/sinks/get-started.adoc +++ b/modules/ROOT/partials/sinks/get-started.adoc @@ -28,5 +28,5 @@ You can pass the configuration directly or with a configuration file. .Example configuration data structure [source,json,subs="+attributes"] ---- -include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.adoc[] +include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.json[] ---- \ No newline at end of file diff --git a/modules/ROOT/partials/sources/get-started.adoc b/modules/ROOT/partials/sources/get-started.adoc index 4aa4970..63cde39 100644 --- a/modules/ROOT/partials/sources/get-started.adoc +++ b/modules/ROOT/partials/sources/get-started.adoc @@ -28,5 +28,5 @@ You can pass the configuration directly or with a configuration file. .Example configuration data structure [source,json,subs="+attributes"] ---- -include::common:streaming:example$connectors/sources/{connectorType}/sample-data.adoc[] +include::common:streaming:example$connectors/sources/{connectorType}/sample-data.json[] ---- \ No newline at end of file From e2310f26b8147ffe03de0d96e60e33032ac75a2c Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Tue, 3 Feb 2026 07:58:23 -0800 Subject: [PATCH 11/11] once more --- modules/ROOT/partials/sinks/get-started.adoc | 2 +- modules/ROOT/partials/sources/get-started.adoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ROOT/partials/sinks/get-started.adoc b/modules/ROOT/partials/sinks/get-started.adoc index 534eff3..e4acdf4 100644 --- a/modules/ROOT/partials/sinks/get-started.adoc +++ b/modules/ROOT/partials/sinks/get-started.adoc @@ -26,7 +26,7 @@ You can pass the configuration directly or with a configuration file. ---- + .Example configuration data structure -[source,json,subs="+attributes"] +[source,json] ---- include::common:streaming:example$connectors/sinks/{connectorType}/sample-data.json[] ---- \ No newline at end of file diff --git a/modules/ROOT/partials/sources/get-started.adoc b/modules/ROOT/partials/sources/get-started.adoc index 63cde39..fb4cd25 100644 --- a/modules/ROOT/partials/sources/get-started.adoc +++ b/modules/ROOT/partials/sources/get-started.adoc @@ -26,7 +26,7 @@ You can pass the configuration directly or with a configuration file. ---- + .Example configuration data structure -[source,json,subs="+attributes"] +[source,json] ---- include::common:streaming:example$connectors/sources/{connectorType}/sample-data.json[] ---- \ No newline at end of file