diff --git a/charts/debezium/values-azure.yaml b/charts/debezium/values-azure.yaml index 1ac6954ae..81b82c67d 100644 --- a/charts/debezium/values-azure.yaml +++ b/charts/debezium/values-azure.yaml @@ -107,6 +107,63 @@ connect: "value.converter.schemas.enable": "true", } } + + sqlserverconnector_meta: { + "name": "debezium-odse-meta-tables-connector-v041825", + "config": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.hostname": "tf-cdc-nbs6-sql-managed-instance.cdf5734b998e.database.windows.net", + "database.port": "1433", + "database.user": "nbs_ods", + "database.password": "ods", + "database.dbname": "nbs_odse", + "database.names": "nbs_odse", + "database.server.name": "odse", + "database.history.kafka.bootstrap.servers": "wn0-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn1-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn2-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092", + "database.history.kafka.topic": "dbhistory.database_server_name.database_name", + # Uncomment following to manually bypass the sqlserver agent status query results + #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", + "database.trustServerCertificate": "true", + "include.schema.changes": "true", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "true", + "producer.max.request.size": "10000000", #10MB + "producer.message.max.bytes": "10000000", #10MB + "snapshot.mode": "initial", + "schema.history.internal.kafka.topic": "odse-schema-history", + "schema.history.internal.kafka.bootstrap.servers": "wn0-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn1-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092,wn2-dev-hd.dsnw3omymczu3cea2rtxoridtc.bx.internal.cloudapp.net:9092", + "table.include.list": "dbo.Page_cond_mapping, dbo.NBS_page, dbo.NBS_ui_metadata, dbo.NBS_rdb_metadata", + "tasks.max": "1", + "topic.prefix": "cdc", + "topic.creation.default.replication.factor": 2, + "topic.creation.default.partitions": 10, + "topic.creation.default.cleanup.policy": "compact", + "time.precision.mode": "connect", + "transforms": "dropPrefix, convertTimezone, unwrap, convertTimestampsConfig_add_time, + convertTimestampsConfig_last_chg_time, convertTimestampsConfig_record_status_time", + "transforms.dropPrefix.regex": "cdc\\.NBS_ODSE\\.dbo\\.(.+)", + "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.dropPrefix.replacement": "nrt_odse_$1", + "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", + "transforms.convertTimezone.converted.timezone": "UTC", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.convertTimestampsConfig_add_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_add_time.target.type": "string", + "transforms.convertTimestampsConfig_add_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_add_time.field": "add_time", + "transforms.convertTimestampsConfig_last_chg_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_last_chg_time.target.type": "string", + "transforms.convertTimestampsConfig_last_chg_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_last_chg_time.field": "last_chg_time", + "transforms.convertTimestampsConfig_record_status_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_record_status_time.target.type": "string", + "transforms.convertTimestampsConfig_record_status_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_record_status_time.field": "status_time", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "true", + } + } + sqlserverconnector_srte: { "name": "debezium-srte-connector-v013125", "config": { diff --git a/charts/debezium/values-feature.yaml b/charts/debezium/values-feature.yaml index 5f6f3ec21..c54f2d795 100644 --- a/charts/debezium/values-feature.yaml +++ b/charts/debezium/values-feature.yaml @@ -107,6 +107,62 @@ connect: } } + sqlserverconnector_meta: { + "name": "debezium-odse-meta-tables-connector-v041825", + "config": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.hostname": "nbs-db.private-feature.nbspreview.com", + "database.port": "1433", + "database.user": "", + "database.password": "", + "database.dbname": "nbs_odse", + "database.names": "nbs_odse", + "database.server.name": "odse", + "database.history.kafka.bootstrap.servers": "b-2.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.topic": "dbhistory.database_server_name.database_name", + # Uncomment following to manually bypass the sqlserver agent status query results + #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", + "database.trustServerCertificate": "true", + "include.schema.changes": "true", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "true", + "producer.max.request.size": "10000000", #10MB + "producer.message.max.bytes": "10000000", #10MB + "snapshot.mode": "initial", + "schema.history.internal.kafka.topic": "odse-schema-history", + "schema.history.internal.kafka.bootstrap.servers": "b-2.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092", + "table.include.list": "dbo.Page_cond_mapping, dbo.NBS_page, dbo.NBS_ui_metadata, dbo.NBS_rdb_metadata", + "tasks.max": "1", + "topic.prefix": "cdc", + "topic.creation.default.replication.factor": 2, + "topic.creation.default.partitions": 10, + "topic.creation.default.cleanup.policy": "compact", + "time.precision.mode": "connect", + "transforms": "dropPrefix, convertTimezone, unwrap, convertTimestampsConfig_add_time, + convertTimestampsConfig_last_chg_time, convertTimestampsConfig_record_status_time", + "transforms.dropPrefix.regex": "cdc\\.NBS_ODSE\\.dbo\\.(.+)", + "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.dropPrefix.replacement": "nrt_odse_$1", + "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", + "transforms.convertTimezone.converted.timezone": "UTC", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.convertTimestampsConfig_add_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_add_time.target.type": "string", + "transforms.convertTimestampsConfig_add_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_add_time.field": "add_time", + "transforms.convertTimestampsConfig_last_chg_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_last_chg_time.target.type": "string", + "transforms.convertTimestampsConfig_last_chg_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_last_chg_time.field": "last_chg_time", + "transforms.convertTimestampsConfig_record_status_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_record_status_time.target.type": "string", + "transforms.convertTimestampsConfig_record_status_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_record_status_time.field": "status_time", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "true", + } + } + sqlserverconnector_srte: { "name": "debezium-srte-connector-v013125", "config": { diff --git a/charts/debezium/values-int1.yaml b/charts/debezium/values-int1.yaml index 6588061e4..f07c3e529 100644 --- a/charts/debezium/values-int1.yaml +++ b/charts/debezium/values-int1.yaml @@ -109,6 +109,64 @@ connect: } } + sqlserverconnector_meta: { + "name": "debezium-odse-meta-tables-connector-v041825", + "config": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.hostname": "nbs-db.private-int1.nbspreview.com", + "database.port": "1433", + "database.user": "", + "database.password": "", + "database.dbname": "nbs_odse", + "database.names": "nbs_odse", + "database.server.name": "odse", + "database.history.kafka.bootstrap.servers": "b-2.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092, + b-1.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.topic": "dbhistory.database_server_name.database_name", + # Uncomment following to manually bypass the sqlserver agent status query results + #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", + "database.trustServerCertificate": "true", + "include.schema.changes": "true", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "true", + "producer.max.request.size": "10000000", #10MB + "producer.message.max.bytes": "10000000", #10MB + "snapshot.mode": "initial", + "schema.history.internal.kafka.topic": "odse-schema-history", + "schema.history.internal.kafka.bootstrap.servers": "b-2.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092, + b-1.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092", + "table.include.list": "dbo.Page_cond_mapping, dbo.NBS_page, dbo.NBS_ui_metadata, dbo.NBS_rdb_metadata", + "tasks.max": "1", + "topic.prefix": "cdc", + "topic.creation.default.replication.factor": 2, + "topic.creation.default.partitions": 10, + "topic.creation.default.cleanup.policy": "compact", + "time.precision.mode": "connect", + "transforms": "dropPrefix, convertTimezone, unwrap, convertTimestampsConfig_add_time, + convertTimestampsConfig_last_chg_time, convertTimestampsConfig_record_status_time", + "transforms.dropPrefix.regex": "cdc\\.NBS_ODSE\\.dbo\\.(.+)", + "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.dropPrefix.replacement": "nrt_odse_$1", + "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", + "transforms.convertTimezone.converted.timezone": "UTC", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.convertTimestampsConfig_add_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_add_time.target.type": "string", + "transforms.convertTimestampsConfig_add_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_add_time.field": "add_time", + "transforms.convertTimestampsConfig_last_chg_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_last_chg_time.target.type": "string", + "transforms.convertTimestampsConfig_last_chg_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_last_chg_time.field": "last_chg_time", + "transforms.convertTimestampsConfig_record_status_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_record_status_time.target.type": "string", + "transforms.convertTimestampsConfig_record_status_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_record_status_time.field": "status_time", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "true", + } + } + sqlserverconnector_srte: { "name": "debezium-srte-connector-v013125", "config": { diff --git a/charts/debezium/values-test1.yaml b/charts/debezium/values-test1.yaml index 40583bb89..d44c0997b 100644 --- a/charts/debezium/values-test1.yaml +++ b/charts/debezium/values-test1.yaml @@ -107,6 +107,62 @@ connect: } } + sqlserverconnector_meta: { + "name": "debezium-odse-meta-tables-connector-v041825", + "config": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.hostname": "nbs-db.private-test.nbspreview.com", + "database.port": "1433", + "database.user": "", + "database.password": "", + "database.dbname": "nbs_odse", + "database.names": "nbs_odse", + "database.server.name": "odse", + "database.history.kafka.bootstrap.servers": "b-1.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092,b-2.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092", + "database.history.kafka.topic": "dbhistory.database_server_name.database_name", + # Uncomment following to manually bypass the sqlserver agent status query results + #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", + "database.trustServerCertificate": "true", + "include.schema.changes": "true", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "true", + "producer.max.request.size": "10000000", #10MB + "producer.message.max.bytes": "10000000", #10MB + "snapshot.mode": "initial", + "schema.history.internal.kafka.topic": "odse-schema-history", + "schema.history.internal.kafka.bootstrap.servers": "b-1.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092,b-2.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092", + "table.include.list": "dbo.Page_cond_mapping, dbo.NBS_page, dbo.NBS_ui_metadata, dbo.NBS_rdb_metadata", + "tasks.max": "1", + "topic.prefix": "cdc", + "topic.creation.default.replication.factor": 2, + "topic.creation.default.partitions": 10, + "topic.creation.default.cleanup.policy": "compact", + "time.precision.mode": "connect", + "transforms": "dropPrefix, convertTimezone, unwrap, convertTimestampsConfig_add_time, + convertTimestampsConfig_last_chg_time, convertTimestampsConfig_record_status_time", + "transforms.dropPrefix.regex": "cdc\\.NBS_ODSE\\.dbo\\.(.+)", + "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.dropPrefix.replacement": "nrt_odse_$1", + "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", + "transforms.convertTimezone.converted.timezone": "UTC", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.convertTimestampsConfig_add_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_add_time.target.type": "string", + "transforms.convertTimestampsConfig_add_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_add_time.field": "add_time", + "transforms.convertTimestampsConfig_last_chg_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_last_chg_time.target.type": "string", + "transforms.convertTimestampsConfig_last_chg_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_last_chg_time.field": "last_chg_time", + "transforms.convertTimestampsConfig_record_status_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_record_status_time.target.type": "string", + "transforms.convertTimestampsConfig_record_status_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_record_status_time.field": "status_time", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "true", + } + } + sqlserverconnector_srte: { "name": "debezium-srte-connector-v013125", "config": { diff --git a/charts/debezium/values.yaml b/charts/debezium/values.yaml index 603ea9997..58f2ec20a 100644 --- a/charts/debezium/values.yaml +++ b/charts/debezium/values.yaml @@ -109,6 +109,62 @@ connect: } } + sqlserverconnector_meta: { + "name": "debezium-odse-meta-tables-connector-v041825", + "config": { + "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", + "database.hostname": "nbs-db.private-EXAMPLE_DOMAIN", + "database.port": "1433", + "database.user": "EXAMPLE_ODSE_DB_USER", + "database.password": "EXAMPLE_ODSE_DB_USER_PASSWORD", + "database.dbname": "nbs_odse", + "database.names": "nbs_odse", + "database.server.name": "odse", + "database.history.kafka.bootstrap.servers": "EXAMPLE_MSK_KAFKA_ENDPOINT", + "database.history.kafka.topic": "dbhistory.database_server_name.database_name", + # Uncomment following to manually bypass the sqlserver agent status query results + #"database.sqlserver.agent.status.query": "select dbo.IsSqlAgentRunning()", + "database.trustServerCertificate": "true", + "include.schema.changes": "true", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "true", + "producer.max.request.size": "10000000", #10MB + "producer.message.max.bytes": "10000000", #10MB + "snapshot.mode": "initial", + "schema.history.internal.kafka.topic": "odse-schema-history", + "schema.history.internal.kafka.bootstrap.servers": "EXAMPLE_MSK_KAFKA_ENDPOINT", + "table.include.list": "dbo.Page_cond_mapping, dbo.NBS_page, dbo.NBS_ui_metadata, dbo.NBS_rdb_metadata", + "tasks.max": "1", + "topic.prefix": "cdc", + "topic.creation.default.replication.factor": 2, + "topic.creation.default.partitions": 10, + "topic.creation.default.cleanup.policy": "compact", + "time.precision.mode": "connect", + "transforms": "dropPrefix, convertTimezone, unwrap, convertTimestampsConfig_add_time, + convertTimestampsConfig_last_chg_time, convertTimestampsConfig_record_status_time", + "transforms.dropPrefix.regex": "cdc\\.NBS_ODSE\\.dbo\\.(.+)", + "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", + "transforms.dropPrefix.replacement": "nrt_odse_$1", + "transforms.convertTimezone.type": "io.debezium.transforms.TimezoneConverter", + "transforms.convertTimezone.converted.timezone": "UTC", + "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", + "transforms.convertTimestampsConfig_add_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_add_time.target.type": "string", + "transforms.convertTimestampsConfig_add_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_add_time.field": "add_time", + "transforms.convertTimestampsConfig_last_chg_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_last_chg_time.target.type": "string", + "transforms.convertTimestampsConfig_last_chg_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_last_chg_time.field": "last_chg_time", + "transforms.convertTimestampsConfig_record_status_time.type": "org.apache.kafka.connect.transforms.TimestampConverter$Value", + "transforms.convertTimestampsConfig_record_status_time.target.type": "string", + "transforms.convertTimestampsConfig_record_status_time.format": "yyyy-MM-dd HH:mm:ss.SSS", + "transforms.convertTimestampsConfig_record_status_time.field": "status_time", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "true", + } + } + sqlserverconnector_srte: { "name": "debezium-srte-connector-v013125", "config": {