From 0143d697f109fd61b975ec83ff335b3dc765e212 Mon Sep 17 00:00:00 2001 From: GavinXiao <2749984520@qq.com> Date: Mon, 24 Nov 2025 17:28:34 +0800 Subject: [PATCH 1/5] feat(TAP-8714): Batch consumption of events becomes processing of individual consumption --- connectors-common/connector-core/pom.xml | 2 +- connectors-common/hive-core/pom.xml | 2 +- .../connector/kafka/KafkaSRService.java | 3 +- .../tapdata/connector/kafka/KafkaService.java | 41 +++---- .../connector/kafka/util/BatchPusher.java | 8 +- .../java/io/tapdata/common/MqService.java | 5 +- connectors-common/mysql-core/pom.xml | 2 +- .../tapdata/connector/mysql/MysqlReader.java | 36 ++---- connectors-common/pom.xml | 4 +- connectors-common/postgres-core/pom.xml | 2 +- .../postgres/cdc/AbstractWalLogMiner.java | 8 +- .../postgres/cdc/PostgresCdcRunner.java | 28 ++--- .../connector/postgres/cdc/WalLogMiner.java | 19 +--- .../connector/postgres/cdc/WalLogMinerV2.java | 19 +--- .../connector/postgres/cdc/WalLogMinerV3.java | 22 +--- .../connector/postgres/cdc/WalPgtoMiner.java | 19 +--- connectors-common/read-partition/pom.xml | 4 +- .../connector/activemq/ActivemqConnector.java | 8 +- .../connector/activemq/ActivemqService.java | 29 ++--- .../src/main/resources/spec_activemq.json | 12 +- .../aliyun-adb-postgres-connector/pom.xml | 2 +- .../resources/aliyun-adb-postgres-spec.json | 12 +- connectors/aliyun-mongodb-connector/pom.xml | 2 +- .../main/resources/aliyun-mongodb-spec.json | 12 +- .../aliyun-rds-mariadb-connector/pom.xml | 2 +- .../resources/aliyun-rds-mariadb-spec.json | 12 +- connectors/aliyun-rds-mysql-connector/pom.xml | 2 +- .../main/resources/aliyun-rds-mysql-spec.json | 12 +- .../aliyun-rds-postgres-connector/pom.xml | 2 +- .../resources/aliyun-rds-postgres-spec.json | 12 +- connectors/aws-clickhouse-connector/pom.xml | 2 +- .../src/main/resources/aws_clickhouse.json | 12 +- connectors/aws-rds-mysql-connector/pom.xml | 2 +- .../main/resources/aws-rds-mysql-spec.json | 12 +- connectors/azure-cosmosdb-connector/pom.xml | 2 +- .../main/resources/azure-cosmosdb-spec.json | 12 +- connectors/clickhouse-connector/pom.xml | 2 +- .../src/main/resources/spec_clickhouse.json | 12 +- connectors/highgo-connector/pom.xml | 2 +- .../connector/postgres/HighgoConnector.java | 9 +- .../src/main/resources/spec_highgo.json | 12 +- connectors/kafka-connector/pom.xml | 2 +- .../connector/kafka/KafkaConnector.java | 8 +- .../src/main/resources/spec_kafka.json | 12 +- connectors/mariadb-connector/pom.xml | 2 +- .../src/main/resources/spec_mariadb.json | 12 +- connectors/mongodb-connector/pom.xml | 2 +- .../io/tapdata/mongodb/MongodbConnector.java | 20 ++-- .../mongodb/reader/MongodbStreamReader.java | 4 +- .../mongodb/reader/MongodbV4StreamReader.java | 29 +---- .../reader/v3/MongodbV3StreamReader.java | 86 +++++--------- .../src/main/resources/spec.json | 12 +- .../tapdata/mongodb/MongodbConnectorTest.java | 106 +++++++++--------- connectors/mysql-connector/pom.xml | 2 +- .../connector/mysql/MysqlConnector.java | 8 +- .../src/main/resources/mysql-spec.json | 12 +- connectors/pom.xml | 4 +- connectors/postgres-connector/pom.xml | 2 +- .../connector/postgres/PostgresConnector.java | 22 ++-- .../src/main/resources/spec_postgres.json | 12 +- .../connector/rabbitmq/RabbitmqConnector.java | 8 +- .../connector/rabbitmq/RabbitmqService.java | 36 +++--- .../src/main/resources/spec_rabbitmq.json | 12 +- .../connector/rocketmq/RocketmqConnector.java | 8 +- .../connector/rocketmq/RocketmqService.java | 28 ++--- .../src/main/resources/spec_rocketmq.json | 12 +- connectors/tidb-connector/pom.xml | 2 +- .../tapdata/connector/tidb/TidbConnector.java | 6 +- .../analyse/AnalyseTapEventFromDDLObject.java | 10 +- .../cdc/process/thread/ProcessHandler.java | 6 +- .../cdc/process/thread/TapEventManager.java | 8 +- .../src/main/resources/spec_tidb.json | 12 +- .../connector/tidb/TidbConnectorTest.java | 12 +- .../AnalyseTapEventFromDDLObjectTest.java | 8 +- .../process/thread/TapEventManagerTest.java | 17 ++- connectors/vastbase-connector/pom.xml | 2 +- .../connector/postgres/VastbaseConnector.java | 9 +- .../src/main/resources/spec_vastbase.json | 12 +- 78 files changed, 499 insertions(+), 488 deletions(-) diff --git a/connectors-common/connector-core/pom.xml b/connectors-common/connector-core/pom.xml index b04913318..09d07ce95 100644 --- a/connectors-common/connector-core/pom.xml +++ b/connectors-common/connector-core/pom.xml @@ -33,7 +33,7 @@ io.tapdata tapdata-pdk-runner - 2.0-SNAPSHOT + 2.5-SNAPSHOT test diff --git a/connectors-common/hive-core/pom.xml b/connectors-common/hive-core/pom.xml index f64472c12..a98dd637a 100644 --- a/connectors-common/hive-core/pom.xml +++ b/connectors-common/hive-core/pom.xml @@ -18,7 +18,7 @@ 8 1.0-SNAPSHOT - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/KafkaSRService.java b/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/KafkaSRService.java index f46690ce0..3455c0619 100644 --- a/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/KafkaSRService.java +++ b/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/KafkaSRService.java @@ -31,6 +31,7 @@ import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import java.io.BufferedReader; import java.io.IOException; @@ -313,7 +314,7 @@ public void consumeOne(TapTable tapTable, int eventBatchSize, BiConsumer tableList, int eventBatchSize, BiConsumer, Object> eventsOffsetConsumer) { + public void streamConsume(List tableList, StreamReadOneByOneConsumer eventsOffsetConsumer) { throw new CoreException("The schemaRegister function is not supported as the source for the time being. "); } diff --git a/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/KafkaService.java b/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/KafkaService.java index ab1295802..41f9edbed 100644 --- a/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/KafkaService.java +++ b/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/KafkaService.java @@ -47,6 +47,7 @@ import org.apache.kafka.common.TopicPartitionInfo; import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.header.internals.RecordHeaders; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import javax.script.Invocable; import javax.script.ScriptEngine; @@ -641,13 +642,13 @@ public void consumeOne(TapTable tapTable, int eventBatchSize, BiConsumer consumerRecord : consumerRecords) { makeMessage(consumerRecord, tableName, list::add); if (list.size() >= eventBatchSize) { - syncEventSubmit(list, eventsOffsetConsumer); + eventsOffsetConsumer.accept(list, TapSimplify.list()); list = TapSimplify.list(); } } } if (EmptyKit.isNotEmpty(list)) { - syncEventSubmit(list, eventsOffsetConsumer); + eventsOffsetConsumer.accept(list, TapSimplify.list()); } } catch (Exception e) { throwable.set(e); @@ -669,37 +670,27 @@ public void consumeOne(TapTable tapTable, int eventBatchSize, BiConsumer eventList, BiConsumer, Object> eventsOffsetConsumer) { - eventsOffsetConsumer.accept(eventList, TapSimplify.list()); - } - @Override - public void streamConsume(List tableList, Object offset, int eventBatchSize, BiConsumer, Object> eventsOffsetConsumer) { + public void streamConsume(List tableList, Object offset, StreamReadOneByOneConsumer eventsOffsetConsumer) { consuming.set(true); int maxDelay = 500; KafkaConfig kafkaConfig = (KafkaConfig) mqConfig; ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration((kafkaConfig), connectorId, true); try (KafkaConsumer kafkaConsumer = new KafkaConsumer<>(consumerConfiguration.build())) { KafkaOffset streamOffset = KafkaOffsetUtils.setConsumerByOffset(kafkaConsumer, tableList, offset, consuming); - try (BatchPusher batchPusher = new BatchPusher( - tapEvents -> eventsOffsetConsumer.accept(tapEvents, streamOffset.clone()) - ).batchSize(eventBatchSize).maxDelay(maxDelay)) { - // 将初始化的 offset 推送到目标，让指定时间的增量任务下次启动时拿到 offset - Optional.of(new HeartbeatEvent()).ifPresent(event -> { - event.setTime(System.currentTimeMillis()); - batchPusher.add(event); - }); + // 将初始化的 offset 推送到目标，让指定时间的增量任务下次启动时拿到 offset + Optional.of(new HeartbeatEvent()).ifPresent(event -> { + event.setTime(System.currentTimeMillis()); + eventsOffsetConsumer.accept(event, streamOffset.clone()); + }); - // 消费数据 - while (consuming.get()) { - ConsumerRecords consumerRecords = kafkaConsumer.poll(Duration.ofSeconds(2L)); - if (consumerRecords.isEmpty()) { - batchPusher.checkAndSummit(); - } else { - for (ConsumerRecord consumerRecord : consumerRecords) { - streamOffset.addTopicOffset(consumerRecord); // 推进 offset - makeMessage(consumerRecord, consumerRecord.topic(), batchPusher::add); - } + // 消费数据 + while (consuming.get()) { + ConsumerRecords consumerRecords = kafkaConsumer.poll(Duration.ofSeconds(2L)); + if (!consumerRecords.isEmpty()) { + for (ConsumerRecord consumerRecord : consumerRecords) { + streamOffset.addTopicOffset(consumerRecord); // 推进 offset + makeMessage(consumerRecord, consumerRecord.topic(), e -> eventsOffsetConsumer.accept(e, streamOffset.clone())); } } } diff --git a/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/util/BatchPusher.java b/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/util/BatchPusher.java index cc7646f05..238e9717e 100644 --- a/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/util/BatchPusher.java +++ b/connectors-common/kafka-core/src/main/java/io/tapdata/connector/kafka/util/BatchPusher.java @@ -11,7 +11,6 @@ */ public class BatchPusher implements AutoCloseable { - private int batchSize = 100; private int maxDelay = 2000; private long lastTime; private final Consumer> submitConsumer; @@ -23,11 +22,6 @@ public BatchPusher(Consumer> submitConsumer) { this.submitConsumer = submitConsumer; } - public BatchPusher batchSize(int batchSize) { - this.batchSize = batchSize; - return this; - } - public BatchPusher maxDelay(int maxDelay) { this.maxDelay = maxDelay; return this; @@ -39,7 +33,7 @@ public void add(T record) { } public void checkAndSummit() { - if (batchList.size() >= batchSize || (System.currentTimeMillis() - lastTime > maxDelay && !batchList.isEmpty())) { + if (System.currentTimeMillis() - lastTime > maxDelay && !batchList.isEmpty()) { summit(); } } diff --git a/connectors-common/mq-core/src/main/java/io/tapdata/common/MqService.java b/connectors-common/mq-core/src/main/java/io/tapdata/common/MqService.java index c2aac3144..c2dbd9292 100644 --- a/connectors-common/mq-core/src/main/java/io/tapdata/common/MqService.java +++ b/connectors-common/mq-core/src/main/java/io/tapdata/common/MqService.java @@ -8,6 +8,7 @@ import io.tapdata.pdk.apis.entity.TestItem; import io.tapdata.pdk.apis.entity.WriteListResult; import io.tapdata.pdk.apis.functions.connection.ConnectionCheckItem; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import java.util.List; import java.util.function.BiConsumer; @@ -43,11 +44,11 @@ default void consumeOne(TapTable tapTable, int eventBatchSize, BiConsumer tableList, int eventBatchSize, BiConsumer, Object> eventsOffsetConsumer) throws Throwable { + default void streamConsume(List tableList, StreamReadOneByOneConsumer eventsOffsetConsumer) throws Throwable { throw new UnsupportedOperationException(); } - default void streamConsume(List tableList, Object offset, int eventBatchSize, BiConsumer, Object> eventsOffsetConsumer) { + default void streamConsume(List tableList, Object offset, StreamReadOneByOneConsumer eventsOffsetConsumer) { throw new UnsupportedOperationException(); } } diff --git a/connectors-common/mysql-core/pom.xml b/connectors-common/mysql-core/pom.xml index 82d6621ee..817076814 100644 --- a/connectors-common/mysql-core/pom.xml +++ b/connectors-common/mysql-core/pom.xml @@ -33,7 +33,7 @@ 8 8.0.33 1.5.4.Final - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT 1.0-SNAPSHOT diff --git a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java index 14e661a73..7eab47b33 100644 --- a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java +++ b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java @@ -43,7 +43,7 @@ import io.tapdata.kit.EmptyKit; import io.tapdata.kit.ErrorKit; import io.tapdata.kit.StringKit; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.TapAdvanceFilter; import org.apache.commons.collections4.CollectionUtils; @@ -88,7 +88,7 @@ public class MysqlReader implements Closeable { private final Supplier isAlive; protected final MysqlJdbcContextV2 mysqlJdbcContext; private EmbeddedEngine embeddedEngine; - protected StreamReadConsumer streamReadConsumer; + protected StreamReadOneByOneConsumer streamReadConsumer; private LinkedBlockingQueue eventQueue; private ScheduledExecutorService mysqlSchemaHistoryMonitor; protected KVReadOnlyMap tapTableMap; @@ -221,9 +221,8 @@ public void readWithFilter(TapConnectorContext tapConnectorContext, TapTable tap } public void readBinlog(TapConnectorContext tapConnectorContext, List tables, - Object offset, int batchSize, DDLParserType ddlParserType, LinkedBlockingQueue eventQueue, Map extraConfig) throws Throwable { + Object offset, DDLParserType ddlParserType, LinkedBlockingQueue eventQueue, Map extraConfig) throws Throwable { try { - batchSize = Math.max(batchSize, MIN_BATCH_SIZE); initDebeziumServerName(tapConnectorContext); this.tapTableMap = tapConnectorContext.getTableMap(); this.ddlParserType = ddlParserType; @@ -275,8 +274,8 @@ public void readBinlog(TapConnectorContext tapConnectorContext, List tab .with("database.history.store.only.monitored.tables.ddl", true) .with("database.history.store.only.captured.tables.ddl", true) .with(MySqlConnectorConfig.SNAPSHOT_LOCKING_MODE, MySqlConnectorConfig.SnapshotLockingMode.NONE) - .with("max.queue.size", batchSize * 8) - .with("max.batch.size", batchSize) + .with("max.queue.size", mysqlConfig.getMaximumQueueSize()) + .with("max.batch.size", mysqlConfig.getMaximumQueueSize() / 8) .with(MySqlConnectorConfig.SERVER_ID, randomServerId()) .with("time.precision.mode", "adaptive_time_microseconds") // .with("converters", "time") @@ -369,7 +368,7 @@ public void taskStarted() { } public void readBinlog(TapConnectorContext tapConnectorContext, List tables, - Object offset, int batchSize, DDLParserType ddlParserType, StreamReadConsumer consumer, HashMap contextMapForMasterSlave) throws Throwable { + Object offset, DDLParserType ddlParserType, StreamReadOneByOneConsumer consumer, HashMap contextMapForMasterSlave) throws Throwable { MysqlUtil.buildMasterNode(mysqlConfig, contextMapForMasterSlave); try { initDebeziumServerName(tapConnectorContext); @@ -513,7 +512,7 @@ public void taskStarted() { } }) .using((numberOfMessagesSinceLastCommit, timeSinceLastCommit) -> - numberOfMessagesSinceLastCommit >= batchSize || timeSinceLastCommit.getSeconds() >= 5) + timeSinceLastCommit.getSeconds() >= 5) .using((result, message, throwable) -> { tapConnectorContext.configContext(); if (result) { @@ -672,13 +671,11 @@ private void consumeRecords(List sourceRecords, DebeziumEngine.Rec } } if (CollectionUtils.isNotEmpty(mysqlStreamEvents)) { - List tapEvents = new ArrayList<>(); MysqlStreamOffset mysqlStreamOffset = null; for (MysqlStreamEvent mysqlStreamEvent : mysqlStreamEvents) { - tapEvents.add(mysqlStreamEvent.getTapEvent()); mysqlStreamOffset = mysqlStreamEvent.getMysqlStreamOffset(); + streamReadConsumer.accept(mysqlStreamEvent.getTapEvent(), mysqlStreamOffset); } - streamReadConsumer.accept(tapEvents, mysqlStreamOffset); } } @@ -688,27 +685,18 @@ protected void sourceRecordConsumer(SourceRecord record) { } if (null == record || null == record.value()) return; Schema valueSchema = record.valueSchema(); - List mysqlStreamEvents = new ArrayList<>(); if (null != valueSchema.field("op")) { MysqlStreamEvent mysqlStreamEvent = wrapDML(record); - Optional.ofNullable(mysqlStreamEvent).ifPresent(mysqlStreamEvents::add); + Optional.ofNullable(mysqlStreamEvent).ifPresent(e -> streamReadConsumer.accept(e.getTapEvent(), e.getMysqlStreamOffset())); } else if (null != valueSchema.field("ddl")) { - mysqlStreamEvents = wrapDDL(record); + List mysqlStreamEvents = wrapDDL(record); + mysqlStreamEvents.forEach(e -> streamReadConsumer.accept(e.getTapEvent(), e.getMysqlStreamOffset())); } else if ("io.debezium.connector.common.Heartbeat".equals(valueSchema.name())) { Optional.ofNullable((Struct) record.value()) .map(value -> value.getInt64("ts_ms")) .map(TapSimplify::heartbeatEvent) .map(heartbeatEvent -> new MysqlStreamEvent(heartbeatEvent, getMysqlStreamOffset(record))) - .ifPresent(mysqlStreamEvents::add); - } - if (CollectionUtils.isNotEmpty(mysqlStreamEvents)) { - List tapEvents = new ArrayList<>(); - MysqlStreamOffset mysqlStreamOffset = null; - for (MysqlStreamEvent mysqlStreamEvent : mysqlStreamEvents) { - tapEvents.add(mysqlStreamEvent.getTapEvent()); - mysqlStreamOffset = mysqlStreamEvent.getMysqlStreamOffset(); - } - streamReadConsumer.accept(tapEvents, mysqlStreamOffset); + .ifPresent(e -> streamReadConsumer.accept(e.getTapEvent(), e.getMysqlStreamOffset())); } } diff --git a/connectors-common/pom.xml b/connectors-common/pom.xml index fb3509acf..1ef79d453 100644 --- a/connectors-common/pom.xml +++ b/connectors-common/pom.xml @@ -28,8 +28,8 @@ ${project.artifactId}-v${project.version} 8 2.1-SNAPSHOT - 2.0.1-SNAPSHOT - 2.0.1-SNAPSHOT + 2.0.5-SNAPSHOT + 2.0.5-SNAPSHOT 1.0-SNAPSHOT 5.8.1 1.8.1 diff --git a/connectors-common/postgres-core/pom.xml b/connectors-common/postgres-core/pom.xml index 24fd0f92b..25033cfde 100644 --- a/connectors-common/postgres-core/pom.xml +++ b/connectors-common/postgres-core/pom.xml @@ -21,7 +21,7 @@ 1.0-SNAPSHOT 1.5.4.Final 1.0-SNAPSHOT - 2.0.1-SNAPSHOT + 2.0.5-SNAPSHOT 1.2.83 diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/AbstractWalLogMiner.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/AbstractWalLogMiner.java index bf81703d5..e48bf14f2 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/AbstractWalLogMiner.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/AbstractWalLogMiner.java @@ -17,7 +17,7 @@ import io.tapdata.entity.utils.cache.KVReadOnlyMap; import io.tapdata.kit.EmptyKit; import io.tapdata.kit.StringKit; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import java.math.BigDecimal; import java.sql.ResultSet; @@ -35,8 +35,7 @@ public abstract class AbstractWalLogMiner { protected final PostgresJdbcContext postgresJdbcContext; protected final Log tapLogger; - protected StreamReadConsumer consumer; - protected int recordSize; + protected StreamReadOneByOneConsumer consumer; protected List tableList; protected boolean filterSchema; private Map dataTypeMap; @@ -95,9 +94,8 @@ public AbstractWalLogMiner withWalLogDirectory(String walLogDirectory) { public abstract void startMiner(Supplier isAlive) throws Throwable; - public AbstractWalLogMiner registerConsumer(StreamReadConsumer consumer, int recordSize) { + public AbstractWalLogMiner registerConsumer(StreamReadOneByOneConsumer consumer) { this.consumer = consumer; - this.recordSize = recordSize; return this; } diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/PostgresCdcRunner.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/PostgresCdcRunner.java index f3f51840e..83313e674 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/PostgresCdcRunner.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/PostgresCdcRunner.java @@ -26,7 +26,7 @@ import io.tapdata.entity.utils.cache.KVReadOnlyMap; import io.tapdata.kit.EmptyKit; import io.tapdata.kit.NumberKit; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectorContext; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; @@ -57,8 +57,7 @@ public class PostgresCdcRunner extends DebeziumCdcRunner { private final TapConnectorContext connectorContext; private PostgresDebeziumConfig postgresDebeziumConfig; private PostgresOffset postgresOffset; - private int recordSize; - private StreamReadConsumer consumer; + private StreamReadOneByOneConsumer consumer; private final AtomicReference throwableAtomicReference = new AtomicReference<>(); protected TimeZone timeZone; private String dropTransactionId = null; @@ -145,8 +144,7 @@ public AtomicReference getThrowable() { return throwableAtomicReference; } - public void registerConsumer(StreamReadConsumer consumer, int recordSize) { - this.recordSize = recordSize; + public void registerConsumer(StreamReadOneByOneConsumer consumer) { this.consumer = consumer; //build debezium engine this.engine = (EmbeddedEngine) EmbeddedEngine.create() @@ -168,7 +166,7 @@ public void taskStopped() { // .using(Clock.SYSTEM) // .notifying(this::consumeRecord) .using((numberOfMessagesSinceLastCommit, timeSinceLastCommit) -> - numberOfMessagesSinceLastCommit >= recordSize || timeSinceLastCommit.getSeconds() >= 5) + timeSinceLastCommit.getSeconds() >= 5) .notifying(this::consumeRecords).using((result, message, throwable) -> { if (result) { if (StringUtils.isNotBlank(message)) { @@ -195,7 +193,6 @@ public void taskStopped() { @Override public void consumeRecords(List sourceRecords, DebeziumEngine.RecordCommitter committer) throws InterruptedException { super.consumeRecords(sourceRecords, committer); - List eventList = TapSimplify.list(); Map offset = null; for (SourceRecord sr : sourceRecords) { try { @@ -207,7 +204,7 @@ public void consumeRecords(List sourceRecords, DebeziumEngine.Reco continue; } if ("io.debezium.connector.common.Heartbeat".equals(sr.valueSchema().name())) { - eventList.add(new HeartbeatEvent().init().referenceTime(((Struct) sr.value()).getInt64("ts_ms"))); + consumer.accept(new HeartbeatEvent().init().referenceTime(((Struct) sr.value()).getInt64("ts_ms")), postgresOffset); continue; } else if (EmptyKit.isNull(sr.valueSchema().field("op"))) { continue; @@ -260,22 +257,13 @@ public void consumeRecords(List sourceRecords, DebeziumEngine.Reco event.setNamespaces(Lists.newArrayList(schema, table)); } } - eventList.add(event); - if (eventList.size() >= recordSize) { - PostgresOffset postgresOffset = new PostgresOffset(); - postgresOffset.setSourceOffset(TapSimplify.toJson(offset)); - consumer.accept(eventList, postgresOffset); - eventList = TapSimplify.list(); - } + PostgresOffset postgresOffset = new PostgresOffset(); + postgresOffset.setSourceOffset(TapSimplify.toJson(offset)); + consumer.accept(event, postgresOffset); } catch (StopConnectorException | StopEngineException ex) { throw ex; } } - if (EmptyKit.isNotEmpty(eventList)) { - PostgresOffset postgresOffset = new PostgresOffset(); - postgresOffset.setSourceOffset(TapSimplify.toJson(offset)); - consumer.accept(eventList, postgresOffset); - } } private DataMap getMapFromStruct(Struct struct) { diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMiner.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMiner.java index 38d91ffb1..6f69c71df 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMiner.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMiner.java @@ -3,21 +3,16 @@ import io.tapdata.common.concurrent.ConcurrentProcessor; import io.tapdata.common.concurrent.TapExecutors; import io.tapdata.connector.postgres.PostgresJdbcContext; -import io.tapdata.entity.event.TapEvent; import io.tapdata.entity.logger.Log; import io.tapdata.entity.simplify.TapSimplify; import io.tapdata.exception.TapPdkOffsetOutOfLogEx; import io.tapdata.kit.EmptyKit; import java.sql.*; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import java.util.stream.Collectors; -import static io.tapdata.base.ConnectorBase.list; public class WalLogMiner extends AbstractWalLogMiner { @@ -64,23 +59,11 @@ public void startMiner(Supplier isAlive) throws Throwable { } Thread t = new Thread(() -> { consumer.streamReadStarted(); - NormalRedo lastRedo = null; - AtomicReference> events = new AtomicReference<>(list()); while (isAlive.get()) { try { NormalRedo redo = concurrentProcessor.get(2, TimeUnit.SECONDS); if (EmptyKit.isNotNull(redo)) { - lastRedo = redo; - events.get().add(createEvent(redo)); - if (events.get().size() >= recordSize) { - consumer.accept(events.get(), redo.getCdcSequenceStr()); - events.set(new ArrayList<>()); - } - } else { - if (events.get().size() > 0) { - consumer.accept(events.get(), lastRedo.getCdcSequenceStr()); - events.set(new ArrayList<>()); - } + consumer.accept(createEvent(redo), redo.getCdcSequenceStr()); } } catch (Exception e) { threadException.set(e); diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMinerV2.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMinerV2.java index e467fbd56..faaa286bc 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMinerV2.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMinerV2.java @@ -3,7 +3,6 @@ import io.tapdata.common.concurrent.ConcurrentProcessor; import io.tapdata.common.concurrent.TapExecutors; import io.tapdata.connector.postgres.PostgresJdbcContext; -import io.tapdata.entity.event.TapEvent; import io.tapdata.entity.event.control.HeartbeatEvent; import io.tapdata.entity.logger.Log; import io.tapdata.entity.simplify.TapSimplify; @@ -21,8 +20,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static io.tapdata.base.ConnectorBase.list; - public class WalLogMinerV2 extends AbstractWalLogMiner { private String startLsn; @@ -46,26 +43,14 @@ public void startMiner(Supplier isAlive) throws Throwable { ConcurrentProcessor concurrentProcessor = TapExecutors.createSimple(8, 32, "wal-miner"); Thread t = new Thread(() -> { consumer.streamReadStarted(); - NormalRedo lastRedo = null; - AtomicReference> events = new AtomicReference<>(list()); while (isAlive.get()) { try { NormalRedo redo = concurrentProcessor.get(2, TimeUnit.SECONDS); if (EmptyKit.isNotNull(redo)) { if (EmptyKit.isNotNull(redo.getOperation())) { - lastRedo = redo; - events.get().add(createEvent(redo)); - if (events.get().size() >= recordSize) { - consumer.accept(events.get(), redo.getCdcSequenceStr()); - events.set(new ArrayList<>()); - } + consumer.accept(createEvent(redo), redo.getCdcSequenceStr()); } else { - consumer.accept(Collections.singletonList(new HeartbeatEvent().init().referenceTime(System.currentTimeMillis())), redo.getCdcSequenceStr()); - } - } else { - if (events.get().size() > 0) { - consumer.accept(events.get(), lastRedo.getCdcSequenceStr()); - events.set(new ArrayList<>()); + consumer.accept(new HeartbeatEvent().init().referenceTime(System.currentTimeMillis()), redo.getCdcSequenceStr()); } } } catch (Exception e) { diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMinerV3.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMinerV3.java index 605eee1ce..42a987fd2 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMinerV3.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalLogMinerV3.java @@ -3,7 +3,6 @@ import io.tapdata.common.concurrent.ConcurrentProcessor; import io.tapdata.common.concurrent.TapExecutors; import io.tapdata.connector.postgres.PostgresJdbcContext; -import io.tapdata.entity.event.TapEvent; import io.tapdata.entity.event.control.HeartbeatEvent; import io.tapdata.entity.logger.Log; import io.tapdata.entity.simplify.TapSimplify; @@ -15,16 +14,11 @@ import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import java.util.stream.Collectors; -import static io.tapdata.base.ConnectorBase.list; - public class WalLogMinerV3 extends AbstractWalLogMiner { private String timestamp; @@ -42,26 +36,14 @@ public void startMiner(Supplier isAlive) throws Throwable { ConcurrentProcessor concurrentProcessor = TapExecutors.createSimple(8, 32, "wal-miner"); Thread t = new Thread(() -> { consumer.streamReadStarted(); - NormalRedo lastRedo = null; - AtomicReference> events = new AtomicReference<>(list()); while (isAlive.get()) { try { NormalRedo redo = concurrentProcessor.get(2, TimeUnit.SECONDS); if (EmptyKit.isNotNull(redo)) { if (EmptyKit.isNotNull(redo.getOperation())) { - lastRedo = redo; - events.get().add(createEvent(redo)); - if (events.get().size() >= recordSize) { - consumer.accept(events.get(), redo.getCdcSequenceStr()); - events.set(new ArrayList<>()); - } + consumer.accept(createEvent(redo), redo.getCdcSequenceStr()); } else { - consumer.accept(Collections.singletonList(new HeartbeatEvent().init().referenceTime(System.currentTimeMillis())), redo.getCdcSequenceStr()); - } - } else { - if (events.get().size() > 0) { - consumer.accept(events.get(), lastRedo.getCdcSequenceStr()); - events.set(new ArrayList<>()); + consumer.accept(new HeartbeatEvent().init().referenceTime(System.currentTimeMillis()), redo.getCdcSequenceStr()); } } } catch (Exception e) { diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalPgtoMiner.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalPgtoMiner.java index 1a278bbc6..43fa1d17c 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalPgtoMiner.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/WalPgtoMiner.java @@ -1,12 +1,10 @@ package io.tapdata.connector.postgres.cdc; import com.alibaba.fastjson.JSONObject; -import io.tapdata.base.ConnectorBase; import io.tapdata.common.concurrent.ConcurrentProcessor; import io.tapdata.common.concurrent.TapExecutors; import io.tapdata.common.sqlparser.ResultDO; import io.tapdata.connector.postgres.PostgresJdbcContext; -import io.tapdata.entity.event.TapEvent; import io.tapdata.entity.event.control.HeartbeatEvent; import io.tapdata.entity.logger.Log; import io.tapdata.entity.simplify.TapSimplify; @@ -18,7 +16,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; public class WalPgtoMiner extends AbstractWalLogMiner { @@ -65,26 +62,14 @@ public void startMiner(Supplier isAlive) throws Throwable { try (ConcurrentProcessor concurrentProcessor = TapExecutors.createSimple(8, 32, "wal-miner")) { Thread t = new Thread(() -> { consumer.streamReadStarted(); - NormalRedo lastRedo = null; - AtomicReference> events = new AtomicReference<>(ConnectorBase.list()); while (isAlive.get()) { try { NormalRedo redo = concurrentProcessor.get(2, TimeUnit.SECONDS); if (EmptyKit.isNotNull(redo)) { if (EmptyKit.isNotNull(redo.getOperation())) { - lastRedo = redo; - events.get().add(createEvent(redo)); - if (events.get().size() >= recordSize) { - consumer.accept(events.get(), redo.getCdcSequenceStr()); - events.set(new ArrayList<>()); - } + consumer.accept(createEvent(redo), redo.getCdcSequenceStr()); } else { - consumer.accept(Collections.singletonList(new HeartbeatEvent().init().referenceTime(System.currentTimeMillis())), redo.getCdcSequenceStr()); - } - } else { - if (!events.get().isEmpty()) { - consumer.accept(events.get(), lastRedo.getCdcSequenceStr()); - events.set(new ArrayList<>()); + consumer.accept(new HeartbeatEvent().init().referenceTime(System.currentTimeMillis()), redo.getCdcSequenceStr()); } } } catch (Exception e) { diff --git a/connectors-common/read-partition/pom.xml b/connectors-common/read-partition/pom.xml index fc153b3bf..671303bad 100644 --- a/connectors-common/read-partition/pom.xml +++ b/connectors-common/read-partition/pom.xml @@ -20,13 +20,13 @@ io.tapdata tapdata-api - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT provided io.tapdata tapdata-pdk-api - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT provided diff --git a/connectors/activemq-connector/src/main/java/io/tapdata/connector/activemq/ActivemqConnector.java b/connectors/activemq-connector/src/main/java/io/tapdata/connector/activemq/ActivemqConnector.java index 9a7bc7109..11cf7031c 100644 --- a/connectors/activemq-connector/src/main/java/io/tapdata/connector/activemq/ActivemqConnector.java +++ b/connectors/activemq-connector/src/main/java/io/tapdata/connector/activemq/ActivemqConnector.java @@ -12,7 +12,7 @@ import io.tapdata.entity.schema.value.TapTimeValue; import io.tapdata.entity.simplify.TapSimplify; import io.tapdata.pdk.apis.annotations.TapConnectorClass; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.ConnectionOptions; @@ -61,7 +61,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec connectorFunctions.supportConnectionCheckFunction(this::checkConnection); connectorFunctions.supportWriteRecord(this::writeRecord); connectorFunctions.supportBatchRead(this::batchRead); - connectorFunctions.supportStreamRead(this::streamRead); + connectorFunctions.supportOneByOneStreamRead(this::streamRead); connectorFunctions.supportTimestampToStreamOffset(this::timestampToStreamOffset); } @@ -102,8 +102,8 @@ private void batchRead(TapConnectorContext tapConnectorContext, TapTable tapTabl activemqService.consumeOne(tapTable, eventBatchSize, eventsOffsetConsumer); } - private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, int recordSize, StreamReadConsumer consumer) throws Throwable { - activemqService.streamConsume(tableList, recordSize, consumer); + private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, StreamReadOneByOneConsumer consumer) throws Throwable { + activemqService.streamConsume(tableList, consumer); } private Object timestampToStreamOffset(TapConnectorContext connectorContext, Long offsetStartTime) { diff --git a/connectors/activemq-connector/src/main/java/io/tapdata/connector/activemq/ActivemqService.java b/connectors/activemq-connector/src/main/java/io/tapdata/connector/activemq/ActivemqService.java index 98175b618..71581c0eb 100644 --- a/connectors/activemq-connector/src/main/java/io/tapdata/connector/activemq/ActivemqService.java +++ b/connectors/activemq-connector/src/main/java/io/tapdata/connector/activemq/ActivemqService.java @@ -23,6 +23,7 @@ import org.apache.activemq.ActiveMQConnectionFactory; import org.apache.activemq.command.ActiveMQQueue; import org.apache.activemq.command.ActiveMQTextMessage; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import javax.jms.Queue; import javax.jms.*; @@ -221,7 +222,7 @@ public void consumeOne(TapTable tapTable, int eventBatchSize, BiConsumer= eventBatchSize) { eventsOffsetConsumer.accept(list, TapSimplify.list()); list = TapSimplify.list(); @@ -234,18 +235,17 @@ public void consumeOne(TapTable tapTable, int eventBatchSize, BiConsumer tableList, int eventBatchSize, BiConsumer, Object> eventsOffsetConsumer) throws Throwable { + public void streamConsume(List tableList, StreamReadOneByOneConsumer eventsOffsetConsumer) throws Throwable { consuming.set(true); Session session = activemqConnection.createSession(false, Session.AUTO_ACKNOWLEDGE); List> tablesList = Lists.partition(tableList, (tableList.size() - 1) / concurrency + 1); executorService = Executors.newFixedThreadPool(tablesList.size()); CountDownLatch countDownLatch = new CountDownLatch(tablesList.size()); tablesList.forEach(tables -> executorService.submit(() -> { - List list = TapSimplify.list(); Map consumerMap = new HashMap<>(); int err = 0; while (consuming.get() && err < 10) { - for (String tableName : tables) { + for (final String tableName : tables) { try { MessageConsumer messageConsumer = consumerMap.get(tableName); if (EmptyKit.isNull(messageConsumer)) { @@ -257,11 +257,7 @@ public void streamConsume(List tableList, int eventBatchSize, BiConsumer if (EmptyKit.isNull(message)) { continue; } - makeMessage(message, list, tableName); - if (list.size() >= eventBatchSize) { - eventsOffsetConsumer.accept(list, TapSimplify.list()); - list = TapSimplify.list(); - } + Optional.ofNullable(makeMessage(message, tableName)).ifPresent(e -> eventsOffsetConsumer.accept(e, TapSimplify.list())); } catch (Exception e) { TapLogger.error(TAG, "error occur when consume queue: {}", tableName, e); err++; @@ -269,9 +265,6 @@ public void streamConsume(List tableList, int eventBatchSize, BiConsumer } TapSimplify.sleep(50); } - if (EmptyKit.isNotEmpty(list)) { - eventsOffsetConsumer.accept(list, TapSimplify.list()); - } consumerMap.forEach((key, value) -> { try { value.close(); @@ -293,19 +286,17 @@ public void streamConsume(List tableList, int eventBatchSize, BiConsumer session.close(); } - private void makeMessage(Message message, List list, String tableName) throws JMSException { + private TapRecordEvent makeMessage(Message message, String tableName) throws JMSException { TextMessage textMessage = (TextMessage) message; Map data = jsonParser.fromJson(textMessage.getText(), Map.class); switch (MqOp.fromValue(textMessage.getStringProperty("mqOp"))) { case INSERT: - list.add(new TapInsertRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis())); - break; + return new TapInsertRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis()); case UPDATE: - list.add(new TapUpdateRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis())); - break; + return new TapUpdateRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis()); case DELETE: - list.add(new TapDeleteRecordEvent().init().table(tableName).before(data).referenceTime(System.currentTimeMillis())); - break; + return new TapDeleteRecordEvent().init().table(tableName).before(data).referenceTime(System.currentTimeMillis()); } + return null; } } diff --git a/connectors/activemq-connector/src/main/resources/spec_activemq.json b/connectors/activemq-connector/src/main/resources/spec_activemq.json index 0b5c39737..eaa2391ad 100644 --- a/connectors/activemq-connector/src/main/resources/spec_activemq.json +++ b/connectors/activemq-connector/src/main/resources/spec_activemq.json @@ -4,7 +4,17 @@ "icon": "icons/activemq.png", "doc": "${doc}", "id": "activemq", - "tags": ["Database"] + "tags": ["Database"], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "connection": { diff --git a/connectors/aliyun-adb-postgres-connector/pom.xml b/connectors/aliyun-adb-postgres-connector/pom.xml index fde69c624..6b14c53dc 100644 --- a/connectors/aliyun-adb-postgres-connector/pom.xml +++ b/connectors/aliyun-adb-postgres-connector/pom.xml @@ -22,7 +22,7 @@ - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/aliyun-adb-postgres-connector/src/main/resources/aliyun-adb-postgres-spec.json b/connectors/aliyun-adb-postgres-connector/src/main/resources/aliyun-adb-postgres-spec.json index 4375d3c57..40583a352 100644 --- a/connectors/aliyun-adb-postgres-connector/src/main/resources/aliyun-adb-postgres-spec.json +++ b/connectors/aliyun-adb-postgres-connector/src/main/resources/aliyun-adb-postgres-spec.json @@ -4,7 +4,17 @@ "icon": "icons/aliyun_adb_postgres.png", "doc" : "${doc}", "id": "aliyun-adb-postgres", - "tags": ["Database"] + "tags": ["Database"], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities":[ diff --git a/connectors/aliyun-mongodb-connector/pom.xml b/connectors/aliyun-mongodb-connector/pom.xml index 2527d6187..fd7ba5d06 100644 --- a/connectors/aliyun-mongodb-connector/pom.xml +++ b/connectors/aliyun-mongodb-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/aliyun-mongodb-connector/src/main/resources/aliyun-mongodb-spec.json b/connectors/aliyun-mongodb-connector/src/main/resources/aliyun-mongodb-spec.json index 84831a6a2..d77f2e351 100644 --- a/connectors/aliyun-mongodb-connector/src/main/resources/aliyun-mongodb-spec.json +++ b/connectors/aliyun-mongodb-connector/src/main/resources/aliyun-mongodb-spec.json @@ -4,7 +4,17 @@ "icon": "icons/aliyun-mongodb.png", "doc" : "${doc}", "tags" : ["schema-free","Database"], - "id": "aliyun-db-mongodb" + "id": "aliyun-db-mongodb", + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities": [ diff --git a/connectors/aliyun-rds-mariadb-connector/pom.xml b/connectors/aliyun-rds-mariadb-connector/pom.xml index c55ae7ffa..b1eda00e7 100644 --- a/connectors/aliyun-rds-mariadb-connector/pom.xml +++ b/connectors/aliyun-rds-mariadb-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/aliyun-rds-mariadb-connector/src/main/resources/aliyun-rds-mariadb-spec.json b/connectors/aliyun-rds-mariadb-connector/src/main/resources/aliyun-rds-mariadb-spec.json index 1dbe312ad..c7634d2dd 100644 --- a/connectors/aliyun-rds-mariadb-connector/src/main/resources/aliyun-rds-mariadb-spec.json +++ b/connectors/aliyun-rds-mariadb-connector/src/main/resources/aliyun-rds-mariadb-spec.json @@ -4,7 +4,17 @@ "icon": "icons/aliyun_rds_mariadb.png", "id": "aliyun-rds-mariadb", "doc": "${doc}", - "tags": ["Database"] + "tags": ["Database"], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities": [ diff --git a/connectors/aliyun-rds-mysql-connector/pom.xml b/connectors/aliyun-rds-mysql-connector/pom.xml index 899de4eac..b370499f7 100644 --- a/connectors/aliyun-rds-mysql-connector/pom.xml +++ b/connectors/aliyun-rds-mysql-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/aliyun-rds-mysql-connector/src/main/resources/aliyun-rds-mysql-spec.json b/connectors/aliyun-rds-mysql-connector/src/main/resources/aliyun-rds-mysql-spec.json index 40756d30d..960dd8d9a 100644 --- a/connectors/aliyun-rds-mysql-connector/src/main/resources/aliyun-rds-mysql-spec.json +++ b/connectors/aliyun-rds-mysql-connector/src/main/resources/aliyun-rds-mysql-spec.json @@ -4,7 +4,17 @@ "icon": "icons/aliyun_rds_mysql.png", "id": "aliyun-rds-mysql", "doc": "${doc}", - "tags": ["Database","ssl"] + "tags": ["Database","ssl"], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities": [ diff --git a/connectors/aliyun-rds-postgres-connector/pom.xml b/connectors/aliyun-rds-postgres-connector/pom.xml index cba9f8240..d2882f881 100644 --- a/connectors/aliyun-rds-postgres-connector/pom.xml +++ b/connectors/aliyun-rds-postgres-connector/pom.xml @@ -23,7 +23,7 @@ - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/aliyun-rds-postgres-connector/src/main/resources/aliyun-rds-postgres-spec.json b/connectors/aliyun-rds-postgres-connector/src/main/resources/aliyun-rds-postgres-spec.json index adb52dd66..91bcd5794 100644 --- a/connectors/aliyun-rds-postgres-connector/src/main/resources/aliyun-rds-postgres-spec.json +++ b/connectors/aliyun-rds-postgres-connector/src/main/resources/aliyun-rds-postgres-spec.json @@ -4,7 +4,17 @@ "icon": "icons/aliyun_rds_postgres.png", "doc" : "${doc}", "id": "aliyun-rds-postgres", - "tags": ["Database"] + "tags": ["Database"], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities":[ diff --git a/connectors/aws-clickhouse-connector/pom.xml b/connectors/aws-clickhouse-connector/pom.xml index c567a9aa9..6993350f6 100644 --- a/connectors/aws-clickhouse-connector/pom.xml +++ b/connectors/aws-clickhouse-connector/pom.xml @@ -19,7 +19,7 @@ 1.0-SNAPSHOT 3.12.0 31.0.1-jre - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/aws-clickhouse-connector/src/main/resources/aws_clickhouse.json b/connectors/aws-clickhouse-connector/src/main/resources/aws_clickhouse.json index 993a0ef02..e4cdb54d9 100644 --- a/connectors/aws-clickhouse-connector/src/main/resources/aws_clickhouse.json +++ b/connectors/aws-clickhouse-connector/src/main/resources/aws_clickhouse.json @@ -6,7 +6,17 @@ "doc": "${doc}", "tags": [ "Database" - ] + ], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities": [ diff --git a/connectors/aws-rds-mysql-connector/pom.xml b/connectors/aws-rds-mysql-connector/pom.xml index cf779b9a9..6cd87fa61 100644 --- a/connectors/aws-rds-mysql-connector/pom.xml +++ b/connectors/aws-rds-mysql-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/aws-rds-mysql-connector/src/main/resources/aws-rds-mysql-spec.json b/connectors/aws-rds-mysql-connector/src/main/resources/aws-rds-mysql-spec.json index cbaf71b6e..b16bb26de 100644 --- a/connectors/aws-rds-mysql-connector/src/main/resources/aws-rds-mysql-spec.json +++ b/connectors/aws-rds-mysql-connector/src/main/resources/aws-rds-mysql-spec.json @@ -4,7 +4,17 @@ "icon": "icons/aws_rds_mysql.png", "id": "aws-rds-mysql", "doc": "${doc}", - "tags": ["Database"] + "tags": ["Database"], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities": [ diff --git a/connectors/azure-cosmosdb-connector/pom.xml b/connectors/azure-cosmosdb-connector/pom.xml index e37151837..13125f917 100644 --- a/connectors/azure-cosmosdb-connector/pom.xml +++ b/connectors/azure-cosmosdb-connector/pom.xml @@ -19,7 +19,7 @@ 8 - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/azure-cosmosdb-connector/src/main/resources/azure-cosmosdb-spec.json b/connectors/azure-cosmosdb-connector/src/main/resources/azure-cosmosdb-spec.json index ac3cfa05d..f35cd8c4f 100644 --- a/connectors/azure-cosmosdb-connector/src/main/resources/azure-cosmosdb-spec.json +++ b/connectors/azure-cosmosdb-connector/src/main/resources/azure-cosmosdb-spec.json @@ -7,7 +7,17 @@ "schema-free", "Database" ], - "id": "azure-cosmosdb" + "id": "azure-cosmosdb", + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities": [ diff --git a/connectors/clickhouse-connector/pom.xml b/connectors/clickhouse-connector/pom.xml index 8d9147681..0773fff8a 100644 --- a/connectors/clickhouse-connector/pom.xml +++ b/connectors/clickhouse-connector/pom.xml @@ -19,7 +19,7 @@ 1.0-SNAPSHOT 3.12.0 31.0.1-jre - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/clickhouse-connector/src/main/resources/spec_clickhouse.json b/connectors/clickhouse-connector/src/main/resources/spec_clickhouse.json index bac4e3817..0acf2fa36 100644 --- a/connectors/clickhouse-connector/src/main/resources/spec_clickhouse.json +++ b/connectors/clickhouse-connector/src/main/resources/spec_clickhouse.json @@ -6,7 +6,17 @@ "doc": "${doc}", "tags": [ "Database" - ] + ], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities": [ diff --git a/connectors/highgo-connector/pom.xml b/connectors/highgo-connector/pom.xml index 6aec28edf..dbbd66358 100644 --- a/connectors/highgo-connector/pom.xml +++ b/connectors/highgo-connector/pom.xml @@ -15,7 +15,7 @@ 8 - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT 1.5.4.Final diff --git a/connectors/highgo-connector/src/main/java/io/tapdata/connector/postgres/HighgoConnector.java b/connectors/highgo-connector/src/main/java/io/tapdata/connector/postgres/HighgoConnector.java index 3e66d1c4a..fdf367b68 100644 --- a/connectors/highgo-connector/src/main/java/io/tapdata/connector/postgres/HighgoConnector.java +++ b/connectors/highgo-connector/src/main/java/io/tapdata/connector/postgres/HighgoConnector.java @@ -20,7 +20,6 @@ import io.tapdata.entity.event.ddl.table.TapNewFieldEvent; import io.tapdata.entity.event.dml.TapRecordEvent; import io.tapdata.entity.schema.TapField; -import io.tapdata.entity.schema.TapIndex; import io.tapdata.entity.schema.TapTable; import io.tapdata.entity.schema.type.TapType; import io.tapdata.entity.schema.value.*; @@ -34,7 +33,7 @@ import io.tapdata.kit.ErrorKit; import io.tapdata.kit.StringKit; import io.tapdata.pdk.apis.annotations.TapConnectorClass; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.ConnectionOptions; @@ -110,7 +109,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec // source connectorFunctions.supportBatchCount(this::batchCount); connectorFunctions.supportBatchRead(this::batchReadWithoutOffset); - connectorFunctions.supportStreamRead(this::streamRead); + connectorFunctions.supportOneByOneStreamRead(this::streamRead); connectorFunctions.supportTimestampToStreamOffset(this::timestampToStreamOffset); // query connectorFunctions.supportQueryByFilter(this::queryByFilter); @@ -396,11 +395,11 @@ protected void writeRecord(TapConnectorContext connectorContext, List tableList, Object offsetState, int recordSize, StreamReadConsumer consumer) throws Throwable { + private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, StreamReadOneByOneConsumer consumer) throws Throwable { cdcRunner = new PostgresCdcRunner(postgresJdbcContext, nodeContext); testReplicateIdentity(nodeContext.getTableMap()); buildSlot(nodeContext, true); - cdcRunner.useSlot(slotName.toString()).watch(tableList).offset(offsetState).registerConsumer(consumer, recordSize); + cdcRunner.useSlot(slotName.toString()).watch(tableList).offset(offsetState).registerConsumer(consumer); cdcRunner.startCdcRunner(); if (EmptyKit.isNotNull(cdcRunner) && EmptyKit.isNotNull(cdcRunner.getThrowable().get())) { Throwable throwable = ErrorKit.getLastCause(cdcRunner.getThrowable().get()); diff --git a/connectors/highgo-connector/src/main/resources/spec_highgo.json b/connectors/highgo-connector/src/main/resources/spec_highgo.json index 220e21c8e..9ee445c56 100644 --- a/connectors/highgo-connector/src/main/resources/spec_highgo.json +++ b/connectors/highgo-connector/src/main/resources/spec_highgo.json @@ -4,7 +4,17 @@ "icon": "icons/highgo.svg", "doc" : "${doc}", "id": "highgo", - "tags": ["Database", "ssl", "doubleActive"] + "tags": ["Database", "ssl", "doubleActive"], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities":[ diff --git a/connectors/kafka-connector/pom.xml b/connectors/kafka-connector/pom.xml index 9073eaaed..738b0b445 100644 --- a/connectors/kafka-connector/pom.xml +++ b/connectors/kafka-connector/pom.xml @@ -12,7 +12,7 @@ kafka-connector - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/kafka-connector/src/main/java/io/tapdata/connector/kafka/KafkaConnector.java b/connectors/kafka-connector/src/main/java/io/tapdata/connector/kafka/KafkaConnector.java index fdf90c094..986e1cb3f 100644 --- a/connectors/kafka-connector/src/main/java/io/tapdata/connector/kafka/KafkaConnector.java +++ b/connectors/kafka-connector/src/main/java/io/tapdata/connector/kafka/KafkaConnector.java @@ -19,7 +19,7 @@ import io.tapdata.entity.schema.value.TapTimeValue; import io.tapdata.kit.EmptyKit; import io.tapdata.pdk.apis.annotations.TapConnectorClass; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.Capability; @@ -143,7 +143,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec connectorFunctions.supportConnectionCheckFunction(this::checkConnection); connectorFunctions.supportWriteRecord(this::writeRecord); connectorFunctions.supportBatchRead(this::batchRead); - connectorFunctions.supportStreamRead(this::streamRead); + connectorFunctions.supportOneByOneStreamRead(this::streamRead); connectorFunctions.supportTimestampToStreamOffset(this::timestampToStreamOffset); connectorFunctions.supportNewFieldFunction(this::fieldDDLHandler); @@ -310,9 +310,9 @@ private void batchRead(TapConnectorContext tapConnectorContext, TapTable tapTabl } } - private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, int recordSize, StreamReadConsumer consumer) { + private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, StreamReadOneByOneConsumer consumer) { try { - kafkaService.streamConsume(tableList, offsetState, recordSize, consumer); + kafkaService.streamConsume(tableList, offsetState, consumer); } catch (Throwable e) { kafkaExceptionCollector.collectTerminateByServer(e); kafkaExceptionCollector.collectUserPwdInvalid(kafkaConfig.getMqUsername(), e); diff --git a/connectors/kafka-connector/src/main/resources/spec_kafka.json b/connectors/kafka-connector/src/main/resources/spec_kafka.json index a57dc1f37..5b938fe48 100644 --- a/connectors/kafka-connector/src/main/resources/spec_kafka.json +++ b/connectors/kafka-connector/src/main/resources/spec_kafka.json @@ -7,7 +7,17 @@ "tags": [ "Database", "schema-free" - ] + ], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "supportDDL": { diff --git a/connectors/mariadb-connector/pom.xml b/connectors/mariadb-connector/pom.xml index 0f25688ec..8423db40c 100644 --- a/connectors/mariadb-connector/pom.xml +++ b/connectors/mariadb-connector/pom.xml @@ -16,7 +16,7 @@ 4.0.3 1.5.4.Final 4.4 - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/mariadb-connector/src/main/resources/spec_mariadb.json b/connectors/mariadb-connector/src/main/resources/spec_mariadb.json index b7aae1fbc..036c250d2 100644 --- a/connectors/mariadb-connector/src/main/resources/spec_mariadb.json +++ b/connectors/mariadb-connector/src/main/resources/spec_mariadb.json @@ -6,7 +6,17 @@ "doc": "${doc}", "tags": [ "Database", "ssl" - ] + ], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities":[ diff --git a/connectors/mongodb-connector/pom.xml b/connectors/mongodb-connector/pom.xml index b4fc0b6bb..adf53fca6 100644 --- a/connectors/mongodb-connector/pom.xml +++ b/connectors/mongodb-connector/pom.xml @@ -13,7 +13,7 @@ 8 - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/MongodbConnector.java b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/MongodbConnector.java index 9e889f707..15cad69cf 100644 --- a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/MongodbConnector.java +++ b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/MongodbConnector.java @@ -39,7 +39,7 @@ import io.tapdata.mongodb.writer.MongodbWriter; import io.tapdata.partition.DatabaseReadPartitionSplitter; import io.tapdata.pdk.apis.annotations.TapConnectorClass; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.*; @@ -553,7 +553,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec connectorFunctions.supportBatchCount(this::batchCount); connectorFunctions.supportCreateIndex(this::createIndex); connectorFunctions.supportCreateTableV2(this::createTableV2); - connectorFunctions.supportStreamRead(this::streamRead); + connectorFunctions.supportOneByOneStreamRead(this::streamRead); connectorFunctions.supportTimestampToStreamOffset(this::streamOffset); connectorFunctions.supportErrorHandleFunction(this::errorHandle); @@ -1619,22 +1619,22 @@ protected Object streamOffset(TapConnectorContext connectorContext, Long offsetS * @param connectorContext // * @param offset * // * @param consumer */ - protected void streamRead(TapConnectorContext connectorContext, List tableList, Object offset, int eventBatchSize, StreamReadConsumer consumer) { + protected void streamRead(TapConnectorContext connectorContext, List tableList, Object offset, StreamReadOneByOneConsumer consumer) { int size = tableList.size(); MongoCdcOffset mongoCdcOffset = MongoCdcOffset.fromOffset(offset); - streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), eventBatchSize, consumer); + streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), consumer); if (size == tableList.size() || !tableList.isEmpty()) { if (mongodbStreamReader == null) { mongodbStreamReader = createStreamReader(); } mongodbStreamReader.onStart(mongoConfig); - doStreamRead(mongodbStreamReader, connectorContext, tableList, mongoCdcOffset.getCdcOffset(), eventBatchSize, consumer); + doStreamRead(mongodbStreamReader, connectorContext, tableList, mongoCdcOffset.getCdcOffset(), consumer); } } - protected void doStreamRead(MongodbStreamReader streamReader, TapConnectorContext connectorContext, List tableList, Object offset, int eventBatchSize, StreamReadConsumer consumer) { + protected void doStreamRead(MongodbStreamReader streamReader, TapConnectorContext connectorContext, List tableList, Object offset, StreamReadOneByOneConsumer consumer) { try { - streamReader.read(connectorContext, tableList, offset, eventBatchSize, consumer); + streamReader.read(connectorContext, tableList, offset, consumer); } catch (Exception e) { exceptionCollector.collectTerminateByServer(e); exceptionCollector.collectWritePrivileges(e); @@ -1648,7 +1648,7 @@ protected void doStreamRead(MongodbStreamReader streamReader, TapConnectorContex } } - protected void streamReadOpLog(TapConnectorContext connectorContext, List tableList, Object offset, int eventBatchSize, StreamReadConsumer consumer) { + protected void streamReadOpLog(TapConnectorContext connectorContext, List tableList, Object offset, StreamReadOneByOneConsumer consumer) { String database = mongoConfig.getDatabase(); if (StreamWithOpLogCollection.OP_LOG_DB.equals(database) && tableList.contains(StreamWithOpLogCollection.OP_LOG_COLLECTION)) { connectorContext.getLog().info("Start read oplog collection, db: local"); @@ -1658,7 +1658,7 @@ protected void streamReadOpLog(TapConnectorContext connectorContext, List doStreamRead(opLogStreamReader, connectorContext, list(StreamWithOpLogCollection.OP_LOG_COLLECTION), offset, eventBatchSize, consumer)); + sourceRunnerFuture = sourceRunner.submit(() -> doStreamRead(opLogStreamReader, connectorContext, list(StreamWithOpLogCollection.OP_LOG_COLLECTION), offset, consumer)); } } } diff --git a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/MongodbStreamReader.java b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/MongodbStreamReader.java index cefd9f403..a6bc9a68b 100644 --- a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/MongodbStreamReader.java +++ b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/MongodbStreamReader.java @@ -1,7 +1,7 @@ package io.tapdata.mongodb.reader; import io.tapdata.mongodb.entity.MongodbConfig; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectorContext; import java.util.List; @@ -13,7 +13,7 @@ public interface MongodbStreamReader { void onStart(MongodbConfig mongodbConfig); - void read(TapConnectorContext connectorContext, List tableList, Object offset, int eventBatchSize, StreamReadConsumer consumer) throws Exception; + void read(TapConnectorContext connectorContext, List tableList, Object offset, StreamReadOneByOneConsumer consumer) throws Exception; Object streamOffset(Long offsetStartTime); diff --git a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/MongodbV4StreamReader.java b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/MongodbV4StreamReader.java index c2425ff21..803c01933 100644 --- a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/MongodbV4StreamReader.java +++ b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/MongodbV4StreamReader.java @@ -24,7 +24,7 @@ import io.tapdata.mongodb.MongodbUtil; import io.tapdata.mongodb.entity.MongodbConfig; import io.tapdata.mongodb.util.MongodbLookupUtil; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectorContext; import org.apache.commons.collections4.MapUtils; import org.bson.*; @@ -34,7 +34,6 @@ import org.bson.io.ByteBufferBsonInput; import java.nio.ByteBuffer; -import java.time.Instant; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -89,7 +88,7 @@ public void onStart(MongodbConfig mongodbConfig) { } @Override - public void read(TapConnectorContext connectorContext, List tableList, Object offset, int eventBatchSize, StreamReadConsumer consumer) throws Exception { + public void read(TapConnectorContext connectorContext, List tableList, Object offset, StreamReadOneByOneConsumer consumer) throws Exception { openChangeStreamPreAndPostImages(tableList); if (Boolean.TRUE.equals(mongodbConfig.getDoubleActive())) { tableList.add("_tap_double_active"); @@ -129,33 +128,11 @@ public void read(TapConnectorContext connectorContext, List tableList, O AtomicReference throwableAtomicReference = new AtomicReference<>(); try (final MongoChangeStreamCursor> streamCursor = changeStream.cursor()) { consumeStreamEventThread = new Thread(() -> { - List events = list(); - OffsetEvent lastOffsetEvent = null; - long lastSendTime = System.currentTimeMillis(); - // Calculate time window based on batch size - // If batch size <= 500, use fixed 50ms window - // If batch size > 500, use dynamic window = batch size / 10 (ms) - long timeWindowMs = eventBatchSize <= 500 ? 50 : eventBatchSize / 10; while (running.get()) { try { OffsetEvent event = concurrentProcessor.get(10, TimeUnit.MILLISECONDS); if (EmptyKit.isNotNull(event)) { - lastOffsetEvent = event; - events.add(event.getEvent()); - // Check batch size OR time window - if (events.size() >= eventBatchSize || - (System.currentTimeMillis() - lastSendTime > timeWindowMs)) { - consumer.accept(events, event.getOffset()); - events.clear(); - lastSendTime = System.currentTimeMillis(); - } - } else { - // Send remaining events when queue is empty - if (!events.isEmpty() && lastOffsetEvent != null) { - consumer.accept(events, lastOffsetEvent.getOffset()); - events.clear(); - lastSendTime = System.currentTimeMillis(); - } + consumer.accept(event.getEvent(), event.getOffset()); } } catch (Exception e) { throwableAtomicReference.set(e); diff --git a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/v3/MongodbV3StreamReader.java b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/v3/MongodbV3StreamReader.java index 16e7e87d2..643356f8c 100644 --- a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/v3/MongodbV3StreamReader.java +++ b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/v3/MongodbV3StreamReader.java @@ -9,7 +9,6 @@ import com.mongodb.client.MongoCursor; import com.mongodb.client.model.Filters; import io.tapdata.entity.event.TapBaseEvent; -import io.tapdata.entity.event.TapEvent; import io.tapdata.entity.event.dml.TapUpdateRecordEvent; import io.tapdata.entity.event.control.HeartbeatEvent; import io.tapdata.entity.logger.TapLogger; @@ -21,7 +20,7 @@ import io.tapdata.mongodb.reader.MongodbStreamReader; import io.tapdata.mongodb.util.IntervalReport; import io.tapdata.mongodb.util.MongodbLookupUtil; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectorContext; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; @@ -65,8 +64,6 @@ public class MongodbV3StreamReader implements MongodbStreamReader { private ThreadPoolExecutor replicaSetReadThreadPool; - private LinkedBlockingDeque tapEventQueue; - private Exception error; private KVMap globalStateMap; @@ -85,17 +82,13 @@ public void onStart(MongodbConfig mongodbConfig) { } @Override - public void read(TapConnectorContext connectorContext, List tableList, Object offset, int eventBatchSize, StreamReadConsumer consumer) throws Exception { + public void read(TapConnectorContext connectorContext, List tableList, Object offset, StreamReadOneByOneConsumer consumer) throws Exception { if (CollectionUtils.isNotEmpty(tableList)) { for (String tableName : tableList) { namespaces.add(new StringBuilder(mongodbConfig.getDatabase()).append(".").append(tableName).toString()); } } - if (tapEventQueue == null) { - tapEventQueue = new LinkedBlockingDeque<>(eventBatchSize); - } - if (this.globalStateMap == null) { this.globalStateMap = connectorContext.getGlobalStateMap(); } @@ -117,15 +110,12 @@ protected void report(BsonTimestamp bsonTimestamp) throws InterruptedException { while (running.get()) { HeartbeatEvent heartbeatEvent = new HeartbeatEvent().referenceTime(bsonTimestamp.getTime() * 1000L); heartbeatEvent.setTime(heartbeatEvent.getReferenceTime()); - if (tapEventQueue.offer( - new TapEventOffset( + TapEventOffset tapEventOffset = new TapEventOffset( heartbeatEvent, new MongoV3StreamOffset(bsonTimestamp.getTime(), bsonTimestamp.getInc()), replicaSetName - ), - 3, - TimeUnit.SECONDS - )) { + ); + if (!accept(tapEventOffset, consumer)) { break; } } @@ -136,7 +126,7 @@ protected void report(BsonTimestamp bsonTimestamp) throws InterruptedException { if (running.get()) { try { Thread.currentThread().setName("replicaSet-read-thread-" + replicaSetName); - readFromOplog(connectorContext, replicaSetName, intervalReport, mongodbURI, eventBatchSize, consumer); + readFromOplog(connectorContext, replicaSetName, intervalReport, mongodbURI, consumer); } catch (Exception e) { running.compareAndSet(true, false); TapLogger.error(TAG, "read oplog event from {} failed {}", replicaSetName, e.getMessage(), e); @@ -147,28 +137,8 @@ protected void report(BsonTimestamp bsonTimestamp) throws InterruptedException { } } - List tapEvents = new ArrayList<>(eventBatchSize); - long lastConsumeTs = 0L; while (running.get()) { - try { - final TapEventOffset tapEventOffset = tapEventQueue.poll(3, TimeUnit.SECONDS); - if (tapEventOffset != null) { - tapEvents.add(tapEventOffset.getTapEvent()); - this.offset.put(tapEventOffset.getReplicaSetName(), tapEventOffset.getOffset()); - } - - long consumeInterval = System.currentTimeMillis() - lastConsumeTs; - if (tapEvents.size() >= eventBatchSize - || consumeInterval > CONSUME_TIME_OUT) { - Map newOffset = new ConcurrentHashMap<>(this.offset); - consumer.accept(tapEvents, newOffset); - tapEvents = new ArrayList<>(eventBatchSize); - lastConsumeTs = System.currentTimeMillis(); - } - } catch (InterruptedException e) { - TapLogger.info("Stream polling failed: {}", e.getMessage(), e); - break; - } + // } if (error != null) { @@ -176,6 +146,20 @@ protected void report(BsonTimestamp bsonTimestamp) throws InterruptedException { } } + boolean accept(TapEventOffset tapEventOffset, StreamReadOneByOneConsumer consumer) { + try { + if (tapEventOffset != null) { + Map newOffset = new ConcurrentHashMap<>(this.offset); + consumer.accept(tapEventOffset.getTapEvent(), newOffset); + this.offset.put(tapEventOffset.getReplicaSetName(), tapEventOffset.getOffset()); + } + return true; + } catch (Exception e) { + TapLogger.info("Stream polling failed: {}", e.getMessage(), e); + return false; + } + } + @Override public Object streamOffset(Long offsetStartTime) { if (offsetStartTime == null) { @@ -209,7 +193,7 @@ public void onDestroy() { } } - private void readFromOplog(TapConnectorContext connectorContext, String replicaSetName, IntervalReport intervalReport, String mongodbURI, int eventBatchSize, StreamReadConsumer consumer) { + private void readFromOplog(TapConnectorContext connectorContext, String replicaSetName, IntervalReport intervalReport, String mongodbURI, StreamReadOneByOneConsumer consumer) { Bson filter = null; BsonTimestamp startTs = null; @@ -245,7 +229,6 @@ private void readFromOplog(TapConnectorContext connectorContext, String replicaS return; } -// List tapEvents = new ArrayList<>(eventBatchSize); // todo exception retry while (running.get()) { filter = Filters.and(Filters.gte("ts", startTs), fromMigrateFilter); @@ -271,35 +254,18 @@ private void readFromOplog(TapConnectorContext connectorContext, String replicaS tapBaseEvent.setReferenceTime((long) (bsonTimestamp.getTime()) * 1000); while (running.get()) { - if (tapEventQueue.offer( - new TapEventOffset( + TapEventOffset tapEventOffset = new TapEventOffset( tapBaseEvent, new MongoV3StreamOffset(bsonTimestamp.getTime(), bsonTimestamp.getInc()), replicaSetName - ), - 3, - TimeUnit.SECONDS - )) { + ); + if (!accept(tapEventOffset, consumer)) { + running.compareAndSet(true, false); break; } } -// Map snapshotOffset = new ConcurrentHashMap<>(offset); -// tapEvents.add(tapEvent); - -// snapshotOffset.put(replicaSetName, new MongoV3StreamOffset(bsonTimestamp.getTime(), bsonTimestamp.getInc())); -// if (tapEvents.size() % eventBatchSize == 0) { -// consumer.accept(tapEvents, snapshotOffset); -// tapEvents = new ArrayList<>(); -// } startTs = bsonTimestamp; } -// else { -// if (tapEvents.size() > 0) { -// consumer.accept(tapEvents); -// tapEvents = new ArrayList<>(); -// } -// Thread.sleep(500L); -// } } } catch (InterruptedException | MongoInterruptedException e) { running.compareAndSet(true, false); diff --git a/connectors/mongodb-connector/src/main/resources/spec.json b/connectors/mongodb-connector/src/main/resources/spec.json index 23a86a272..4880d295d 100644 --- a/connectors/mongodb-connector/src/main/resources/spec.json +++ b/connectors/mongodb-connector/src/main/resources/spec.json @@ -4,7 +4,17 @@ "icon": "icons/mongodb.png", "doc" : "${doc}", "tags" : ["schema-free","Database","doubleActive"], - "id": "mongodb" + "id": "mongodb", + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities": [ diff --git a/connectors/mongodb-connector/src/test/java/io/tapdata/mongodb/MongodbConnectorTest.java b/connectors/mongodb-connector/src/test/java/io/tapdata/mongodb/MongodbConnectorTest.java index 95e5795f2..bc2946fd1 100644 --- a/connectors/mongodb-connector/src/test/java/io/tapdata/mongodb/MongodbConnectorTest.java +++ b/connectors/mongodb-connector/src/test/java/io/tapdata/mongodb/MongodbConnectorTest.java @@ -21,6 +21,7 @@ import io.tapdata.mongodb.reader.MongodbStreamReader; import io.tapdata.mongodb.reader.StreamWithOpLogCollection; import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.ExecuteResult; @@ -208,7 +209,7 @@ class StreamReadTest { List tableList; Object offset; int eventBatchSize; - StreamReadConsumer consumer; + StreamReadOneByOneConsumer consumer; MongoCdcOffset mongoCdcOffset; @BeforeEach @@ -217,70 +218,70 @@ void init() { tableList = mock(List.class); offset = 0L; eventBatchSize = 100; - consumer = mock(StreamReadConsumer.class); + consumer = mock(StreamReadOneByOneConsumer.class); when(tableList.size()).thenReturn(1, 1); - doNothing().when(mongodbConnector).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), eventBatchSize, consumer); + doNothing().when(mongodbConnector).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), consumer); when(tableList.isEmpty()).thenReturn(false); when(mongodbConnector.createStreamReader()).thenReturn(mongodbStreamReader); - doNothing().when(mongodbConnector).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, eventBatchSize, consumer); - doCallRealMethod().when(mongodbConnector).streamRead(connectorContext, tableList, offset, eventBatchSize, consumer); + doNothing().when(mongodbConnector).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, consumer); + doCallRealMethod().when(mongodbConnector).streamRead(connectorContext, tableList, offset, consumer); } @Test void testNormal() { - Assertions.assertDoesNotThrow(() -> mongodbConnector.streamRead(connectorContext, tableList, offset, eventBatchSize, consumer)); + Assertions.assertDoesNotThrow(() -> mongodbConnector.streamRead(connectorContext, tableList, offset, consumer)); verify(tableList, times(2)).size(); - verify(mongodbConnector).streamReadOpLog(connectorContext, tableList, null, eventBatchSize, consumer); - verify(mongodbConnector, times(0)).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), eventBatchSize, consumer); + verify(mongodbConnector).streamReadOpLog(connectorContext, tableList, null, consumer); + verify(mongodbConnector, times(0)).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), consumer); verify(tableList, times(0)).isEmpty(); verify(mongodbConnector, times(0)).createStreamReader(); - verify(mongodbConnector).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, eventBatchSize, consumer); + verify(mongodbConnector).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, consumer); } @Test void testAfterReadOpLog() { when(tableList.size()).thenReturn(1, 2); - Assertions.assertDoesNotThrow(() -> mongodbConnector.streamRead(connectorContext, tableList, offset, eventBatchSize, consumer)); + Assertions.assertDoesNotThrow(() -> mongodbConnector.streamRead(connectorContext, tableList, offset, consumer)); verify(tableList, times(2)).size(); - verify(mongodbConnector).streamReadOpLog(connectorContext, tableList, null, eventBatchSize, consumer); - verify(mongodbConnector, times(0)).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), eventBatchSize, consumer); + verify(mongodbConnector).streamReadOpLog(connectorContext, tableList, null, consumer); + verify(mongodbConnector, times(0)).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), consumer); verify(tableList).isEmpty(); verify(mongodbConnector, times(0)).createStreamReader(); - verify(mongodbConnector).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, eventBatchSize, consumer); + verify(mongodbConnector).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, consumer); } @Test void testMongodbStreamReaderIsNull() { ReflectionTestUtils.setField(mongodbConnector, "mongodbStreamReader", null); - Assertions.assertDoesNotThrow(() -> mongodbConnector.streamRead(connectorContext, tableList, offset, eventBatchSize, consumer)); + Assertions.assertDoesNotThrow(() -> mongodbConnector.streamRead(connectorContext, tableList, offset, consumer)); verify(tableList, times(2)).size(); - verify(mongodbConnector).streamReadOpLog(connectorContext, tableList, null, eventBatchSize, consumer); - verify(mongodbConnector, times(0)).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), eventBatchSize, consumer); + verify(mongodbConnector).streamReadOpLog(connectorContext, tableList, null, consumer); + verify(mongodbConnector, times(0)).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), consumer); verify(tableList, times(0)).isEmpty(); verify(mongodbConnector, times(1)).createStreamReader(); - verify(mongodbConnector).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, eventBatchSize, consumer); + verify(mongodbConnector).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, consumer); } @Test void testOffsetIsMongoOffset() { offset = mongoCdcOffset; - doCallRealMethod().when(mongodbConnector).streamRead(connectorContext, tableList, offset, eventBatchSize, consumer); - Assertions.assertDoesNotThrow(() -> mongodbConnector.streamRead(connectorContext, tableList, offset, eventBatchSize, consumer)); + doCallRealMethod().when(mongodbConnector).streamRead(connectorContext, tableList, offset, consumer); + Assertions.assertDoesNotThrow(() -> mongodbConnector.streamRead(connectorContext, tableList, offset, consumer)); verify(tableList, times(2)).size(); - verify(mongodbConnector, times(0)).streamReadOpLog(connectorContext, tableList, null, eventBatchSize, consumer); - verify(mongodbConnector).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), eventBatchSize, consumer); + verify(mongodbConnector, times(0)).streamReadOpLog(connectorContext, tableList, null, consumer); + verify(mongodbConnector).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), consumer); verify(tableList, times(0)).isEmpty(); verify(mongodbConnector, times(0)).createStreamReader(); - verify(mongodbConnector).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, eventBatchSize, consumer); + verify(mongodbConnector).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, consumer); } @Test void testOnlyReadOpLogCollection() { when(tableList.size()).thenReturn(1, 2); when(tableList.isEmpty()).thenReturn(true); - Assertions.assertDoesNotThrow(() -> mongodbConnector.streamRead(connectorContext, tableList, offset, eventBatchSize, consumer)); + Assertions.assertDoesNotThrow(() -> mongodbConnector.streamRead(connectorContext, tableList, offset, consumer)); verify(tableList, times(2)).size(); - verify(mongodbConnector).streamReadOpLog(connectorContext, tableList, null, eventBatchSize, consumer); - verify(mongodbConnector, times(0)).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), eventBatchSize, consumer); + verify(mongodbConnector).streamReadOpLog(connectorContext, tableList, null, consumer); + verify(mongodbConnector, times(0)).streamReadOpLog(connectorContext, tableList, mongoCdcOffset.getOpLogOffset(), consumer); verify(tableList).isEmpty(); verify(mongodbConnector, times(0)).createStreamReader(); - verify(mongodbConnector, times(0)).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, eventBatchSize, consumer); + verify(mongodbConnector, times(0)).doStreamRead(mongodbStreamReader, connectorContext, tableList, 0L, consumer); } } @@ -289,7 +290,7 @@ class DoStreamTest { List tableList; Object offset; int eventBatchSize; - StreamReadConsumer consumer; + StreamReadOneByOneConsumer consumer; MongoCdcOffset mongoCdcOffset; MongodbStreamReader streamReader; @BeforeEach @@ -299,20 +300,20 @@ void init() throws Exception { tableList = mock(List.class); offset = 0L; eventBatchSize = 100; - consumer = mock(StreamReadConsumer.class); + consumer = mock(StreamReadOneByOneConsumer.class); when(tableList.size()).thenReturn(1, 1); - doNothing().when(streamReader).read(connectorContext, tableList, offset, eventBatchSize, consumer); + doNothing().when(streamReader).read(connectorContext, tableList, offset, consumer); doNothing().when(streamReader).onDestroy(); doNothing().when(mongodbConnector).errorHandle(any(Exception.class), any(TapConnectorContext.class)); doNothing().when(log).debug(anyString(), anyString()); - doCallRealMethod().when(mongodbConnector).doStreamRead(streamReader, connectorContext, tableList, offset, eventBatchSize, consumer); + doCallRealMethod().when(mongodbConnector).doStreamRead(streamReader, connectorContext, tableList, offset, consumer); } @Test void testNormal() throws Exception { - Assertions.assertDoesNotThrow(() -> mongodbConnector.doStreamRead(streamReader, connectorContext, tableList, offset, eventBatchSize, consumer)); - verify(streamReader, times(1)).read(connectorContext, tableList, offset, eventBatchSize, consumer); + Assertions.assertDoesNotThrow(() -> mongodbConnector.doStreamRead(streamReader, connectorContext, tableList, offset, consumer)); + verify(streamReader, times(1)).read(connectorContext, tableList, offset, consumer); verify(streamReader, times(0)).onDestroy(); verify(mongodbConnector, times(0)).errorHandle(any(Exception.class), any(TapConnectorContext.class)); verify(log, times(0)).debug(anyString(), anyString()); @@ -321,9 +322,9 @@ void testNormal() throws Exception { void testException() throws Exception { doAnswer(a -> { throw new Exception("error"); - }).when(streamReader).read(connectorContext, tableList, offset, eventBatchSize, consumer); - Assertions.assertDoesNotThrow(() -> mongodbConnector.doStreamRead(streamReader, connectorContext, tableList, offset, eventBatchSize, consumer)); - verify(streamReader, times(1)).read(connectorContext, tableList, offset, eventBatchSize, consumer); + }).when(streamReader).read(connectorContext, tableList, offset, consumer); + Assertions.assertDoesNotThrow(() -> mongodbConnector.doStreamRead(streamReader, connectorContext, tableList, offset, consumer)); + verify(streamReader, times(1)).read(connectorContext, tableList, offset, consumer); verify(streamReader, times(1)).onDestroy(); verify(mongodbConnector, times(1)).errorHandle(any(Exception.class), any(TapConnectorContext.class)); verify(log, times(0)).debug(anyString(), anyString()); @@ -332,12 +333,12 @@ void testException() throws Exception { void testDestroyException() throws Exception { doAnswer(a -> { throw new Exception("error"); - }).when(streamReader).read(connectorContext, tableList, offset, eventBatchSize, consumer); + }).when(streamReader).read(connectorContext, tableList, offset, consumer); doAnswer(a -> { throw new Exception("failed"); }).when(streamReader).onDestroy(); - Assertions.assertDoesNotThrow(() -> mongodbConnector.doStreamRead(streamReader, connectorContext, tableList, offset, eventBatchSize, consumer)); - verify(streamReader, times(1)).read(connectorContext, tableList, offset, eventBatchSize, consumer); + Assertions.assertDoesNotThrow(() -> mongodbConnector.doStreamRead(streamReader, connectorContext, tableList, offset, consumer)); + verify(streamReader, times(1)).read(connectorContext, tableList, offset, consumer); verify(streamReader, times(1)).onDestroy(); verify(mongodbConnector, times(1)).errorHandle(any(Exception.class), any(TapConnectorContext.class)); verify(log, times(1)).debug(anyString(), anyString()); @@ -349,7 +350,7 @@ class StreamReadOpLogTest { List tableList; Object offset; int eventBatchSize; - StreamReadConsumer consumer; + StreamReadOneByOneConsumer consumer; MongoCdcOffset mongoCdcOffset; @BeforeEach void init() { @@ -357,7 +358,7 @@ void init() { tableList = mock(List.class); offset = 0L; eventBatchSize = 100; - consumer = mock(StreamReadConsumer.class); + consumer = mock(StreamReadOneByOneConsumer.class); when(mongoConfig.getDatabase()).thenReturn("test"); when(tableList.contains(StreamWithOpLogCollection.OP_LOG_COLLECTION)).thenReturn(false); @@ -369,14 +370,13 @@ void init() { any(TapConnectorContext.class), anyList(), any(), - anyInt(), - any(StreamReadConsumer.class)); - doCallRealMethod().when(mongodbConnector).streamReadOpLog(connectorContext, tableList, 0L, eventBatchSize, consumer); + any(StreamReadOneByOneConsumer.class)); + doCallRealMethod().when(mongodbConnector).streamReadOpLog(connectorContext, tableList, 0L, consumer); } @Test void testNotReadOpLog() { - Assertions.assertDoesNotThrow(() -> mongodbConnector.streamReadOpLog(connectorContext, tableList, 0L, eventBatchSize, consumer)); + Assertions.assertDoesNotThrow(() -> mongodbConnector.streamReadOpLog(connectorContext, tableList, 0L, consumer)); verify(mongoConfig).getDatabase(); verify(tableList, times(0)).contains(StreamWithOpLogCollection.OP_LOG_COLLECTION); verify(connectorContext, times(0)).getLog(); @@ -388,13 +388,12 @@ void testNotReadOpLog() { any(TapConnectorContext.class), anyList(), any(), - anyInt(), - any(StreamReadConsumer.class)); + any(StreamReadOneByOneConsumer.class)); } @Test void testNotReadOpLog2() { when(mongoConfig.getDatabase()).thenReturn("local"); - Assertions.assertDoesNotThrow(() -> mongodbConnector.streamReadOpLog(connectorContext, tableList, 0L, eventBatchSize, consumer)); + Assertions.assertDoesNotThrow(() -> mongodbConnector.streamReadOpLog(connectorContext, tableList, 0L, consumer)); verify(mongoConfig).getDatabase(); verify(tableList, times(1)).contains(StreamWithOpLogCollection.OP_LOG_COLLECTION); verify(connectorContext, times(0)).getLog(); @@ -406,14 +405,13 @@ void testNotReadOpLog2() { any(TapConnectorContext.class), anyList(), any(), - anyInt(), - any(StreamReadConsumer.class)); + any(StreamReadOneByOneConsumer.class)); } @Test void testReadOpLog() { when(mongoConfig.getDatabase()).thenReturn("local"); when(tableList.contains(StreamWithOpLogCollection.OP_LOG_COLLECTION)).thenReturn(true); - Assertions.assertDoesNotThrow(() -> mongodbConnector.streamReadOpLog(connectorContext, tableList, 0L, eventBatchSize, consumer)); + Assertions.assertDoesNotThrow(() -> mongodbConnector.streamReadOpLog(connectorContext, tableList, 0L, consumer)); verify(mongoConfig).getDatabase(); verify(tableList, times(1)).contains(StreamWithOpLogCollection.OP_LOG_COLLECTION); verify(connectorContext, times(1)).getLog(); @@ -425,8 +423,7 @@ void testReadOpLog() { any(TapConnectorContext.class), anyList(), any(), - anyInt(), - any(StreamReadConsumer.class)); + any(StreamReadOneByOneConsumer.class)); } @Test void testReadOpLogWithNullMongodbOpLogStreamV3Reader() { @@ -436,7 +433,7 @@ void testReadOpLogWithNullMongodbOpLogStreamV3Reader() { ReflectionTestUtils.setField(mongodbConnector, "opLogStreamReader", null); when(mongoConfig.getDatabase()).thenReturn("local"); when(tableList.contains(StreamWithOpLogCollection.OP_LOG_COLLECTION)).thenReturn(true); - Assertions.assertDoesNotThrow(() -> mongodbConnector.streamReadOpLog(connectorContext, tableList, 0L, eventBatchSize, consumer)); + Assertions.assertDoesNotThrow(() -> mongodbConnector.streamReadOpLog(connectorContext, tableList, 0L, consumer)); verify(mongoConfig).getDatabase(); verify(tableList, times(1)).contains(StreamWithOpLogCollection.OP_LOG_COLLECTION); verify(connectorContext, times(1)).getLog(); @@ -449,8 +446,7 @@ void testReadOpLogWithNullMongodbOpLogStreamV3Reader() { any(TapConnectorContext.class), anyList(), any(), - anyInt(), - any(StreamReadConsumer.class)); + any(StreamReadOneByOneConsumer.class)); } } } diff --git a/connectors/mysql-connector/pom.xml b/connectors/mysql-connector/pom.xml index 2076f4998..4b36c0eb2 100644 --- a/connectors/mysql-connector/pom.xml +++ b/connectors/mysql-connector/pom.xml @@ -23,7 +23,7 @@ 8 - 2.0.2-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java b/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java index c4ea09ee5..6b1b8a9b0 100644 --- a/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java +++ b/connectors/mysql-connector/src/main/java/io/tapdata/connector/mysql/MysqlConnector.java @@ -41,7 +41,7 @@ import io.tapdata.kit.EmptyKit; import io.tapdata.partition.DatabaseReadPartitionSplitter; import io.tapdata.pdk.apis.annotations.TapConnectorClass; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.*; @@ -271,7 +271,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec connectorFunctions.supportClearTable(this::clearTable); connectorFunctions.supportBatchCount(this::batchCount); connectorFunctions.supportBatchRead(this::batchReadWithoutOffset); - connectorFunctions.supportStreamRead(this::streamRead); + connectorFunctions.supportOneByOneStreamRead(this::streamRead); connectorFunctions.supportTimestampToStreamOffset(this::timestampToStreamOffset); connectorFunctions.supportQueryByAdvanceFilter(this::queryByAdvanceFilterWithOffset); connectorFunctions.supportCountByPartitionFilterFunction(this::countByAdvanceFilter); @@ -834,9 +834,9 @@ protected Set dateFields(TapTable tapTable) { } - private void streamRead(TapConnectorContext tapConnectorContext, List tables, Object offset, int batchSize, StreamReadConsumer consumer) throws Throwable { + private void streamRead(TapConnectorContext tapConnectorContext, List tables, Object offset, StreamReadOneByOneConsumer consumer) throws Throwable { throwNonSupportWhenLightInit(); - mysqlReader.readBinlog(tapConnectorContext, tables, offset, batchSize, DDLParserType.MYSQL_CCJ_SQL_PARSER, consumer, contextMapForMasterSlave); + mysqlReader.readBinlog(tapConnectorContext, tables, offset, DDLParserType.MYSQL_CCJ_SQL_PARSER, consumer, contextMapForMasterSlave); } diff --git a/connectors/mysql-connector/src/main/resources/mysql-spec.json b/connectors/mysql-connector/src/main/resources/mysql-spec.json index 5ac971f18..9813ff24d 100644 --- a/connectors/mysql-connector/src/main/resources/mysql-spec.json +++ b/connectors/mysql-connector/src/main/resources/mysql-spec.json @@ -4,7 +4,17 @@ "icon": "icons/mysql.png", "id": "mysql", "doc": "${doc}", - "tags": ["Database", "ssl", "doubleActive", "disableForeignKey"] + "tags": ["Database", "ssl", "doubleActive", "disableForeignKey"], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities": [ diff --git a/connectors/pom.xml b/connectors/pom.xml index 1d81e8bd4..dafc82317 100644 --- a/connectors/pom.xml +++ b/connectors/pom.xml @@ -75,8 +75,8 @@ ${project.artifactId}-v${project.version} 8 - 2.0-SNAPSHOT - 2.0.0-SNAPSHOT + 2.5-SNAPSHOT + 2.0.5-SNAPSHOT 1.0-SNAPSHOT 5.8.1 1.8.1 diff --git a/connectors/postgres-connector/pom.xml b/connectors/postgres-connector/pom.xml index 742ca1dc4..c37f0041a 100644 --- a/connectors/postgres-connector/pom.xml +++ b/connectors/postgres-connector/pom.xml @@ -16,7 +16,7 @@ 8 42.7.5 - 2.0.4-SNAPSHOT + 2.0.5-SNAPSHOT diff --git a/connectors/postgres-connector/src/main/java/io/tapdata/connector/postgres/PostgresConnector.java b/connectors/postgres-connector/src/main/java/io/tapdata/connector/postgres/PostgresConnector.java index 692bd99d7..360959df2 100644 --- a/connectors/postgres-connector/src/main/java/io/tapdata/connector/postgres/PostgresConnector.java +++ b/connectors/postgres-connector/src/main/java/io/tapdata/connector/postgres/PostgresConnector.java @@ -39,7 +39,7 @@ import io.tapdata.exception.TapCodeException; import io.tapdata.kit.*; import io.tapdata.pdk.apis.annotations.TapConnectorClass; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.*; @@ -147,7 +147,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec // source connectorFunctions.supportBatchCount(this::batchCount); connectorFunctions.supportBatchRead(this::batchReadWithoutOffset); - connectorFunctions.supportStreamRead(this::streamRead); + connectorFunctions.supportOneByOneStreamRead(this::streamRead); connectorFunctions.supportTimestampToStreamOffset(this::timestampToStreamOffset); // query connectorFunctions.supportQueryByFilter(this::queryByFilter); @@ -251,7 +251,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec connectorFunctions.supportTransactionRollbackFunction(this::rollbackTransaction); connectorFunctions.supportQueryHashByAdvanceFilterFunction(this::queryTableHash); connectorFunctions.supportQueryPartitionTablesByParentName(this::discoverPartitionInfoByParentName); - connectorFunctions.supportStreamReadMultiConnectionFunction(this::streamReadMultiConnection); + connectorFunctions.supportStreamReadMultiConnectionOneByOneFunction(this::streamReadMultiConnection); connectorFunctions.supportExportEventSqlFunction(this::exportEventSql); } @@ -602,27 +602,27 @@ protected void afterInitialSync(TapConnectorContext connectorContext, TapTable t }); } - private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, int recordSize, StreamReadConsumer consumer) throws Throwable { + private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, StreamReadOneByOneConsumer consumer) throws Throwable { if ("walminer".equals(postgresConfig.getLogPluginName())) { if (EmptyKit.isNotEmpty(postgresConfig.getPgtoHost())) { new WalPgtoMiner(postgresJdbcContext, firstConnectorId, tapLogger) .watch(tableList, nodeContext.getTableMap()) .offset(offsetState) - .registerConsumer(consumer, recordSize) + .registerConsumer(consumer) .startMiner(this::isAlive); } else { new WalLogMinerV2(postgresJdbcContext, tapLogger) .watch(tableList, nodeContext.getTableMap()) .withWalLogDirectory(getWalDirectory()) .offset(offsetState) - .registerConsumer(consumer, recordSize) + .registerConsumer(consumer) .startMiner(this::isAlive); } } else { cdcRunner = new PostgresCdcRunner(postgresJdbcContext, nodeContext); testReplicateIdentity(nodeContext.getTableMap()); buildSlot(nodeContext, true); - cdcRunner.useSlot(slotName.toString()).watch(tableList).offset(offsetState).registerConsumer(consumer, recordSize); + cdcRunner.useSlot(slotName.toString()).watch(tableList).offset(offsetState).registerConsumer(consumer); cdcRunner.startCdcRunner(); if (EmptyKit.isNotNull(cdcRunner) && EmptyKit.isNotNull(cdcRunner.getThrowable().get())) { Throwable throwable = ErrorKit.getLastCause(cdcRunner.getThrowable().get()); @@ -654,7 +654,7 @@ private void flushOffset(TapConnectorContext connectorContext, Object offset) { } } - private void streamReadMultiConnection(TapConnectorContext nodeContext, List connectionConfigWithTables, Object offsetState, int batchSize, StreamReadConsumer consumer) throws Throwable { + private void streamReadMultiConnection(TapConnectorContext nodeContext, List connectionConfigWithTables, Object offsetState, StreamReadOneByOneConsumer consumer) throws Throwable { Map> schemaTableMap = new HashMap<>(); for (ConnectionConfigWithTables withTables : connectionConfigWithTables) { if (null == withTables.getConnectionConfig()) @@ -682,21 +682,21 @@ private void streamReadMultiConnection(TapConnectorContext nodeContext, List tableList, Object offsetState, int recordSize, StreamReadConsumer consumer) throws Throwable { - rabbitmqService.streamConsume(tableList, recordSize, consumer); + private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, StreamReadOneByOneConsumer consumer) throws Throwable { + rabbitmqService.streamConsume(tableList, consumer); } private Object timestampToStreamOffset(TapConnectorContext connectorContext, Long offsetStartTime) { diff --git a/connectors/rabbitmq-connector/src/main/java/io/tapdata/connector/rabbitmq/RabbitmqService.java b/connectors/rabbitmq-connector/src/main/java/io/tapdata/connector/rabbitmq/RabbitmqService.java index 050bd6c41..b41587c70 100644 --- a/connectors/rabbitmq-connector/src/main/java/io/tapdata/connector/rabbitmq/RabbitmqService.java +++ b/connectors/rabbitmq-connector/src/main/java/io/tapdata/connector/rabbitmq/RabbitmqService.java @@ -29,6 +29,7 @@ import io.tapdata.pdk.apis.entity.TestItem; import io.tapdata.pdk.apis.entity.WriteListResult; import io.tapdata.pdk.apis.functions.connection.ConnectionCheckItem; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import java.io.IOException; import java.util.HashMap; @@ -238,7 +239,15 @@ public void consumeOne(TapTable tapTable, int eventBatchSize, BiConsumer list = TapSimplify.list(); AtomicLong time = new AtomicLong(System.currentTimeMillis()); - DefaultConsumer consumer = mqConsumer(channel, list, tableName, eventBatchSize, time, eventsOffsetConsumer); + DefaultConsumer consumer = mqConsumer(channel, tableName, (e, o) -> list.add(e), () -> { + if (list.size() >= eventBatchSize) { + List subList = TapSimplify.list(); + subList.addAll(list); + eventsOffsetConsumer.accept(subList, TapSimplify.list()); + time.set(System.currentTimeMillis()); + list.clear(); + } + }); channel.queueDeclare(tableName, true, false, false, null); channel.basicConsume(tableName, consumer); while (consuming.get() && System.currentTimeMillis() - time.get() < 10000) { @@ -260,14 +269,13 @@ protected void exception() { } @Override - public void streamConsume(List tableList, int eventBatchSize, BiConsumer, Object> eventsOffsetConsumer) throws Throwable { + public void streamConsume(List tableList, StreamReadOneByOneConsumer eventsOffsetConsumer) throws Throwable { consuming.set(true); atomicReference.set(null); - List list = TapSimplify.list(); try (Channel channel = rabbitmqConnection.createChannel()) { for (String tableName : tableList) { AtomicLong time = new AtomicLong(System.currentTimeMillis()); - DefaultConsumer consumer = mqConsumer(channel, list, tableName, eventBatchSize, time, eventsOffsetConsumer); + DefaultConsumer consumer = mqConsumer(channel, tableName, eventsOffsetConsumer::accept, () -> time.set(System.currentTimeMillis())); channel.queueDeclare(tableName, true, false, false, null); channel.basicConsume(tableName, consumer); } @@ -277,14 +285,10 @@ public void streamConsume(List tableList, int eventBatchSize, BiConsumer } } catch (Exception e) { throw new CoreException(e, e.getMessage()); - } finally { - if (EmptyKit.isNotEmpty(list)) { - eventsOffsetConsumer.accept(list, TapSimplify.list()); - } } } - protected DefaultConsumer mqConsumer(Channel channel, List list, String tableName, int eventBatchSize, AtomicLong time, BiConsumer, Object> eventsOffsetConsumer) { + protected DefaultConsumer mqConsumer(Channel channel, String tableName, BiConsumer consumer, Runnable after) { return new DefaultConsumer(channel) { @Override public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) throws IOException { @@ -294,22 +298,16 @@ public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProp Map data = parse(properties, body); switch (mqOp) { case UPDATE: - list.add(new TapUpdateRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis())); + consumer.accept(new TapUpdateRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis()), TapSimplify.list()); break; case DELETE: - list.add(new TapDeleteRecordEvent().init().table(tableName).before(data).referenceTime(System.currentTimeMillis())); + consumer.accept(new TapDeleteRecordEvent().init().table(tableName).before(data).referenceTime(System.currentTimeMillis()), TapSimplify.list()); break; default: - list.add(new TapInsertRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis())); + consumer.accept(new TapInsertRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis()), TapSimplify.list()); } channel.basicAck(envelope.getDeliveryTag(), false); - if (list.size() >= eventBatchSize) { - List subList = TapSimplify.list(); - subList.addAll(list); - eventsOffsetConsumer.accept(subList, TapSimplify.list()); - time.set(System.currentTimeMillis()); - list.clear(); - } + after.run(); } catch (Exception e) { atomicReference.set(e); throw e; diff --git a/connectors/rabbitmq-connector/src/main/resources/spec_rabbitmq.json b/connectors/rabbitmq-connector/src/main/resources/spec_rabbitmq.json index 94e74f3b4..a6b5187df 100644 --- a/connectors/rabbitmq-connector/src/main/resources/spec_rabbitmq.json +++ b/connectors/rabbitmq-connector/src/main/resources/spec_rabbitmq.json @@ -4,7 +4,17 @@ "icon": "icons/rabbitmq.png", "doc": "${doc}", "id": "rabbitmq", - "tags": ["Database"] + "tags": ["Database"], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "connection": { diff --git a/connectors/rocketmq-connector/src/main/java/io/tapdata/connector/rocketmq/RocketmqConnector.java b/connectors/rocketmq-connector/src/main/java/io/tapdata/connector/rocketmq/RocketmqConnector.java index ac0e04ef3..b1af0a956 100644 --- a/connectors/rocketmq-connector/src/main/java/io/tapdata/connector/rocketmq/RocketmqConnector.java +++ b/connectors/rocketmq-connector/src/main/java/io/tapdata/connector/rocketmq/RocketmqConnector.java @@ -13,7 +13,7 @@ import io.tapdata.entity.schema.value.TapTimeValue; import io.tapdata.entity.simplify.TapSimplify; import io.tapdata.pdk.apis.annotations.TapConnectorClass; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.ConnectionOptions; @@ -65,7 +65,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec connectorFunctions.supportCreateTableV2(this::createTableV2); connectorFunctions.supportWriteRecord(this::writeRecord); connectorFunctions.supportBatchRead(this::batchRead); - connectorFunctions.supportStreamRead(this::streamRead); + connectorFunctions.supportOneByOneStreamRead(this::streamRead); connectorFunctions.supportTimestampToStreamOffset(this::timestampToStreamOffset); } @@ -121,8 +121,8 @@ private void batchRead(TapConnectorContext tapConnectorContext, TapTable tapTabl rocketmqService.consumeOne(tapTable, eventBatchSize, eventsOffsetConsumer); } - private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, int recordSize, StreamReadConsumer consumer) throws Throwable { - rocketmqService.streamConsume(tableList, recordSize, consumer); + private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, StreamReadOneByOneConsumer consumer) throws Throwable { + rocketmqService.streamConsume(tableList, consumer); } private Object timestampToStreamOffset(TapConnectorContext connectorContext, Long offsetStartTime) { diff --git a/connectors/rocketmq-connector/src/main/java/io/tapdata/connector/rocketmq/RocketmqService.java b/connectors/rocketmq-connector/src/main/java/io/tapdata/connector/rocketmq/RocketmqService.java index 9ecbd28a6..e63d4c93f 100644 --- a/connectors/rocketmq-connector/src/main/java/io/tapdata/connector/rocketmq/RocketmqService.java +++ b/connectors/rocketmq-connector/src/main/java/io/tapdata/connector/rocketmq/RocketmqService.java @@ -33,6 +33,7 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; @@ -357,7 +358,7 @@ public void consumeOne(TapTable tapTable, int eventBatchSize, BiConsumer= eventBatchSize) { eventsOffsetConsumer.accept(list, TapSimplify.list()); list = TapSimplify.list(); @@ -372,29 +373,26 @@ public void consumeOne(TapTable tapTable, int eventBatchSize, BiConsumer list, String tableName) { + private TapRecordEvent makeMessage(MessageExt message, String tableName) { Map data = jsonParser.fromJsonBytes(message.getBody(), Map.class); switch (MqOp.fromValue(message.getUserProperty("mqOp"))) { case INSERT: - list.add(new TapInsertRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis())); - break; + return new TapInsertRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis()); case UPDATE: - list.add(new TapUpdateRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis())); - break; + return new TapUpdateRecordEvent().init().table(tableName).after(data).referenceTime(System.currentTimeMillis()); case DELETE: - list.add(new TapDeleteRecordEvent().init().table(tableName).before(data).referenceTime(System.currentTimeMillis())); - break; + return new TapDeleteRecordEvent().init().table(tableName).before(data).referenceTime(System.currentTimeMillis()); } + return null; } @Override - public void streamConsume(List tableList, int eventBatchSize, BiConsumer, Object> eventsOffsetConsumer) { + public void streamConsume(List tableList, StreamReadOneByOneConsumer eventsOffsetConsumer) { consuming.set(true); List> tablesList = Lists.partition(tableList, (tableList.size() - 1) / concurrency + 1); executorService = Executors.newFixedThreadPool(tablesList.size()); CountDownLatch countDownLatch = new CountDownLatch(tablesList.size()); tablesList.forEach(tables -> executorService.submit(() -> { - List list = TapSimplify.list(); Map consumerMap = new HashMap<>(); int err = 0; while (consuming.get() && err < 10) { @@ -422,11 +420,8 @@ public void streamConsume(List tableList, int eventBatchSize, BiConsumer continue; } for (MessageExt message : messageList) { - makeMessage(message, list, tableName); - if (list.size() >= eventBatchSize) { - eventsOffsetConsumer.accept(list, TapSimplify.list()); - list = TapSimplify.list(); - } + TapRecordEvent e = makeMessage(message, tableName); + eventsOffsetConsumer.accept(e, TapSimplify.list()); } } catch (Exception e) { TapLogger.error(TAG, "error occur when consume queue: {}", tableName, e); @@ -435,9 +430,6 @@ public void streamConsume(List tableList, int eventBatchSize, BiConsumer } TapSimplify.sleep(50); } - if (EmptyKit.isNotEmpty(list)) { - eventsOffsetConsumer.accept(list, TapSimplify.list()); - } consumerMap.forEach((key, value) -> { try { value.shutdown(); diff --git a/connectors/rocketmq-connector/src/main/resources/spec_rocketmq.json b/connectors/rocketmq-connector/src/main/resources/spec_rocketmq.json index 0bc9cb906..5e8d79575 100644 --- a/connectors/rocketmq-connector/src/main/resources/spec_rocketmq.json +++ b/connectors/rocketmq-connector/src/main/resources/spec_rocketmq.json @@ -4,7 +4,17 @@ "icon": "icons/rocketmq.png", "doc": "${doc}", "id": "rocketmq", - "tags": ["Database"] + "tags": ["Database"], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "connection": { diff --git a/connectors/tidb-connector/pom.xml b/connectors/tidb-connector/pom.xml index fd82cf1ba..ee3e84f2f 100644 --- a/connectors/tidb-connector/pom.xml +++ b/connectors/tidb-connector/pom.xml @@ -19,7 +19,7 @@ 1.0-SNAPSHOT 1.0-SNAPSHOT 3.12.0 - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT 1.7.15 2.17.1 diff --git a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/TidbConnector.java b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/TidbConnector.java index e546dacbe..c5c87b409 100644 --- a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/TidbConnector.java +++ b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/TidbConnector.java @@ -35,7 +35,7 @@ import io.tapdata.kit.EmptyKit; import io.tapdata.kit.ErrorKit; import io.tapdata.pdk.apis.annotations.TapConnectorClass; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.*; @@ -133,7 +133,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec // source functions connectorFunctions.supportBatchCount(this::batchCount); connectorFunctions.supportBatchRead(this::batchReadWithoutOffset); - connectorFunctions.supportStreamRead(this::streamRead); + connectorFunctions.supportOneByOneStreamRead(this::streamRead); connectorFunctions.supportTimestampToStreamOffset(this::timestampToStreamOffset); connectorFunctions.supportGetTableInfoFunction(this::getTableInfo); connectorFunctions.supportRunRawCommandFunction(this::runRawCommand); @@ -152,7 +152,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec codecRegistry.registerFromTapValue(TapYearValue.class, TapValue::getOriginValue); } - protected void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, int recordSize, StreamReadConsumer consumer) { + protected void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, StreamReadOneByOneConsumer consumer) { String feedId = genericFeedId(nodeContext.getStateMap()); String cdcServer = String.valueOf(Optional.ofNullable(nodeContext.getStateMap().get(ProcessHandler.CDC_SERVER)).orElse("127.0.0.1:8300")); nodeContext.getLog().info("Source timezone: {}", timezone.toZoneId().toString()); diff --git a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObject.java b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObject.java index 1e4f4e4ac..5fed6222d 100644 --- a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObject.java +++ b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObject.java @@ -10,7 +10,7 @@ import io.tapdata.entity.logger.Log; import io.tapdata.entity.schema.TapTable; import io.tapdata.entity.utils.cache.KVReadOnlyMap; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import java.util.ArrayList; import java.util.List; @@ -19,14 +19,14 @@ public class AnalyseTapEventFromDDLObject implements AnalyseRecord> { private final KVReadOnlyMap tapTableMap; private static final DDLWrapperConfig DDL_WRAPPER_CONFIG = CCJBaseDDLWrapper.CCJDDLWrapperConfig.create().split("`"); - StreamReadConsumer consumer; + StreamReadOneByOneConsumer consumer; Map> offset; public AnalyseTapEventFromDDLObject(KVReadOnlyMap tapTableMap) { this.tapTableMap = tapTableMap; } - public AnalyseTapEventFromDDLObject withConsumer(StreamReadConsumer consumer, Map> offset) { + public AnalyseTapEventFromDDLObject withConsumer(StreamReadOneByOneConsumer consumer, Map> offset) { this.consumer = consumer; this.offset = offset; return this; @@ -44,9 +44,7 @@ public List analyse(DDLObject ddlObject, AnalyseColumnFilter { tapDDLEvent.setTime(System.currentTimeMillis()); - List l = new ArrayList<>(); - l.add(tapDDLEvent); - this.consumer.accept(l, offset); + this.consumer.accept(tapDDLEvent, offset); log.debug("Read DDL: {}, about to be packaged as some event(s)", ddlSql); } ); diff --git a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/ProcessHandler.java b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/ProcessHandler.java index 6e9bebe74..df6bf941c 100644 --- a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/ProcessHandler.java +++ b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/ProcessHandler.java @@ -10,7 +10,7 @@ import io.tapdata.connector.tidb.util.pojo.Sink; import io.tapdata.entity.error.CoreException; import io.tapdata.entity.logger.Log; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectorContext; import org.apache.commons.collections4.CollectionUtils; @@ -44,11 +44,11 @@ public final class ProcessHandler implements Activity { final Log log; final String basePath; - public static ProcessHandler of(ProcessInfo processInfo, StreamReadConsumer consumer) { + public static ProcessHandler of(ProcessInfo processInfo, StreamReadOneByOneConsumer consumer) { return new ProcessHandler(processInfo, consumer); } - public ProcessHandler(ProcessInfo processInfo, StreamReadConsumer consumer) { + public ProcessHandler(ProcessInfo processInfo, StreamReadOneByOneConsumer consumer) { this.processInfo = processInfo; this.tableVersionMap = new ConcurrentHashMap<>(); this.log = processInfo.nodeContext.getLog(); diff --git a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManager.java b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManager.java index 147352aa7..d45e52dcf 100644 --- a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManager.java +++ b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManager.java @@ -10,7 +10,7 @@ import io.tapdata.entity.schema.TapTable; import io.tapdata.entity.simplify.TapSimplify; import io.tapdata.entity.utils.cache.KVReadOnlyMap; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import org.apache.commons.collections4.CollectionUtils; import java.util.*; @@ -21,13 +21,13 @@ class TapEventManager { final AnalyseTapEventFromDDLObject ddlEventParser; final ProcessHandler handler; final AtomicReference throwableCollector; - final StreamReadConsumer consumer; + final StreamReadOneByOneConsumer consumer; final Log log; final KVReadOnlyMap tableMap; Map> offset; - protected TapEventManager(ProcessHandler handler, StreamReadConsumer consumer) { + protected TapEventManager(ProcessHandler handler, StreamReadOneByOneConsumer consumer) { this.handler = handler; this.throwableCollector = handler.processInfo.throwableCollector; this.consumer = consumer; @@ -90,7 +90,7 @@ protected void handleDML(DMLObject dmlObject) { List tapRecordEvents = dmlEventParser.analyse(dmlObject, null, log); if (CollectionUtils.isNotEmpty(tapRecordEvents)) { TiOffset.updateNotNull(offset, table, es, tableVersion); - consumer.accept(tapRecordEvents, this.offset); + tapRecordEvents.forEach(event -> consumer.accept(event, this.offset)); } } diff --git a/connectors/tidb-connector/src/main/resources/spec_tidb.json b/connectors/tidb-connector/src/main/resources/spec_tidb.json index 49a616f33..ac6d34ab2 100644 --- a/connectors/tidb-connector/src/main/resources/spec_tidb.json +++ b/connectors/tidb-connector/src/main/resources/spec_tidb.json @@ -6,7 +6,17 @@ "doc": "${doc}", "tags": [ "Database", "ssl" - ] + ], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities":[ diff --git a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/TidbConnectorTest.java b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/TidbConnectorTest.java index 6f387b246..6dc3c42a0 100644 --- a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/TidbConnectorTest.java +++ b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/TidbConnectorTest.java @@ -17,7 +17,7 @@ import io.tapdata.entity.utils.DataMap; import io.tapdata.entity.utils.cache.KVMap; import io.tapdata.kit.EmptyKit; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.functions.ConnectorFunctions; @@ -176,7 +176,7 @@ class StreamReadTest { List tableList; Object offsetState; int recordSize; - StreamReadConsumer consumer; + StreamReadOneByOneConsumer consumer; KVMap kvMap; ProcessHandler handler; @@ -186,7 +186,7 @@ void init() throws Throwable { tableList = mock(List.class); offsetState = 0L; recordSize =1; - consumer = mock(StreamReadConsumer.class); + consumer = mock(StreamReadOneByOneConsumer.class); kvMap = mock(KVMap.class); doNothing().when(handler).doActivity(); doNothing().when(handler).close(); @@ -197,13 +197,13 @@ void init() throws Throwable { when(kvMap.get(ProcessHandler.CDC_SERVER)).thenReturn(""); doNothing().when(log).info(anyString(), anyString()); - doCallRealMethod().when(connector).streamRead(nodeContext, tableList, offsetState, recordSize, consumer); + doCallRealMethod().when(connector).streamRead(nodeContext, tableList, offsetState, consumer); } @Test void testNormal() throws Exception { try(MockedStatic ph = mockStatic(ProcessHandler.class)) { - ph.when(() -> ProcessHandler.of(any(ProcessHandler.ProcessInfo.class), any(StreamReadConsumer.class))).thenReturn(handler); - Assertions.assertDoesNotThrow(() -> connector.streamRead(nodeContext, tableList, offsetState, recordSize, consumer)); + ph.when(() -> ProcessHandler.of(any(ProcessHandler.ProcessInfo.class), any(StreamReadOneByOneConsumer.class))).thenReturn(handler); + Assertions.assertDoesNotThrow(() -> connector.streamRead(nodeContext, tableList, offsetState, consumer)); verify(connector).genericFeedId(kvMap); verify(kvMap).get(ProcessHandler.CDC_SERVER); verify(nodeContext, times(2)).getStateMap(); diff --git a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObjectTest.java b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObjectTest.java index 211902ec4..a90feb1ba 100644 --- a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObjectTest.java +++ b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObjectTest.java @@ -11,7 +11,7 @@ import io.tapdata.entity.logger.Log; import io.tapdata.entity.schema.TapTable; import io.tapdata.entity.utils.cache.KVReadOnlyMap; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -33,16 +33,16 @@ class AnalyseTapEventFromDDLObjectTest { AnalyseTapEventFromDDLObject analyseTapEventFromDDLObject; KVReadOnlyMap tapTableMap; - StreamReadConsumer consumer; + StreamReadOneByOneConsumer consumer; Map> offset; @BeforeEach void init() { tapTableMap = mock(KVReadOnlyMap.class); - consumer = mock(StreamReadConsumer.class); + consumer = mock(StreamReadOneByOneConsumer.class); offset = mock(Map.class); analyseTapEventFromDDLObject = new AnalyseTapEventFromDDLObject(tapTableMap).withConsumer(consumer, offset); - doNothing().when(consumer).accept(anyList(), anyMap()); + doNothing().when(consumer).accept(any(), anyMap()); } @Nested diff --git a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManagerTest.java b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManagerTest.java index 1ceb95451..8e9485732 100644 --- a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManagerTest.java +++ b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManagerTest.java @@ -9,7 +9,7 @@ import io.tapdata.entity.logger.Log; import io.tapdata.entity.schema.TapTable; import io.tapdata.entity.utils.cache.KVReadOnlyMap; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -22,7 +22,6 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; @@ -45,7 +44,7 @@ class TapEventManagerTest { AnalyseTapEventFromDDLObject ddlEventParser; ProcessHandler handler; AtomicReference throwableCollector; - StreamReadConsumer consumer; + StreamReadOneByOneConsumer consumer; KVReadOnlyMap tableMap; @BeforeEach void setUp() { @@ -54,7 +53,7 @@ void setUp() { ddlEventParser = mock(AnalyseTapEventFromDDLObject.class); handler = mock(ProcessHandler.class); throwableCollector = new AtomicReference<>(); - consumer = mock(StreamReadConsumer.class); + consumer = mock(StreamReadOneByOneConsumer.class); tableMap = mock(KVReadOnlyMap.class); log = mock(Log.class); manager = mock(TapEventManager.class); @@ -188,7 +187,7 @@ void init() { dmlObject.setTable("table"); dmlObject.setTableVersion(1L); doNothing().when(log).debug(anyString(), anyString(), anyLong(), anyLong(), anyString()); - doNothing().when(consumer).accept(anyList(), anyMap()); + doNothing().when(consumer).accept(any(), anyMap()); doCallRealMethod().when(manager).handleDML(dmlObject); } @Test @@ -202,7 +201,7 @@ void testNormal() { Assertions.assertDoesNotThrow(() -> manager.handleDML(dmlObject)); verify(log, times(0)).debug(anyString(), anyString(), anyLong(), anyLong(), anyString()); verify(dmlEventParser, times(1)).analyse(dmlObject, null, log); - verify(consumer, times(0)).accept(anyList(), anyMap()); + verify(consumer, times(0)).accept(any(), anyMap()); } } @Test @@ -218,7 +217,7 @@ void testNormal1() { Assertions.assertDoesNotThrow(() -> manager.handleDML(dmlObject)); verify(log, times(0)).debug(anyString(), anyString(), anyLong(), anyLong(), anyString()); verify(dmlEventParser, times(1)).analyse(dmlObject, null, log); - verify(consumer, times(1)).accept(anyList(), anyMap()); + verify(consumer, times(1)).accept(any(), anyMap()); } } @Test @@ -231,7 +230,7 @@ void test1() { Assertions.assertDoesNotThrow(() -> manager.handleDML(dmlObject)); verify(log, times(1)).warn(anyString(), anyString(), anyLong(), anyLong(), anyString()); verify(dmlEventParser, times(0)).analyse(dmlObject, null, log); - verify(consumer, times(0)).accept(anyList(), anyMap()); + verify(consumer, times(0)).accept(any(), anyMap()); } } @Test @@ -244,7 +243,7 @@ void test2() { Assertions.assertDoesNotThrow(() -> manager.handleDML(dmlObject)); verify(log, times(0)).debug(anyString(), anyString(), anyLong(), anyLong(), anyString()); verify(dmlEventParser, times(1)).analyse(dmlObject, null, log); - verify(consumer, times(0)).accept(anyList(), anyMap()); + verify(consumer, times(0)).accept(any(), anyMap()); } } } diff --git a/connectors/vastbase-connector/pom.xml b/connectors/vastbase-connector/pom.xml index e0dd296ef..76c6661d0 100644 --- a/connectors/vastbase-connector/pom.xml +++ b/connectors/vastbase-connector/pom.xml @@ -15,7 +15,7 @@ 8 - 2.0.0-SNAPSHOT + 2.0.5-SNAPSHOT 1.5.4.Final diff --git a/connectors/vastbase-connector/src/main/java/io/tapdata/connector/postgres/VastbaseConnector.java b/connectors/vastbase-connector/src/main/java/io/tapdata/connector/postgres/VastbaseConnector.java index cd85b6e02..aec429b29 100644 --- a/connectors/vastbase-connector/src/main/java/io/tapdata/connector/postgres/VastbaseConnector.java +++ b/connectors/vastbase-connector/src/main/java/io/tapdata/connector/postgres/VastbaseConnector.java @@ -16,7 +16,6 @@ import io.tapdata.entity.event.ddl.table.TapNewFieldEvent; import io.tapdata.entity.event.dml.TapRecordEvent; import io.tapdata.entity.schema.TapField; -import io.tapdata.entity.schema.TapIndex; import io.tapdata.entity.schema.TapTable; import io.tapdata.entity.schema.type.TapType; import io.tapdata.entity.schema.value.*; @@ -30,7 +29,7 @@ import io.tapdata.kit.ErrorKit; import io.tapdata.kit.StringKit; import io.tapdata.pdk.apis.annotations.TapConnectorClass; -import io.tapdata.pdk.apis.consumer.StreamReadConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.ConnectionOptions; @@ -111,7 +110,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec // source connectorFunctions.supportBatchCount(this::batchCount); connectorFunctions.supportBatchRead(this::batchReadWithoutOffset); - connectorFunctions.supportStreamRead(this::streamRead); + connectorFunctions.supportOneByOneStreamRead(this::streamRead); connectorFunctions.supportTimestampToStreamOffset(this::timestampToStreamOffset); // query connectorFunctions.supportQueryByFilter(this::queryByFilter); @@ -396,11 +395,11 @@ protected void writeRecord(TapConnectorContext connectorContext, List tableList, Object offsetState, int recordSize, StreamReadConsumer consumer) throws Throwable { + private void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, StreamReadOneByOneConsumer consumer) throws Throwable { cdcRunner = new PostgresCdcRunner(postgresJdbcContext, nodeContext); testReplicateIdentity(nodeContext.getTableMap()); buildSlot(nodeContext, true); - cdcRunner.useSlot(slotName.toString()).watch(tableList).offset(offsetState).registerConsumer(consumer, recordSize); + cdcRunner.useSlot(slotName.toString()).watch(tableList).offset(offsetState).registerConsumer(consumer); cdcRunner.startCdcRunner(); if (EmptyKit.isNotNull(cdcRunner) && EmptyKit.isNotNull(cdcRunner.getThrowable().get())) { Throwable throwable = ErrorKit.getLastCause(cdcRunner.getThrowable().get()); diff --git a/connectors/vastbase-connector/src/main/resources/spec_vastbase.json b/connectors/vastbase-connector/src/main/resources/spec_vastbase.json index 2fe5de9f2..111d9b625 100644 --- a/connectors/vastbase-connector/src/main/resources/spec_vastbase.json +++ b/connectors/vastbase-connector/src/main/resources/spec_vastbase.json @@ -4,7 +4,17 @@ "icon": "icons/vastbase.png", "doc" : "${doc}", "id": "vastbase", - "tags": ["Database", "ssl", "doubleActive"] + "tags": ["Database", "ssl", "doubleActive"], + "autoAccumulateBatch": { + "batchRead": { + "open": false, + "maxDelayMs": "1000" + }, + "increaseRead": { + "open": true, + "maxDelayMs": "1000" + } + } }, "configOptions": { "capabilities":[ From fc9048d629e0298ceef01e02269ab039035a9c8c Mon Sep 17 00:00:00 2001 From: GavinXiao <2749984520@qq.com> Date: Tue, 25 Nov 2025 10:12:41 +0800 Subject: [PATCH 2/5] fix(TAP-8714): wait 500ms when main thread do wait sub thread, upload api version --- .../debezium-connector-postgres/pom.xml | 2 +- .../mongodb/reader/v3/MongodbV3StreamReader.java | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/connectors-common/debezium-bucket/debezium-connector-postgres/pom.xml b/connectors-common/debezium-bucket/debezium-connector-postgres/pom.xml index 85c963f6a..e8f7ce00d 100644 --- a/connectors-common/debezium-bucket/debezium-connector-postgres/pom.xml +++ b/connectors-common/debezium-bucket/debezium-connector-postgres/pom.xml @@ -101,7 +101,7 @@ io.tapdata tapdata-pdk-api - 2.0.1-SNAPSHOT + 2.0.5-SNAPSHOT provided diff --git a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/v3/MongodbV3StreamReader.java b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/v3/MongodbV3StreamReader.java index 643356f8c..fd3f6afaa 100644 --- a/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/v3/MongodbV3StreamReader.java +++ b/connectors/mongodb-connector/src/main/java/io/tapdata/mongodb/reader/v3/MongodbV3StreamReader.java @@ -129,6 +129,7 @@ protected void report(BsonTimestamp bsonTimestamp) throws InterruptedException { readFromOplog(connectorContext, replicaSetName, intervalReport, mongodbURI, consumer); } catch (Exception e) { running.compareAndSet(true, false); + this.notifyAll(); TapLogger.error(TAG, "read oplog event from {} failed {}", replicaSetName, e.getMessage(), e); error = e; } @@ -138,7 +139,11 @@ protected void report(BsonTimestamp bsonTimestamp) throws InterruptedException { } while (running.get()) { - // + try { + this.wait(500L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } if (error != null) { @@ -175,6 +180,7 @@ public Object streamOffset(Long offsetStartTime) { @Override public void onDestroy() { running.compareAndSet(true, false); + this.notifyAll(); if (mongoClient != null) { mongoClient.close(); } @@ -226,6 +232,7 @@ private void readFromOplog(TapConnectorContext connectorContext, String replicaS } } catch (MongoInterruptedException e) { running.compareAndSet(true, false); + this.notifyAll(); return; } @@ -261,6 +268,7 @@ private void readFromOplog(TapConnectorContext connectorContext, String replicaS ); if (!accept(tapEventOffset, consumer)) { running.compareAndSet(true, false); + this.notifyAll(); break; } } @@ -269,6 +277,7 @@ private void readFromOplog(TapConnectorContext connectorContext, String replicaS } } catch (InterruptedException | MongoInterruptedException e) { running.compareAndSet(true, false); + this.notifyAll(); } } } From 934335f683e6a3eca62a666ccd266e1ca30c0adc Mon Sep 17 00:00:00 2001 From: GavinXiao <2749984520@qq.com> Date: Tue, 2 Dec 2025 10:33:26 +0800 Subject: [PATCH 3/5] fix(quickAPI): npe of query list --- .../common/postman/entity/params/Url.java | 149 +++++++++++------- 1 file changed, 89 insertions(+), 60 deletions(-) diff --git a/connectors-common/api-loader-core/src/main/java/io/tapdata/common/postman/entity/params/Url.java b/connectors-common/api-loader-core/src/main/java/io/tapdata/common/postman/entity/params/Url.java index c428d3011..e30b4ba37 100644 --- a/connectors-common/api-loader-core/src/main/java/io/tapdata/common/postman/entity/params/Url.java +++ b/connectors-common/api-loader-core/src/main/java/io/tapdata/common/postman/entity/params/Url.java @@ -2,16 +2,23 @@ import io.tapdata.common.postman.enums.PostParam; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; public class Url { String raw; - List host; - List path; - List> query; - List> variable; - public Url copyOne(){ + List host = new ArrayList<>(); + List path = new ArrayList<>(); + List> query = new ArrayList<>(); + List> variable = new ArrayList<>(); + + public Url copyOne() { Url url = new Url(); url.raw(this.raw); url.host(this.host); @@ -21,13 +28,15 @@ public Url copyOne(){ return url; } - public static Url create(){ + public static Url create() { return new Url(); } - public static Url create(String url){ + + public static Url create(String url) { return Url.create().raw(url); } - public static Url create(Map map){ + + public static Url create(Map map) { try { if (Objects.isNull(map)) return Url.create(); Object rawObj = map.get(PostParam.RAW); @@ -37,100 +46,120 @@ public static Url create(Map map){ Object variableObj = map.get(PostParam.VARIABLE); String raw = Objects.isNull(rawObj) ? null : (String) rawObj; - List host = Objects.isNull(hostObj) ? null : (List) hostObj; - List path = Objects.isNull(pathObj) ? null : (List) pathObj; - List> query = Objects.isNull(queryObj) ? null : (List>) queryObj; - List> variable = Objects.isNull(variableObj) ? null : (List>) variableObj; - return Url.create().raw(raw).host(host).path(path).query(query).variable(variable); - }catch (Exception e){ + Url url = Url.create().raw(raw); + if (hostObj instanceof Collection) { + url.host().addAll((Collection) hostObj); + } + if (pathObj instanceof Collection) { + url.path().addAll((Collection) pathObj); + } + if (queryObj instanceof Collection) { + url.query().addAll((Collection>) queryObj); + } + if (variableObj instanceof Collection) { + url.variable().addAll((Collection>) variableObj); + } + return url; + } catch (Exception e) { return Url.create(); } } - public static Url create(Object urlObj){ - if ( urlObj instanceof String) { + + public static Url create(Object urlObj) { + if (urlObj instanceof String) { return Url.create((String) urlObj); - }else if(urlObj instanceof Map){ - Map map = (Map) urlObj; + } else if (urlObj instanceof Map) { + Map map = (Map) urlObj; return Url.create(map); } return Url.create(); } - public String raw(){ + + public String raw() { return this.raw; } - public Url raw(String raw){ + + public Url raw(String raw) { this.raw = raw; return this; } - public List host(){ + + public List host() { return this.host; } - public Url host(List host){ - this.host = host; + + public Url host(List host) { + this.host = Optional.ofNullable(host).orElse(new ArrayList<>()); return this; } - public List path(){ - return this.path; + + public List path() { + return Optional.ofNullable(this.path).orElse(new ArrayList<>()); } - public Url path(List path){ - this.path = path; + + public Url path(List path) { + this.path = Optional.ofNullable(path).orElse(new ArrayList<>()); return this; } - public List> query(){ - return this.query; + + public List> query() { + return Optional.ofNullable(this.query).orElse(new ArrayList<>()); } - public Url query(List> query){ - this.query = query; + + public Url query(List> query) { + this.query = Optional.ofNullable(query).orElse(new ArrayList<>()); return this; } - public List> variable(){ - return this.variable; + + public List> variable() { + return Optional.ofNullable(this.variable).orElse(new ArrayList<>()); } - public Url variable(List> variable){ - this.variable = variable; + + public Url variable(List> variable) { + this.variable = Optional.ofNullable(variable).orElse(new ArrayList<>()); return this; } public Url variableAssignment(Map params) { final String[] rawBack = {this.raw}; - List> queryBack = new ArrayList<>(); - Map> varMap = Objects.isNull(this.variable) ? new HashMap<>(): this.variable.stream().collect(Collectors.toMap(var -> String.valueOf(var.get(PostParam.KEY)), var -> var, (v1, v2) -> v2)); - Map> queryMap = Objects.isNull(this.query) ? new HashMap<>(): this.query.stream().collect(Collectors.toMap(var -> String.valueOf(var.get(PostParam.KEY)), var -> var, (v1, v2) -> v2)); - varMap.forEach((var,varValue)->{ - Object attributeParamValue = Objects.isNull(params)? null: params.get(var); - if (attributeParamValue instanceof Map){ - attributeParamValue = ((Map)attributeParamValue).get(PostParam.VALUE); + List> queryBack = new ArrayList<>(); + Map> varMap = Objects.isNull(this.variable) ? new HashMap<>() : this.variable.stream().collect(Collectors.toMap(var -> String.valueOf(var.get(PostParam.KEY)), var -> var, (v1, v2) -> v2)); + Map> queryMap = Objects.isNull(this.query) ? new HashMap<>() : this.query.stream().collect(Collectors.toMap(var -> String.valueOf(var.get(PostParam.KEY)), var -> var, (v1, v2) -> v2)); + varMap.forEach((var, varValue) -> { + Object attributeParamValue = Objects.isNull(params) ? null : params.get(var); + if (attributeParamValue instanceof Map) { + attributeParamValue = ((Map) attributeParamValue).get(PostParam.VALUE); } Object valueObj = varValue.get(PostParam.VALUE); - String val = Objects.isNull(valueObj)? "": String.valueOf(valueObj); - if (Objects.nonNull(attributeParamValue) ){ + String val = Objects.isNull(valueObj) ? "" : String.valueOf(valueObj); + if (Objects.nonNull(attributeParamValue)) { val = String.valueOf(attributeParamValue); } - rawBack[0] = rawBack[0].replaceAll(":"+var,val); + rawBack[0] = rawBack[0].replaceAll(":" + var, val); }); - queryMap.forEach((var,varValue)->{ - Object attributeParamValue = Objects.isNull(params)? null: params.get(var); - if (attributeParamValue instanceof Map){ - attributeParamValue = ((Map)attributeParamValue).get(PostParam.VALUE); + queryMap.forEach((var, varValue) -> { + Object attributeParamValue = Objects.isNull(params) ? null : params.get(var); + if (attributeParamValue instanceof Map) { + attributeParamValue = ((Map) attributeParamValue).get(PostParam.VALUE); } Object valueObj = varValue.get(PostParam.VALUE); - String val = Objects.isNull(valueObj)? "": String.valueOf(valueObj); - if (Objects.nonNull(attributeParamValue)){ + String val = Objects.isNull(valueObj) ? "" : String.valueOf(valueObj); + if (Objects.nonNull(attributeParamValue)) { val = String.valueOf(attributeParamValue); } - Map backMap = new HashMap<>(); - backMap.put(PostParam.KEY,var); - backMap.put(PostParam.VALUE,val); - backMap.put(PostParam.DESCRIPTION,varValue.get(PostParam.DESCRIPTION)); + Map backMap = new HashMap<>(); + backMap.put(PostParam.KEY, var); + backMap.put(PostParam.VALUE, val); + backMap.put(PostParam.DESCRIPTION, varValue.get(PostParam.DESCRIPTION)); queryBack.add(backMap); }); if (Objects.nonNull(params) && !params.isEmpty()) { - for (Map.Entry varEntry : params.entrySet()) { + for (Map.Entry varEntry : params.entrySet()) { Object value = varEntry.getValue(); - if (value instanceof Map){ - value = ((Map)value).get(PostParam.VALUE); + if (value instanceof Map) { + value = ((Map) value).get(PostParam.VALUE); } - rawBack[0] = rawBack[0].replaceAll("\\{\\{"+varEntry.getKey()+"}}",Objects.isNull(value)?"":String.valueOf(value)); + rawBack[0] = rawBack[0].replaceAll("\\{\\{" + varEntry.getKey() + "}}", Objects.isNull(value) ? "" : String.valueOf(value)); } } return Url.create().query(queryBack).variable(this.variable).raw(rawBack[0]).host(this.host).path(this.path); From 8dbd2e59f4fdd749baebaf01df105e3f452e5ea7 Mon Sep 17 00:00:00 2001 From: GavinXiao <2749984520@qq.com> Date: Tue, 2 Dec 2025 11:35:16 +0800 Subject: [PATCH 4/5] feat(TAP-8714): add cdcAcceptBatch and cdcAcceptDelaySeconds for mysql to read cdc log and commit event --- .../tapdata/connector/mysql/MysqlReader.java | 4 +- .../connector/mysql/config/MysqlConfig.java | 28 +++++++++ .../src/main/resources/mysql-spec.json | 60 ++++++++++++++++++- 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java index 7eab47b33..42c50d8d0 100644 --- a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java +++ b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/MysqlReader.java @@ -502,6 +502,8 @@ public void readBinlog(TapConnectorContext tapConnectorContext, List tab .append("\n")); configStr.append("}"); tapLogger.info(configStr.toString()); + final int commitBatchSize = mysqlConfig.fixValue(mysqlConfig.getCdcAcceptBatch(), 100, 1000); + final int commitDelaySeconds = mysqlConfig.fixValue(mysqlConfig.getCdcAcceptDelaySeconds(), 1, 60); embeddedEngine = (EmbeddedEngine) new EmbeddedEngine.BuilderImpl() .using(configuration) .notifying(this::consumeRecords) @@ -512,7 +514,7 @@ public void taskStarted() { } }) .using((numberOfMessagesSinceLastCommit, timeSinceLastCommit) -> - timeSinceLastCommit.getSeconds() >= 5) + numberOfMessagesSinceLastCommit >= commitBatchSize || timeSinceLastCommit.getSeconds() >= commitDelaySeconds) .using((result, message, throwable) -> { tapConnectorContext.configContext(); if (result) { diff --git a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java index 814c86c0a..8873f6fe7 100644 --- a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java +++ b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java @@ -203,4 +203,32 @@ public void setMaximumQueueSize(Integer maximumQueueSize) { this.maximumQueueSize = maximumQueueSize; } + int cdcAcceptBatch = 100; //使用io.debezium.embedded.EmbeddedEngine默认的commit批数100 + int cdcAcceptDelaySeconds = 1; //使用io.debezium.embedded.EmbeddedEngine默认的commit延迟1秒 + + public int getCdcAcceptBatch() { + return cdcAcceptBatch; + } + + public void setCdcAcceptBatch(int cdcAcceptBatch) { + this.cdcAcceptBatch = cdcAcceptBatch; + } + + public int getCdcAcceptDelaySeconds() { + return cdcAcceptDelaySeconds; + } + + public void setCdcAcceptDelaySeconds(int cdcAcceptDelaySeconds) { + this.cdcAcceptDelaySeconds = cdcAcceptDelaySeconds; + } + + public int fixValue(int val, int min, int max) { + if (val <= 0) { + val = min; + } + if (val > max) { + val = max; + } + return val; + } } diff --git a/connectors/mysql-connector/src/main/resources/mysql-spec.json b/connectors/mysql-connector/src/main/resources/mysql-spec.json index 9813ff24d..090bd4c26 100644 --- a/connectors/mysql-connector/src/main/resources/mysql-spec.json +++ b/connectors/mysql-connector/src/main/resources/mysql-spec.json @@ -496,6 +496,48 @@ } } ] + }, + "cdcAcceptBatch": { + "type": "string", + "title": "${cdcAcceptBatch}", + "default": 100, + "x-index": 20, + "x-decorator": "FormItem", + "x-component": "InputNumber", + "x-decorator-props": { + "tooltip": "${cdcAcceptBatchTip}" + }, + "x-reactions": [ + { + "dependencies": ["$inputs"], + "fulfill": { + "state": { + "display": "{{!$deps[0].length ? \"visible\":\"hidden\"}}" + } + } + } + ] + }, + "cdcAcceptDelaySeconds": { + "type": "string", + "title": "${cdcAcceptDelaySeconds}", + "default": 1, + "x-index": 30, + "x-decorator": "FormItem", + "x-component": "InputNumber", + "x-decorator-props": { + "tooltip": "${cdcAcceptDelaySecondsTip}" + }, + "x-reactions": [ + { + "dependencies": ["$inputs"], + "fulfill": { + "state": { + "display": "{{!$deps[0].length ? \"visible\":\"hidden\"}}" + } + } + } + ] } } } @@ -537,7 +579,11 @@ "batchReadThreadSize": "Batch read thread size", "maximumQueueSize": "Maximum queue size", "maximumQueueSizeTip": "The queue size for reading incremental data in MySQL. If the downstream synchronization is slow or individual records in the table are too large, please lower this setting.", - "lowerCaseTableNames": "Lower Case TableNames" + "lowerCaseTableNames": "Lower Case TableNames", + "cdcAcceptBatch": "Incremental event commit batch size", + "cdcAcceptBatchTip": "The number of batches in the commit rule for incremental log reading can reduce commit frequency and improve CDC performance by saving a batch of log reads for submission. The default batch count in the policy is 100", + "cdcAcceptDelaySeconds": "Incremental event commit delay (seconds)", + "cdcAcceptDelaySecondsTip": "The commit delay in the commit rule for incremental log reading can reduce commit frequency and improve CDC performance by saving a certain amount of time for log reading. The default delay is 1 second" }, "zh_CN": { "host": "地址", @@ -574,7 +620,11 @@ "batchReadThreadSize": "批量读取线程数", "maximumQueueSize": "最大队列大小", "maximumQueueSizeTip": "MySql读取增量数据队列大小，如果下游同步较慢或表的单条数据过大，请调低此配置。", - "lowerCaseTableNames": "大小写敏感" + "lowerCaseTableNames": "大小写敏感", + "cdcAcceptBatch": "从日志读取增量事件提交批数", + "cdcAcceptBatchTip": "增量日志读取的提交规则中的批数，日志读取攒一批后提交可减少Commit频率，提高cdc性能，默认策略中批次数为100", + "cdcAcceptDelaySeconds": "从日志读取增量事件提交延时（秒）", + "cdcAcceptDelaySecondsTip": "增量日志读取的提交规则中的提交延时，日志读取攒一定时间后提交可减少Commit频率，提高cdc性能，默认延时1秒" }, "zh_TW": { "host": "地址", @@ -611,7 +661,11 @@ "batchReadThreadSize": "批量讀取線程數", "maximumQueueSize": "最大隊列大小", "maximumQueueSizeTip": "MySql 讀取增量數據隊列大小。如果下游同步較慢或表的單條數據過大，請調低此配置。", - "lowerCaseTableNames": "大小寫敏感" + "lowerCaseTableNames": "大小寫敏感", + "cdcAcceptBatch": "從日誌讀取增量事件提交批數", + "cdcAcceptBatchTip": "增量日誌讀取的提交規則中的批數，日誌讀取攢一批後提交可减少Commit頻率，提高cdc效能，默認策略中批次數為100", + "cdcAcceptDelaySeconds": "從日誌讀取增量事件提交延時（秒）", + "cdcAcceptDelaySecondsTip": "增量日誌讀取的提交規則中的提交延時，日誌讀取攢一定時間後提交可减少Commit頻率，提高cdc效能，默認延時1秒" } }, "dataTypes": { From a56da70a636734be9d55adf6418029e7b1d38f98 Mon Sep 17 00:00:00 2001 From: GavinXiao <2749984520@qq.com> Date: Thu, 4 Dec 2025 21:38:13 +0800 Subject: [PATCH 5/5] fix(TAP-8714): rallback tidb and not do commit event one by one to engine --- .../connector/mysql/config/MysqlConfig.java | 29 --------- .../postgres/cdc/PostgresCdcRunner.java | 4 +- .../io/tapdata/common/CommonDbConfig.java | 29 +++++++++ .../src/main/resources/spec_postgres.json | 63 ++++++++++++++++++- connectors/tidb-connector/pom.xml | 2 +- .../tapdata/connector/tidb/TidbConnector.java | 6 +- .../analyse/AnalyseTapEventFromDDLObject.java | 10 +-- .../cdc/process/thread/ProcessHandler.java | 6 +- .../cdc/process/thread/TapEventManager.java | 8 +-- .../src/main/resources/spec_tidb.json | 12 +--- .../connector/tidb/TidbConnectorTest.java | 12 ++-- .../AnalyseTapEventFromDDLObjectTest.java | 8 +-- .../process/thread/TapEventManagerTest.java | 16 ++--- 13 files changed, 128 insertions(+), 77 deletions(-) diff --git a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java index 8873f6fe7..d35d5e408 100644 --- a/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java +++ b/connectors-common/mysql-core/src/main/java/io/tapdata/connector/mysql/config/MysqlConfig.java @@ -202,33 +202,4 @@ public Integer getMaximumQueueSize() { public void setMaximumQueueSize(Integer maximumQueueSize) { this.maximumQueueSize = maximumQueueSize; } - - int cdcAcceptBatch = 100; //使用io.debezium.embedded.EmbeddedEngine默认的commit批数100 - int cdcAcceptDelaySeconds = 1; //使用io.debezium.embedded.EmbeddedEngine默认的commit延迟1秒 - - public int getCdcAcceptBatch() { - return cdcAcceptBatch; - } - - public void setCdcAcceptBatch(int cdcAcceptBatch) { - this.cdcAcceptBatch = cdcAcceptBatch; - } - - public int getCdcAcceptDelaySeconds() { - return cdcAcceptDelaySeconds; - } - - public void setCdcAcceptDelaySeconds(int cdcAcceptDelaySeconds) { - this.cdcAcceptDelaySeconds = cdcAcceptDelaySeconds; - } - - public int fixValue(int val, int min, int max) { - if (val <= 0) { - val = min; - } - if (val > max) { - val = max; - } - return val; - } } diff --git a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/PostgresCdcRunner.java b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/PostgresCdcRunner.java index 83313e674..d88161472 100644 --- a/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/PostgresCdcRunner.java +++ b/connectors-common/postgres-core/src/main/java/io/tapdata/connector/postgres/cdc/PostgresCdcRunner.java @@ -146,6 +146,8 @@ public AtomicReference getThrowable() { public void registerConsumer(StreamReadOneByOneConsumer consumer) { this.consumer = consumer; + final int commitBatchSize = postgresConfig.fixValue(postgresConfig.getCdcAcceptBatch(), 100, 1000); + final int commitDelaySeconds = postgresConfig.fixValue(postgresConfig.getCdcAcceptDelaySeconds(), 1, 60); //build debezium engine this.engine = (EmbeddedEngine) EmbeddedEngine.create() .using(postgresDebeziumConfig.create()) @@ -166,7 +168,7 @@ public void taskStopped() { // .using(Clock.SYSTEM) // .notifying(this::consumeRecord) .using((numberOfMessagesSinceLastCommit, timeSinceLastCommit) -> - timeSinceLastCommit.getSeconds() >= 5) + numberOfMessagesSinceLastCommit >= commitBatchSize || timeSinceLastCommit.getSeconds() >= commitDelaySeconds) .notifying(this::consumeRecords).using((result, message, throwable) -> { if (result) { if (StringUtils.isNotBlank(message)) { diff --git a/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConfig.java b/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConfig.java index 96745280a..8b468c195 100644 --- a/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConfig.java +++ b/connectors-common/sql-core/src/main/java/io/tapdata/common/CommonDbConfig.java @@ -463,4 +463,33 @@ public int getMaxIndexNameLength() { public void setMaxIndexNameLength(int maxIndexNameLength) { this.maxIndexNameLength = maxIndexNameLength; } + + int cdcAcceptBatch = 100; //使用io.debezium.embedded.EmbeddedEngine默认的commit批数100 + int cdcAcceptDelaySeconds = 1; //使用io.debezium.embedded.EmbeddedEngine默认的commit延迟1秒 + + public int getCdcAcceptBatch() { + return cdcAcceptBatch; + } + + public void setCdcAcceptBatch(int cdcAcceptBatch) { + this.cdcAcceptBatch = cdcAcceptBatch; + } + + public int getCdcAcceptDelaySeconds() { + return cdcAcceptDelaySeconds; + } + + public void setCdcAcceptDelaySeconds(int cdcAcceptDelaySeconds) { + this.cdcAcceptDelaySeconds = cdcAcceptDelaySeconds; + } + + public int fixValue(int val, int min, int max) { + if (val <= 0) { + val = min; + } + if (val > max) { + val = max; + } + return val; + } } diff --git a/connectors/postgres-connector/src/main/resources/spec_postgres.json b/connectors/postgres-connector/src/main/resources/spec_postgres.json index a2a472b1f..a12d784a6 100644 --- a/connectors/postgres-connector/src/main/resources/spec_postgres.json +++ b/connectors/postgres-connector/src/main/resources/spec_postgres.json @@ -833,6 +833,48 @@ } } ] + }, + "cdcAcceptBatch": { + "type": "string", + "title": "${cdcAcceptBatch}", + "default": 100, + "x-index": 30, + "x-decorator": "FormItem", + "x-component": "InputNumber", + "x-decorator-props": { + "tooltip": "${cdcAcceptBatchTip}" + }, + "x-reactions": [ + { + "dependencies": ["$inputs"], + "fulfill": { + "state": { + "display": "{{!$deps[0].length ? \"visible\":\"hidden\"}}" + } + } + } + ] + }, + "cdcAcceptDelaySeconds": { + "type": "string", + "title": "${cdcAcceptDelaySeconds}", + "default": 1, + "x-index": 40, + "x-decorator": "FormItem", + "x-component": "InputNumber", + "x-decorator-props": { + "tooltip": "${cdcAcceptDelaySecondsTip}" + }, + "x-reactions": [ + { + "dependencies": ["$inputs"], + "fulfill": { + "state": { + "display": "{{!$deps[0].length ? \"visible\":\"hidden\"}}" + } + } + } + ] } } } @@ -901,7 +943,12 @@ "splitUpdatePk": "Split update unique key", "walLogSize": "Wal log total size(MB)", "clearIdleSlot": "Clear idle replication slot", - "walLogPercent": "Wal log usage rate" + "walLogPercent": "Wal log usage rate", + "cdcAcceptBatch": "Incremental event commit batch size", + "cdcAcceptBatchTip": "The number of batches in the commit rule for incremental log reading can reduce commit frequency and improve CDC performance by saving a batch of log reads for submission. The default batch count in the policy is 100", + "cdcAcceptDelaySeconds": "Incremental event commit delay (seconds)", + "cdcAcceptDelaySecondsTip": "The commit delay in the commit rule for incremental log reading can reduce commit frequency and improve CDC performance by saving a certain amount of time for log reading. The default delay is 1 second" + }, "zh_CN": { "doc": "docs/postgres_zh_CN.md", @@ -965,7 +1012,12 @@ "splitUpdatePk": "修改唯一键拆分", "walLogSize": "Wal日志总大小(MB)", "clearIdleSlot": "清理空闲复制槽", - "walLogPercent": "Wal日志占用率" + "walLogPercent": "Wal日志占用率", + "cdcAcceptBatch": "从日志读取增量事件提交批数", + "cdcAcceptBatchTip": "增量日志读取的提交规则中的批数，日志读取攒一批后提交可减少Commit频率，提高cdc性能，默认策略中批次数为100", + "cdcAcceptDelaySeconds": "从日志读取增量事件提交延时（秒）", + "cdcAcceptDelaySecondsTip": "增量日志读取的提交规则中的提交延时，日志读取攒一定时间后提交可减少Commit频率，提高cdc性能，默认延时1秒" + }, "zh_TW": { "doc": "docs/postgres_zh_TW.md", @@ -1029,7 +1081,12 @@ "splitUpdatePk": "修改唯一鍵拆分", "walLogSize": "Wal日誌總大小(MB)", "clearIdleSlot": "清理空閒複製槽", - "walLogPercent": "Wal日志占用率" + "walLogPercent": "Wal日志占用率", + "cdcAcceptBatch": "從日誌讀取增量事件提交批數", + "cdcAcceptBatchTip": "增量日誌讀取的提交規則中的批數，日誌讀取攢一批後提交可减少Commit頻率，提高cdc效能，默認策略中批次數為100", + "cdcAcceptDelaySeconds": "從日誌讀取增量事件提交延時（秒）", + "cdcAcceptDelaySecondsTip": "增量日誌讀取的提交規則中的提交延時，日誌讀取攢一定時間後提交可减少Commit頻率，提高cdc效能，默認延時1秒" + } }, "dataTypes": { diff --git a/connectors/tidb-connector/pom.xml b/connectors/tidb-connector/pom.xml index ee3e84f2f..fd82cf1ba 100644 --- a/connectors/tidb-connector/pom.xml +++ b/connectors/tidb-connector/pom.xml @@ -19,7 +19,7 @@ 1.0-SNAPSHOT 1.0-SNAPSHOT 3.12.0 - 2.0.5-SNAPSHOT + 2.0.0-SNAPSHOT 1.7.15 2.17.1 diff --git a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/TidbConnector.java b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/TidbConnector.java index c5c87b409..e546dacbe 100644 --- a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/TidbConnector.java +++ b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/TidbConnector.java @@ -35,7 +35,7 @@ import io.tapdata.kit.EmptyKit; import io.tapdata.kit.ErrorKit; import io.tapdata.pdk.apis.annotations.TapConnectorClass; -import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.*; @@ -133,7 +133,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec // source functions connectorFunctions.supportBatchCount(this::batchCount); connectorFunctions.supportBatchRead(this::batchReadWithoutOffset); - connectorFunctions.supportOneByOneStreamRead(this::streamRead); + connectorFunctions.supportStreamRead(this::streamRead); connectorFunctions.supportTimestampToStreamOffset(this::timestampToStreamOffset); connectorFunctions.supportGetTableInfoFunction(this::getTableInfo); connectorFunctions.supportRunRawCommandFunction(this::runRawCommand); @@ -152,7 +152,7 @@ public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodec codecRegistry.registerFromTapValue(TapYearValue.class, TapValue::getOriginValue); } - protected void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, StreamReadOneByOneConsumer consumer) { + protected void streamRead(TapConnectorContext nodeContext, List tableList, Object offsetState, int recordSize, StreamReadConsumer consumer) { String feedId = genericFeedId(nodeContext.getStateMap()); String cdcServer = String.valueOf(Optional.ofNullable(nodeContext.getStateMap().get(ProcessHandler.CDC_SERVER)).orElse("127.0.0.1:8300")); nodeContext.getLog().info("Source timezone: {}", timezone.toZoneId().toString()); diff --git a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObject.java b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObject.java index 5fed6222d..1e4f4e4ac 100644 --- a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObject.java +++ b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObject.java @@ -10,7 +10,7 @@ import io.tapdata.entity.logger.Log; import io.tapdata.entity.schema.TapTable; import io.tapdata.entity.utils.cache.KVReadOnlyMap; -import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadConsumer; import java.util.ArrayList; import java.util.List; @@ -19,14 +19,14 @@ public class AnalyseTapEventFromDDLObject implements AnalyseRecord> { private final KVReadOnlyMap tapTableMap; private static final DDLWrapperConfig DDL_WRAPPER_CONFIG = CCJBaseDDLWrapper.CCJDDLWrapperConfig.create().split("`"); - StreamReadOneByOneConsumer consumer; + StreamReadConsumer consumer; Map> offset; public AnalyseTapEventFromDDLObject(KVReadOnlyMap tapTableMap) { this.tapTableMap = tapTableMap; } - public AnalyseTapEventFromDDLObject withConsumer(StreamReadOneByOneConsumer consumer, Map> offset) { + public AnalyseTapEventFromDDLObject withConsumer(StreamReadConsumer consumer, Map> offset) { this.consumer = consumer; this.offset = offset; return this; @@ -44,7 +44,9 @@ public List analyse(DDLObject ddlObject, AnalyseColumnFilter { tapDDLEvent.setTime(System.currentTimeMillis()); - this.consumer.accept(tapDDLEvent, offset); + List l = new ArrayList<>(); + l.add(tapDDLEvent); + this.consumer.accept(l, offset); log.debug("Read DDL: {}, about to be packaged as some event(s)", ddlSql); } ); diff --git a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/ProcessHandler.java b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/ProcessHandler.java index df6bf941c..6e9bebe74 100644 --- a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/ProcessHandler.java +++ b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/ProcessHandler.java @@ -10,7 +10,7 @@ import io.tapdata.connector.tidb.util.pojo.Sink; import io.tapdata.entity.error.CoreException; import io.tapdata.entity.logger.Log; -import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadConsumer; import io.tapdata.pdk.apis.context.TapConnectorContext; import org.apache.commons.collections4.CollectionUtils; @@ -44,11 +44,11 @@ public final class ProcessHandler implements Activity { final Log log; final String basePath; - public static ProcessHandler of(ProcessInfo processInfo, StreamReadOneByOneConsumer consumer) { + public static ProcessHandler of(ProcessInfo processInfo, StreamReadConsumer consumer) { return new ProcessHandler(processInfo, consumer); } - public ProcessHandler(ProcessInfo processInfo, StreamReadOneByOneConsumer consumer) { + public ProcessHandler(ProcessInfo processInfo, StreamReadConsumer consumer) { this.processInfo = processInfo; this.tableVersionMap = new ConcurrentHashMap<>(); this.log = processInfo.nodeContext.getLog(); diff --git a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManager.java b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManager.java index d45e52dcf..147352aa7 100644 --- a/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManager.java +++ b/connectors/tidb-connector/src/main/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManager.java @@ -10,7 +10,7 @@ import io.tapdata.entity.schema.TapTable; import io.tapdata.entity.simplify.TapSimplify; import io.tapdata.entity.utils.cache.KVReadOnlyMap; -import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadConsumer; import org.apache.commons.collections4.CollectionUtils; import java.util.*; @@ -21,13 +21,13 @@ class TapEventManager { final AnalyseTapEventFromDDLObject ddlEventParser; final ProcessHandler handler; final AtomicReference throwableCollector; - final StreamReadOneByOneConsumer consumer; + final StreamReadConsumer consumer; final Log log; final KVReadOnlyMap tableMap; Map> offset; - protected TapEventManager(ProcessHandler handler, StreamReadOneByOneConsumer consumer) { + protected TapEventManager(ProcessHandler handler, StreamReadConsumer consumer) { this.handler = handler; this.throwableCollector = handler.processInfo.throwableCollector; this.consumer = consumer; @@ -90,7 +90,7 @@ protected void handleDML(DMLObject dmlObject) { List tapRecordEvents = dmlEventParser.analyse(dmlObject, null, log); if (CollectionUtils.isNotEmpty(tapRecordEvents)) { TiOffset.updateNotNull(offset, table, es, tableVersion); - tapRecordEvents.forEach(event -> consumer.accept(event, this.offset)); + consumer.accept(tapRecordEvents, this.offset); } } diff --git a/connectors/tidb-connector/src/main/resources/spec_tidb.json b/connectors/tidb-connector/src/main/resources/spec_tidb.json index ac6d34ab2..49a616f33 100644 --- a/connectors/tidb-connector/src/main/resources/spec_tidb.json +++ b/connectors/tidb-connector/src/main/resources/spec_tidb.json @@ -6,17 +6,7 @@ "doc": "${doc}", "tags": [ "Database", "ssl" - ], - "autoAccumulateBatch": { - "batchRead": { - "open": false, - "maxDelayMs": "1000" - }, - "increaseRead": { - "open": true, - "maxDelayMs": "1000" - } - } + ] }, "configOptions": { "capabilities":[ diff --git a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/TidbConnectorTest.java b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/TidbConnectorTest.java index 6dc3c42a0..6f387b246 100644 --- a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/TidbConnectorTest.java +++ b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/TidbConnectorTest.java @@ -17,7 +17,7 @@ import io.tapdata.entity.utils.DataMap; import io.tapdata.entity.utils.cache.KVMap; import io.tapdata.kit.EmptyKit; -import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadConsumer; import io.tapdata.pdk.apis.context.TapConnectionContext; import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.functions.ConnectorFunctions; @@ -176,7 +176,7 @@ class StreamReadTest { List tableList; Object offsetState; int recordSize; - StreamReadOneByOneConsumer consumer; + StreamReadConsumer consumer; KVMap kvMap; ProcessHandler handler; @@ -186,7 +186,7 @@ void init() throws Throwable { tableList = mock(List.class); offsetState = 0L; recordSize =1; - consumer = mock(StreamReadOneByOneConsumer.class); + consumer = mock(StreamReadConsumer.class); kvMap = mock(KVMap.class); doNothing().when(handler).doActivity(); doNothing().when(handler).close(); @@ -197,13 +197,13 @@ void init() throws Throwable { when(kvMap.get(ProcessHandler.CDC_SERVER)).thenReturn(""); doNothing().when(log).info(anyString(), anyString()); - doCallRealMethod().when(connector).streamRead(nodeContext, tableList, offsetState, consumer); + doCallRealMethod().when(connector).streamRead(nodeContext, tableList, offsetState, recordSize, consumer); } @Test void testNormal() throws Exception { try(MockedStatic ph = mockStatic(ProcessHandler.class)) { - ph.when(() -> ProcessHandler.of(any(ProcessHandler.ProcessInfo.class), any(StreamReadOneByOneConsumer.class))).thenReturn(handler); - Assertions.assertDoesNotThrow(() -> connector.streamRead(nodeContext, tableList, offsetState, consumer)); + ph.when(() -> ProcessHandler.of(any(ProcessHandler.ProcessInfo.class), any(StreamReadConsumer.class))).thenReturn(handler); + Assertions.assertDoesNotThrow(() -> connector.streamRead(nodeContext, tableList, offsetState, recordSize, consumer)); verify(connector).genericFeedId(kvMap); verify(kvMap).get(ProcessHandler.CDC_SERVER); verify(nodeContext, times(2)).getStateMap(); diff --git a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObjectTest.java b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObjectTest.java index a90feb1ba..211902ec4 100644 --- a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObjectTest.java +++ b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/analyse/AnalyseTapEventFromDDLObjectTest.java @@ -11,7 +11,7 @@ import io.tapdata.entity.logger.Log; import io.tapdata.entity.schema.TapTable; import io.tapdata.entity.utils.cache.KVReadOnlyMap; -import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadConsumer; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -33,16 +33,16 @@ class AnalyseTapEventFromDDLObjectTest { AnalyseTapEventFromDDLObject analyseTapEventFromDDLObject; KVReadOnlyMap tapTableMap; - StreamReadOneByOneConsumer consumer; + StreamReadConsumer consumer; Map> offset; @BeforeEach void init() { tapTableMap = mock(KVReadOnlyMap.class); - consumer = mock(StreamReadOneByOneConsumer.class); + consumer = mock(StreamReadConsumer.class); offset = mock(Map.class); analyseTapEventFromDDLObject = new AnalyseTapEventFromDDLObject(tapTableMap).withConsumer(consumer, offset); - doNothing().when(consumer).accept(any(), anyMap()); + doNothing().when(consumer).accept(anyList(), anyMap()); } @Nested diff --git a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManagerTest.java b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManagerTest.java index 8e9485732..8f04af339 100644 --- a/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManagerTest.java +++ b/connectors/tidb-connector/src/test/java/io/tapdata/connector/tidb/cdc/process/thread/TapEventManagerTest.java @@ -9,7 +9,7 @@ import io.tapdata.entity.logger.Log; import io.tapdata.entity.schema.TapTable; import io.tapdata.entity.utils.cache.KVReadOnlyMap; -import io.tapdata.pdk.apis.consumer.StreamReadOneByOneConsumer; +import io.tapdata.pdk.apis.consumer.StreamReadConsumer; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -44,7 +44,7 @@ class TapEventManagerTest { AnalyseTapEventFromDDLObject ddlEventParser; ProcessHandler handler; AtomicReference throwableCollector; - StreamReadOneByOneConsumer consumer; + StreamReadConsumer consumer; KVReadOnlyMap tableMap; @BeforeEach void setUp() { @@ -53,7 +53,7 @@ void setUp() { ddlEventParser = mock(AnalyseTapEventFromDDLObject.class); handler = mock(ProcessHandler.class); throwableCollector = new AtomicReference<>(); - consumer = mock(StreamReadOneByOneConsumer.class); + consumer = mock(StreamReadConsumer.class); tableMap = mock(KVReadOnlyMap.class); log = mock(Log.class); manager = mock(TapEventManager.class); @@ -187,7 +187,7 @@ void init() { dmlObject.setTable("table"); dmlObject.setTableVersion(1L); doNothing().when(log).debug(anyString(), anyString(), anyLong(), anyLong(), anyString()); - doNothing().when(consumer).accept(any(), anyMap()); + doNothing().when(consumer).accept(anyList(), anyMap()); doCallRealMethod().when(manager).handleDML(dmlObject); } @Test @@ -201,7 +201,7 @@ void testNormal() { Assertions.assertDoesNotThrow(() -> manager.handleDML(dmlObject)); verify(log, times(0)).debug(anyString(), anyString(), anyLong(), anyLong(), anyString()); verify(dmlEventParser, times(1)).analyse(dmlObject, null, log); - verify(consumer, times(0)).accept(any(), anyMap()); + verify(consumer, times(0)).accept(anyList(), anyMap()); } } @Test @@ -217,7 +217,7 @@ void testNormal1() { Assertions.assertDoesNotThrow(() -> manager.handleDML(dmlObject)); verify(log, times(0)).debug(anyString(), anyString(), anyLong(), anyLong(), anyString()); verify(dmlEventParser, times(1)).analyse(dmlObject, null, log); - verify(consumer, times(1)).accept(any(), anyMap()); + verify(consumer, times(1)).accept(anyList(), anyMap()); } } @Test @@ -230,7 +230,7 @@ void test1() { Assertions.assertDoesNotThrow(() -> manager.handleDML(dmlObject)); verify(log, times(1)).warn(anyString(), anyString(), anyLong(), anyLong(), anyString()); verify(dmlEventParser, times(0)).analyse(dmlObject, null, log); - verify(consumer, times(0)).accept(any(), anyMap()); + verify(consumer, times(0)).accept(anyList(), anyMap()); } } @Test @@ -243,7 +243,7 @@ void test2() { Assertions.assertDoesNotThrow(() -> manager.handleDML(dmlObject)); verify(log, times(0)).debug(anyString(), anyString(), anyLong(), anyLong(), anyString()); verify(dmlEventParser, times(1)).analyse(dmlObject, null, log); - verify(consumer, times(0)).accept(any(), anyMap()); + verify(consumer, times(0)).accept(anyList(), anyMap()); } } }