Skip to content

Commit 02e7182

Browse files
ruchirvaninasdaqruchirvaninasdaq
authored
Merge pull request #36 from Nasdaq/0.4.0
Reusing Consumer Group
2 parents 9274c03 + 51f9559 commit 02e7182

5 files changed

Lines changed: 89 additions & 24 deletions

File tree

‎ncds-sdk/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>com.nasdaq.ncds</groupId>
99
<artifactId>ncds</artifactId>
10-
<version>0.3.0</version>
10+
<version>0.4.0</version>
1111
<relativePath>../pom.xml</relativePath>
1212
</parent>
1313

‎ncds-sdk/src/main/java/com/nasdaq/ncdsclient/consumer/NasdaqKafkaAvroConsumer.java‎

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.apache.kafka.clients.consumer.ConsumerConfig;
1313
import org.apache.kafka.clients.consumer.KafkaConsumer;
1414
import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
15+
import org.apache.kafka.clients.consumer.OffsetResetStrategy;
1516
import org.apache.kafka.common.TopicPartition;
1617
import org.apache.kafka.common.serialization.StringDeserializer;
1718

@@ -82,8 +83,12 @@ public KafkaConsumer getKafkaConsumer(String streamName) throws Exception {
8283
if (kafkaSchema == null) {
8384
throw new Exception("Kafka Schema not Found for Stream: " + streamName);
8485
}
85-
kafkaConsumer = getConsumer(kafkaSchema);
86-
kafkaConsumer.subscribe(Collections.singletonList(streamName + ".stream"));
86+
kafkaConsumer = getConsumer(kafkaSchema, streamName);
87+
TopicPartition topicPartition = new TopicPartition(streamName + ".stream",0);
88+
kafkaConsumer.assign(Collections.singletonList(topicPartition));
89+
if(kafkaProps.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).equals(OffsetResetStrategy.EARLIEST.toString().toLowerCase())) {
90+
return seekToMidNight(topicPartition);
91+
}
8792
}
8893
catch (Exception e) {
8994
throw (e);
@@ -106,7 +111,7 @@ public KafkaConsumer getKafkaConsumer(String streamName, long timestamp) throws
106111
if (kafkaSchema == null) {
107112
throw new Exception("Kafka Schema not Found for Stream: " + streamName);
108113
}
109-
kafkaConsumer = getConsumer(kafkaSchema);
114+
kafkaConsumer = getConsumer(kafkaSchema, streamName);
110115
TopicPartition topicPartition = new TopicPartition(streamName + ".stream",0);
111116
kafkaConsumer.assign(Collections.singleton(topicPartition));
112117

@@ -137,7 +142,7 @@ public KafkaConsumer getKafkaConsumer(String streamName, long timestamp) throws
137142
*/
138143

139144

140-
public KafkaAvroConsumer getConsumer(Schema avroSchema) throws Exception {
145+
public KafkaAvroConsumer getConsumer(Schema avroSchema, String streamName) throws Exception {
141146
try {
142147
if(!IsItJunit.isJUnitTest()) {
143148
ConfigProperties.resolveAndExportToSystemProperties(securityProps);
@@ -147,9 +152,9 @@ public KafkaAvroConsumer getConsumer(Schema avroSchema) throws Exception {
147152
kafkaProps.put("key.deserializer", StringDeserializer.class.getName());
148153
kafkaProps.put("value.deserializer", AvroDeserializer.class.getName());
149154
if(!kafkaProps.containsKey(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)) {
150-
kafkaProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
155+
kafkaProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, OffsetResetStrategy.EARLIEST.toString().toLowerCase());
151156
}
152-
kafkaProps.put(ConsumerConfig.GROUP_ID_CONFIG, this.clientID + "_" +getDate() + "_" + UUID.randomUUID().toString());
157+
kafkaProps.put(ConsumerConfig.GROUP_ID_CONFIG, this.clientID + "_" + streamName + "_" + getDate());
153158
ConfigProperties.resolve(kafkaProps);
154159
return new KafkaAvroConsumer(kafkaProps, avroSchema);
155160
}
@@ -201,8 +206,12 @@ public KafkaConsumer getNewsConsumer(String topic) throws Exception {
201206
if (newsSchema == null) {
202207
throw new Exception("News Schema not Found ");
203208
}
204-
kafkaConsumer = getConsumer(newsSchema);
205-
kafkaConsumer.subscribe(Collections.singletonList(topic+".stream"));
209+
kafkaConsumer = getConsumer(newsSchema, topic);
210+
TopicPartition topicPartition = new TopicPartition(topic + ".stream",0);
211+
kafkaConsumer.assign(Collections.singletonList(topicPartition));
212+
if(kafkaProps.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).equals(OffsetResetStrategy.EARLIEST.toString().toLowerCase())) {
213+
return seekToMidNight(topicPartition);
214+
}
206215
return kafkaConsumer;
207216
}
208217
catch (Exception e){
@@ -217,4 +226,32 @@ private String getDate(){
217226
String date = dateformat.format(new Date());
218227
return date;
219228
}
229+
230+
private KafkaConsumer seekToMidNight(TopicPartition topicPartition){
231+
Map<TopicPartition,Long> timestmaps = new HashMap();
232+
timestmaps.put(topicPartition , getTodayMidNightTimeStamp());
233+
Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes = kafkaConsumer.offsetsForTimes(timestmaps);
234+
OffsetAndTimestamp offsetAndTimestamp = null;
235+
if (offsetsForTimes != null && (offsetAndTimestamp = offsetsForTimes.get(topicPartition)) != null) {
236+
kafkaConsumer.seek(topicPartition, offsetAndTimestamp.offset());
237+
} else {
238+
kafkaConsumer.seekToBeginning(Collections.singleton(topicPartition));
239+
}
240+
return kafkaConsumer;
241+
}
242+
243+
private long getTodayMidNightTimeStamp(){
244+
245+
TimeZone timeZone = TimeZone.getTimeZone("America/New_York");
246+
247+
Calendar today = Calendar.getInstance(timeZone);
248+
today.set(Calendar.HOUR_OF_DAY, 0);
249+
today.set(Calendar.MINUTE, 0);
250+
today.set(Calendar.SECOND, 0);
251+
252+
long timestampFromMidnight = today.getTimeInMillis();
253+
254+
return timestampFromMidnight;
255+
}
256+
220257
}

‎ncds-sdk/src/main/java/com/nasdaq/ncdsclient/internal/ReadSchemaTopic.java‎

Lines changed: 40 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,8 @@
55
import io.strimzi.kafka.oauth.common.ConfigProperties;
66
import org.apache.avro.Schema;
77
import org.apache.avro.generic.GenericRecord;
8-
import org.apache.kafka.clients.consumer.ConsumerConfig;
9-
import org.apache.kafka.clients.consumer.ConsumerRecord;
10-
import org.apache.kafka.clients.consumer.ConsumerRecords;
11-
import org.apache.kafka.clients.consumer.KafkaConsumer;
8+
import org.apache.kafka.clients.consumer.*;
9+
import org.apache.kafka.common.TopicPartition;
1210
import org.apache.kafka.common.serialization.StringSerializer;
1311

1412
import java.time.Duration;
@@ -34,7 +32,6 @@ public ReadSchemaTopic(){
3432

3533
public Schema readSchema(String topic) throws Exception {
3634
KafkaConsumer schemaConsumer= getConsumer("Control-"+getClientID(securityProps));
37-
schemaConsumer.subscribe(Collections.singletonList(controlSchemaName));
3835
Duration sec = Duration.ofSeconds(10);
3936
Schema messageSchema = null;
4037
ConsumerRecord<String,GenericRecord> lastRecord=null;
@@ -92,7 +89,6 @@ public Set<String> getTopics() throws Exception{
9289
Set<String> topics = new HashSet<String>();
9390

9491
KafkaConsumer schemaConsumer= getConsumer("Control-"+getClientID(securityProps));
95-
schemaConsumer.subscribe(Collections.singletonList(controlSchemaName));
9692
Duration sec = Duration.ofSeconds(10);
9793
while (true) {
9894
ConsumerRecords<String, GenericRecord> schemaRecords = schemaConsumer.poll(sec);
@@ -130,34 +126,66 @@ private KafkaAvroConsumer getConsumer(String cleindId) throws Exception {
130126
}
131127

132128
Schema.Parser parser = new Schema.Parser();
133-
controlMessageSchema = parser.parse(ClassLoader.getSystemResourceAsStream("ControlMessageSchema.avsc"));
129+
//controlMessageSchema = parser.parse(ClassLoader.getSystemResourceAsStream("ControlMessageSchema.avsc"));
130+
controlMessageSchema = parser.parse(this.getClass().getResourceAsStream("/ControlMessageSchema.avsc"));
134131

135132
if (IsItJunit.isJUnitTest()) {
136133
kafkaProps = KafkaConfigLoader.loadConfig();
137134
}
138135
kafkaProps.put("key.deserializer", StringSerializer.class.getName());
139136
kafkaProps.put("value.deserializer", AvroDeserializer.class.getName());
140-
kafkaProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
141-
kafkaProps.put(ConsumerConfig.GROUP_ID_CONFIG, cleindId + "_" + UUID.randomUUID().toString());
137+
kafkaProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, OffsetResetStrategy.EARLIEST.toString().toLowerCase());
138+
kafkaProps.put(ConsumerConfig.GROUP_ID_CONFIG, cleindId);
142139
kafkaProps.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, "5048576");
143140
ConfigProperties.resolve(kafkaProps);
144141
}
145142
catch (Exception e) {
146143
throw e;
147144
}
148-
return new KafkaAvroConsumer(kafkaProps, controlMessageSchema);
149145

146+
KafkaAvroConsumer kafkaAvroConsumer = new KafkaAvroConsumer(kafkaProps, controlMessageSchema);
147+
TopicPartition topicPartition = new TopicPartition(controlSchemaName,0);
148+
kafkaAvroConsumer.assign(Collections.singletonList(topicPartition));
149+
return seekTo7DaysBack(kafkaAvroConsumer, topicPartition);
150150
}
151151

152152
private Schema internalSchema (String topic) throws Exception {
153153
try {
154154
final Schema topicSchema;
155155
Schema.Parser parser = new Schema.Parser();
156-
topicSchema = parser.parse(ClassLoader.getSystemResourceAsStream("schemas/" + topic + ".avsc"));
156+
topicSchema = parser.parse(this.getClass().getResourceAsStream("schemas/" + topic + ".avsc"));
157157
return topicSchema;
158158
} catch (Exception e){
159159
throw new Exception("SCHEMA NOT FOUND FOR TOPIC: "+ topic);
160160
}
161161
}
162162

163-
}
163+
private KafkaAvroConsumer seekTo7DaysBack(KafkaAvroConsumer kafkaAvroConsumer, TopicPartition topicPartition){
164+
Map<TopicPartition,Long> timestmaps = new HashMap();
165+
timestmaps.put(topicPartition , getTodayMidNightTimeStamp());
166+
Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes = kafkaAvroConsumer.offsetsForTimes(timestmaps);
167+
OffsetAndTimestamp offsetAndTimestamp = null;
168+
if (offsetsForTimes != null && (offsetAndTimestamp = offsetsForTimes.get(topicPartition)) != null) {
169+
kafkaAvroConsumer.seek(topicPartition, offsetAndTimestamp.offset());
170+
} else {
171+
kafkaAvroConsumer.seekToBeginning(Collections.singleton(topicPartition));
172+
}
173+
return kafkaAvroConsumer;
174+
}
175+
176+
private long getTodayMidNightTimeStamp(){
177+
178+
TimeZone timeZone = TimeZone.getTimeZone("America/New_York");
179+
180+
Calendar today = Calendar.getInstance(timeZone);
181+
today.add(Calendar.DATE, -7);
182+
today.set(Calendar.HOUR_OF_DAY, 0);
183+
today.set(Calendar.MINUTE, 0);
184+
today.set(Calendar.SECOND, 0);
185+
186+
long timestampFromMidnight = today.getTimeInMillis();
187+
188+
return timestampFromMidnight;
189+
}
190+
191+
}

‎ncdssdk-client/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>com.nasdaq.ncds</groupId>
99
<artifactId>ncds</artifactId>
10-
<version>0.3.0</version>
10+
<version>0.4.0</version>
1111
</parent>
1212

1313
<artifactId>ncdssdk-client</artifactId>
@@ -19,7 +19,7 @@
1919
<dependency>
2020
<groupId>com.nasdaq.ncds</groupId>
2121
<artifactId>ncds-sdk</artifactId>
22-
<version>0.3.0</version>
22+
<version>0.4.0</version>
2323
<scope>compile</scope>
2424
</dependency>
2525
</dependencies>

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<modelVersion>4.0.0</modelVersion>
66
<groupId>com.nasdaq.ncds</groupId>
77
<artifactId>ncds</artifactId>
8-
<version>0.3.0</version>
8+
<version>0.4.0</version>
99
<packaging>pom</packaging>
1010
<name>Nasdaq Cloud Data Service </name>
1111

0 commit comments

Comments
 (0)