From 5e275a546395ea5d991cf9f1c51692dd48a199c3 Mon Sep 17 00:00:00 2001 From: Surabhi Date: Tue, 7 Sep 2021 15:40:55 +0530 Subject: [PATCH 1/2] telemetry logs --- .../utils/telemetry/LogTelemetryBuilder.java | 125 ++++++++++++++++-- .../utils/telemetry/LogTelemetryLayout.java | 2 +- .../utils/telemetry/LogTelemetryMessage.java | 26 +++- .../telemetry/util/TelemetryEventNames.java | 23 ++++ .../telemetry/util/TelemetryEventTypes.java | 22 +++ .../utils/telemetry/util/TelemetryEvents.java | 4 +- 6 files changed, 184 insertions(+), 18 deletions(-) create mode 100644 src/main/java/com/uci/utils/telemetry/util/TelemetryEventNames.java create mode 100644 src/main/java/com/uci/utils/telemetry/util/TelemetryEventTypes.java diff --git a/src/main/java/com/uci/utils/telemetry/LogTelemetryBuilder.java b/src/main/java/com/uci/utils/telemetry/LogTelemetryBuilder.java index 617673c..8657eed 100644 --- a/src/main/java/com/uci/utils/telemetry/LogTelemetryBuilder.java +++ b/src/main/java/com/uci/utils/telemetry/LogTelemetryBuilder.java @@ -1,51 +1,150 @@ package com.uci.utils.telemetry; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import com.uci.utils.telemetry.dto.Actor; import com.uci.utils.telemetry.dto.Context; +import com.uci.utils.telemetry.dto.Producer; import com.uci.utils.telemetry.dto.Telemetry; +import com.uci.utils.telemetry.util.TelemetryEventNames; +import com.uci.utils.telemetry.util.TelemetryEventTypes; import com.uci.utils.telemetry.util.TelemetryEvents; public class LogTelemetryBuilder { private static final String TELEMETRY_IMPL_VERSION = "3.0"; - private static final String QUESTION_TELEMETRY_IMPL_VERSION = "1.0"; + private static final String LOG_TELEMETRY_IMPL_VERSION = "1.0"; private static final String ACTOR_TYPE_SYSTEM = "System"; + private static final String ACTOR_TYPE_USER = "User"; private static final String DIKSHA_ORG = "DIKSHA"; + private static final String pageId = "bot-screen"; private static final String LOG_EVENT_MID = "fb3db9abceb578d8acbc812cdbd9c931"; //represents the MD5 Hash of "Individual Question-Responses - Survey/Questionnaire" public String build( String message, - String level) { + TelemetryEventNames eventName, + String botOrg, + String channel, + String provider, + String producerID, + String userID) { + + /* Event related parameters */ + Map oData = getEData(eventName); + + /* Channel Name */ + String channelName = (botOrg != null && botOrg.equalsIgnoreCase("Anonymous")) ? DIKSHA_ORG : botOrg; + + /* Environment */ + String env = channel != null && !channel.isEmpty() + ? (provider != null && !provider.isEmpty() ? channel + "." + provider : channel) + : (provider != null && !provider.isEmpty() ? provider : null); + + /* Conversation Details */ +// List> cdata = campaignService.get; +// Map map1 = new HashMap<>(); +// map1.put("ConversationOwner", conversationOwnerID); +// Map map2 = new HashMap<>(); +// map2.put("Conversation", assessment.getBotID().toString()); +// cdata.add(map1); +// cdata.add(map2); + + Map rollup = new HashMap<>(); + rollup.put("l1", "ConversationOwner"); + rollup.put("l2", "Conversation"); + + /* Context */ Context context = Context.builder() - .channel(DIKSHA_ORG) - .env("sys") + .channel(channelName) + .env(env) + .pdata(Producer.builder() + .id("prod.uci.diksha") + .pid(producerID).ver(LOG_TELEMETRY_IMPL_VERSION) + .build() + ) +// .did(userID) +// .cdata(cdata) + .rollup(rollup) .build(); + /* Edata */ Map edata = new HashMap<>(); - edata.put("type", "system"); - edata.put("level", level); + edata.put("type", oData.get("edataType").toString()); + edata.put("level", "INFO"); + edata.put("pageid", pageId); edata.put("message", message); -// if(params != null) { -// edata.put("params", params); -// } + /* Telemetry Object */ Telemetry telemetry = Telemetry.builder(). - eid(TelemetryEvents.LOG.getName()). + eid(oData.get("eid").toString()). ets(System.currentTimeMillis()). ver(TELEMETRY_IMPL_VERSION). mid(LOG_EVENT_MID) .actor(Actor.builder() - .type(ACTOR_TYPE_SYSTEM) -// .id() + .type(oData.get("actorType").toString()) + .id(userID) .build()) .context(context) -// .object(object) .edata(edata) .build(); return Telemetry.getTelemetryRequestData(telemetry); } + + private Map getEData(TelemetryEventNames event) { + Map edata = new HashMap(); + edata.put("eid", ""); + edata.put("edataType", ""); + edata.put("actorType", ""); + if(event != null) { + switch (event) { + case STARTCONVERSATION: + edata.put("eid", TelemetryEvents.START.getName()); + edata.put("edataType", TelemetryEventTypes.CONVERSATION.getName()); + edata.put("actorType", ACTOR_TYPE_USER); + break; + case ENDCONVERSATION: + edata.put("eid", TelemetryEvents.START.getName()); + edata.put("edataType", TelemetryEventTypes.CONVERSATION.getName()); + edata.put("actorType", ACTOR_TYPE_USER); + break; + case SWITCHCONVERSATIONLOGIC: + edata.put("eid", TelemetryEvents.INTERACT.getName()); + edata.put("edataType", TelemetryEventTypes.SWITCHCONVERSATION.getName()); + edata.put("actorType", ACTOR_TYPE_USER); + break; + case PRETRANSFORMER: + edata.put("eid", TelemetryEvents.LOG.getName()); + edata.put("edataType", TelemetryEventTypes.PRETRANSFORMER.getName()); + break; + case POSTTRANSFORMER: + edata.put("eid", TelemetryEvents.LOG.getName()); + edata.put("edataType", TelemetryEventTypes.POSTTRANSFORMER.getName()); + break; + case SENT: + edata.put("eid", TelemetryEvents.AUDIT.getName()); + edata.put("edataType", TelemetryEventTypes.SENT.getName()); + break; + case DELIVERED: + edata.put("eid", TelemetryEvents.AUDIT.getName()); + edata.put("edataType", TelemetryEventTypes.DELIVERED.getName()); + break; + case READ: + edata.put("eid", TelemetryEvents.AUDIT.getName()); + edata.put("edataType", TelemetryEventTypes.READ.getName()); + edata.put("actorType", ACTOR_TYPE_USER); + break; + case AUDITEXCEPTIONS: + edata.put("eid", TelemetryEvents.ERROR.getName()); + edata.put("edataType", TelemetryEventTypes.EXCEPTION.getName()); + break; + default: + break; + } + } + return edata; + } } diff --git a/src/main/java/com/uci/utils/telemetry/LogTelemetryLayout.java b/src/main/java/com/uci/utils/telemetry/LogTelemetryLayout.java index c9c7807..2c10ade 100644 --- a/src/main/java/com/uci/utils/telemetry/LogTelemetryLayout.java +++ b/src/main/java/com/uci/utils/telemetry/LogTelemetryLayout.java @@ -22,7 +22,7 @@ public String toSerializable( LogEvent event ) { String message = event.getMessage().getFormattedMessage(); LogTelemetryBuilder object = new LogTelemetryBuilder(); - String msgBody = object.build(message, event.getLevel().toString()); + String msgBody = object.build(message, null, "", "", "", "", ""); JSONObject jsonBody = new JSONObject(msgBody); return jsonBody.toString(); diff --git a/src/main/java/com/uci/utils/telemetry/LogTelemetryMessage.java b/src/main/java/com/uci/utils/telemetry/LogTelemetryMessage.java index 1ea3403..83fec37 100644 --- a/src/main/java/com/uci/utils/telemetry/LogTelemetryMessage.java +++ b/src/main/java/com/uci/utils/telemetry/LogTelemetryMessage.java @@ -3,18 +3,38 @@ import org.apache.logging.log4j.message.Message; import org.json.JSONObject; +import com.uci.utils.telemetry.util.TelemetryEventNames; + public class LogTelemetryMessage implements Message { private final String requestBody; - - public LogTelemetryMessage(String requestBody) { + private final TelemetryEventNames eventName; + private String botOrg = ""; + private String channel = ""; + private String provider = ""; + private String producerID = ""; + private String userID = ""; + private String campaign = ""; + + public LogTelemetryMessage(String requestBody, TelemetryEventNames eventName, String botOrg, String channel, String provider, String producerID, String userID) { + this.requestBody = requestBody; + this.eventName = eventName; + this.botOrg = botOrg; + this.channel = channel; + this.provider = provider; + this.producerID = producerID; + this.userID = userID; + } + + public LogTelemetryMessage(String requestBody, TelemetryEventNames eventName) { this.requestBody = requestBody; + this.eventName = eventName; } @Override public String getFormattedMessage() { LogTelemetryBuilder object = new LogTelemetryBuilder(); - String msgBody = object.build(requestBody, "INFO"); + String msgBody = object.build(requestBody, eventName, botOrg, channel, provider, producerID, userID); JSONObject jsonBody = new JSONObject(msgBody); return jsonBody.toString(); diff --git a/src/main/java/com/uci/utils/telemetry/util/TelemetryEventNames.java b/src/main/java/com/uci/utils/telemetry/util/TelemetryEventNames.java new file mode 100644 index 0000000..c9415c2 --- /dev/null +++ b/src/main/java/com/uci/utils/telemetry/util/TelemetryEventNames.java @@ -0,0 +1,23 @@ +package com.uci.utils.telemetry.util; + +public enum TelemetryEventNames { + STARTCONVERSATION("Start of a Conversation"), + ENDCONVERSATION("End of a Conversation"), + SWITCHCONVERSATIONLOGIC("Switch from one Conversation Logic to another"), + PRETRANSFORMER("Pre transformer"), + POSTTRANSFORMER("Post transformer"), + SENT("Sent"), + DELIVERED("Delivered"), + READ("Read"), + AUDITEXCEPTIONS("Audit - Exceptions"); + + private String name; + + TelemetryEventNames(String name) { + this.name = name; + } + + public String getName() { + return name; + } +} diff --git a/src/main/java/com/uci/utils/telemetry/util/TelemetryEventTypes.java b/src/main/java/com/uci/utils/telemetry/util/TelemetryEventTypes.java new file mode 100644 index 0000000..188a31e --- /dev/null +++ b/src/main/java/com/uci/utils/telemetry/util/TelemetryEventTypes.java @@ -0,0 +1,22 @@ +package com.uci.utils.telemetry.util; + +public enum TelemetryEventTypes { + CONVERSATION("conversation"), + SWITCHCONVERSATION("switch-conversation"), + PRETRANSFORMER("pre-transformer"), + POSTTRANSFORMER("post-transformer"), + SENT("sent"), + DELIVERED("delivered"), + READ("read"), + EXCEPTION("Exception"); + + private String name; + + TelemetryEventTypes(String name) { + this.name = name; + } + + public String getName() { + return name; + } +} diff --git a/src/main/java/com/uci/utils/telemetry/util/TelemetryEvents.java b/src/main/java/com/uci/utils/telemetry/util/TelemetryEvents.java index 603c425..e6264f0 100644 --- a/src/main/java/com/uci/utils/telemetry/util/TelemetryEvents.java +++ b/src/main/java/com/uci/utils/telemetry/util/TelemetryEvents.java @@ -10,7 +10,9 @@ public enum TelemetryEvents { ASSESS("ASSESS"), SEARCH("SEARCH"), LOG("LOG"), - ERROR("ERROR"); + ERROR("ERROR"), + START("START"), + INTERACT("INTERACT"); private String name; TelemetryEvents(String name) { From 5ce803063daff30c34ceb605f0534fbc5623cdbc Mon Sep 17 00:00:00 2001 From: Surabhi Date: Mon, 13 Sep 2021 15:31:25 +0530 Subject: [PATCH 2/2] telemetry logs to kafka topic with conversation id and conversation owner id --- .../java/com/uci/utils/CampaignService.java | 28 +++++++++++++++++-- .../utils/telemetry/LogTelemetryBuilder.java | 27 +++++++++++------- .../utils/telemetry/LogTelemetryLayout.java | 2 +- .../utils/telemetry/LogTelemetryMessage.java | 23 ++++++++++++++- 4 files changed, 65 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/uci/utils/CampaignService.java b/src/main/java/com/uci/utils/CampaignService.java index ee340bf..612828d 100644 --- a/src/main/java/com/uci/utils/CampaignService.java +++ b/src/main/java/com/uci/utils/CampaignService.java @@ -89,6 +89,7 @@ public Application getCampaignFromName(String campaignName) throws Exception { * @return Application */ public Mono getCampaignFromNameTransformer(String campaignName) { +// System.out.println(campaignName); return webClient.get() .uri(builder -> builder.path("admin/v1/bot/search/").queryParam("name", campaignName).queryParam("match", true).build()) .retrieve() @@ -97,12 +98,23 @@ public Mono getCampaignFromNameTransformer(String campaignName) { @Override public JsonNode apply(String response) { if (response != null) { - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = new ObjectMapper(); try { - return mapper.readTree(response).get("data").get(0); - } catch (JsonProcessingException e) { + JsonNode root = mapper.readTree(response); + String responseCode = root.path("responseCode").asText(); + if(isApiResponseOk(responseCode)) { + return root.path("result").get("data").get(0); + } + return null; + } catch (JsonProcessingException jsonMappingException) { return null; } +// ObjectMapper mapper = new ObjectMapper(); +// try { +// return mapper.readTree(response).get("data").get(0); +// } catch (JsonProcessingException e) { +// return null; +// } } return null; } @@ -223,5 +235,15 @@ public Application getCampaignFromNameESamwad(String campaignName) { } return currentApplication; } + + /** + * Check if response code sent in api response is ok + * + * @param responseCode + * @return Boolean + */ + private Boolean isApiResponseOk(String responseCode) { + return responseCode.equals("OK"); + } } diff --git a/src/main/java/com/uci/utils/telemetry/LogTelemetryBuilder.java b/src/main/java/com/uci/utils/telemetry/LogTelemetryBuilder.java index 8657eed..4cc5c3e 100644 --- a/src/main/java/com/uci/utils/telemetry/LogTelemetryBuilder.java +++ b/src/main/java/com/uci/utils/telemetry/LogTelemetryBuilder.java @@ -30,7 +30,9 @@ public String build( String channel, String provider, String producerID, - String userID) { + String userID, + String conversationId, + String conversationOwnerId) { /* Event related parameters */ Map oData = getEData(eventName); @@ -44,13 +46,13 @@ public String build( : (provider != null && !provider.isEmpty() ? provider : null); /* Conversation Details */ -// List> cdata = campaignService.get; -// Map map1 = new HashMap<>(); -// map1.put("ConversationOwner", conversationOwnerID); -// Map map2 = new HashMap<>(); -// map2.put("Conversation", assessment.getBotID().toString()); -// cdata.add(map1); -// cdata.add(map2); + List> cdata = new ArrayList(); + Map map1 = new HashMap<>(); + map1.put("ConversationOwner", conversationOwnerId); + Map map2 = new HashMap<>(); + map2.put("Conversation", conversationId); + cdata.add(map1); + cdata.add(map2); Map rollup = new HashMap<>(); rollup.put("l1", "ConversationOwner"); @@ -65,8 +67,8 @@ public String build( .pid(producerID).ver(LOG_TELEMETRY_IMPL_VERSION) .build() ) -// .did(userID) -// .cdata(cdata) + .did(userID) + .cdata(cdata) .rollup(rollup) .build(); @@ -76,6 +78,11 @@ public String build( edata.put("level", "INFO"); edata.put("pageid", pageId); edata.put("message", message); + + /* If actor type not user, replace used id with empty string */ + if(!oData.get("actorType").toString().equals(ACTOR_TYPE_USER)) { + userID = ""; + } /* Telemetry Object */ Telemetry telemetry = Telemetry.builder(). diff --git a/src/main/java/com/uci/utils/telemetry/LogTelemetryLayout.java b/src/main/java/com/uci/utils/telemetry/LogTelemetryLayout.java index 2c10ade..406484c 100644 --- a/src/main/java/com/uci/utils/telemetry/LogTelemetryLayout.java +++ b/src/main/java/com/uci/utils/telemetry/LogTelemetryLayout.java @@ -22,7 +22,7 @@ public String toSerializable( LogEvent event ) { String message = event.getMessage().getFormattedMessage(); LogTelemetryBuilder object = new LogTelemetryBuilder(); - String msgBody = object.build(message, null, "", "", "", "", ""); + String msgBody = object.build(message, null, "", "", "", "", "", "", ""); JSONObject jsonBody = new JSONObject(msgBody); return jsonBody.toString(); diff --git a/src/main/java/com/uci/utils/telemetry/LogTelemetryMessage.java b/src/main/java/com/uci/utils/telemetry/LogTelemetryMessage.java index 83fec37..0857011 100644 --- a/src/main/java/com/uci/utils/telemetry/LogTelemetryMessage.java +++ b/src/main/java/com/uci/utils/telemetry/LogTelemetryMessage.java @@ -1,8 +1,14 @@ package com.uci.utils.telemetry; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + import org.apache.logging.log4j.message.Message; import org.json.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.uci.utils.CampaignService; import com.uci.utils.telemetry.util.TelemetryEventNames; public class LogTelemetryMessage implements Message { @@ -14,6 +20,21 @@ public class LogTelemetryMessage implements Message { private String producerID = ""; private String userID = ""; private String campaign = ""; + private String conversationId = ""; + private String conversationOwnerId = ""; +// private CampaignService campaignService; + + public LogTelemetryMessage(String requestBody, TelemetryEventNames eventName, String botOrg, String channel, String provider, String producerID, String userID, String conversationId, String conversationOwnerId) { + this.requestBody = requestBody; + this.eventName = eventName; + this.botOrg = botOrg; + this.channel = channel; + this.provider = provider; + this.producerID = producerID; + this.userID = userID; + this.conversationId = conversationId; + this.conversationOwnerId = conversationOwnerId; + } public LogTelemetryMessage(String requestBody, TelemetryEventNames eventName, String botOrg, String channel, String provider, String producerID, String userID) { this.requestBody = requestBody; @@ -34,7 +55,7 @@ public LogTelemetryMessage(String requestBody, TelemetryEventNames eventName) { public String getFormattedMessage() { LogTelemetryBuilder object = new LogTelemetryBuilder(); - String msgBody = object.build(requestBody, eventName, botOrg, channel, provider, producerID, userID); + String msgBody = object.build(requestBody, eventName, botOrg, channel, provider, producerID, userID, conversationId, conversationOwnerId); JSONObject jsonBody = new JSONObject(msgBody); return jsonBody.toString();