From 9b6180a053c2b8875152232b0f2aac3c2cabec43 Mon Sep 17 00:00:00 2001 From: Shrijeet Paliwal Date: Thu, 26 Jun 2014 18:17:27 -0700 Subject: [PATCH] Uograde jackson and jersey Change upgrades jackson to 2.x and jersey 1.8.x. Jackson 2.x has moved to new namespace, the changes in java files are mostly fixing imports based on new name space. Additionally CustomDeserializerFactory has been removed, the new approach uses Module for adding custom deserializers. --- hraven-core/pom.xml | 37 +++++++++++++------ .../main/java/com/twitter/hraven/AppKey.java | 5 ++- .../java/com/twitter/hraven/AppSummary.java | 3 +- .../twitter/hraven/ClientObjectMapper.java | 26 ++++++------- .../java/com/twitter/hraven/CounterMap.java | 3 +- .../main/java/com/twitter/hraven/Flow.java | 6 +-- .../main/java/com/twitter/hraven/FlowKey.java | 5 ++- .../java/com/twitter/hraven/HdfsStats.java | 2 +- .../java/com/twitter/hraven/HdfsStatsKey.java | 4 +- .../java/com/twitter/hraven/JobDetails.java | 6 +-- .../main/java/com/twitter/hraven/JobId.java | 5 ++- .../main/java/com/twitter/hraven/JobKey.java | 5 ++- .../com/twitter/hraven/QualifiedJobId.java | 5 ++- .../com/twitter/hraven/QualifiedPathKey.java | 3 +- .../java/com/twitter/hraven/TaskDetails.java | 8 ++-- .../main/java/com/twitter/hraven/TaskKey.java | 7 ++-- .../hraven/rest/ObjectMapperProvider.java | 17 +++++---- .../hraven/rest/client/HRavenRestClient.java | 4 +- .../hraven/rest/client/UrlDataLoader.java | 3 +- .../com/twitter/hraven/util/JSONUtil.java | 16 ++++---- .../com/twitter/hraven/TestJsonSerde.java | 28 +++++++------- hraven-etl/pom.xml | 27 ++++---------- .../etl/JobHistoryFileParserHadoop2.java | 11 +++--- pom.xml | 6 +-- 24 files changed, 126 insertions(+), 116 deletions(-) diff --git a/hraven-core/pom.xml b/hraven-core/pom.xml index 452a3d9..52c5ce9 100644 --- a/hraven-core/pom.xml +++ b/hraven-core/pom.xml @@ -29,7 +29,7 @@ 0.9.16-SNAPSHOT hRaven - core Core components of hRaven, including model classes and data access layer - + 6.1.26 @@ -277,8 +277,12 @@ oro oro + + org.codehaus.jackson + jackson-mapper-asl + - + org.apache.hbase hbase @@ -334,23 +338,28 @@ ${jersey.version} - org.codehaus.jackson - jackson-core-asl + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-core ${jackson.version} - org.codehaus.jackson - jackson-mapper-asl + com.fasterxml.jackson.core + jackson-databind ${jackson.version} - org.codehaus.jackson - jackson-jaxrs + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider ${jackson.version} - org.codehaus.jackson - jackson-xc + com.fasterxml.jackson.module + jackson-module-jaxb-annotations ${jackson.version} @@ -370,8 +379,14 @@ hadoop-test ${hadoop.version} test + + + org.codehaus.jackson + jackson-mapper-asl + + - + org.apache.hbase hbase diff --git a/hraven-core/src/main/java/com/twitter/hraven/AppKey.java b/hraven-core/src/main/java/com/twitter/hraven/AppKey.java index 53e8fce..ad4e1e0 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/AppKey.java +++ b/hraven-core/src/main/java/com/twitter/hraven/AppKey.java @@ -16,10 +16,11 @@ package com.twitter.hraven; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.commons.lang.builder.CompareToBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; + public class AppKey implements Comparable { diff --git a/hraven-core/src/main/java/com/twitter/hraven/AppSummary.java b/hraven-core/src/main/java/com/twitter/hraven/AppSummary.java index 0e9a8df..827cad6 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/AppSummary.java +++ b/hraven-core/src/main/java/com/twitter/hraven/AppSummary.java @@ -19,7 +19,8 @@ import java.util.HashSet; import java.util.Set; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + import com.twitter.hraven.datasource.ProcessingException; diff --git a/hraven-core/src/main/java/com/twitter/hraven/ClientObjectMapper.java b/hraven-core/src/main/java/com/twitter/hraven/ClientObjectMapper.java index c18e05c..0343fa4 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/ClientObjectMapper.java +++ b/hraven-core/src/main/java/com/twitter/hraven/ClientObjectMapper.java @@ -18,13 +18,14 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonToken; -import org.codehaus.jackson.map.DeserializationContext; -import org.codehaus.jackson.map.JsonDeserializer; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.deser.CustomDeserializerFactory; -import org.codehaus.jackson.map.deser.StdDeserializerProvider; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.core.Version; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; + /** * Custom Jackson ObjectMapper factory that knows how to deserialize json back into objects. This @@ -38,12 +39,11 @@ public class ClientObjectMapper { public static ObjectMapper createCustomMapper() { ObjectMapper result = new ObjectMapper(); - CustomDeserializerFactory deserializerFactory = new CustomDeserializerFactory(); - - deserializerFactory.addSpecificMapping(Configuration.class, new ConfigurationDeserializer()); - deserializerFactory.addSpecificMapping(CounterMap.class, new CounterDeserializer()); - - result.setDeserializerProvider(new StdDeserializerProvider(deserializerFactory)); + SimpleModule module = new SimpleModule("Custom JsonDeserializer", new Version(1, 0, 0, + "FINAL", null, null)); + module.addDeserializer(Configuration.class, new ConfigurationDeserializer()); + module.addDeserializer(CounterMap.class, new CounterDeserializer()); + result.registerModule(module); return result; } diff --git a/hraven-core/src/main/java/com/twitter/hraven/CounterMap.java b/hraven-core/src/main/java/com/twitter/hraven/CounterMap.java index 017a282..825c459 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/CounterMap.java +++ b/hraven-core/src/main/java/com/twitter/hraven/CounterMap.java @@ -21,7 +21,8 @@ import java.util.NoSuchElementException; import java.util.Set; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + @JsonSerialize( include=JsonSerialize.Inclusion.NON_NULL diff --git a/hraven-core/src/main/java/com/twitter/hraven/Flow.java b/hraven-core/src/main/java/com/twitter/hraven/Flow.java index bed215c..7fea42a 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/Flow.java +++ b/hraven-core/src/main/java/com/twitter/hraven/Flow.java @@ -18,9 +18,9 @@ import org.apache.commons.lang.builder.CompareToBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.hadoop.hbase.util.Bytes; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.ArrayList; import java.util.List; diff --git a/hraven-core/src/main/java/com/twitter/hraven/FlowKey.java b/hraven-core/src/main/java/com/twitter/hraven/FlowKey.java index ad87e04..1b75888 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/FlowKey.java +++ b/hraven-core/src/main/java/com/twitter/hraven/FlowKey.java @@ -17,8 +17,9 @@ import org.apache.commons.lang.builder.CompareToBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + public class FlowKey extends AppKey implements Comparable { diff --git a/hraven-core/src/main/java/com/twitter/hraven/HdfsStats.java b/hraven-core/src/main/java/com/twitter/hraven/HdfsStats.java index b846a7c..b6b2af6 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/HdfsStats.java +++ b/hraven-core/src/main/java/com/twitter/hraven/HdfsStats.java @@ -20,7 +20,7 @@ import org.apache.commons.lang.builder.CompareToBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.hadoop.hbase.client.Result; -import org.codehaus.jackson.map.annotate.JsonSerialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; /** diff --git a/hraven-core/src/main/java/com/twitter/hraven/HdfsStatsKey.java b/hraven-core/src/main/java/com/twitter/hraven/HdfsStatsKey.java index 6a16bfb..794d31e 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/HdfsStatsKey.java +++ b/hraven-core/src/main/java/com/twitter/hraven/HdfsStatsKey.java @@ -16,8 +16,8 @@ package com.twitter.hraven; import org.apache.commons.lang.builder.CompareToBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; /** * Represents a unique hdfs stats record diff --git a/hraven-core/src/main/java/com/twitter/hraven/JobDetails.java b/hraven-core/src/main/java/com/twitter/hraven/JobDetails.java index ed51a60..af76baf 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/JobDetails.java +++ b/hraven-core/src/main/java/com/twitter/hraven/JobDetails.java @@ -28,11 +28,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; import org.apache.commons.lang.NotImplementedException; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.twitter.hraven.datasource.JobHistoryService; /** diff --git a/hraven-core/src/main/java/com/twitter/hraven/JobId.java b/hraven-core/src/main/java/com/twitter/hraven/JobId.java index 887a0a5..5b308d7 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/JobId.java +++ b/hraven-core/src/main/java/com/twitter/hraven/JobId.java @@ -17,8 +17,9 @@ import org.apache.commons.lang.builder.CompareToBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; /** * Job identifier with individual elements of the jobtracker assigned ID parsed diff --git a/hraven-core/src/main/java/com/twitter/hraven/JobKey.java b/hraven-core/src/main/java/com/twitter/hraven/JobKey.java index 8a9d427..6e0a3d4 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/JobKey.java +++ b/hraven-core/src/main/java/com/twitter/hraven/JobKey.java @@ -17,8 +17,9 @@ import org.apache.commons.lang.builder.CompareToBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; /** * Represents the row key for a given job. Row keys are stored as: username ! diff --git a/hraven-core/src/main/java/com/twitter/hraven/QualifiedJobId.java b/hraven-core/src/main/java/com/twitter/hraven/QualifiedJobId.java index 36f762f..149a6ee 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/QualifiedJobId.java +++ b/hraven-core/src/main/java/com/twitter/hraven/QualifiedJobId.java @@ -15,8 +15,9 @@ */ package com.twitter.hraven; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; /** * The job ID should be relatively unique, unless two clusters start at the same diff --git a/hraven-core/src/main/java/com/twitter/hraven/QualifiedPathKey.java b/hraven-core/src/main/java/com/twitter/hraven/QualifiedPathKey.java index 34fdea3..e1a915d 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/QualifiedPathKey.java +++ b/hraven-core/src/main/java/com/twitter/hraven/QualifiedPathKey.java @@ -18,7 +18,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.CompareToBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.codehaus.jackson.map.annotate.JsonSerialize; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; /** * The cluster + path qualifier for hdfs stats diff --git a/hraven-core/src/main/java/com/twitter/hraven/TaskDetails.java b/hraven-core/src/main/java/com/twitter/hraven/TaskDetails.java index 2e65e5b..d94bf33 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/TaskDetails.java +++ b/hraven-core/src/main/java/com/twitter/hraven/TaskDetails.java @@ -22,11 +22,11 @@ import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.util.Bytes; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.twitter.hraven.datasource.JobHistoryService; /** diff --git a/hraven-core/src/main/java/com/twitter/hraven/TaskKey.java b/hraven-core/src/main/java/com/twitter/hraven/TaskKey.java index 479500d..e9dff08 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/TaskKey.java +++ b/hraven-core/src/main/java/com/twitter/hraven/TaskKey.java @@ -17,9 +17,10 @@ import org.apache.commons.lang.builder.CompareToBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; /** diff --git a/hraven-core/src/main/java/com/twitter/hraven/rest/ObjectMapperProvider.java b/hraven-core/src/main/java/com/twitter/hraven/rest/ObjectMapperProvider.java index 0776b41..58a4049 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/rest/ObjectMapperProvider.java +++ b/hraven-core/src/main/java/com/twitter/hraven/rest/ObjectMapperProvider.java @@ -15,6 +15,7 @@ */ package com.twitter.hraven.rest; + import java.io.IOException; import java.util.Iterator; import java.util.Map; @@ -23,17 +24,17 @@ import javax.ws.rs.ext.Provider; import org.apache.hadoop.conf.Configuration; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.Version; -import org.codehaus.jackson.map.JsonSerializer; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.SerializationConfig.Feature; -import org.codehaus.jackson.map.SerializerProvider; -import org.codehaus.jackson.map.module.SimpleModule; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.Version; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.base.Predicate; import com.twitter.hraven.AppSummary; import com.twitter.hraven.Constants; @@ -65,7 +66,7 @@ public ObjectMapper getContext(Class type) { public static ObjectMapper createCustomMapper() { ObjectMapper result = new ObjectMapper(); - result.configure(Feature.INDENT_OUTPUT, true); + result.configure(SerializationFeature.INDENT_OUTPUT, true); SimpleModule module = new SimpleModule("hRavenModule", new Version(0, 4, 0, null)); addJobMappings(module); module.addSerializer(Flow.class, new FlowSerializer()); diff --git a/hraven-core/src/main/java/com/twitter/hraven/rest/client/HRavenRestClient.java b/hraven-core/src/main/java/com/twitter/hraven/rest/client/HRavenRestClient.java index a60b459..0a68061 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/rest/client/HRavenRestClient.java +++ b/hraven-core/src/main/java/com/twitter/hraven/rest/client/HRavenRestClient.java @@ -27,9 +27,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.twitter.hraven.Flow; import com.twitter.hraven.JobDetails; import com.twitter.hraven.TaskDetails; diff --git a/hraven-core/src/main/java/com/twitter/hraven/rest/client/UrlDataLoader.java b/hraven-core/src/main/java/com/twitter/hraven/rest/client/UrlDataLoader.java index 7131b58..3b73305 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/rest/client/UrlDataLoader.java +++ b/hraven-core/src/main/java/com/twitter/hraven/rest/client/UrlDataLoader.java @@ -8,8 +8,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.codehaus.jackson.type.TypeReference; + +import com.fasterxml.jackson.core.type.TypeReference; import com.twitter.hraven.util.JSONUtil; class UrlDataLoader { diff --git a/hraven-core/src/main/java/com/twitter/hraven/util/JSONUtil.java b/hraven-core/src/main/java/com/twitter/hraven/util/JSONUtil.java index 364600f..7fefb5c 100644 --- a/hraven-core/src/main/java/com/twitter/hraven/util/JSONUtil.java +++ b/hraven-core/src/main/java/com/twitter/hraven/util/JSONUtil.java @@ -20,11 +20,10 @@ import java.io.PrintWriter; import java.io.Writer; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.SerializationConfig; -import org.codehaus.jackson.type.TypeReference; - +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.twitter.hraven.ClientObjectMapper; import com.twitter.hraven.rest.ObjectMapperProvider; @@ -44,8 +43,8 @@ public class JSONUtil { public static void writeJson(Writer writer, Object object) throws IOException { ObjectMapper om = ObjectMapperProvider.createCustomMapper(); - om.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); - om.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false); + om.configure(SerializationFeature.INDENT_OUTPUT, true); + om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); writer.write(om.writeValueAsString(object)); writer.write("\n"); @@ -58,8 +57,7 @@ public static void writeJson(String fileName, Object object) throws IOException public static Object readJson(InputStream inputStream, TypeReference type) throws IOException { ObjectMapper om = ClientObjectMapper.createCustomMapper(); - om.getDeserializationConfig().set(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, - false); + om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); return om.readValue(inputStream, type); } } diff --git a/hraven-core/src/test/java/com/twitter/hraven/TestJsonSerde.java b/hraven-core/src/test/java/com/twitter/hraven/TestJsonSerde.java index 6ebb41d..1d4d7c1 100644 --- a/hraven-core/src/test/java/com/twitter/hraven/TestJsonSerde.java +++ b/hraven-core/src/test/java/com/twitter/hraven/TestJsonSerde.java @@ -15,10 +15,12 @@ */ package com.twitter.hraven; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.twitter.hraven.Flow; -import com.twitter.hraven.JobDetails; import com.twitter.hraven.datasource.HRavenTestUtil; import com.twitter.hraven.datasource.JobHistoryByIdService; import com.twitter.hraven.datasource.JobHistoryService; @@ -32,10 +34,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.HTable; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.SerializationConfig; -import org.codehaus.jackson.type.TypeReference; import org.junit.BeforeClass; import org.junit.Test; @@ -87,9 +85,9 @@ public void testJsonSerializationFlowStatsJobDetails() throws Exception { // serialize flows into json ObjectMapper om = ObjectMapperProvider.createCustomMapper(); - om.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); - om.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false); - om.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); + om.configure(SerializationFeature.INDENT_OUTPUT, true); + om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); ByteArrayOutputStream f = new ByteArrayOutputStream(); om.writeValue(f, actualFlows); ByteArrayInputStream is = new ByteArrayInputStream(f.toByteArray()); @@ -132,9 +130,9 @@ public void testSerializationContext() throws Exception { new SerializationContext(SerializationContext.DetailLevel.EVERYTHING, new SerializationContext.ConfigurationFilter(serializedKeys))); ObjectMapper om = ObjectMapperProvider.createCustomMapper(); - om.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); - om.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false); - om.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); + om.configure(SerializationFeature.INDENT_OUTPUT, true); + om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); ByteArrayOutputStream f = new ByteArrayOutputStream(); om.writeValue(f, actualFlow); ByteArrayInputStream is = new ByteArrayInputStream(f.toByteArray()); @@ -153,9 +151,9 @@ public void testSerializationContext() throws Exception { new SerializationContext(SerializationContext.DetailLevel.EVERYTHING, new SerializationContext.RegexConfigurationFilter(patterns))); om = ObjectMapperProvider.createCustomMapper(); - om.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); - om.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false); - om.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); + om.configure(SerializationFeature.INDENT_OUTPUT, true); + om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); f = new ByteArrayOutputStream(); om.writeValue(f, actualFlow); is = new ByteArrayInputStream(f.toByteArray()); diff --git a/hraven-etl/pom.xml b/hraven-etl/pom.xml index 796391e..e38dc35 100644 --- a/hraven-etl/pom.xml +++ b/hraven-etl/pom.xml @@ -30,7 +30,7 @@ hRaven - etl jar ETL map reduce jobs and supporting components for data loading - + diff --git a/hraven-etl/src/main/java/com/twitter/hraven/etl/JobHistoryFileParserHadoop2.java b/hraven-etl/src/main/java/com/twitter/hraven/etl/JobHistoryFileParserHadoop2.java index 34bfb06..ba40e5d 100644 --- a/hraven-etl/src/main/java/com/twitter/hraven/etl/JobHistoryFileParserHadoop2.java +++ b/hraven-etl/src/main/java/com/twitter/hraven/etl/JobHistoryFileParserHadoop2.java @@ -26,7 +26,6 @@ import org.apache.avro.io.Decoder; import org.apache.avro.io.DecoderFactory; import org.apache.commons.configuration.ConversionException; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -35,9 +34,9 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.mapred.JobHistoryCopy.RecordTypes; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; import com.google.common.collect.Maps; import com.twitter.hraven.Constants; @@ -123,11 +122,11 @@ public class JobHistoryFileParserHadoop2 extends JobHistoryFileParserBase { private static final String TYPE_LONG = "long"; private static final String TYPE_STRING = "String"; /** only acls in the job history file seem to be of this type: map of strings */ - private static final String TYPE_MAP_STRINGS = "{\"type\":\"map\",\"values\":\"string\"}"; + private static final String TYPE_MAP_STRINGS = "{\"values\":\"string\",\"type\":\"map\"}"; /** * vMemKbytes, clockSplit, physMemKbytes, cpuUsages are arrays of ints See MAPREDUCE-5432 */ - private static final String TYPE_ARRAY_INTS = "{\"type\":\"array\",\"items\":\"int\"}"; + private static final String TYPE_ARRAY_INTS = "{\"items\":\"int\",\"type\":\"array\"}"; /** this is part of {@link org.apache.hadoop.mapreduce.jobhistory.TaskFailedEvent.java} */ private static final String NULL_STRING = "[\"null\",\"string\"]"; diff --git a/pom.xml b/pom.xml index 35bee67..c1d122a 100644 --- a/pom.xml +++ b/pom.xml @@ -132,8 +132,8 @@ 1.8.5 1.2.15 14.0 - 1.12 - 1.9.6 + 1.18.1 + 2.3.3 @@ -169,7 +169,7 @@ 0.9 - + maven-assembly-plugin