Skip to content

fix for the HashMap concurrent bug in Hive #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
haitaoyao wants to merge 1 commit into
base: trunk
Choose a base branch
from
Open

fix for the HashMap concurrent bug in Hive #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions ...c/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,27 @@
package org.apache.hadoop.hive.serde2.lazy.objectinspector;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.Text;

/**
* ObjectInspectorFactory is the primary way to create new ObjectInspector
* instances.
*
*
* SerDe classes should call the static functions in this library to create an
* ObjectInspector to return to the caller of SerDe2.getObjectInspector().
*
*
* The reason of having caches here is that ObjectInspectors do not have an
* internal state - so ObjectInspectors with the same construction parameters
* should result in exactly the same ObjectInspector.
*/
public final class LazyObjectInspectorFactory {

static HashMap<ArrayList<Object>, LazySimpleStructObjectInspector> cachedLazySimpleStructObjectInspector =
new HashMap<ArrayList<Object>, LazySimpleStructObjectInspector>();
static ConcurrentHashMap<ArrayList<Object>, LazySimpleStructObjectInspector> cachedLazySimpleStructObjectInspector =
new ConcurrentHashMap<ArrayList<Object>, LazySimpleStructObjectInspector>();

public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector(
List<String> structFieldNames,
Expand Down Expand Up @@ -78,7 +78,8 @@ public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector
return result;
}

static HashMap<ArrayList<Object>, LazyListObjectInspector> cachedLazySimpleListObjectInspector = new HashMap<ArrayList<Object>, LazyListObjectInspector>();
static ConcurrentHashMap<ArrayList<Object>, LazyListObjectInspector> cachedLazySimpleListObjectInspector =
new ConcurrentHashMap<ArrayList<Object>, LazyListObjectInspector>();

public static LazyListObjectInspector getLazySimpleListObjectInspector(
ObjectInspector listElementObjectInspector, byte separator,
Expand All @@ -99,7 +100,8 @@ public static LazyListObjectInspector getLazySimpleListObjectInspector(
return result;
}

static HashMap<ArrayList<Object>, LazyMapObjectInspector> cachedLazySimpleMapObjectInspector = new HashMap<ArrayList<Object>, LazyMapObjectInspector>();
static ConcurrentHashMap<ArrayList<Object>, LazyMapObjectInspector> cachedLazySimpleMapObjectInspector =
new ConcurrentHashMap<ArrayList<Object>, LazyMapObjectInspector>();

public static LazyMapObjectInspector getLazySimpleMapObjectInspector(
ObjectInspector mapKeyObjectInspector,
Expand All @@ -125,9 +127,9 @@ public static LazyMapObjectInspector getLazySimpleMapObjectInspector(
return result;
}

static HashMap<List<Object>, LazyUnionObjectInspector>
static ConcurrentHashMap<List<Object>, LazyUnionObjectInspector>
cachedLazyUnionObjectInspector =
new HashMap<List<Object>, LazyUnionObjectInspector>();
new ConcurrentHashMap<List<Object>, LazyUnionObjectInspector>();

public static LazyUnionObjectInspector getLazyUnionObjectInspector(
List<ObjectInspector> ois, byte separator, Text nullSequence,
Expand Down
15 changes: 9 additions & 6 deletions ...pache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,26 +18,27 @@
package org.apache.hadoop.hive.serde2.lazybinary.objectinspector;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;

/**
* ObjectInspectorFactory is the primary way to create new ObjectInspector
* instances.
*
*
* SerDe classes should call the static functions in this library to create an
* ObjectInspector to return to the caller of SerDe2.getObjectInspector().
*
*
* The reason of having caches here is that ObjectInspectors do not have an
* internal state - so ObjectInspectors with the same construction parameters
* should result in exactly the same ObjectInspector.
*/

public final class LazyBinaryObjectInspectorFactory {

static HashMap<ArrayList<Object>, LazyBinaryStructObjectInspector> cachedLazyBinaryStructObjectInspector = new HashMap<ArrayList<Object>, LazyBinaryStructObjectInspector>();
static ConcurrentHashMap<ArrayList<Object>, LazyBinaryStructObjectInspector> cachedLazyBinaryStructObjectInspector =
new ConcurrentHashMap<ArrayList<Object>, LazyBinaryStructObjectInspector>();

public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector(
List<String> structFieldNames,
Expand Down Expand Up @@ -65,7 +66,8 @@ public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector
return result;
}

static HashMap<ArrayList<Object>, LazyBinaryListObjectInspector> cachedLazyBinaryListObjectInspector = new HashMap<ArrayList<Object>, LazyBinaryListObjectInspector>();
static ConcurrentHashMap<ArrayList<Object>, LazyBinaryListObjectInspector> cachedLazyBinaryListObjectInspector =
new ConcurrentHashMap<ArrayList<Object>, LazyBinaryListObjectInspector>();

public static LazyBinaryListObjectInspector getLazyBinaryListObjectInspector(
ObjectInspector listElementObjectInspector) {
Expand All @@ -80,7 +82,8 @@ public static LazyBinaryListObjectInspector getLazyBinaryListObjectInspector(
return result;
}

static HashMap<ArrayList<Object>, LazyBinaryMapObjectInspector> cachedLazyBinaryMapObjectInspector = new HashMap<ArrayList<Object>, LazyBinaryMapObjectInspector>();
static ConcurrentHashMap<ArrayList<Object>, LazyBinaryMapObjectInspector> cachedLazyBinaryMapObjectInspector =
new ConcurrentHashMap<ArrayList<Object>, LazyBinaryMapObjectInspector>();

public static LazyBinaryMapObjectInspector getLazyBinaryMapObjectInspector(
ObjectInspector mapKeyObjectInspector,
Expand Down
27 changes: 16 additions & 11 deletions serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,20 @@
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;

/**
* ObjectInspectorFactory is the primary way to create new ObjectInspector
* instances.
*
*
* SerDe classes should call the static functions in this library to create an
* ObjectInspector to return to the caller of SerDe2.getObjectInspector().
*
*
* The reason of having caches here is that ObjectInspector is because
* ObjectInspectors do not have an internal state - so ObjectInspectors with the
* same construction parameters should result in exactly the same
Expand All @@ -60,7 +60,7 @@ public enum ObjectInspectorOptions {
JAVA, THRIFT, PROTOCOL_BUFFERS
};

private static HashMap<Type, ObjectInspector> objectInspectorCache = new HashMap<Type, ObjectInspector>();
private static ConcurrentHashMap<Type, ObjectInspector> objectInspectorCache = new ConcurrentHashMap<Type, ObjectInspector>();

public static ObjectInspector getReflectionObjectInspector(Type t,
ObjectInspectorOptions options) {
Expand Down Expand Up @@ -191,7 +191,8 @@ private static ObjectInspector getReflectionObjectInspectorNoCache(Type t,
return oi;
}

static HashMap<ObjectInspector, StandardListObjectInspector> cachedStandardListObjectInspector = new HashMap<ObjectInspector, StandardListObjectInspector>();
static ConcurrentHashMap<ObjectInspector, StandardListObjectInspector> cachedStandardListObjectInspector =
new ConcurrentHashMap<ObjectInspector, StandardListObjectInspector>();

public static StandardListObjectInspector getStandardListObjectInspector(
ObjectInspector listElementObjectInspector) {
Expand All @@ -210,7 +211,8 @@ public static StandardConstantListObjectInspector getStandardConstantListObjectI
}


static HashMap<List<ObjectInspector>, StandardMapObjectInspector> cachedStandardMapObjectInspector = new HashMap<List<ObjectInspector>, StandardMapObjectInspector>();
static ConcurrentHashMap<List<ObjectInspector>, StandardMapObjectInspector> cachedStandardMapObjectInspector =
new ConcurrentHashMap<List<ObjectInspector>, StandardMapObjectInspector>();

public static StandardMapObjectInspector getStandardMapObjectInspector(
ObjectInspector mapKeyObjectInspector,
Expand All @@ -236,9 +238,9 @@ public static StandardConstantMapObjectInspector getStandardConstantMapObjectIns
mapValueObjectInspector, constantValue);
}

static HashMap<List<ObjectInspector>, StandardUnionObjectInspector>
static ConcurrentHashMap<List<ObjectInspector>, StandardUnionObjectInspector>
cachedStandardUnionObjectInspector =
new HashMap<List<ObjectInspector>, StandardUnionObjectInspector>();
new ConcurrentHashMap<List<ObjectInspector>, StandardUnionObjectInspector>();

public static StandardUnionObjectInspector getStandardUnionObjectInspector(
List<ObjectInspector> unionObjectInspectors) {
Expand All @@ -251,7 +253,8 @@ public static StandardUnionObjectInspector getStandardUnionObjectInspector(
return result;
}

static HashMap<ArrayList<List<?>>, StandardStructObjectInspector> cachedStandardStructObjectInspector = new HashMap<ArrayList<List<?>>, StandardStructObjectInspector>();
static ConcurrentHashMap<ArrayList<List<?>>, StandardStructObjectInspector> cachedStandardStructObjectInspector =
new ConcurrentHashMap<ArrayList<List<?>>, StandardStructObjectInspector>();

public static StandardStructObjectInspector getStandardStructObjectInspector(
List<String> structFieldNames,
Expand All @@ -277,7 +280,8 @@ public static StandardStructObjectInspector getStandardStructObjectInspector(
return result;
}

static HashMap<List<StructObjectInspector>, UnionStructObjectInspector> cachedUnionStructObjectInspector = new HashMap<List<StructObjectInspector>, UnionStructObjectInspector>();
static ConcurrentHashMap<List<StructObjectInspector>, UnionStructObjectInspector> cachedUnionStructObjectInspector =
new ConcurrentHashMap<List<StructObjectInspector>, UnionStructObjectInspector>();

public static UnionStructObjectInspector getUnionStructObjectInspector(
List<StructObjectInspector> structObjectInspectors) {
Expand All @@ -290,7 +294,8 @@ public static UnionStructObjectInspector getUnionStructObjectInspector(
return result;
}

static HashMap<ArrayList<Object>, ColumnarStructObjectInspector> cachedColumnarStructObjectInspector = new HashMap<ArrayList<Object>, ColumnarStructObjectInspector>();
static ConcurrentHashMap<ArrayList<Object>, ColumnarStructObjectInspector> cachedColumnarStructObjectInspector =
new ConcurrentHashMap<ArrayList<Object>, ColumnarStructObjectInspector>();

public static ColumnarStructObjectInspector getColumnarStructObjectInspector(
List<String> structFieldNames,
Expand Down