From fd96226c20adf08f86e3cb6ba4b802cd88ec6dd6 Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Tue, 12 Jul 2016 17:09:42 -0700 Subject: [PATCH 01/16] Preparing for release --- giraph-accumulo/pom.xml | 2 +- giraph-block-app-8/pom.xml | 2 +- giraph-block-app/pom.xml | 2 +- giraph-core/pom.xml | 2 +- giraph-debugger/pom.xml | 2 +- giraph-dist/pom.xml | 2 +- giraph-examples/pom.xml | 4 ++-- giraph-gora/pom.xml | 2 +- giraph-hbase/pom.xml | 2 +- giraph-hcatalog/pom.xml | 2 +- giraph-rexster/giraph-kibble/pom.xml | 2 +- giraph-rexster/giraph-rexster-io/pom.xml | 2 +- giraph-rexster/pom.xml | 2 +- pom.xml | 2 +- src/site/xdoc/rexster.xml | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/giraph-accumulo/pom.xml b/giraph-accumulo/pom.xml index 9f4bfb238..ed7df42bf 100644 --- a/giraph-accumulo/pom.xml +++ b/giraph-accumulo/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-accumulo jar diff --git a/giraph-block-app-8/pom.xml b/giraph-block-app-8/pom.xml index 6645eed30..0896f5ce5 100644 --- a/giraph-block-app-8/pom.xml +++ b/giraph-block-app-8/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-block-app-8 jar diff --git a/giraph-block-app/pom.xml b/giraph-block-app/pom.xml index 3524e195f..77e7c848f 100644 --- a/giraph-block-app/pom.xml +++ b/giraph-block-app/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-block-app jar diff --git a/giraph-core/pom.xml b/giraph-core/pom.xml index 27cb27a03..80068392b 100644 --- a/giraph-core/pom.xml +++ b/giraph-core/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-core jar diff --git a/giraph-debugger/pom.xml b/giraph-debugger/pom.xml index bc66635c3..c11de0fb4 100644 --- a/giraph-debugger/pom.xml +++ b/giraph-debugger/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-debugger jar diff --git a/giraph-dist/pom.xml b/giraph-dist/pom.xml index 310f97f1e..3d09f8f9f 100644 --- a/giraph-dist/pom.xml +++ b/giraph-dist/pom.xml @@ -24,7 +24,7 @@ org.apache.giraph giraph-parent - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-dist diff --git a/giraph-examples/pom.xml b/giraph-examples/pom.xml index 934f984dd..a53efe945 100644 --- a/giraph-examples/pom.xml +++ b/giraph-examples/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-examples jar @@ -429,7 +429,7 @@ under the License. org.apache.giraph giraph-core - 1.2.0-SNAPSHOT + 1.2.0-RC0 test-jar diff --git a/giraph-gora/pom.xml b/giraph-gora/pom.xml index b6c1047c2..16ac000ef 100644 --- a/giraph-gora/pom.xml +++ b/giraph-gora/pom.xml @@ -16,7 +16,7 @@ org.apache.giraph giraph-parent - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-gora jar diff --git a/giraph-hbase/pom.xml b/giraph-hbase/pom.xml index 1334e8298..98449df90 100644 --- a/giraph-hbase/pom.xml +++ b/giraph-hbase/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-hbase jar diff --git a/giraph-hcatalog/pom.xml b/giraph-hcatalog/pom.xml index 68acdf93e..3f8746375 100644 --- a/giraph-hcatalog/pom.xml +++ b/giraph-hcatalog/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-hcatalog jar diff --git a/giraph-rexster/giraph-kibble/pom.xml b/giraph-rexster/giraph-kibble/pom.xml index dca47b0d1..1981a8cf2 100644 --- a/giraph-rexster/giraph-kibble/pom.xml +++ b/giraph-rexster/giraph-kibble/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-rexster - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-kibble diff --git a/giraph-rexster/giraph-rexster-io/pom.xml b/giraph-rexster/giraph-rexster-io/pom.xml index beba69c4f..42e32e7d5 100644 --- a/giraph-rexster/giraph-rexster-io/pom.xml +++ b/giraph-rexster/giraph-rexster-io/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-rexster - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-rexster-io jar diff --git a/giraph-rexster/pom.xml b/giraph-rexster/pom.xml index 5e90d6c9b..5cb86f1b9 100644 --- a/giraph-rexster/pom.xml +++ b/giraph-rexster/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-SNAPSHOT + 1.2.0-RC0 giraph-rexster diff --git a/pom.xml b/pom.xml index 45cbca9e6..417be59c3 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ under the License. org.apache.giraph giraph-parent pom - 1.2.0-SNAPSHOT + 1.2.0-RC0 Apache Giraph Parent http://giraph.apache.org/ diff --git a/src/site/xdoc/rexster.xml b/src/site/xdoc/rexster.xml index a8a95f840..adf531de0 100644 --- a/src/site/xdoc/rexster.xml +++ b/src/site/xdoc/rexster.xml @@ -61,7 +61,7 @@ # Configuration export REXSTER_VERSION=2.4.0 export HADOOP_VERSION=1.0.2 -export GIRAPH_VERSION=1.2.0-SNAPSHOT +export GIRAPH_VERSION=1.2.0-RC0 export GIRAPH_DIR=/path/to/giraph export REXSTER_DIR=/path/to/rexster export HADOOP_DIR=/path/to/hadoop From da11fb33218be30bd19de8340651700f55db04d0 Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Tue, 12 Jul 2016 17:25:30 -0700 Subject: [PATCH 02/16] Finbugs issues. Preparing for release --- .../org/apache/giraph/ooc/persistence/InMemoryDataAccessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/persistence/InMemoryDataAccessor.java b/giraph-core/src/main/java/org/apache/giraph/ooc/persistence/InMemoryDataAccessor.java index 4eca0f1a9..3376ce55e 100644 --- a/giraph-core/src/main/java/org/apache/giraph/ooc/persistence/InMemoryDataAccessor.java +++ b/giraph-core/src/main/java/org/apache/giraph/ooc/persistence/InMemoryDataAccessor.java @@ -94,7 +94,7 @@ public boolean dataExist(int threadId, DataIndex index) { /** * {@link DataOutputWrapper} implementation for {@link InMemoryDataAccessor} */ - public class InMemoryDataOutputWrapper implements DataOutputWrapper { + public static class InMemoryDataOutputWrapper implements DataOutputWrapper { /** Output to write data to */ private final BigDataOutput output; /** Size of output at the moment it was created */ From 26509d6ab7aab0920f47e3ba817507bc92e1349b Mon Sep 17 00:00:00 2001 From: Maja Kabiljo Date: Wed, 13 Jul 2016 11:05:48 -0700 Subject: [PATCH 03/16] [GIRAPH-1091] Fix SimpleRangePartitionFactoryTest Summary: SimpleRangePartitionFactoryTest relied on old logic for calculating number of partitions and got broken with GIRAPH-1082. Test Plan: Ran the test Differential Revision: https://reviews.facebook.net/D60747 --- .../partition/SimpleRangePartitionFactoryTest.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/giraph-core/src/test/java/org/apache/giraph/partition/SimpleRangePartitionFactoryTest.java b/giraph-core/src/test/java/org/apache/giraph/partition/SimpleRangePartitionFactoryTest.java index 4bb132bda..67863dc0f 100644 --- a/giraph-core/src/test/java/org/apache/giraph/partition/SimpleRangePartitionFactoryTest.java +++ b/giraph-core/src/test/java/org/apache/giraph/partition/SimpleRangePartitionFactoryTest.java @@ -38,9 +38,11 @@ /** Test {@link org.apache.giraph.partition.SimpleLongRangePartitionerFactory}. */ public class SimpleRangePartitionFactoryTest { - private void testRange(int numWorkers, int keySpaceSize, int allowedWorkerDiff, boolean emptyWorkers) { + private void testRange(int numWorkers, int numPartitions, int keySpaceSize, + int allowedWorkerDiff, boolean emptyWorkers) { Configuration conf = new Configuration(); conf.setLong(GiraphConstants.PARTITION_VERTEX_KEY_SPACE_SIZE, keySpaceSize); + GiraphConstants.USER_PARTITION_COUNT.set(conf, numPartitions); SimpleLongRangePartitionerFactory factory = new SimpleLongRangePartitionerFactory(); factory.setConf(new ImmutableClassesGiraphConfiguration(conf)); @@ -106,14 +108,14 @@ private void checkMapping(int[] mapping, int allowedDiff, boolean emptyWorkers) @Test public void testLongRangePartitionerFactory() { // perfect distribution - testRange(10, 100000, 0, false); - testRange(1000, 100000, 0, false); + testRange(10, 1000, 100000, 0, false); + testRange(1000, 50000, 100000, 0, false); // perfect distribution even when max is hit, and max is not divisible by #workers - testRange(8949, 100023, 0, false); - testRange(1949, 211111, 0, false); + testRange(8949, (50000 / 8949) * 8949, 100023, 0, false); + testRange(1949, (50000 / 1949) * 1949, 211111, 0, false); // imperfect distribution - because there are more workers than max partitions. - testRange(194942, 211111, 1, true); + testRange(194942, 50000, 211111, 1, true); } } From 3c933792fba709dd878f840fa9094f8884c1f0a9 Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Wed, 13 Jul 2016 14:38:02 -0700 Subject: [PATCH 04/16] GIRAPH-1092 TestCollections.testLargeBasicList fails with OOM Summary: This test case requires too much memory to run in Jenkins. Talked to Sergey Pupyrev and we decided to disable it. Test Plan: none Reviewers: majakabiljo, maja.kabiljo, spupyrev Reviewed By: spupyrev Differential Revision: https://reviews.facebook.net/D60753 --- .../src/test/java/org/apache/giraph/types/TestCollections.java | 1 + 1 file changed, 1 insertion(+) diff --git a/giraph-core/src/test/java/org/apache/giraph/types/TestCollections.java b/giraph-core/src/test/java/org/apache/giraph/types/TestCollections.java index 97d6a4164..12b5dfc41 100644 --- a/giraph-core/src/test/java/org/apache/giraph/types/TestCollections.java +++ b/giraph-core/src/test/java/org/apache/giraph/types/TestCollections.java @@ -53,6 +53,7 @@ public void testLargeBasicSet() { } @Test + @Ignore("this test requires 1G to run") public void testLargeBasicList() { int capacity = 123456789; WArrayList longSet = LongTypeOps.INSTANCE.createArrayList(capacity); From df8f128e26f9b3ff24445e7332d375f04bb4d75c Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Fri, 15 Jul 2016 14:22:59 -0700 Subject: [PATCH 05/16] [GIRAPH-1095] Performance regression after GIRAPH-1068 Summary: Need to pass some missing parameters to zookeeper Test Plan: run a few jobs Reviewers: dionysis.logothetis, heslami, majakabiljo, maja.kabiljo Reviewed By: maja.kabiljo Differential Revision: https://reviews.facebook.net/D60831 --- .../java/org/apache/giraph/conf/GiraphConstants.java | 12 +++++++++++- .../java/org/apache/giraph/zk/ZooKeeperManager.java | 8 ++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java index ee67bedfe..b38426122 100644 --- a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java +++ b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java @@ -1003,7 +1003,8 @@ public interface GiraphConstants { BooleanConfOption KEEP_ZOOKEEPER_DATA = new BooleanConfOption("giraph.keepZooKeeperData", false, "Keep the zookeeper output for debugging? Default is to remove it."); - + /** Default ZooKeeper snap count. */ + int DEFAULT_ZOOKEEPER_SNAP_COUNT = 50000; /** Default ZooKeeper tick time. */ int DEFAULT_ZOOKEEPER_TICK_TIME = 6000; /** Default ZooKeeper maximum client connections. */ @@ -1021,6 +1022,15 @@ public interface GiraphConstants { new IntConfOption("giraph.zkMaxSessionTimeout", MINUTES.toMillis(15), "ZooKeeper maximum session timeout"); + /** ZooKeeper force sync */ + BooleanConfOption ZOOKEEPER_FORCE_SYNC = + new BooleanConfOption("giraph.zKForceSync", false, + "ZooKeeper force sync"); + + /** ZooKeeper skip ACLs */ + BooleanConfOption ZOOKEEPER_SKIP_ACL = + new BooleanConfOption("giraph.ZkSkipAcl", true, "ZooKeeper skip ACLs"); + /** * Whether to use SASL with DIGEST and Hadoop Job Tokens to authenticate * and authorize Netty BSP Clients to Servers. diff --git a/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java b/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java index c54e7b287..097172d27 100644 --- a/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java +++ b/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java @@ -437,12 +437,20 @@ private void generateZooKeeperConfig() { LOG.info("generateZooKeeperConfigFile: Make directory of " + zkDirFile.getName() + " = " + mkDirRet); } + /** Set zookeeper system properties */ + System.setProperty("zookeeper.snapCount", + Integer.toString(GiraphConstants.DEFAULT_ZOOKEEPER_SNAP_COUNT)); + System.setProperty("zookeeper.forceSync", + GiraphConstants.ZOOKEEPER_FORCE_SYNC.get(conf) ? "yes" : "no"); + System.setProperty("zookeeper.skipACL", + GiraphConstants.ZOOKEEPER_SKIP_ACL.get(conf) ? "yes" : "no"); config.setDataDir(zkDir); config.setDataLogDir(zkDir); config.setClientPortAddress(new InetSocketAddress(zkBasePort)); config.setMinSessionTimeout(conf.getZooKeeperMinSessionTimeout()); config.setMaxSessionTimeout(conf.getZooKeeperMaxSessionTimeout()); + } /** From 1fb6a5c5bcf095045151a7ce7be0c129d4bd06eb Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Mon, 18 Jul 2016 17:30:04 -0700 Subject: [PATCH 06/16] GIRAPH-1097 Fix TestOutOfCore.testOutOfCoreLocalDiskAccessor Summary: On my laptop it failed because of an NPE in WorkerSuperstepMetrics. I tracked it down and found that it is triggered from the branch of code that prints out metrics. We don't normally print out metrics in unit tests, so I'd expect this feature doesn't exist or not functional in hadoop_1. I'll try to disable it, to see how jenkins reacts. Test Plan: mvn test -pl giraph-examples -am -Dtest=TestOutOfCore -DfailIfNoTests=false -Phadoop_1 Reviewers: maja.kabiljo, dionysis.logothetis, heslami Reviewed By: heslami Differential Revision: https://reviews.facebook.net/D60873 --- .../src/test/java/org/apache/giraph/TestOutOfCore.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/giraph-examples/src/test/java/org/apache/giraph/TestOutOfCore.java b/giraph-examples/src/test/java/org/apache/giraph/TestOutOfCore.java index e49754106..a9d6a65d7 100644 --- a/giraph-examples/src/test/java/org/apache/giraph/TestOutOfCore.java +++ b/giraph-examples/src/test/java/org/apache/giraph/TestOutOfCore.java @@ -18,7 +18,6 @@ package org.apache.giraph; -import org.apache.giraph.comm.netty.NettyClient; import org.apache.giraph.conf.GiraphConfiguration; import org.apache.giraph.conf.GiraphConstants; import org.apache.giraph.examples.GeneratedVertexReader; @@ -59,7 +58,6 @@ public void prepareTest() { SimplePageRankComputation.SimplePageRankWorkerContext.class); conf.setMasterComputeClass( SimplePageRankComputation.SimplePageRankMasterCompute.class); - GiraphConstants.METRICS_ENABLE.set(conf, true); GiraphConstants.USER_PARTITION_COUNT.set(conf, NUM_PARTITIONS); GiraphConstants.USE_OUT_OF_CORE_GRAPH.set(conf, true); GiraphConstants.MAX_PARTITIONS_IN_MEMORY.set(conf, NUM_PARTITIONS_IN_MEMORY); @@ -97,7 +95,7 @@ private void runTest() getTempPath(getCallingMethodName())); // Overwrite the number of vertices set in BspCase GeneratedVertexReader.READER_VERTICES.set(conf, 200); - assertTrue(job.run(true)); + assertTrue(job.run(false)); if (!runningInDistributedMode()) { double maxPageRank = SimplePageRankComputation.SimplePageRankWorkerContext.getFinalMax(); From 3da24f119fd0ea4b586de62459245fb2294bd154 Mon Sep 17 00:00:00 2001 From: Maja Kabiljo Date: Fri, 12 Aug 2016 14:57:53 -0700 Subject: [PATCH 07/16] GIRAPH-1105: Fix number of open requests in FacebookConfiguration Test Plan: This was significantly better in some experiments, but we can investigate more in the future Differential Revision: https://reviews.facebook.net/D62019 --- .../main/java/org/apache/giraph/conf/FacebookConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/giraph-core/src/main/java/org/apache/giraph/conf/FacebookConfiguration.java b/giraph-core/src/main/java/org/apache/giraph/conf/FacebookConfiguration.java index d30215f5c..0ba04f369 100644 --- a/giraph-core/src/main/java/org/apache/giraph/conf/FacebookConfiguration.java +++ b/giraph-core/src/main/java/org/apache/giraph/conf/FacebookConfiguration.java @@ -130,7 +130,7 @@ public void configure(GiraphConfiguration conf) { // Limit number of open requests to 2000 NettyClient.LIMIT_NUMBER_OF_OPEN_REQUESTS.setIfUnset(conf, true); - StaticFlowControl.MAX_NUMBER_OF_OPEN_REQUESTS.setIfUnset(conf, 2000); + StaticFlowControl.MAX_NUMBER_OF_OPEN_REQUESTS.setIfUnset(conf, 100); // Pooled allocator in netty is faster GiraphConstants.NETTY_USE_POOLED_ALLOCATOR.setIfUnset(conf, true); // Turning off auto read is faster From 13b85590ff0d482702ee5dbfcd02ab57e17e24b4 Mon Sep 17 00:00:00 2001 From: Hassan Eslami Date: Tue, 26 Jul 2016 11:27:24 -0700 Subject: [PATCH 08/16] Fixing Giraph pom.xml to reflect new project committers Summary: Fixed the list of project committers. Please review your information and let me know if I should change anything. This will be the first diff that I'll be committing all by myself, more like a test to see my username is gone through Apache's internal :-) Test Plan: N/A Reviewers: ikabiljo, pavanka, avery.ching, sergey.edunov Reviewed By: sergey.edunov Differential Revision: https://reviews.facebook.net/D61197 --- pom.xml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pom.xml b/pom.xml index 417be59c3..cb7c995fa 100644 --- a/pom.xml +++ b/pom.xml @@ -218,6 +218,38 @@ under the License. Facebook http://www.facebook.com + + edunov + Sergey Edunov + edunov@apache.org + -8 + Facebook + http://www.facebook.com + + + pavanka + Pavan Kumar + pavanka@apache.org + -8 + Facebook + http://www.facebook.com + + + ikabiljo + Igor Kabiljo + ikabiljo@apache.org + -8 + Facebook + http://www.facebook.com + + + heslami + Hassan Eslami + heslami@apache.org + -8 + Facebook, UIUC + http://www.facebook.com + From 7e9ba5312a941ec13ec900989c4a8e229ff034c9 Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Wed, 20 Jul 2016 10:20:36 -0700 Subject: [PATCH 09/16] GIRAPH-1098 Job may get stuck if zookeeper port fixed and is in use Test Plan: mvn clean verify -Phadoop_facebook Reviewers: majakabiljo, dionysis.logothetis, maja.kabiljo Reviewed By: maja.kabiljo Differential Revision: https://reviews.facebook.net/D60945 --- .../org/apache/giraph/zk/InProcessZooKeeperRunner.java | 9 ++------- .../main/java/org/apache/giraph/zk/ZooKeeperManager.java | 2 +- .../main/java/org/apache/giraph/zk/ZooKeeperRunner.java | 4 +++- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/giraph-core/src/main/java/org/apache/giraph/zk/InProcessZooKeeperRunner.java b/giraph-core/src/main/java/org/apache/giraph/zk/InProcessZooKeeperRunner.java index 4f15f3a22..cee2c7885 100644 --- a/giraph-core/src/main/java/org/apache/giraph/zk/InProcessZooKeeperRunner.java +++ b/giraph-core/src/main/java/org/apache/giraph/zk/InProcessZooKeeperRunner.java @@ -47,13 +47,8 @@ public class InProcessZooKeeperRunner private QuorumRunner quorumRunner = new QuorumRunner(); @Override - public int start(String zkDir, final ZookeeperConfig config) { - try { - return quorumRunner.start(config); - } catch (IOException e) { - LOG.error("Unable to start zookeeper", e); - } - return -1; + public int start(String zkDir, ZookeeperConfig config) throws IOException { + return quorumRunner.start(config); } @Override diff --git a/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java b/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java index 097172d27..b8438821b 100644 --- a/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java +++ b/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java @@ -457,7 +457,7 @@ private void generateZooKeeperConfig() { * If this task has been selected, online a ZooKeeper server. Otherwise, * wait until this task knows that the ZooKeeper servers have been onlined. */ - public void onlineZooKeeperServer() { + public void onlineZooKeeperServer() throws IOException { if (zkServerTask == taskPartition) { File zkDirFile = new File(this.zkDir); try { diff --git a/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperRunner.java b/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperRunner.java index 279704713..9d6a33220 100644 --- a/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperRunner.java +++ b/giraph-core/src/main/java/org/apache/giraph/zk/ZooKeeperRunner.java @@ -19,6 +19,8 @@ import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable; +import java.io.IOException; + /** * ZooKeeper wrapper interface. * Implementation should provide a way to start, stop and cleanup @@ -33,7 +35,7 @@ public interface ZooKeeperRunner extends ImmutableClassesGiraphConfigurable { * @param config zookeeper configuration * @return port zookeeper runs on */ - int start(String zkDir, ZookeeperConfig config); + int start(String zkDir, ZookeeperConfig config) throws IOException; /** * Stops zookeeper. From 931d198784a6eabf8cc746f3cc07edeb63b6cd01 Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Fri, 9 Sep 2016 10:43:28 -0700 Subject: [PATCH 10/16] hbase_fix --- giraph-hbase/pom.xml | 19 +- .../io/hbase/HBaseVertexOutputFormat.java | 5 +- .../edgemarker/TableEdgeOutputFormat.java | 5 +- pom.xml | 175 ++++-------------- 4 files changed, 59 insertions(+), 145 deletions(-) diff --git a/giraph-hbase/pom.xml b/giraph-hbase/pom.xml index 98449df90..580edcc3b 100644 --- a/giraph-hbase/pom.xml +++ b/giraph-hbase/pom.xml @@ -170,9 +170,16 @@ under the License. org.apache.hbase - hbase + hbase-common + + + org.apache.hbase + hbase-client + + + org.apache.hbase + hbase-server - org.slf4j @@ -193,7 +200,13 @@ under the License. org.apache.hbase - hbase + hbase-common + test-jar + test + + + org.apache.hbase + hbase-server test-jar test diff --git a/giraph-hbase/src/main/java/org/apache/giraph/io/hbase/HBaseVertexOutputFormat.java b/giraph-hbase/src/main/java/org/apache/giraph/io/hbase/HBaseVertexOutputFormat.java index f875c19fc..87bc74c4d 100644 --- a/giraph-hbase/src/main/java/org/apache/giraph/io/hbase/HBaseVertexOutputFormat.java +++ b/giraph-hbase/src/main/java/org/apache/giraph/io/hbase/HBaseVertexOutputFormat.java @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.giraph.io.VertexOutputFormat; import org.apache.giraph.io.VertexWriter; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableOutputFormat; import org.apache.hadoop.io.Writable; @@ -81,7 +82,7 @@ public abstract static class HBaseVertexWriter< /** Context */ private TaskAttemptContext context; /** Record writer instance */ - private RecordWriter recordWriter; + private RecordWriter recordWriter; /** * Sets up base table output format and creates a record writer. @@ -122,7 +123,7 @@ public void close(TaskAttemptContext context) * @return Record writer to be used for writing. */ public RecordWriter getRecordWriter() { + Mutation> getRecordWriter() { return recordWriter; } diff --git a/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/edgemarker/TableEdgeOutputFormat.java b/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/edgemarker/TableEdgeOutputFormat.java index aa95f9615..08ee03f04 100644 --- a/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/edgemarker/TableEdgeOutputFormat.java +++ b/giraph-hbase/src/test/java/org/apache/giraph/io/hbase/edgemarker/TableEdgeOutputFormat.java @@ -20,11 +20,11 @@ import org.apache.giraph.io.hbase.HBaseVertexOutputFormat; import org.apache.giraph.graph.Vertex; import org.apache.giraph.io.VertexWriter; +import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -62,7 +62,8 @@ public TableEdgeVertexWriter(TaskAttemptContext context) public void writeVertex( Vertex vertex) throws IOException, InterruptedException { - RecordWriter writer = getRecordWriter(); + RecordWriter + writer = getRecordWriter(); byte[] rowBytes = vertex.getId().getBytes(); Put put = new Put(rowBytes); Text value = vertex.getValue(); diff --git a/pom.xml b/pom.xml index cb7c995fa..31c4fad6c 100644 --- a/pom.xml +++ b/pom.xml @@ -333,7 +333,7 @@ under the License. 6.5.4 2.0.2 18.0 - 0.94.16 + 1.2.2 0.5.0-incubating 0.11.0 0.26 @@ -392,7 +392,7 @@ under the License. 2.8 ${giraph.maven.dependency.plugin.skip} - true + false true @@ -1384,19 +1384,27 @@ under the License. org.apache.hadoop hadoop-auth + ${hadoop.version} + provided + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + org.apache.hadoop + hadoop-mapreduce-client-common + ${hadoop.version} + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided - - org.apache.hadoop - hadoop-common - - - org.apache.hadoop - hadoop-mapreduce-client-common - - - org.apache.hadoop - hadoop-mapreduce-client-core - @@ -2108,96 +2116,22 @@ under the License. - org.apache.hadoop - hadoop-auth - ${hadoop.version} - provided - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - provided - - - org.apache.hadoop - hadoop-mapreduce-client-common - ${hadoop.version} + org.apache.hbase + hbase-client + ${dep.hbase.version} provided - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} + org.apache.hbase + hbase-server + ${dep.hbase.version} provided org.apache.hbase - hbase + hbase-common ${dep.hbase.version} provided - - - com.google.protobuf - protobuf-java - - - com.sun.jersey - jersey-server - - - commons-httpclient - commons-httpclient - - - commons-lang - commons-lang - - - commons-logging - commons-logging - - - log4j - log4j - - - org.apache.avro - avro - - - org.apache.hadoop - hadoop-core - - - org.apache.thrift - thrift - - - org.mortbay.jetty - servlet-api-2.5 - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jsp-api-2.1 - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - javax.ws.rs - jsr311-api - - @@ -2290,52 +2224,17 @@ under the License. org.apache.hbase - hbase + hbase-common + test-jar + ${dep.hbase.version} + test + + + org.apache.hbase + hbase-server test-jar ${dep.hbase.version} test - - - com.sun.jersey - jersey-server - - - commons-httpclient - commons-httpclient - - - commons-logging - commons-logging - - - log4j - log4j - - - org.apache.hadoop - avro - - - org.apache.hadoop - hadoop-core - - - org.apache.thrift - thrift - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - javax.ws.rs - jsr311-api - - org.mockito From 1bb23dfaa9a441452688627e567ae5d5aabf6deb Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Wed, 21 Sep 2016 11:04:34 -0700 Subject: [PATCH 11/16] GIRAPH-1094 Remove hbase from hadoop_1 Summary: Hadoop_1 and current versions of hbase are incompatible. Removing support for HBASE from Hadoop_1 profile Test Plan: mvn clean package -Phadoop_1 -fae Reviewers: majakabiljo, maja.kabiljo, dionysis.logothetis Reviewed By: maja.kabiljo, dionysis.logothetis Differential Revision: https://reviews.facebook.net/D64197 --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 31c4fad6c..21d6f225b 100644 --- a/pom.xml +++ b/pom.xml @@ -1044,7 +1044,6 @@ under the License. hadoop_1 giraph-accumulo - giraph-hbase giraph-hcatalog giraph-gora giraph-rexster From 3f331bad55db03275120996517bc886c9a45cc18 Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Wed, 21 Sep 2016 14:47:10 -0700 Subject: [PATCH 12/16] GIRAPH-1094 remove hbase1 from distribution for hadoop_1 Summary: Missed that part in the last diff. Test Plan: mvn clean package -Phadoop_2 -fae then checked that giraph-hbase.jar is in the distribution mvn clean package -Phadoop_1 -fae then checked that giraph-hbase.jar is not in the distribution Reviewers: maja.kabiljo, majakabiljo, dionysis.logothetis Reviewed By: dionysis.logothetis Differential Revision: https://reviews.facebook.net/D64203 --- giraph-dist/pom.xml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/giraph-dist/pom.xml b/giraph-dist/pom.xml index 3d09f8f9f..21ae80c27 100644 --- a/giraph-dist/pom.xml +++ b/giraph-dist/pom.xml @@ -46,6 +46,16 @@ + + + hadoop_2 + + + org.apache.giraph + giraph-hbase + + + @@ -87,10 +97,6 @@ org.apache.giraph giraph-core - - org.apache.giraph - giraph-hbase - org.apache.giraph giraph-accumulo From 7a38416982f98a34e054af0ef5ef62e903796ac1 Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Wed, 28 Sep 2016 15:31:11 -0700 Subject: [PATCH 13/16] Update version to 1.2.0 --- giraph-accumulo/pom.xml | 2 +- giraph-block-app-8/pom.xml | 2 +- giraph-block-app/pom.xml | 2 +- giraph-core/pom.xml | 2 +- giraph-debugger/pom.xml | 2 +- giraph-dist/pom.xml | 2 +- giraph-examples/pom.xml | 4 ++-- giraph-gora/pom.xml | 2 +- giraph-hbase/pom.xml | 2 +- giraph-hcatalog/pom.xml | 2 +- giraph-rexster/giraph-kibble/pom.xml | 2 +- giraph-rexster/giraph-rexster-io/pom.xml | 2 +- giraph-rexster/pom.xml | 2 +- pom.xml | 2 +- 14 files changed, 15 insertions(+), 15 deletions(-) diff --git a/giraph-accumulo/pom.xml b/giraph-accumulo/pom.xml index ed7df42bf..b8d1195b6 100644 --- a/giraph-accumulo/pom.xml +++ b/giraph-accumulo/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-RC0 + 1.2.0 giraph-accumulo jar diff --git a/giraph-block-app-8/pom.xml b/giraph-block-app-8/pom.xml index 0896f5ce5..0b8d1c88f 100644 --- a/giraph-block-app-8/pom.xml +++ b/giraph-block-app-8/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-RC0 + 1.2.0 giraph-block-app-8 jar diff --git a/giraph-block-app/pom.xml b/giraph-block-app/pom.xml index 77e7c848f..8152871e6 100644 --- a/giraph-block-app/pom.xml +++ b/giraph-block-app/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-RC0 + 1.2.0 giraph-block-app jar diff --git a/giraph-core/pom.xml b/giraph-core/pom.xml index 80068392b..df7aaad45 100644 --- a/giraph-core/pom.xml +++ b/giraph-core/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-RC0 + 1.2.0 giraph-core jar diff --git a/giraph-debugger/pom.xml b/giraph-debugger/pom.xml index c11de0fb4..3b53a5f75 100644 --- a/giraph-debugger/pom.xml +++ b/giraph-debugger/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-RC0 + 1.2.0 giraph-debugger jar diff --git a/giraph-dist/pom.xml b/giraph-dist/pom.xml index 21ae80c27..dcbc75bcc 100644 --- a/giraph-dist/pom.xml +++ b/giraph-dist/pom.xml @@ -24,7 +24,7 @@ org.apache.giraph giraph-parent - 1.2.0-RC0 + 1.2.0 giraph-dist diff --git a/giraph-examples/pom.xml b/giraph-examples/pom.xml index a53efe945..4fc6485d6 100644 --- a/giraph-examples/pom.xml +++ b/giraph-examples/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-RC0 + 1.2.0 giraph-examples jar @@ -429,7 +429,7 @@ under the License. org.apache.giraph giraph-core - 1.2.0-RC0 + 1.2.0 test-jar diff --git a/giraph-gora/pom.xml b/giraph-gora/pom.xml index 16ac000ef..34a705800 100644 --- a/giraph-gora/pom.xml +++ b/giraph-gora/pom.xml @@ -16,7 +16,7 @@ org.apache.giraph giraph-parent - 1.2.0-RC0 + 1.2.0 giraph-gora jar diff --git a/giraph-hbase/pom.xml b/giraph-hbase/pom.xml index 580edcc3b..6773cb13a 100644 --- a/giraph-hbase/pom.xml +++ b/giraph-hbase/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-RC0 + 1.2.0 giraph-hbase jar diff --git a/giraph-hcatalog/pom.xml b/giraph-hcatalog/pom.xml index 3f8746375..a92fcafd0 100644 --- a/giraph-hcatalog/pom.xml +++ b/giraph-hcatalog/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-RC0 + 1.2.0 giraph-hcatalog jar diff --git a/giraph-rexster/giraph-kibble/pom.xml b/giraph-rexster/giraph-kibble/pom.xml index 1981a8cf2..dc6d26b16 100644 --- a/giraph-rexster/giraph-kibble/pom.xml +++ b/giraph-rexster/giraph-kibble/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-rexster - 1.2.0-RC0 + 1.2.0 giraph-kibble diff --git a/giraph-rexster/giraph-rexster-io/pom.xml b/giraph-rexster/giraph-rexster-io/pom.xml index 42e32e7d5..35d006adf 100644 --- a/giraph-rexster/giraph-rexster-io/pom.xml +++ b/giraph-rexster/giraph-rexster-io/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.giraph giraph-rexster - 1.2.0-RC0 + 1.2.0 giraph-rexster-io jar diff --git a/giraph-rexster/pom.xml b/giraph-rexster/pom.xml index 5cb86f1b9..11c55f7b6 100644 --- a/giraph-rexster/pom.xml +++ b/giraph-rexster/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.giraph giraph-parent - 1.2.0-RC0 + 1.2.0 giraph-rexster diff --git a/pom.xml b/pom.xml index 21d6f225b..bc4d14ccf 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ under the License. org.apache.giraph giraph-parent pom - 1.2.0-RC0 + 1.2.0 Apache Giraph Parent http://giraph.apache.org/ From 7ea61447fc4839106a86bf889a9853899708bbc7 Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Wed, 5 Oct 2016 15:05:58 -0700 Subject: [PATCH 14/16] GIRAPH-1118 - Giraph-gora and Giraph-rexster test cases fail in release-1.2 Test Plan: mvn clean verify -Phadoop_facebook mvn clean install -Phadoop_1 mvn clean install -Phadoop_2 Reviewers: majakabiljo, dionysis.logothetis, maja.kabiljo Reviewed By: maja.kabiljo Differential Revision: https://reviews.facebook.net/D64683 --- giraph-dist/pom.xml | 4 --- .../io/gora/TestGoraVertexInputFormat.java | 16 +++++------ .../TestRexsterLongDoubleFloatIOFormat.java | 27 ++++++++++++++----- pom.xml | 2 -- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/giraph-dist/pom.xml b/giraph-dist/pom.xml index dcbc75bcc..b9d57cef2 100644 --- a/giraph-dist/pom.xml +++ b/giraph-dist/pom.xml @@ -123,9 +123,5 @@ - - org.apache.giraph - giraph-rexster-io - diff --git a/giraph-gora/src/test/java/org/apache/giraph/io/gora/TestGoraVertexInputFormat.java b/giraph-gora/src/test/java/org/apache/giraph/io/gora/TestGoraVertexInputFormat.java index b42ab497c..25440e1c2 100644 --- a/giraph-gora/src/test/java/org/apache/giraph/io/gora/TestGoraVertexInputFormat.java +++ b/giraph-gora/src/test/java/org/apache/giraph/io/gora/TestGoraVertexInputFormat.java @@ -26,7 +26,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import org.apache.giraph.conf.GiraphConfiguration; @@ -92,14 +94,12 @@ public void getTestDb() throws Exception { results = InternalVertexRunner.run(conf, new String[0], new String[0]); Assert.assertNotNull(results); Assert.assertEquals(3, ((ArrayList)results).size()); - if (results instanceof Collection - & (((Collection)results).size() == 3)) { - Assert.assertEquals("10\t0.0", - ((ArrayList)results).get(0).toString()); - Assert.assertEquals("1\t0.0", - ((ArrayList)results).get(1).toString()); - Assert.assertEquals("100\t0.0", - ((ArrayList)results).get(2).toString()); + Collections.sort((ArrayList)results); + String[] correct = new String[] {"1\t0.0", "10\t0.0", "100\t0.0"}; + Arrays.sort(correct); + for (int i = 0; i < correct.length; i++) { + Assert.assertEquals(correct[i], + ((ArrayList) results).get(i).toString()); } } diff --git a/giraph-rexster/giraph-rexster-io/src/test/java/org/apache/giraph/rexster/io/formats/TestRexsterLongDoubleFloatIOFormat.java b/giraph-rexster/giraph-rexster-io/src/test/java/org/apache/giraph/rexster/io/formats/TestRexsterLongDoubleFloatIOFormat.java index 5008608e9..40f992e3f 100644 --- a/giraph-rexster/giraph-rexster-io/src/test/java/org/apache/giraph/rexster/io/formats/TestRexsterLongDoubleFloatIOFormat.java +++ b/giraph-rexster/giraph-rexster-io/src/test/java/org/apache/giraph/rexster/io/formats/TestRexsterLongDoubleFloatIOFormat.java @@ -54,6 +54,7 @@ import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -70,6 +71,13 @@ /** * This test suit is intended to extensively test Rexster I/O Format * together with the Kibble for such a goal. + * + * + * Note: this is a very simple test case: load data into rexster + * and then read it using giraph. And reverse: load data using giraph + * and then read it using rexster. The graph that is being loaded + * is always the same, but the output we receive is actually different + * for different underlying formats. Why? Probably because of some bugs. */ public class TestRexsterLongDoubleFloatIOFormat { /** temporary directory */ @@ -85,6 +93,8 @@ public class TestRexsterLongDoubleFloatIOFormat { @BeforeClass public static void initialSetup() throws Exception { + //In case there were previous runs that failed + deleteDbs(); startRexsterServer(); insertDbData(); } @@ -106,6 +116,7 @@ public void testEmptyDbInputGremlin() throws Exception { testDbInput(EMPTYDB, true, true); } + @Ignore @Test public void testTgDbInput() throws Exception { testDbInput(DATABASES[0], false, false); @@ -139,16 +150,19 @@ public void testOrientDbInputGremlin() throws Exception { testDbInput(DATABASES[2], false, true); } + @Ignore @Test public void testTgDbOutput() throws Exception { testDbOutput("empty" + DATABASES[0]); } + @Ignore @Test public void testNeoDbOutput() throws Exception { testDbOutput("empty" + DATABASES[1]); } + @Ignore @Test public void testOrientDbOutput() throws Exception { testDbOutput("empty" + DATABASES[2]); @@ -156,7 +170,6 @@ public void testOrientDbOutput() throws Exception { private void testDbInput(String name, boolean isEmpty, boolean isGramlin) throws Exception { - GiraphConfiguration conf = new GiraphConfiguration(); GIRAPH_REXSTER_HOSTNAME.set(conf, "127.0.0.1"); GIRAPH_REXSTER_PORT.set(conf, 18182); @@ -174,6 +187,7 @@ private void testDbInput(String name, boolean isEmpty, boolean isGramlin) Iterable results = InternalVertexRunner.run(conf, new String[0], new String[0]); + if (isEmpty) { boolean empty = false; if (results != null) { @@ -182,10 +196,10 @@ private void testDbInput(String name, boolean isEmpty, boolean isGramlin) } else { empty = true; } - assert empty; + Assert.assertTrue(empty); return; } else { - assert results != null; + Assert.assertNotNull(results); } URL url = this.getClass().getResource(name + "-output.json"); @@ -319,7 +333,7 @@ private static void insertDbData() throws Exception { int responseCode = conn.getResponseCode(); if (responseCode != 200) { throw new RuntimeException("Unable to insert data in " + DATABASES[i] + - " code: " + responseCode ); + " code: " + responseCode + "\nresponse: " + conn.getResponseMessage()); } BufferedReader in = new BufferedReader( new InputStreamReader(conn.getInputStream())); @@ -359,7 +373,7 @@ private static void insertDbData() throws Exception { responseCode = conn.getResponseCode(); if (responseCode != 200) { throw new RuntimeException("Unable to insert data in " + DATABASES[i] + - " code: " + responseCode ); + " code: " + responseCode + "\nresponse: " + conn.getResponseMessage()); } in = new BufferedReader(new InputStreamReader(conn.getInputStream())); response = new StringBuffer(); @@ -425,7 +439,6 @@ private ArrayList getRexsterContent(String name) throws Exception { } br.close(); is.close(); - results = new JSONObject(json.toString()); JSONArray edges = results.getJSONArray("results"); for (int i = 0; i < edges.length(); ++i) { @@ -465,7 +478,7 @@ protected void checkResult(ArrayList expected, found = true; } } - assert found; + Assert.assertTrue("expected: " + expected + " result: " + result, found); } } diff --git a/pom.xml b/pom.xml index bc4d14ccf..458861f1d 100644 --- a/pom.xml +++ b/pom.xml @@ -1046,7 +1046,6 @@ under the License. giraph-accumulo giraph-hcatalog giraph-gora - giraph-rexster giraph-dist @@ -1360,7 +1359,6 @@ under the License. giraph-hbase giraph-hcatalog giraph-gora - giraph-rexster giraph-dist From 0cab55b4ef4b3e0d287a68c1c16f624dc8f6319c Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Thu, 6 Oct 2016 11:13:49 -0700 Subject: [PATCH 15/16] GIRAPH-1118 - Giraph-gora and Giraph-rexster test cases fail in release-1.2 Test Plan: mvn clean verify -Phadoop_facebook rm -rf ~/.m2/repository/org/apache/giraph mvn clean install -Phadoop_1 rm -rf ~/.m2/repository/org/apache/giraph mvn clean install -Phadoop_2 Reviewers: maja.kabiljo, majakabiljo, dionysis.logothetis Reviewed By: dionysis.logothetis Differential Revision: https://reviews.facebook.net/D64719 --- giraph-dist/pom.xml | 10 ---------- pom.xml | 4 ++++ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/giraph-dist/pom.xml b/giraph-dist/pom.xml index b9d57cef2..4ceecf433 100644 --- a/giraph-dist/pom.xml +++ b/giraph-dist/pom.xml @@ -113,15 +113,5 @@ org.apache.giraph giraph-gora - - org.apache.giraph - giraph-kibble - - - javax.servlet - javax.servlet-api - - - diff --git a/pom.xml b/pom.xml index 458861f1d..fdab2bb11 100644 --- a/pom.xml +++ b/pom.xml @@ -1087,6 +1087,10 @@ under the License. tomcat jasper-runtime + + com.sun.jersey + jersey-json + From 19316327027f8ea89f995d9b1ebbd9bca6a85ff7 Mon Sep 17 00:00:00 2001 From: Sergey Edunov Date: Tue, 11 Oct 2016 16:49:16 -0700 Subject: [PATCH 16/16] Fixing RAT checks for Apache Giraph release Test Plan: mvn apache-rat:check -Phadoop_2 mvn apache-rat:check -Phadoop_1 mvn clean verify -Phadoop_facebook Reviewers: maja.kabiljo, majakabiljo, dionysis.logothetis Reviewed By: dionysis.logothetis Differential Revision: https://reviews.facebook.net/D64917 --- .../algo/TestMultiSeedBreadthFirstSearch.java | 17 +++++++++++++++++ .../library/pagerank/PageRankTest.java | 17 +++++++++++++++++ .../kryo/KryoWritableWrapperJava8Test.java | 17 +++++++++++++++++ .../MultipleSimultanousMutationsTest.java | 18 ++++++++++++++++++ giraph-debugger/.gitignore | 8 -------- giraph-debugger/README.md | 15 +++++++++++++++ giraph-debugger/giraph-debug | 15 +++++++++++++++ .../org/apache/giraph/debugger/gui/css/app.css | 17 +++++++++++++++++ .../giraph/debugger/gui/css/valpanel.css | 17 +++++++++++++++++ .../org/apache/giraph/debugger/gui/index.html | 17 +++++++++++++++++ .../apache/giraph/debugger/gui/js/debugger.js | 17 +++++++++++++++++ .../giraph/debugger/gui/js/editor.core.js | 17 +++++++++++++++++ .../giraph/debugger/gui/js/editor.utils.js | 17 +++++++++++++++++ .../org/apache/giraph/debugger/gui/js/utils.js | 17 +++++++++++++++++ .../debugger/gui/js/utils.sampleGraphs.js | 17 +++++++++++++++++ .../apache/giraph/debugger/gui/js/valpanel.js | 17 +++++++++++++++++ giraph-gora/pom.xml | 7 +++++++ pom.xml | 8 ++++++-- 18 files changed, 265 insertions(+), 10 deletions(-) delete mode 100644 giraph-debugger/.gitignore diff --git a/giraph-block-app-8/src/test/java/org/apache/giraph/block_app/library/algo/TestMultiSeedBreadthFirstSearch.java b/giraph-block-app-8/src/test/java/org/apache/giraph/block_app/library/algo/TestMultiSeedBreadthFirstSearch.java index c3aae071d..51eedb8ae 100644 --- a/giraph-block-app-8/src/test/java/org/apache/giraph/block_app/library/algo/TestMultiSeedBreadthFirstSearch.java +++ b/giraph-block-app-8/src/test/java/org/apache/giraph/block_app/library/algo/TestMultiSeedBreadthFirstSearch.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.giraph.block_app.library.algo; import org.apache.giraph.block_app.framework.BlockUtils; diff --git a/giraph-block-app-8/src/test/java/org/apache/giraph/block_app/library/pagerank/PageRankTest.java b/giraph-block-app-8/src/test/java/org/apache/giraph/block_app/library/pagerank/PageRankTest.java index f8ce74dd3..7b19d93ef 100644 --- a/giraph-block-app-8/src/test/java/org/apache/giraph/block_app/library/pagerank/PageRankTest.java +++ b/giraph-block-app-8/src/test/java/org/apache/giraph/block_app/library/pagerank/PageRankTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.giraph.block_app.library.pagerank; import org.apache.giraph.block_app.framework.BlockUtils; diff --git a/giraph-block-app-8/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperJava8Test.java b/giraph-block-app-8/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperJava8Test.java index d8b4cc119..1f528f005 100644 --- a/giraph-block-app-8/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperJava8Test.java +++ b/giraph-block-app-8/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperJava8Test.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.giraph.writable.kryo; import java.io.IOException; diff --git a/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/MultipleSimultanousMutationsTest.java b/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/MultipleSimultanousMutationsTest.java index d442d5d0f..97fd5e309 100644 --- a/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/MultipleSimultanousMutationsTest.java +++ b/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/MultipleSimultanousMutationsTest.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.giraph.block_app.framework; import org.apache.giraph.block_app.framework.api.BlockWorkerSendApi; diff --git a/giraph-debugger/.gitignore b/giraph-debugger/.gitignore deleted file mode 100644 index c2a737f66..000000000 --- a/giraph-debugger/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Maven stuffs -/target - -# Eclipse files -/.classpath -/.project -/.settings -/.checkstyle diff --git a/giraph-debugger/README.md b/giraph-debugger/README.md index 4e620410e..694350e76 100644 --- a/giraph-debugger/README.md +++ b/giraph-debugger/README.md @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Giraph Debugger ## Overview diff --git a/giraph-debugger/giraph-debug b/giraph-debugger/giraph-debug index 53a7c930d..8408ee6a7 100644 --- a/giraph-debugger/giraph-debug +++ b/giraph-debugger/giraph-debug @@ -1,4 +1,19 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # giraph-debug -- a script for launching Giraph jar with our debugger # # To debug your Giraph computation, simply run: diff --git a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/css/app.css b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/css/app.css index cbaf5ad35..5dd5c087c 100644 --- a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/css/app.css +++ b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/css/app.css @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* * Style for debugger. */ diff --git a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/css/valpanel.css b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/css/valpanel.css index 1bd7da81d..a6935c442 100644 --- a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/css/valpanel.css +++ b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/css/valpanel.css @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ .valpanel-icons-container { height: 30px; width: 100%; diff --git a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/index.html b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/index.html index b58ec988e..a6e21a4d4 100644 --- a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/index.html +++ b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/index.html @@ -1,4 +1,21 @@ + diff --git a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/debugger.js b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/debugger.js index d67f7fceb..b1a9d343c 100644 --- a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/debugger.js +++ b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/debugger.js @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* * Abstracts the debugger controls. */ diff --git a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/editor.core.js b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/editor.core.js index 2536e3bb9..1bb182538 100644 --- a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/editor.core.js +++ b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/editor.core.js @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* * Graph Editor is based on Directed Graph Editor by rkirsling http://bl.ocks.org/rkirsling/5001347. */ diff --git a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/editor.utils.js b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/editor.utils.js index 863a8d983..fdbac044e 100644 --- a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/editor.utils.js +++ b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/editor.utils.js @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* * Sets the size of the graph editing window. * The graph is always centered in the container according to these dimensions. diff --git a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/utils.js b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/utils.js index a0b916751..08baf32fa 100644 --- a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/utils.js +++ b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/utils.js @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* * Utility functions used in other JS files. * Parts of this file are borrowed from others. A comment is placed on top in such cases. diff --git a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/utils.sampleGraphs.js b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/utils.sampleGraphs.js index 4c4d91b27..430ea5f11 100644 --- a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/utils.sampleGraphs.js +++ b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/utils.sampleGraphs.js @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* * Map of sample graphs and their corresponding init handlers. * Format is "" : function() { } diff --git a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/valpanel.js b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/valpanel.js index 98da75101..387de8cf4 100644 --- a/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/valpanel.js +++ b/giraph-debugger/src/main/resources/org/apache/giraph/debugger/gui/js/valpanel.js @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* * ValidationPanel is a class that abstracts the message, vertex * and exception details. It has three view modes - compact, preview and expanded. diff --git a/giraph-gora/pom.xml b/giraph-gora/pom.xml index 34a705800..811a04791 100644 --- a/giraph-gora/pom.xml +++ b/giraph-gora/pom.xml @@ -71,6 +71,13 @@ org.codehaus.mojo findbugs-maven-plugin + + org.apache.rat + apache-rat-plugin + + ${project.basedir}/.. + + diff --git a/pom.xml b/pom.xml index fdab2bb11..be50789c9 100644 --- a/pom.xml +++ b/pom.xml @@ -782,14 +782,18 @@ under the License. .idea/** **/*.iml **/*.ipr + **/*.json **/test/resources/** - giraph-gora/conf/* + giraph-gora/conf/** giraph-rexster/giraph-kibble/src/main/resources/META-INF/services/com.tinkerpop.rexster.extension.RexsterExtension - src/site/resources/images/*.svg + src/site/resources/images/*.svg + giraph-debugger/src/main/resources/org/apache/giraph/debugger/mock/*.vm + giraph-debugger/src/main/protobuf/*.proto + giraph-debugger/gui