From 63469259574a670e37e6b6d578090cb53ce17220 Mon Sep 17 00:00:00 2001 From: Amogh Margoor Date: Thu, 16 Jul 2015 00:22:37 -0400 Subject: [PATCH 1/8] Add map keytoPrepMaterialization to MaterializationService --- .../calcite/materialize/MaterializationActor.java | 3 +++ .../materialize/MaterializationService.java | 15 ++++++++++++--- .../calcite/prepare/CalcitePrepareImpl.java | 7 +++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java index ede840d88856..9b41fa454e1d 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java +++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java @@ -18,6 +18,7 @@ import org.apache.calcite.jdbc.CalciteRootSchema; import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.prepare.Prepare; import org.apache.calcite.rel.type.RelDataType; import com.google.common.collect.HashMultimap; @@ -36,6 +37,8 @@ class MaterializationActor { final Map keyMap = Maps.newHashMap(); + final Map keyPrepareMap = Maps.newHashMap(); + final Map keyBySql = Maps.newHashMap(); final Map keyByTile = Maps.newHashMap(); diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java index b594ccacfd75..e0687e813ee3 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java +++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java @@ -134,14 +134,19 @@ public MaterializationKey defineMaterialization(final CalciteSchema schema, rowType = parse.rowType; } final MaterializationKey key = new MaterializationKey(); + final MaterializationActor.Materialization materialization = new MaterializationActor.Materialization(key, schema.root(), tableEntry, viewSql, rowType); actor.keyMap.put(materialization.key, materialization); actor.keyBySql.put(queryKey, materialization.key); + actor.keyPrepareMap.put(materialization.key, + new Prepare.Materialization(materialization.materializedTable, + materialization.sql)); if (tileKey != null) { actor.keyByTile.put(tileKey, materialization.key); } + return key; } @@ -307,9 +312,13 @@ public List query(CalciteSchema rootSchema) { : actor.keyMap.values()) { if (materialization.rootSchema == rootSchema && materialization.materializedTable != null) { - list.add( - new Prepare.Materialization(materialization.materializedTable, - materialization.sql)); + Prepare.Materialization pm = actor.keyPrepareMap.get(materialization.key); + if (pm == null) { + pm = new Prepare.Materialization(materialization.materializedTable, + materialization.sql); + actor.keyPrepareMap.put(materialization.key, pm); + } + list.add(pm); } } return list; diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index d78f1600f258..19f0fb877eaa 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -831,6 +831,13 @@ protected void populateMaterializations(Context context, // REVIEW: initialize queryRel and tableRel inside MaterializationService, // not here? try { + // Optimization. If queryRel and tableRel are both not null + // then materialization has been populated + if (materialization.queryRel != null + && materialization.tableRel != null) { + return; + } + final CalciteSchema schema = materialization.materializedTable.schema; CalciteCatalogReader catalogReader = new CalciteCatalogReader( From 5134d8ac9b598d0c9947a2499d026197b5d255a7 Mon Sep 17 00:00:00 2001 From: Amogh Margoor Date: Mon, 20 Jul 2015 00:13:44 -0400 Subject: [PATCH 2/8] Prevent Materialization --- .../java/org/apache/calcite/prepare/CalcitePrepareImpl.java | 4 ++-- core/src/main/java/org/apache/calcite/prepare/Prepare.java | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index 19f0fb877eaa..b0fe79b26956 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -834,10 +834,10 @@ protected void populateMaterializations(Context context, // Optimization. If queryRel and tableRel are both not null // then materialization has been populated if (materialization.queryRel != null - && materialization.tableRel != null) { + && materialization.tableRel != null + && materialization.isStarTableIdentified()) { return; } - final CalciteSchema schema = materialization.materializedTable.schema; CalciteCatalogReader catalogReader = new CalciteCatalogReader( diff --git a/core/src/main/java/org/apache/calcite/prepare/Prepare.java b/core/src/main/java/org/apache/calcite/prepare/Prepare.java index 5892fa9ae593..338432713a6a 100644 --- a/core/src/main/java/org/apache/calcite/prepare/Prepare.java +++ b/core/src/main/java/org/apache/calcite/prepare/Prepare.java @@ -580,6 +580,10 @@ public void materialize(RelNode queryRel, this.starRelOptTable = starRelOptTable; assert starRelOptTable.unwrap(StarTable.class) != null; } + + public boolean isStarTableIdentified() { + return !(starRelOptTable == null); + } } } From b0aa390164d1fc3b4926c854e5a0946fa9f9944d Mon Sep 17 00:00:00 2001 From: Amogh Margoor Date: Mon, 20 Jul 2015 00:14:58 -0400 Subject: [PATCH 3/8] Revert "Prevent Materialization" This reverts commit 5134d8ac9b598d0c9947a2499d026197b5d255a7. --- .../java/org/apache/calcite/prepare/CalcitePrepareImpl.java | 4 ++-- core/src/main/java/org/apache/calcite/prepare/Prepare.java | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index b0fe79b26956..19f0fb877eaa 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -834,10 +834,10 @@ protected void populateMaterializations(Context context, // Optimization. If queryRel and tableRel are both not null // then materialization has been populated if (materialization.queryRel != null - && materialization.tableRel != null - && materialization.isStarTableIdentified()) { + && materialization.tableRel != null) { return; } + final CalciteSchema schema = materialization.materializedTable.schema; CalciteCatalogReader catalogReader = new CalciteCatalogReader( diff --git a/core/src/main/java/org/apache/calcite/prepare/Prepare.java b/core/src/main/java/org/apache/calcite/prepare/Prepare.java index 338432713a6a..5892fa9ae593 100644 --- a/core/src/main/java/org/apache/calcite/prepare/Prepare.java +++ b/core/src/main/java/org/apache/calcite/prepare/Prepare.java @@ -580,10 +580,6 @@ public void materialize(RelNode queryRel, this.starRelOptTable = starRelOptTable; assert starRelOptTable.unwrap(StarTable.class) != null; } - - public boolean isStarTableIdentified() { - return !(starRelOptTable == null); - } } } From 5fb8cad19f8bcd222191dd5ca59baca706a11766 Mon Sep 17 00:00:00 2001 From: Amogh Margoor Date: Mon, 20 Jul 2015 00:16:11 -0400 Subject: [PATCH 4/8] Revert "Add map keytoPrepMaterialization to MaterializationService" This reverts commit 63469259574a670e37e6b6d578090cb53ce17220. --- .../calcite/materialize/MaterializationActor.java | 3 --- .../materialize/MaterializationService.java | 15 +++------------ .../calcite/prepare/CalcitePrepareImpl.java | 7 ------- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java index 9b41fa454e1d..ede840d88856 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java +++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java @@ -18,7 +18,6 @@ import org.apache.calcite.jdbc.CalciteRootSchema; import org.apache.calcite.jdbc.CalciteSchema; -import org.apache.calcite.prepare.Prepare; import org.apache.calcite.rel.type.RelDataType; import com.google.common.collect.HashMultimap; @@ -37,8 +36,6 @@ class MaterializationActor { final Map keyMap = Maps.newHashMap(); - final Map keyPrepareMap = Maps.newHashMap(); - final Map keyBySql = Maps.newHashMap(); final Map keyByTile = Maps.newHashMap(); diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java index e0687e813ee3..b594ccacfd75 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java +++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java @@ -134,19 +134,14 @@ public MaterializationKey defineMaterialization(final CalciteSchema schema, rowType = parse.rowType; } final MaterializationKey key = new MaterializationKey(); - final MaterializationActor.Materialization materialization = new MaterializationActor.Materialization(key, schema.root(), tableEntry, viewSql, rowType); actor.keyMap.put(materialization.key, materialization); actor.keyBySql.put(queryKey, materialization.key); - actor.keyPrepareMap.put(materialization.key, - new Prepare.Materialization(materialization.materializedTable, - materialization.sql)); if (tileKey != null) { actor.keyByTile.put(tileKey, materialization.key); } - return key; } @@ -312,13 +307,9 @@ public List query(CalciteSchema rootSchema) { : actor.keyMap.values()) { if (materialization.rootSchema == rootSchema && materialization.materializedTable != null) { - Prepare.Materialization pm = actor.keyPrepareMap.get(materialization.key); - if (pm == null) { - pm = new Prepare.Materialization(materialization.materializedTable, - materialization.sql); - actor.keyPrepareMap.put(materialization.key, pm); - } - list.add(pm); + list.add( + new Prepare.Materialization(materialization.materializedTable, + materialization.sql)); } } return list; diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index 19f0fb877eaa..d78f1600f258 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -831,13 +831,6 @@ protected void populateMaterializations(Context context, // REVIEW: initialize queryRel and tableRel inside MaterializationService, // not here? try { - // Optimization. If queryRel and tableRel are both not null - // then materialization has been populated - if (materialization.queryRel != null - && materialization.tableRel != null) { - return; - } - final CalciteSchema schema = materialization.materializedTable.schema; CalciteCatalogReader catalogReader = new CalciteCatalogReader( From e08969ef16460adefd79b8f0fa51a7ee90a80350 Mon Sep 17 00:00:00 2001 From: Amogh Margoor Date: Mon, 20 Jul 2015 01:10:39 -0400 Subject: [PATCH 5/8] Revert "Revert "Add map keytoPrepMaterialization to MaterializationService"" This reverts commit 5fb8cad19f8bcd222191dd5ca59baca706a11766. --- .../calcite/materialize/MaterializationActor.java | 3 +++ .../materialize/MaterializationService.java | 15 ++++++++++++--- .../calcite/prepare/CalcitePrepareImpl.java | 7 +++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java index ede840d88856..9b41fa454e1d 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java +++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationActor.java @@ -18,6 +18,7 @@ import org.apache.calcite.jdbc.CalciteRootSchema; import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.prepare.Prepare; import org.apache.calcite.rel.type.RelDataType; import com.google.common.collect.HashMultimap; @@ -36,6 +37,8 @@ class MaterializationActor { final Map keyMap = Maps.newHashMap(); + final Map keyPrepareMap = Maps.newHashMap(); + final Map keyBySql = Maps.newHashMap(); final Map keyByTile = Maps.newHashMap(); diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java index b594ccacfd75..e0687e813ee3 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java +++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java @@ -134,14 +134,19 @@ public MaterializationKey defineMaterialization(final CalciteSchema schema, rowType = parse.rowType; } final MaterializationKey key = new MaterializationKey(); + final MaterializationActor.Materialization materialization = new MaterializationActor.Materialization(key, schema.root(), tableEntry, viewSql, rowType); actor.keyMap.put(materialization.key, materialization); actor.keyBySql.put(queryKey, materialization.key); + actor.keyPrepareMap.put(materialization.key, + new Prepare.Materialization(materialization.materializedTable, + materialization.sql)); if (tileKey != null) { actor.keyByTile.put(tileKey, materialization.key); } + return key; } @@ -307,9 +312,13 @@ public List query(CalciteSchema rootSchema) { : actor.keyMap.values()) { if (materialization.rootSchema == rootSchema && materialization.materializedTable != null) { - list.add( - new Prepare.Materialization(materialization.materializedTable, - materialization.sql)); + Prepare.Materialization pm = actor.keyPrepareMap.get(materialization.key); + if (pm == null) { + pm = new Prepare.Materialization(materialization.materializedTable, + materialization.sql); + actor.keyPrepareMap.put(materialization.key, pm); + } + list.add(pm); } } return list; diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index d78f1600f258..19f0fb877eaa 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -831,6 +831,13 @@ protected void populateMaterializations(Context context, // REVIEW: initialize queryRel and tableRel inside MaterializationService, // not here? try { + // Optimization. If queryRel and tableRel are both not null + // then materialization has been populated + if (materialization.queryRel != null + && materialization.tableRel != null) { + return; + } + final CalciteSchema schema = materialization.materializedTable.schema; CalciteCatalogReader catalogReader = new CalciteCatalogReader( From 04551d248d077c1b51e04e7bbfa97c70b805ee32 Mon Sep 17 00:00:00 2001 From: Amogh Margoor Date: Mon, 20 Jul 2015 01:11:20 -0400 Subject: [PATCH 6/8] Revert "Revert "Prevent Materialization"" This reverts commit b0aa390164d1fc3b4926c854e5a0946fa9f9944d. --- .../java/org/apache/calcite/prepare/CalcitePrepareImpl.java | 4 ++-- core/src/main/java/org/apache/calcite/prepare/Prepare.java | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java index 19f0fb877eaa..b0fe79b26956 100644 --- a/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java +++ b/core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java @@ -834,10 +834,10 @@ protected void populateMaterializations(Context context, // Optimization. If queryRel and tableRel are both not null // then materialization has been populated if (materialization.queryRel != null - && materialization.tableRel != null) { + && materialization.tableRel != null + && materialization.isStarTableIdentified()) { return; } - final CalciteSchema schema = materialization.materializedTable.schema; CalciteCatalogReader catalogReader = new CalciteCatalogReader( diff --git a/core/src/main/java/org/apache/calcite/prepare/Prepare.java b/core/src/main/java/org/apache/calcite/prepare/Prepare.java index 5892fa9ae593..338432713a6a 100644 --- a/core/src/main/java/org/apache/calcite/prepare/Prepare.java +++ b/core/src/main/java/org/apache/calcite/prepare/Prepare.java @@ -580,6 +580,10 @@ public void materialize(RelNode queryRel, this.starRelOptTable = starRelOptTable; assert starRelOptTable.unwrap(StarTable.class) != null; } + + public boolean isStarTableIdentified() { + return !(starRelOptTable == null); + } } } From 49cbf89409ad169dad050a85eba42993f65907d3 Mon Sep 17 00:00:00 2001 From: Amogh Margoor Date: Mon, 20 Jul 2015 01:57:20 -0400 Subject: [PATCH 7/8] Fixing bug in MaterializationService --- .../org/apache/calcite/materialize/MaterializationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java index e0687e813ee3..0942157022c6 100644 --- a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java +++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java @@ -65,7 +65,7 @@ public class MaterializationService { private static final ThreadLocal THREAD_INSTANCE = new ThreadLocal() { @Override protected MaterializationService initialValue() { - return new MaterializationService(); + return null; } }; From f2760f234732fa8a83c25a5b7aa97b0a77632fd1 Mon Sep 17 00:00:00 2001 From: Amogh Margoor Date: Mon, 20 Jul 2015 07:52:09 -0400 Subject: [PATCH 8/8] Disabling a test. Not a regression. Will be enabled when we get 1.4 --- .../test/java/org/apache/calcite/test/MaterializationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java index 52a167894153..bda6ebc3699d 100644 --- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java +++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java @@ -718,6 +718,7 @@ private void checkSatisfiable(RexNode e, String s) { checkNoMaterialize(q, q, JdbcTest.HR_MODEL); } + @Ignore("This is a bug in calcite which is fixed is latest master. Will get it when we sync 1.4") @Test public void testJoinMaterialization() { String q = "select *\n" + "from (select * from \"emps\" where \"empid\" < 300)\n"