From 7793f563f138bb9db3c7070c6b741aad558e4559 Mon Sep 17 00:00:00 2001 From: Amogh Margoor Date: Tue, 7 Jul 2015 08:33:40 -0400 Subject: [PATCH 1/3] fix:dev:NEZ-11 Adding UT for NEZ-11 --- .../org/apache/calcite/test/LatticeTest.java | 173 +++++++++++++++++- 1 file changed, 168 insertions(+), 5 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/LatticeTest.java b/core/src/test/java/org/apache/calcite/test/LatticeTest.java index 492b3775d7f9..eba48fb97699 100644 --- a/core/src/test/java/org/apache/calcite/test/LatticeTest.java +++ b/core/src/test/java/org/apache/calcite/test/LatticeTest.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.test; +import org.apache.calcite.materialize.Lattices; import org.apache.calcite.materialize.MaterializationService; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; @@ -42,12 +43,72 @@ import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; /** * Unit test for lattices. */ public class LatticeTest { + private static final String SALES_LATTICE = "{\n" + + " name: 'star',\n" + + " sql: [\n" + + " 'select 1 from \"foodmart\".\"sales_fact_1997\" as \"s\"',\n" + + " 'join \"foodmart\".\"product\" as \"p\" using (\"product_id\")',\n" + + " 'join \"foodmart\".\"time_by_day\" as \"t\" using (\"time_id\")',\n" + + " 'join \"foodmart\".\"product_class\" as \"pc\" on \"p\".\"product_class_id\" = \"pc\".\"product_class_id\"'\n" + + " ],\n" + + " auto: false,\n" + + " algorithm: true,\n" + + " algorithmMaxMillis: 10000,\n" + + " rowCountEstimate: 86837,\n" + + " defaultMeasures: [ {\n" + + " agg: 'count'\n" + + " } ],\n" + + " tiles: [ {\n" + + " dimensions: [ 'the_year', ['t', 'quarter'] ],\n" + + " measures: [ {\n" + + " agg: 'sum',\n" + + " args: 'unit_sales'\n" + + " }, {\n" + + " agg: 'sum',\n" + + " args: 'store_sales'\n" + + " }, {\n" + + " agg: 'count'\n" + + " } ]\n" + + " } ]\n" + + "}\n"; + + private static final String INVENTORY_LATTICE = "{\n" + + " name: 'warehouse',\n" + + " sql: [\n" + + " 'select 1 from \"foodmart\".\"inventory_fact_1997\" as \"s\"',\n" + + " 'join \"foodmart\".\"product\" as \"p\" using (\"product_id\")',\n" + + " 'join \"foodmart\".\"time_by_day\" as \"t\" using (\"time_id\")',\n" + + " 'join \"foodmart\".\"warehouse\" as \"w\" using (\"warehouse_id\")'\n" + + " ],\n" + + " auto: false,\n" + + " algorithm: true,\n" + + " algorithmMaxMillis: 10000,\n" + + " rowCountEstimate: 4070,\n" + + " defaultMeasures: [ {\n" + + " agg: 'count'\n" + + " } ],\n" + + " tiles: [ {\n" + + " dimensions: [ 'the_year', 'warehouse_name'],\n" + + " measures: [ {\n" + + " agg: 'sum',\n" + + " args: 'store_invoice'\n" + + " }, {\n" + + " agg: 'sum',\n" + + " args: 'supply_time'\n" + + " }, {\n" + + " agg: 'sum',\n" + + " args: 'warehouse_cost'\n" + + " } ]\n" + + " } ]\n" + + "}\n"; + private CalciteAssert.AssertThat modelWithLattice(String name, String sql, String... extras) { final StringBuilder buf = new StringBuilder("{ name: '") @@ -309,10 +370,25 @@ public Void apply(String materializationName) { * tiles. * *

Test case for - * CALCITE-428, - * "Use optimization algorithm to suggest which tiles of a lattice to - * materialize". */ + * [CALCITE-428] + * Use optimization algorithm to suggest which tiles of a lattice to + * materialize. */ @Test public void testTileAlgorithm() { + checkTileAlgorithm(FoodMartLatticeStatisticProvider.class.getCanonicalName(), + "EnumerableAggregate(group=[{2, 3}])\n" + + " EnumerableTableScan(table=[[adhoc, m{16, 17, 27, 31}]])"); + } + + @Test public void testTileAlgorithm2() { + // Different explain than above, but note that it still selects columns + // (27, 31). + checkTileAlgorithm(Lattices.class.getCanonicalName() + "#CACHED_SQL", + "EnumerableAggregate(group=[{0, 1}])\n" + + " EnumerableTableScan(table=[[adhoc, m{27, 31, 32, 36, 37}]"); + } + + private void checkTileAlgorithm(String statisticProvider, + String expectedExplain) { MaterializationService.setThreadLocal(); MaterializationService.instance().clear(); foodmartModel( @@ -329,6 +405,9 @@ public Void apply(String materializationName) { + " }, {\n" + " agg: 'count'\n" + " } ],\n" + + " statisticProvider: '" + + statisticProvider + + "',\n" + " tiles: [ {\n" + " dimensions: [ 'the_year', ['t', 'quarter'] ],\n" + " measures: [ ]\n" @@ -337,8 +416,7 @@ public Void apply(String materializationName) { + "from \"foodmart\".\"sales_fact_1997\" as s\n" + "join \"foodmart\".\"time_by_day\" as t using (\"time_id\")\n") .enableMaterializations(true) - .explainContains("EnumerableAggregate(group=[{2, 3}])\n" - + " EnumerableTableScan(table=[[adhoc, m{16, 17, 27, 31}]])") + .explainContains(expectedExplain) .returnsUnordered("the_year=1997; quarter=Q1", "the_year=1997; quarter=Q2", "the_year=1997; quarter=Q3", @@ -529,6 +607,91 @@ private void check(int n) throws IOException { // TODO } + @Test public void testTwoLattices() { + final AtomicInteger counter = new AtomicInteger(); + modelWithLattices(SALES_LATTICE, INVENTORY_LATTICE) + .query("select s.\"unit_sales\", p.\"brand_name\"\n" + + "from \"foodmart\".\"sales_fact_1997\" as s\n" + + "join \"foodmart\".\"product\" as p using (\"product_id\")\n") + .enableMaterializations(true) + .substitutionMatches( + CalciteAssert.checkRel( + "LogicalProject(unit_sales=[$7], brand_name=[$10])\n" + + " LogicalProject(product_id=[$0], time_id=[$1], customer_id=[$2], promotion_id=[$3], store_id=[$4], store_sales=[$5], store_cost=[$6], unit_sales=[$7], product_class_id=[$8], product_id0=[$9], brand_name=[$10], product_name=[$11], SKU=[$12], SRP=[$13], gross_weight=[$14], net_weight=[$15], recyclable_package=[$16], low_fat=[$17], units_per_case=[$18], cases_per_pallet=[$19], shelf_width=[$20], shelf_height=[$21], shelf_depth=[$22])\n" + + " LogicalTableScan(table=[[adhoc, star]])\n", + counter)); + assertThat(counter.intValue(), equalTo(1)); + } + /** Test case for + * [CALCITE-787] + * Wrong assignment of star table to Materialized views */ + @Test public void testOneLatticeOneMV() { + final AtomicInteger counter = new AtomicInteger(); + final Class clazz = + JdbcTest.EmpDeptTableFactory.class; + + final String mv = " materializations: [\n" + + " {\n" + + " table: \"m0\",\n" + + " view: \"m0v\",\n" + + " sql: \"select * from \\\"foodmart\\\".\\\"sales_fact_1997\\\" where \\\"product_id\\\" = 10\" " + + " }\n" + + " ]\n"; + + final String model = + "" + + "{\n" + + " version: '1.0',\n" + + " schemas: [\n" + + JdbcTest.FOODMART_SCHEMA + + ",\n" + + " {\n" + + " name: 'adhoc',\n" + + " tables: [\n" + + " {\n" + + " name: 'EMPLOYEES',\n" + + " type: 'custom',\n" + + " factory: '" + + clazz.getName() + + "',\n" + + " operand: {'foo': true, 'bar': 345}\n" + + " }\n" + + " ],\n" + + " lattices: " + "[" + INVENTORY_LATTICE + + " ]\n" + + " },\n" + + " {\n" + + " name: 'mat',\n" + + mv + + " }\n" + + " ]\n" + + "}"; + + CalciteAssert.model(model) + .withDefaultSchema("foodmart") + .query("select * from \"foodmart\".\"sales_fact_1997\" where \"product_id\" = 10") + .enableMaterializations(true) + .substitutionMatches( + CalciteAssert.checkRel( + "EnumerableTableScan(table=[[mat, m0]])\n", + counter)); + assertThat(counter.intValue(), equalTo(1)); + } + + /** Test case for + * [CALCITE-760] + * Aggregate recommender blows up if row count estimate is too high. */ + @Ignore + @Test public void testLatticeWithBadRowCountEstimate() { + final String lattice = + INVENTORY_LATTICE.replace("rowCountEstimate: 4070,", + "rowCountEstimate: 4074070,"); + assertFalse(lattice.equals(INVENTORY_LATTICE)); + modelWithLattices(lattice) + .query("values 1\n") + .returns("EXPR$0=1\n"); + } + private CalciteAssert.AssertThat foodmartModel(String... extras) { return modelWithLattice("star", "select 1 from \"foodmart\".\"sales_fact_1997\" as \"s\"\n" From 02600ec7bb7862e8e6ccfd8e949e0cbd56c84156 Mon Sep 17 00:00:00 2001 From: Amogh Margoor Date: Tue, 7 Jul 2015 08:34:47 -0400 Subject: [PATCH 2/3] Revert "fix:dev:NEZ-11 Adding UT for NEZ-11" This reverts commit 7793f563f138bb9db3c7070c6b741aad558e4559. --- .../org/apache/calcite/test/LatticeTest.java | 173 +----------------- 1 file changed, 5 insertions(+), 168 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/LatticeTest.java b/core/src/test/java/org/apache/calcite/test/LatticeTest.java index eba48fb97699..492b3775d7f9 100644 --- a/core/src/test/java/org/apache/calcite/test/LatticeTest.java +++ b/core/src/test/java/org/apache/calcite/test/LatticeTest.java @@ -16,7 +16,6 @@ */ package org.apache.calcite.test; -import org.apache.calcite.materialize.Lattices; import org.apache.calcite.materialize.MaterializationService; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; @@ -43,72 +42,12 @@ import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; /** * Unit test for lattices. */ public class LatticeTest { - private static final String SALES_LATTICE = "{\n" - + " name: 'star',\n" - + " sql: [\n" - + " 'select 1 from \"foodmart\".\"sales_fact_1997\" as \"s\"',\n" - + " 'join \"foodmart\".\"product\" as \"p\" using (\"product_id\")',\n" - + " 'join \"foodmart\".\"time_by_day\" as \"t\" using (\"time_id\")',\n" - + " 'join \"foodmart\".\"product_class\" as \"pc\" on \"p\".\"product_class_id\" = \"pc\".\"product_class_id\"'\n" - + " ],\n" - + " auto: false,\n" - + " algorithm: true,\n" - + " algorithmMaxMillis: 10000,\n" - + " rowCountEstimate: 86837,\n" - + " defaultMeasures: [ {\n" - + " agg: 'count'\n" - + " } ],\n" - + " tiles: [ {\n" - + " dimensions: [ 'the_year', ['t', 'quarter'] ],\n" - + " measures: [ {\n" - + " agg: 'sum',\n" - + " args: 'unit_sales'\n" - + " }, {\n" - + " agg: 'sum',\n" - + " args: 'store_sales'\n" - + " }, {\n" - + " agg: 'count'\n" - + " } ]\n" - + " } ]\n" - + "}\n"; - - private static final String INVENTORY_LATTICE = "{\n" - + " name: 'warehouse',\n" - + " sql: [\n" - + " 'select 1 from \"foodmart\".\"inventory_fact_1997\" as \"s\"',\n" - + " 'join \"foodmart\".\"product\" as \"p\" using (\"product_id\")',\n" - + " 'join \"foodmart\".\"time_by_day\" as \"t\" using (\"time_id\")',\n" - + " 'join \"foodmart\".\"warehouse\" as \"w\" using (\"warehouse_id\")'\n" - + " ],\n" - + " auto: false,\n" - + " algorithm: true,\n" - + " algorithmMaxMillis: 10000,\n" - + " rowCountEstimate: 4070,\n" - + " defaultMeasures: [ {\n" - + " agg: 'count'\n" - + " } ],\n" - + " tiles: [ {\n" - + " dimensions: [ 'the_year', 'warehouse_name'],\n" - + " measures: [ {\n" - + " agg: 'sum',\n" - + " args: 'store_invoice'\n" - + " }, {\n" - + " agg: 'sum',\n" - + " args: 'supply_time'\n" - + " }, {\n" - + " agg: 'sum',\n" - + " args: 'warehouse_cost'\n" - + " } ]\n" - + " } ]\n" - + "}\n"; - private CalciteAssert.AssertThat modelWithLattice(String name, String sql, String... extras) { final StringBuilder buf = new StringBuilder("{ name: '") @@ -370,25 +309,10 @@ public Void apply(String materializationName) { * tiles. * *

Test case for - * [CALCITE-428] - * Use optimization algorithm to suggest which tiles of a lattice to - * materialize. */ + * CALCITE-428, + * "Use optimization algorithm to suggest which tiles of a lattice to + * materialize". */ @Test public void testTileAlgorithm() { - checkTileAlgorithm(FoodMartLatticeStatisticProvider.class.getCanonicalName(), - "EnumerableAggregate(group=[{2, 3}])\n" - + " EnumerableTableScan(table=[[adhoc, m{16, 17, 27, 31}]])"); - } - - @Test public void testTileAlgorithm2() { - // Different explain than above, but note that it still selects columns - // (27, 31). - checkTileAlgorithm(Lattices.class.getCanonicalName() + "#CACHED_SQL", - "EnumerableAggregate(group=[{0, 1}])\n" - + " EnumerableTableScan(table=[[adhoc, m{27, 31, 32, 36, 37}]"); - } - - private void checkTileAlgorithm(String statisticProvider, - String expectedExplain) { MaterializationService.setThreadLocal(); MaterializationService.instance().clear(); foodmartModel( @@ -405,9 +329,6 @@ private void checkTileAlgorithm(String statisticProvider, + " }, {\n" + " agg: 'count'\n" + " } ],\n" - + " statisticProvider: '" - + statisticProvider - + "',\n" + " tiles: [ {\n" + " dimensions: [ 'the_year', ['t', 'quarter'] ],\n" + " measures: [ ]\n" @@ -416,7 +337,8 @@ private void checkTileAlgorithm(String statisticProvider, + "from \"foodmart\".\"sales_fact_1997\" as s\n" + "join \"foodmart\".\"time_by_day\" as t using (\"time_id\")\n") .enableMaterializations(true) - .explainContains(expectedExplain) + .explainContains("EnumerableAggregate(group=[{2, 3}])\n" + + " EnumerableTableScan(table=[[adhoc, m{16, 17, 27, 31}]])") .returnsUnordered("the_year=1997; quarter=Q1", "the_year=1997; quarter=Q2", "the_year=1997; quarter=Q3", @@ -607,91 +529,6 @@ private void check(int n) throws IOException { // TODO } - @Test public void testTwoLattices() { - final AtomicInteger counter = new AtomicInteger(); - modelWithLattices(SALES_LATTICE, INVENTORY_LATTICE) - .query("select s.\"unit_sales\", p.\"brand_name\"\n" - + "from \"foodmart\".\"sales_fact_1997\" as s\n" - + "join \"foodmart\".\"product\" as p using (\"product_id\")\n") - .enableMaterializations(true) - .substitutionMatches( - CalciteAssert.checkRel( - "LogicalProject(unit_sales=[$7], brand_name=[$10])\n" - + " LogicalProject(product_id=[$0], time_id=[$1], customer_id=[$2], promotion_id=[$3], store_id=[$4], store_sales=[$5], store_cost=[$6], unit_sales=[$7], product_class_id=[$8], product_id0=[$9], brand_name=[$10], product_name=[$11], SKU=[$12], SRP=[$13], gross_weight=[$14], net_weight=[$15], recyclable_package=[$16], low_fat=[$17], units_per_case=[$18], cases_per_pallet=[$19], shelf_width=[$20], shelf_height=[$21], shelf_depth=[$22])\n" - + " LogicalTableScan(table=[[adhoc, star]])\n", - counter)); - assertThat(counter.intValue(), equalTo(1)); - } - /** Test case for - * [CALCITE-787] - * Wrong assignment of star table to Materialized views */ - @Test public void testOneLatticeOneMV() { - final AtomicInteger counter = new AtomicInteger(); - final Class clazz = - JdbcTest.EmpDeptTableFactory.class; - - final String mv = " materializations: [\n" - + " {\n" - + " table: \"m0\",\n" - + " view: \"m0v\",\n" - + " sql: \"select * from \\\"foodmart\\\".\\\"sales_fact_1997\\\" where \\\"product_id\\\" = 10\" " - + " }\n" - + " ]\n"; - - final String model = - "" - + "{\n" - + " version: '1.0',\n" - + " schemas: [\n" - + JdbcTest.FOODMART_SCHEMA - + ",\n" - + " {\n" - + " name: 'adhoc',\n" - + " tables: [\n" - + " {\n" - + " name: 'EMPLOYEES',\n" - + " type: 'custom',\n" - + " factory: '" - + clazz.getName() - + "',\n" - + " operand: {'foo': true, 'bar': 345}\n" - + " }\n" - + " ],\n" - + " lattices: " + "[" + INVENTORY_LATTICE - + " ]\n" - + " },\n" - + " {\n" - + " name: 'mat',\n" - + mv - + " }\n" - + " ]\n" - + "}"; - - CalciteAssert.model(model) - .withDefaultSchema("foodmart") - .query("select * from \"foodmart\".\"sales_fact_1997\" where \"product_id\" = 10") - .enableMaterializations(true) - .substitutionMatches( - CalciteAssert.checkRel( - "EnumerableTableScan(table=[[mat, m0]])\n", - counter)); - assertThat(counter.intValue(), equalTo(1)); - } - - /** Test case for - * [CALCITE-760] - * Aggregate recommender blows up if row count estimate is too high. */ - @Ignore - @Test public void testLatticeWithBadRowCountEstimate() { - final String lattice = - INVENTORY_LATTICE.replace("rowCountEstimate: 4070,", - "rowCountEstimate: 4074070,"); - assertFalse(lattice.equals(INVENTORY_LATTICE)); - modelWithLattices(lattice) - .query("values 1\n") - .returns("EXPR$0=1\n"); - } - private CalciteAssert.AssertThat foodmartModel(String... extras) { return modelWithLattice("star", "select 1 from \"foodmart\".\"sales_fact_1997\" as \"s\"\n" From 19c7d2ea9354ec05f63c656d8fab585bd0e7edb6 Mon Sep 17 00:00:00 2001 From: Amogh Margoor Date: Tue, 7 Jul 2015 08:53:41 -0400 Subject: [PATCH 3/3] fix:dev:NEZ-11 Adding UT for NEZ-11 --- .../org/apache/calcite/test/LatticeTest.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/core/src/test/java/org/apache/calcite/test/LatticeTest.java b/core/src/test/java/org/apache/calcite/test/LatticeTest.java index 492b3775d7f9..0e79d12026ab 100644 --- a/core/src/test/java/org/apache/calcite/test/LatticeTest.java +++ b/core/src/test/java/org/apache/calcite/test/LatticeTest.java @@ -529,6 +529,62 @@ private void check(int n) throws IOException { // TODO } + /** Test case for + * [CALCITE-787] + * Wrong assignment of star table to Materialized views */ + @Test public void testOneLatticeOneMV() { + final AtomicInteger counter = new AtomicInteger(); + final Class clazz = + JdbcTest.EmpDeptTableFactory.class; + + final String mv = " materializations: [\n" + + " {\n" + + " table: \"m0\",\n" + + " view: \"m0v\",\n" + + " sql: \"select * from \\\"foodmart\\\".\\\"sales_fact_1997\\\" where \\\"product_id\\\" = 10\" " + + " }\n" + + " ]\n"; + + final String model = + "" + + "{\n" + + " version: '1.0',\n" + + " schemas: [\n" + + JdbcTest.FOODMART_SCHEMA + + ",\n" + + " {\n" + + " name: 'adhoc',\n" + + " tables: [\n" + + " {\n" + + " name: 'EMPLOYEES',\n" + + " type: 'custom',\n" + + " factory: '" + + clazz.getName() + + "',\n" + + " operand: {'foo': true, 'bar': 345}\n" + + " }\n" + + " ],\n" + + " lattices: " + "[" + INVENTORY_LATTICE + + " ]\n" + + " },\n" + + " {\n" + + " name: 'mat',\n" + + mv + + " }\n" + + " ]\n" + + "}"; + + CalciteAssert.model(model) + .withDefaultSchema("foodmart") + .query("select * from \"foodmart\".\"sales_fact_1997\" where \"product_id\" = 10") + .enableMaterializations(true) + .substitutionMatches( + CalciteAssert.checkRel( + "EnumerableTableScan(table=[[mat, m0]])\n", + counter)); + assertThat(counter.intValue(), equalTo(1)); + } + private CalciteAssert.AssertThat foodmartModel(String... extras) { return modelWithLattice("star", "select 1 from \"foodmart\".\"sales_fact_1997\" as \"s\"\n"