From 4a2c94312e2b8d596a8cf2c600a3441544b4f5cf Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 20 Apr 2017 19:51:27 +0530 Subject: [PATCH 01/71] CRUD Changes for Virtual Fact --- lens-api/src/main/resources/cube-0.1.xsd | 2 +- .../cube/metadata/CubeMetastoreClient.java | 163 ++++++++++++++++++ .../lens/cube/metadata/CubeTableType.java | 2 +- .../cube/metadata/CubeVirtualFactTable.java | 45 +++++ .../apache/lens/cube/metadata/JAXBUtils.java | 13 ++ .../api/metastore/CubeMetastoreService.java | 47 +++++ .../metastore/CubeMetastoreServiceImpl.java | 67 +++++++ .../server/metastore/MetastoreResource.java | 128 ++++++++++++++ 8 files changed, 465 insertions(+), 2 deletions(-) create mode 100644 lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 68ccc1342..0b570601d 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -182,7 +182,7 @@ - Properties that can be set for a virtual fact are + Properties that can be set for a virtual fact are : 1. cube.fact.query.where.filter : filter string that needs to be added in WHERE clause. This string would be added as an additional filter when the query is being constructed in the cube query writing phase. 2. cube.fact.absolute.start.time: start time of the fact. For queries that ask for time before this, diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index b5c4c89b3..1ac3c8c5a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -99,7 +99,10 @@ private CubeMetastoreClient(HiveConf conf) { private volatile boolean allDimTablesPopulated = false; // map from fact name to fact table private final Map allFactTables = Maps.newConcurrentMap(); + // map from virtual fact name to virtual fact table + private final Map allVirtualFactTables = Maps.newConcurrentMap(); private volatile boolean allFactTablesPopulated = false; + private volatile boolean allVirtualFactTablesPopulated = false; //map from segmentation name to segmentation private final Map allSegmentations = Maps.newConcurrentMap(); private volatile boolean allSegmentationPopulated = false; @@ -374,6 +377,18 @@ public void createCubeFactTable(XFactTable fact) throws LensException { JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables())); } + public void createVirtualFactTable(String cubeName, String factName, String sourceFactName, Double weight, Map properties) + throws LensException { + CubeFactTable sourceFact = getCubeFact(sourceFactName); + Map allProperties = sourceFact.getProperties(); + allProperties.putAll(properties); + + CubeVirtualFactTable factTable = new CubeVirtualFactTable(cubeName, factName, (weight != null ? weight: sourceFact.weight()) , allProperties, getCubeFact(sourceFactName)); + createCubeTable(factTable, null); + // do a get to update cache + getVirtualCubeFact(factName); + + } /** * In-memory storage of {@link PartitionTimeline} objects for each valid @@ -1778,6 +1793,18 @@ public CubeFactTable getFactTable(String tableName) throws LensException { return new CubeFactTable(getTableWithTypeFailFast(tableName, CubeTableType.FACT)); } + /** + * Get {@link CubeVirtualFactTable} object corresponding to the name + * + * @param tableName The virtual fact name + * @return Returns CubeVirtualFactTable if table name passed is a virtual fact table, null otherwise + * @throws LensException + */ + + public CubeVirtualFactTable getVirtualFactTable(String tableName) throws LensException { + return new CubeVirtualFactTable(getTableWithTypeFailFast(tableName, CubeTableType.VIRTUAL_FACT)); + } + public Segmentation getSegmentationTable(String tableName) throws HiveException, LensException { return new Segmentation(getTableWithTypeFailFast(tableName, CubeTableType.SEGMENTATION)); } @@ -1977,6 +2004,36 @@ private CubeFactTable getCubeFact(String tableName, boolean throwException) thro return fact; } + + /** + * Get {@link Cube} object corresponding to the name + * + * @param tableName The virtual fact name + * @return Returns the corresponding virtual fact table object + * @throws LensException if there is no fact by the name + */ + public CubeVirtualFactTable getVirtualCubeFact(String tableName) throws LensException { + return getVirtualCubeFact(tableName, true); + } + private CubeVirtualFactTable getVirtualCubeFact(String tableName, boolean throwException) throws LensException { + tableName = tableName.trim().toLowerCase(); + CubeVirtualFactTable fact = allVirtualFactTables.get(tableName); + if (fact == null) { + synchronized (allVirtualFactTables) { + if (!allVirtualFactTables.containsKey(tableName)) { + Table tbl = getTableWithType(tableName, CubeTableType.VIRTUAL_FACT, throwException); + fact = tbl == null ? null : new CubeVirtualFactTable(tbl); + if (enableCaching && fact != null) { + allVirtualFactTables.put(tableName, fact); + } + } else { + fact = allVirtualFactTables.get(tableName); + } + } + } + return fact; + } + public Segmentation getSegmentation(String segName) throws LensException { return getSegmentation(segName, true); } @@ -2143,6 +2200,34 @@ public Collection getAllFacts() throws LensException { } } + + /** + * Get all virtual facts in metastore + * + * @return List of Virtual Cube Fact Table objects + * @throws LensException + */ + public Collection getAllVirtualFacts() throws LensException { + if (!allVirtualFactTablesPopulated) { + List facts = new ArrayList<>(); + try { + for (String table : getAllHiveTableNames()) { + CubeVirtualFactTable fact = getVirtualCubeFact(table, false); + + if (fact != null) { + facts.add(fact); + } + } + } catch (HiveException e) { + throw new LensException("Could not get all virtual fact tables", e); + } + allVirtualFactTablesPopulated = enableCaching; + return facts; + } else { + return allVirtualFactTables.values(); + } + } + /** * Get all segmentations in metastore * @@ -2209,6 +2294,31 @@ public List getAllFacts(CubeInterface cube) throws LensException return cubeFacts; } + + /** + * Get all virtual fact tables of the cube. + * + * @param cube Cube object + * @return List of virtual fact tables + * @throws LensException + */ + public List getAllVirtualFacts(CubeInterface cube) throws LensException { + String cubeName = null; + if (cube != null) { + if (cube instanceof DerivedCube) { + cube = ((DerivedCube) cube).getParent(); + } + cubeName = cube.getName(); + } + List cubeFacts = new ArrayList<>(); + for (CubeVirtualFactTable fact : getAllVirtualFacts()) { + if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { + cubeFacts.add(fact); + } + } + return cubeFacts; + } + public List getAllSegmentations(CubeInterface cube) throws LensException { String cubeName = null; if (cube != null) { @@ -2315,6 +2425,24 @@ private boolean alterCubeTable(String table, Table hiveTable, AbstractCubeTable return columnsChanged; } + /** + * + * @param table table name + * @param hiveTable hive table + * @param cubeTable lens cube table + * @throws LensException + */ + private void alterCubeVirtualFactTable(String table, Table hiveTable, AbstractCubeTable cubeTable) throws LensException { + + hiveTable.getParameters().putAll(cubeTable.getProperties()); + hiveTable.getTTable().getParameters().putAll(cubeTable.getProperties()); + try { + getClient().alterTable(table, hiveTable, null); + } catch (Exception e) { + throw new LensException(e); + } + } + public void pushHiveTable(Table hiveTable) throws HiveException, LensException { alterHiveTable(hiveTable.getTableName(), hiveTable); } @@ -2441,6 +2569,17 @@ public void dropFact(String factName, boolean cascade) throws LensException { allFactTables.remove(factName.trim().toLowerCase()); } + /** + * Drop a virtual fact + * + * @param factName virtual fact name + * @throws LensException + */ + public void dropVirtualFact(String factName) throws LensException { + // getTableWithTypeFailFast(factName, CubeTableType.VIRTUAL_FACT); + dropHiveTable(factName); + allVirtualFactTables.remove(factName.trim().toLowerCase()); + } public void dropSegmentation(String segName) throws LensException { getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION); @@ -2565,6 +2704,23 @@ public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException { alterSegmentation(cubeSeg.getName(), segmentationFromXSegmentation(cubeSeg)); } + + /** + * Alter a virtual cube fact with new definition + * + * @param factTableName fact table name + * @param cubeVirtualFactTable cube virtual fact table + * + * @throws HiveException + */ + public void alterVirtualCubeFactTable(String factTableName, CubeVirtualFactTable cubeVirtualFactTable) + throws HiveException, LensException { + Table factTbl = getTableWithTypeFailFast(factTableName, CubeTableType.VIRTUAL_FACT); + + alterCubeVirtualFactTable(factTableName, factTbl, cubeVirtualFactTable); + updateVirtualFactCache(factTableName); + } + public void alterSegmentation(String segName, Segmentation seg) throws HiveException, LensException { getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION); @@ -2587,6 +2743,13 @@ private void updateFactCache(String factTableName) throws LensException { } } + + private void updateVirtualFactCache(String factTableName) throws LensException { + if (enableCaching) { + allVirtualFactTables.put(factTableName.trim().toLowerCase(), new CubeVirtualFactTable(refreshTable(factTableName))); + } + } + private void updateDimCache(String dimTblName) throws LensException { if (enableCaching) { allDimTables.put(dimTblName.trim().toLowerCase(), getDimensionTable(refreshTable(dimTblName))); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java index 6dd35b318..85075fba4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java @@ -20,5 +20,5 @@ package org.apache.lens.cube.metadata; public enum CubeTableType { - CUBE, DIMENSION, FACT, DIM_TABLE, STORAGE, SEGMENTATION + CUBE, DIMENSION, FACT, DIM_TABLE, STORAGE, SEGMENTATION, VIRTUAL_FACT } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java new file mode 100644 index 000000000..6544d539d --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata; + +import lombok.Getter; +import org.apache.hadoop.hive.ql.metadata.Table; + +import java.util.*; + +public class CubeVirtualFactTable extends CubeFactTable{ + + @Getter + private CubeFactTable sourceCubeFactTable; + + public CubeVirtualFactTable(Table hiveTable) { + super(hiveTable); + } + + public CubeVirtualFactTable(String cubeName, String factName, double weight, Map properties, CubeFactTable sourceFact) { + super(cubeName, factName, sourceFact.getColumns(), sourceFact.getUpdatePeriods(), weight, properties); + this.sourceCubeFactTable = sourceFact; + } + + @Override + public CubeTableType getTableType() { + return CubeTableType.VIRTUAL_FACT; + } + +} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java index e1e3d167e..a637654a3 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java @@ -749,6 +749,19 @@ public static XFactTable factTableFromCubeFactTable(CubeFactTable cFact) { return fact; } + + public static XVirtualFactTable virtualFactTableFromVirtualCubeFactTable(CubeVirtualFactTable vFact) { + XVirtualFactTable fact = XCF.createXVirtualFactTable(); + fact.setName(vFact.getName()); + fact.setProperties(new XProperties()); + + fact.getProperties().getProperty().addAll(xPropertiesFromMap(vFact.getProperties())); + + fact.setWeight(vFact.weight()); + fact.setCubeName(vFact.getCubeName()); + return fact; + } + public static XSegmentation xsegmentationFromSegmentation(Segmentation cSeg) { XSegmentation seg = XCF.createXSegmentation(); seg.setName(cSeg.getName()); diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java index 28b9d22e7..4bc1bb837 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java @@ -421,6 +421,53 @@ int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, St */ List getAllFactNames(LensSessionHandle sessionid, String cubeName) throws LensException; + /** + * Get virtual fact table given by name + * + * @param sessionid The sessionid + * @param fact The virtual fact table name + * @return {@link XFactTable} + * @throws LensException + */ + XVirtualFactTable getVirtualFactTable(LensSessionHandle sessionid, String fact) throws LensException; + + /** + * Create virtual fact table + * + * @param sessionid The sessionid + * @param fact The virtual fact table definition + * @throws LensException + */ + void createVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable fact) throws LensException; + + /** + * Update/Alter virtual fact table + * + * @param sessionid The sessionid + * @param fact The virtual fact table's new definition + * @throws LensException + */ + void updateVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable fact) throws LensException; + + /** + * Drop virtual fact table. + * + * @param sessionid The sessionid + * @param fact The virtual fact table name + * @throws LensException + */ + void dropVirtualFactTable(LensSessionHandle sessionid, String fact) throws LensException; + + /** + * Get all virtual fact names + * + * @param sessionid The sessionid + * @param cubeName optional filter filter facts by cube name. + * @return List of virtual fact table names + * @throws LensException + */ + List getAllVirtualFactNames(LensSessionHandle sessionid, String cubeName) throws LensException; + /** * Get all storages of fact * diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index 194b380cc..51e045710 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -441,6 +441,73 @@ public List getAllFactNames(LensSessionHandle sessionid, String cubeName } } + @Override + public XVirtualFactTable getVirtualFactTable(LensSessionHandle sessionid, String fact) throws LensException { + try (SessionContext ignored = new SessionContext(sessionid)) { + CubeMetastoreClient msClient = getClient(sessionid); + CubeVirtualFactTable cft = msClient.getVirtualFactTable(fact); + XVirtualFactTable factTable = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cft); + factTable.setSourceFactName(cft.getSourceCubeFactTable().getName()); + return factTable; + } + } + + @Override + public void createVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable fact) throws LensException { + try (SessionContext ignored = new SessionContext(sessionid)){ + getClient(sessionid).createVirtualFactTable(fact.getCubeName(), + fact.getName(), + fact.getSourceFactName(), + fact.getWeight(), + JAXBUtils.mapFromXProperties(fact.getProperties())); + log.info("Created virtual fact table " + fact.getName()); + } + } + + @Override + public void updateVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable fact) throws LensException { + try (SessionContext ignored = new SessionContext(sessionid)){ + CubeVirtualFactTable cubeVirtualFactTable = cubeVirtualFactFromFactTable(fact,getClient(sessionid).getCubeFact(fact.getSourceFactName())); + + getClient(sessionid).alterVirtualCubeFactTable(fact.getName(), cubeVirtualFactTable); + log.info("Updated virtual fact table " + fact.getName()); + } catch (HiveException e) { + throw new LensException(e); + } + } + + public static CubeVirtualFactTable cubeVirtualFactFromFactTable(XVirtualFactTable fact, CubeFactTable sourceFactTable) throws LensException { + //Update properties from source fact + Map allProperties = sourceFactTable.getProperties(); + allProperties.putAll(mapFromXProperties(fact.getProperties())); + return new CubeVirtualFactTable(fact.getCubeName(), fact.getName(), fact.getWeight()!=null? fact.getWeight():sourceFactTable.weight(), allProperties, sourceFactTable); + } + + @Override + public void dropVirtualFactTable(LensSessionHandle sessionid, String fact) throws LensException { + try (SessionContext ignored = new SessionContext(sessionid)){ + getClient(sessionid).dropVirtualFact(fact); + log.info("Dropped virtual fact table " + fact ); + } + } + + @Override + public List getAllVirtualFactNames(LensSessionHandle sessionid, String cubeName) throws LensException { + try (SessionContext ignored = new SessionContext(sessionid)){ + CubeMetastoreClient client = getClient(sessionid); + CubeInterface fact = client.getCube(cubeName); + if (cubeName != null && fact == null) { + throw new LensException("Could not get table: " + cubeName + " as a cube"); + } + Collection facts = client.getAllVirtualFacts(fact); + List factNames = new ArrayList<>(facts.size()); + for (CubeVirtualFactTable cft : facts) { + factNames.add(cft.getName()); + } + return factNames; + } + } + @Override public List getAllSegmentations(LensSessionHandle sessionid, String cubeName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index 7451aa908..b0317651c 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -180,6 +180,20 @@ public void doDelete(LensSessionHandle sessionid, String entityName, Boolean cas getSvc().dropDimensionTable(sessionid, entityName, cascade); } } + }, VIRTUALFACT { + @Override + public List doGetAll(LensSessionHandle sessionid) throws LensException { + return getSvc().getAllVirtualFactNames(sessionid, null); + } + + @Override + public void doDelete(LensSessionHandle sessionid, String entityName, Boolean cascade) throws LensException { + if (cascade == null) { + getSvc().dropVirtualFactTable(sessionid, entityName); + } else { + throw new NotImplementedException(); + } + } }; public abstract List doGetAll(LensSessionHandle sessionid) throws LensException; @@ -664,6 +678,22 @@ public StringList getAllDimensionTablesOfDimension( return new StringList(getSvc().getAllDimTableNames(sessionid, dimensionName)); } + /** + * Get all virtual facts that belong to a cube in the metastore + * + * @param sessionid The sessionid in which user is working + * @param cubeName name of the base cube or derived cube + * @return StringList consisting of all the virtual fact names in the given cube + */ + @GET + @Path("/cubes/{cubeName}/virtualfacts") + public StringList getAllVirtualFactsOfCube( + @QueryParam("sessionid") LensSessionHandle sessionid, @PathParam("cubeName") String cubeName) + throws LensException { + checkSessionId(sessionid); + return new StringList(getSvc().getAllVirtualFactNames(sessionid, cubeName)); + } + /** * Get all facts that belong to a cube in the metastore * @@ -710,6 +740,19 @@ public StringList getAllFacts(@QueryParam("sessionid") LensSessionHandle session return Entity.FACT.getAll(sessionid); } + /** + * Get all virtual fact tables in the metastore in the current database + * + * @param sessionid The sessionid in which user is working + * @return StringList consisting of all virtual fact table names + */ + @GET + @Path("/virtualfacts") + public StringList getAllVirtualFacts(@QueryParam("sessionid") LensSessionHandle sessionid) throws LensException { + checkSessionId(sessionid); + return Entity.VIRTUALFACT.getAll(sessionid); + } + /** * Get all segmentations in the current database @@ -742,6 +785,22 @@ public APIResult deleteAllFacts(@QueryParam("sessionid") LensSessionHandle sessi return Entity.FACT.deleteAll(sessionid, cascade); } + /** + * Delete all virtual fact tables + * + * @param sessionid The sessionid in which user is working + * @param cascade if set to true, all the underlying tables will be dropped, if set to false, only the virtual fact table + * will be dropped + * @return APIResult with state {@link Status#SUCCEEDED} in case of successful delete. APIResult with state {@link + * Status#FAILED} in case of delete failure. APIResult with state {@link Status#PARTIAL} in case of partial delete. + */ + @DELETE + @Path("virtualfacts") + public APIResult deleteAllVirtualFacts(@QueryParam("sessionid") LensSessionHandle sessionid, + @DefaultValue("false") @QueryParam("cascade") boolean cascade) throws LensException { + return Entity.VIRTUALFACT.deleteAll(sessionid, cascade); + } + /** * Delete all segmentations * @@ -771,6 +830,22 @@ public JAXBElement getFactTable(@QueryParam("sessionid") LensSession return X_CUBE_OBJECT_FACTORY.createXFactTable(getSvc().getFactTable(sessionid, factName)); } + /** + * Get the virtual fact table specified by name + * + * @param sessionid The sessionid in which user is working + * @param factName The virtual fact table name + * @return JAXB representation of {@link XFactTable} + */ + @GET + @Path("/virtualfacts/{virtualFactName}") + public JAXBElement getVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, + @PathParam("factName") String factName) + throws LensException { + checkSessionId(sessionid); + return X_CUBE_OBJECT_FACTORY.createXVirtualFactTable(getSvc().getVirtualFactTable(sessionid, factName)); + } + /** * Get the segmentation specified by name * @@ -805,6 +880,25 @@ public APIResult createFactTable(@QueryParam("sessionid") LensSessionHandle sess return success(); } + + /** + * Create a new virtual fact tabble + * + * @param sessionid The sessionid in which user is working + * @param fact The {@link XFactTable} representation of the fact table definition + * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if create was successful. {@link APIResult} with + * state {@link Status#FAILED}, if create has failed + */ + @POST + @Path("/virtualfacts") + public APIResult createVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, XVirtualFactTable fact) + throws LensException { + checkSessionId(sessionid); + log.info("Create virtual fact table"); + getSvc().createVirtualFactTable(sessionid, fact); + return success(); + } + /** * Create a new segmentation * @@ -824,6 +918,25 @@ public APIResult createSegmentation(@QueryParam("sessionid") LensSessionHandle s } + /** + * Update fact table definition + * + * @param sessionid The sessionid in which user is working + * @param factName name of the virtual fact table + * @param fact The {@link XFactTable} representation of the updated virtual fact table definition + * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful. {@link APIResult} with + * state {@link Status#FAILED}, if update has failed + */ + @PUT + @Path("/virtualfacts/{virtualFactName}") + public APIResult updateVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, + @PathParam("factName") String factName, XVirtualFactTable fact) + throws LensException { + checkSessionId(sessionid); + getSvc().updateVirtualFactTable(sessionid, fact); + return success(); + } + /** * Update fact table definition * @@ -880,6 +993,21 @@ public APIResult dropFactTable(@QueryParam("sessionid") LensSessionHandle sessio return Entity.FACT.delete(sessionid, factName, cascade); } + /** + * Drop the virtual fact table, specified by name + * + * @param sessionid The sessionid in which user is working + * @param factName The virtual fact table name + * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if drop was successful. {@link APIResult} with state + * {@link Status#FAILED}, if drop has failed + */ + @DELETE + @Path("/virtualfacts/{virtualFactName}") + public APIResult dropVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, + @PathParam("factName") String factName) + throws LensException { + return Entity.VIRTUALFACT.delete(sessionid, factName); + } /** * Drop the segmentation, specified by name From 0c326b296af846eb6155234c17ad70692416d4f6 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 27 Apr 2017 18:32:52 +0530 Subject: [PATCH 02/71] Test cases for CRUD Virtual Fact --- .../cube/metadata/CubeMetastoreClient.java | 71 ++++-- .../cube/metadata/CubeVirtualFactTable.java | 68 +++-- .../apache/lens/cube/metadata/JAXBUtils.java | 11 +- .../metadata/TestCubeMetastoreClient.java | 76 +++++- .../apache/lens/server/api/util/LensUtil.java | 1 + .../metastore/CubeMetastoreServiceImpl.java | 31 +-- .../server/metastore/MetastoreResource.java | 8 +- .../metastore/TestMetastoreService.java | 233 ++++++++++++++++++ 8 files changed, 439 insertions(+), 60 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 1ac3c8c5a..4de33019c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -101,6 +101,8 @@ private CubeMetastoreClient(HiveConf conf) { private final Map allFactTables = Maps.newConcurrentMap(); // map from virtual fact name to virtual fact table private final Map allVirtualFactTables = Maps.newConcurrentMap(); + // map from virtual fact name to source fact table + private final Map allVirtualFactSourceMappings = Maps.newConcurrentMap(); private volatile boolean allFactTablesPopulated = false; private volatile boolean allVirtualFactTablesPopulated = false; //map from segmentation name to segmentation @@ -377,16 +379,17 @@ public void createCubeFactTable(XFactTable fact) throws LensException { JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables())); } - public void createVirtualFactTable(String cubeName, String factName, String sourceFactName, Double weight, Map properties) - throws LensException { + public void createVirtualFactTable(String cubeName, String factName, String sourceFactName, Double weight, + Map properties) throws LensException { CubeFactTable sourceFact = getCubeFact(sourceFactName); - Map allProperties = sourceFact.getProperties(); + Map allProperties = sourceFact.getProperties(); allProperties.putAll(properties); - CubeVirtualFactTable factTable = new CubeVirtualFactTable(cubeName, factName, (weight != null ? weight: sourceFact.weight()) , allProperties, getCubeFact(sourceFactName)); + CubeVirtualFactTable factTable = new CubeVirtualFactTable(cubeName, factName, + (weight != null ? weight : sourceFact.weight()), allProperties, getCubeFact(sourceFactName)); createCubeTable(factTable, null); // do a get to update cache - getVirtualCubeFact(factName); + getVirtualCubeFact(factName, sourceFactName); } @@ -1662,6 +1665,29 @@ boolean isFactTableForCube(Table tbl, String cube) { .equalsIgnoreCase(cube.toLowerCase()); } + /** + * Is the table name passed a virtual fact table? + * + * @param tableName table name + * @return true if it is cube fact, false otherwise + * @throws HiveException + */ + public boolean isVirtualFactTable(String tableName) throws LensException { + Table tbl = getTable(tableName); + return isVirtualFactTable(tbl); + } + + boolean isVirtualFactTable(Table tbl) { + String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY); + return CubeTableType.VIRTUAL_FACT.name().equals(tableType); + } + + + boolean isVirtualFactTableForCube(Table tbl, String cube) { + return isVirtualFactTable(tbl) && CubeFactTable.getCubeName(tbl.getTableName(), tbl.getParameters()) + .equalsIgnoreCase(cube.toLowerCase()); + } + /** * Is the table name passed a dimension table? * @@ -1802,7 +1828,8 @@ public CubeFactTable getFactTable(String tableName) throws LensException { */ public CubeVirtualFactTable getVirtualFactTable(String tableName) throws LensException { - return new CubeVirtualFactTable(getTableWithTypeFailFast(tableName, CubeTableType.VIRTUAL_FACT)); + return new CubeVirtualFactTable(getTableWithTypeFailFast(tableName, CubeTableType.VIRTUAL_FACT), + getTableWithTypeFailFast(allVirtualFactSourceMappings.get(tableName.trim().toLowerCase()), CubeTableType.FACT)); } public Segmentation getSegmentationTable(String tableName) throws HiveException, LensException { @@ -2009,22 +2036,26 @@ private CubeFactTable getCubeFact(String tableName, boolean throwException) thro * Get {@link Cube} object corresponding to the name * * @param tableName The virtual fact name + * @param sourceFactName The source fact name * @return Returns the corresponding virtual fact table object * @throws LensException if there is no fact by the name */ - public CubeVirtualFactTable getVirtualCubeFact(String tableName) throws LensException { - return getVirtualCubeFact(tableName, true); + public CubeVirtualFactTable getVirtualCubeFact(String tableName, String sourceFactName) throws LensException { + return getVirtualCubeFact(tableName, sourceFactName, true); } - private CubeVirtualFactTable getVirtualCubeFact(String tableName, boolean throwException) throws LensException { + private CubeVirtualFactTable getVirtualCubeFact(String tableName, String sourceFactName, boolean throwException) + throws LensException { tableName = tableName.trim().toLowerCase(); CubeVirtualFactTable fact = allVirtualFactTables.get(tableName); if (fact == null) { synchronized (allVirtualFactTables) { if (!allVirtualFactTables.containsKey(tableName)) { Table tbl = getTableWithType(tableName, CubeTableType.VIRTUAL_FACT, throwException); - fact = tbl == null ? null : new CubeVirtualFactTable(tbl); + Table sourceTbl = getTableWithType(sourceFactName, CubeTableType.FACT, throwException); + fact = (tbl == null || sourceTbl == null) ? null : new CubeVirtualFactTable(tbl, sourceTbl); if (enableCaching && fact != null) { allVirtualFactTables.put(tableName, fact); + allVirtualFactSourceMappings.put(tableName, sourceFactName); } } else { fact = allVirtualFactTables.get(tableName); @@ -2212,8 +2243,11 @@ public Collection getAllVirtualFacts() throws LensExceptio List facts = new ArrayList<>(); try { for (String table : getAllHiveTableNames()) { - CubeVirtualFactTable fact = getVirtualCubeFact(table, false); - + String sourceFactName = allVirtualFactSourceMappings.get(table); + CubeVirtualFactTable fact = null; + if (null != table && null != sourceFactName) { + fact = getVirtualCubeFact(table, allVirtualFactSourceMappings.get(table), false); + } if (fact != null) { facts.add(fact); } @@ -2432,7 +2466,8 @@ private boolean alterCubeTable(String table, Table hiveTable, AbstractCubeTable * @param cubeTable lens cube table * @throws LensException */ - private void alterCubeVirtualFactTable(String table, Table hiveTable, AbstractCubeTable cubeTable) throws LensException { + private void alterCubeVirtualFactTable(String table, Table hiveTable, AbstractCubeTable cubeTable) throws + LensException { hiveTable.getParameters().putAll(cubeTable.getProperties()); hiveTable.getTTable().getParameters().putAll(cubeTable.getProperties()); @@ -2576,9 +2611,10 @@ public void dropFact(String factName, boolean cascade) throws LensException { * @throws LensException */ public void dropVirtualFact(String factName) throws LensException { - // getTableWithTypeFailFast(factName, CubeTableType.VIRTUAL_FACT); + getTableWithTypeFailFast(factName, CubeTableType.VIRTUAL_FACT); dropHiveTable(factName); allVirtualFactTables.remove(factName.trim().toLowerCase()); + allVirtualFactSourceMappings.remove(factName); } public void dropSegmentation(String segName) throws LensException { @@ -2710,11 +2746,10 @@ public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveE * * @param factTableName fact table name * @param cubeVirtualFactTable cube virtual fact table - * * @throws HiveException */ public void alterVirtualCubeFactTable(String factTableName, CubeVirtualFactTable cubeVirtualFactTable) - throws HiveException, LensException { + throws HiveException, LensException { Table factTbl = getTableWithTypeFailFast(factTableName, CubeTableType.VIRTUAL_FACT); alterCubeVirtualFactTable(factTableName, factTbl, cubeVirtualFactTable); @@ -2746,7 +2781,9 @@ private void updateFactCache(String factTableName) throws LensException { private void updateVirtualFactCache(String factTableName) throws LensException { if (enableCaching) { - allVirtualFactTables.put(factTableName.trim().toLowerCase(), new CubeVirtualFactTable(refreshTable(factTableName))); + allVirtualFactTables.put(factTableName.trim().toLowerCase(), + new CubeVirtualFactTable(refreshTable(factTableName), + getHiveTable(allVirtualFactSourceMappings.get(factTableName.trim().toLowerCase())))); } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index 6544d539d..8df40fc47 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -18,28 +18,58 @@ */ package org.apache.lens.cube.metadata; -import lombok.Getter; +import java.util.*; + import org.apache.hadoop.hive.ql.metadata.Table; -import java.util.*; +import lombok.Getter; + +public class CubeVirtualFactTable extends CubeFactTable { + + @Getter + private CubeFactTable sourceCubeFactTable; + + public CubeVirtualFactTable(Table hiveTable, Table sourceHiveTable) { + super(hiveTable); + this.sourceCubeFactTable = new CubeFactTable(sourceHiveTable); + } + + public CubeVirtualFactTable(String cubeName, String factName, double weight, Map properties, + CubeFactTable sourceFact) { + super(cubeName, factName, sourceFact.getColumns(), sourceFact.getUpdatePeriods(), weight, properties); + this.sourceCubeFactTable = sourceFact; + } + + public CubeVirtualFactTable(String cubeName, String factName, Map properties, + CubeFactTable sourceFact) { + super(cubeName, factName, sourceFact.getColumns(), sourceFact.getUpdatePeriods(), sourceFact.weight(), properties); + this.sourceCubeFactTable = sourceFact; + } -public class CubeVirtualFactTable extends CubeFactTable{ + public CubeVirtualFactTable(String cubeName, String factName, CubeFactTable sourceFact) { + super(cubeName, factName, sourceFact.getColumns(), sourceFact.getUpdatePeriods(), sourceFact.weight(), + new HashMap()); + this.sourceCubeFactTable = sourceFact; + } - @Getter - private CubeFactTable sourceCubeFactTable; + @Override + public CubeTableType getTableType() { + return CubeTableType.VIRTUAL_FACT; + } - public CubeVirtualFactTable(Table hiveTable) { - super(hiveTable); - } + @Override + public Set getStorages() { + return this.sourceCubeFactTable.getStorages(); + } - public CubeVirtualFactTable(String cubeName, String factName, double weight, Map properties, CubeFactTable sourceFact) { - super(cubeName, factName, sourceFact.getColumns(), sourceFact.getUpdatePeriods(), weight, properties); - this.sourceCubeFactTable = sourceFact; - } + @Override + public Map> getUpdatePeriods() { + return this.sourceCubeFactTable.getUpdatePeriods(); + } - @Override - public CubeTableType getTableType() { - return CubeTableType.VIRTUAL_FACT; - } + @Override + public String getDataCompletenessTag() { + return this.sourceCubeFactTable.getDataCompletenessTag(); + } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java index a637654a3..c3a5e186a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java @@ -718,6 +718,15 @@ public static CubeFactTable cubeFactFromFactTable(XFactTable fact) throws LensEx mapFromXProperties(fact.getProperties()), storageTablePrefixMap); } + public static CubeVirtualFactTable cubeVirtualFactFromFactTable(XVirtualFactTable fact, CubeFactTable sourceFactTable) + throws LensException { + //Update properties from source fact + Map allProperties = sourceFactTable.getProperties(); + allProperties.putAll(mapFromXProperties(fact.getProperties())); + return new CubeVirtualFactTable(fact.getCubeName(), fact.getName(), + fact.getWeight() != null ? fact.getWeight() : sourceFactTable.weight(), allProperties, sourceFactTable); + } + public static Segmentation segmentationFromXSegmentation(XSegmentation seg) throws LensException { Map props = new HashMap<>(); @@ -749,7 +758,6 @@ public static XFactTable factTableFromCubeFactTable(CubeFactTable cFact) { return fact; } - public static XVirtualFactTable virtualFactTableFromVirtualCubeFactTable(CubeVirtualFactTable vFact) { XVirtualFactTable fact = XCF.createXVirtualFactTable(); fact.setName(vFact.getName()); @@ -758,6 +766,7 @@ public static XVirtualFactTable virtualFactTableFromVirtualCubeFactTable(CubeVir fact.getProperties().getProperty().addAll(xPropertiesFromMap(vFact.getProperties())); fact.setWeight(vFact.weight()); + fact.setSourceFactName(vFact.getSourceCubeFactTable().getName()); fact.setCubeName(vFact.getCubeName()); return fact; } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 3a174990c..ab8380c33 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -135,8 +135,10 @@ private static HashMap getTimePartitionByOffsets(Object... args) { } @BeforeClass - public static void setup() throws HiveException, AlreadyExistsException, LensException { + public static void setup() throws HiveException, AlreadyExistsException, LensException, Exception { + Hive.get().dropDatabase(TestCubeMetastoreClient.class.getSimpleName(), true, true, true); SessionState.start(conf); + Database database = new Database(); database.setName(TestCubeMetastoreClient.class.getSimpleName()); Hive.get(conf).createDatabase(database); @@ -1224,6 +1226,78 @@ public void testCubeFact() throws Exception { assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); } + + @Test(priority = 2) + public void testVirtualCubeFact() throws Exception { + + String sourceFactName = "testMetastoreFact1"; + List factColumns = new ArrayList<>(cubeMeasures.size()); + for (CubeMeasure measure : cubeMeasures) { + factColumns.add(measure.getColumn()); + } + + // add one dimension of the cube + factColumns.add(new FieldSchema("zipcode", "int", "zip")); + + StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, + datePartSingleton, datePartKeySingleton); + + s1.getTblProps().put(MetastoreUtil.getStoragetableStartTimesKey(), "2015, now.day -10 days"); + s1.getTblProps().put(MetastoreUtil.getStoragetableEndTimesKey(), "now.day - 1 day"); + + Map> updatePeriods = getHashMap(c1, Sets.newHashSet(HOURLY, DAILY)); + Map storageTables = getHashMap(c1, s1); + + Map sourceFactPropertiesMap = getHashMap("name1", "value1", "name2", "value2"); + CubeFactTable cubeFact = new CubeFactTable(CUBE_NAME, sourceFactName, factColumns, updatePeriods, 10.0, + sourceFactPropertiesMap); + + // create cube fact + client.createCubeFactTable(CUBE_NAME, sourceFactName, factColumns, updatePeriods, 10.0, + sourceFactPropertiesMap, storageTables); + + String virtualFactName = "testMetastoreVirtualFact"; + + Map virtualFactPropertiesMap = getHashMap("name1", "newvalue1"); + + Map virtualFactAllPropertiesMap = cubeFact.getProperties(); + virtualFactAllPropertiesMap.putAll(virtualFactPropertiesMap); + CubeVirtualFactTable cubeVirtualFact = new CubeVirtualFactTable(CUBE_NAME, virtualFactName, + virtualFactAllPropertiesMap, cubeFact); + + // create virtual cube fact + client.createVirtualFactTable(CUBE_NAME, virtualFactName, sourceFactName, null, virtualFactPropertiesMap); + assertTrue(client.tableExists(virtualFactName)); + Table cubeTbl = client.getHiveTable(virtualFactName); + assertTrue(client.isVirtualFactTable(cubeTbl)); + assertTrue(client.isVirtualFactTableForCube(cubeTbl, CUBE_NAME)); + assertEquals(cubeVirtualFact.weight(), 10.0); + + //get virtual fact + assertEquals(client.getAllVirtualFacts(client.getCube(CUBE_NAME)).get(0).getName(), virtualFactName.toLowerCase()); + assertEquals(client.getAllVirtualFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(), + virtualFactName.toLowerCase()); + + CubeVirtualFactTable cubeFact2 = new CubeVirtualFactTable(cubeTbl, client.getHiveTable(sourceFactName)); + assertTrue(cubeVirtualFact.equals(cubeFact2)); + + //alter virtual fact + Map alterVirtualFactPropertiesMap = getHashMap("name1", "newvalue2", "name3", "value3"); + Map alterVirtualFactAllPropertiesMap = cubeVirtualFact.getProperties(); + alterVirtualFactAllPropertiesMap.putAll(alterVirtualFactPropertiesMap); + cubeVirtualFact = new CubeVirtualFactTable(CUBE_NAME, virtualFactName, alterVirtualFactPropertiesMap, cubeFact); + client.alterVirtualCubeFactTable(virtualFactName, cubeVirtualFact); + cubeFact2 = new CubeVirtualFactTable(cubeTbl, client.getHiveTable(sourceFactName)); + assertEquals(cubeFact2.getProperties().get("name1"), "newvalue2"); + assertEquals(cubeFact2.getProperties().get("name3"), "value3"); + + //drop virtual fact + client.dropVirtualFact(virtualFactName); + client.dropFact(sourceFactName, true); + + assertFalse(client.tableExists(virtualFactName)); + } + @Test(priority = 1) public void testSegmentation() throws Exception { String segmentName = "testMetastoreSegmentation"; diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java index 9d732c1c2..93bed16a2 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java @@ -21,6 +21,7 @@ import java.net.SocketException; import java.net.SocketTimeoutException; import java.util.HashMap; +import java.util.Map; import java.util.Set; import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory; diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index 51e045710..47b795c17 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -454,20 +454,21 @@ public XVirtualFactTable getVirtualFactTable(LensSessionHandle sessionid, String @Override public void createVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable fact) throws LensException { - try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).createVirtualFactTable(fact.getCubeName(), - fact.getName(), - fact.getSourceFactName(), - fact.getWeight(), - JAXBUtils.mapFromXProperties(fact.getProperties())); - log.info("Created virtual fact table " + fact.getName()); - } + try (SessionContext ignored = new SessionContext(sessionid)) { + getClient(sessionid).createVirtualFactTable(fact.getCubeName(), + fact.getName(), + fact.getSourceFactName(), + fact.getWeight(), + JAXBUtils.mapFromXProperties(fact.getProperties())); + log.info("Created virtual fact table " + fact.getName()); } + } @Override public void updateVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable fact) throws LensException { - try (SessionContext ignored = new SessionContext(sessionid)){ - CubeVirtualFactTable cubeVirtualFactTable = cubeVirtualFactFromFactTable(fact,getClient(sessionid).getCubeFact(fact.getSourceFactName())); + try (SessionContext ignored = new SessionContext(sessionid)) { + CubeVirtualFactTable cubeVirtualFactTable = cubeVirtualFactFromFactTable(fact, + getClient(sessionid).getCubeFact(fact.getSourceFactName())); getClient(sessionid).alterVirtualCubeFactTable(fact.getName(), cubeVirtualFactTable); log.info("Updated virtual fact table " + fact.getName()); @@ -476,18 +477,12 @@ public void updateVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTabl } } - public static CubeVirtualFactTable cubeVirtualFactFromFactTable(XVirtualFactTable fact, CubeFactTable sourceFactTable) throws LensException { - //Update properties from source fact - Map allProperties = sourceFactTable.getProperties(); - allProperties.putAll(mapFromXProperties(fact.getProperties())); - return new CubeVirtualFactTable(fact.getCubeName(), fact.getName(), fact.getWeight()!=null? fact.getWeight():sourceFactTable.weight(), allProperties, sourceFactTable); - } @Override public void dropVirtualFactTable(LensSessionHandle sessionid, String fact) throws LensException { - try (SessionContext ignored = new SessionContext(sessionid)){ + try (SessionContext ignored = new SessionContext(sessionid)) { getClient(sessionid).dropVirtualFact(fact); - log.info("Dropped virtual fact table " + fact ); + log.info("Dropped virtual fact table " + fact); } } diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index b0317651c..8dbf472d9 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -789,8 +789,8 @@ public APIResult deleteAllFacts(@QueryParam("sessionid") LensSessionHandle sessi * Delete all virtual fact tables * * @param sessionid The sessionid in which user is working - * @param cascade if set to true, all the underlying tables will be dropped, if set to false, only the virtual fact table - * will be dropped + * @param cascade if set to true, all the underlying tables will be dropped, if set to false, only the virtual fact + * table will be dropped * @return APIResult with state {@link Status#SUCCEEDED} in case of successful delete. APIResult with state {@link * Status#FAILED} in case of delete failure. APIResult with state {@link Status#PARTIAL} in case of partial delete. */ @@ -840,7 +840,7 @@ public JAXBElement getFactTable(@QueryParam("sessionid") LensSession @GET @Path("/virtualfacts/{virtualFactName}") public JAXBElement getVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, - @PathParam("factName") String factName) + @PathParam("virtualFactName") String factName) throws LensException { checkSessionId(sessionid); return X_CUBE_OBJECT_FACTORY.createXVirtualFactTable(getSvc().getVirtualFactTable(sessionid, factName)); @@ -1004,7 +1004,7 @@ public APIResult dropFactTable(@QueryParam("sessionid") LensSessionHandle sessio @DELETE @Path("/virtualfacts/{virtualFactName}") public APIResult dropVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, - @PathParam("factName") String factName) + @PathParam("virtualFactName") String factName) throws LensException { return Entity.VIRTUALFACT.delete(sessionid, factName); } diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index 6d32d23c1..4a20397a9 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -1856,6 +1856,24 @@ private XFactTable createFactTable(String factName, final String cubeName) { return f; } + + private XVirtualFactTable createVirtualFactTable(String factName, String sourceFactName) { + return createVirtualFactTable(factName, "testCube", sourceFactName); + } + + private XVirtualFactTable createVirtualFactTable(String factName, final String cubeName, String sourceFactName) { + XVirtualFactTable f = cubeObjectFactory.createXVirtualFactTable(); + f.setProperties(new XProperties()); + f.setName(factName); + f.setWeight(10.0); + f.setCubeName(cubeName); + f.setSourceFactName(sourceFactName); + + Map properties = LensUtil.getHashMap("foo", "bar"); + f.getProperties().getProperty().addAll(JAXBUtils.xPropertiesFromMap(properties)); + return f; + } + @Test(dataProvider = "mediaTypeData") public void testCreateFactTableWithMultipleUpdatePeriods(MediaType mediaType) throws Exception { @@ -2032,6 +2050,7 @@ public void testCreateFactTableWithMultipleUpdatePeriods(MediaType mediaType) th @Test(dataProvider = "mediaTypeData") public void testCreateFactTable(MediaType mediaType) throws Exception { + final String table = "testCreateFactTable"; final String DB = dbPFX + "testCreateFactTable_DB" + mediaType.getSubtype(); String prevDb = getCurrentDatabase(mediaType); @@ -2221,6 +2240,220 @@ public void testUpdateFactTable(MediaType mediaType) throws Exception { } } + @Test(dataProvider = "mediaTypeData") + public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { + final String table = "testCreateVirtualFactTable"; + final String sourceTable = "testCreateVirtualFactSourceTable"; + final String DB = dbPFX + "testCreateVirtualFactTable_DB" + mediaType.getSubtype(); + String prevDb = getCurrentDatabase(mediaType); + createDatabase(DB, mediaType); + setCurrentDatabase(DB, mediaType); + createStorage("S1", mediaType); + createStorage("S2", mediaType); + try { + + //first create source fact table + XFactTable source = createFactTable(sourceTable); + source.getStorageTables().getStorageTable().add(createStorageTblElement("S1", table, "HOURLY")); + source.getStorageTables().getStorageTable().add(createStorageTblElement("S2", table, "DAILY")); + + APIResult result = target() + .path("metastore") + .path("facts").queryParam("sessionid", lensSessionId) + .request(mediaType) + .post(Entity.entity( + new GenericEntity>(cubeObjectFactory.createXFactTable(source)) { + }, mediaType), + APIResult.class); + assertSuccess(result); + + //now create virtual fact table + XVirtualFactTable f = createVirtualFactTable(table, sourceTable); + + result = target() + .path("metastore") + .path("virtualfacts").queryParam("sessionid", lensSessionId) + .request(mediaType) + .post(Entity.entity( + new GenericEntity>(cubeObjectFactory.createXVirtualFactTable(f)) { + }, mediaType), + APIResult.class); + assertSuccess(result); + + // Get all virtual fact names, this should contain the virtual fact table + StringList virtualFactNames = target().path("metastore/virtualfacts") + .queryParam("sessionid", lensSessionId).request(mediaType).get(StringList.class); + assertTrue(virtualFactNames.getElements().contains(table.toLowerCase())); + + // Get the created table + JAXBElement gotFactElement = target().path("metastore/virtualfacts").path(table) + .queryParam("sessionid", lensSessionId).request(mediaType) + .get(new GenericType>() { + }); + XVirtualFactTable gotFact = gotFactElement.getValue(); + assertTrue(gotFact.getName().equalsIgnoreCase(table)); + assertEquals(gotFact.getWeight(), 10.0); + CubeVirtualFactTable cvf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, JAXBUtils.cubeFactFromFactTable(source)); + + // Check for a column + boolean foundC1 = false; + for (FieldSchema fs : cvf.getColumns()) { + if (fs.getName().equalsIgnoreCase("c1") && fs.getType().equalsIgnoreCase("string")) { + foundC1 = true; + break; + } + } + + //Check for column with start time + Map props = JAXBUtils.mapFromXProperties(gotFact.getProperties()); + assertEquals(props.get(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("c3")), "2016-01-01"); + assertEquals(props.get(MetastoreConstants.FACT_COL_END_TIME_PFX.concat("c3")), "2017-01-01"); + + assertTrue(foundC1); + assertEquals(cvf.getProperties().get("foo"), "bar"); + assertTrue(cvf.getStorages().contains("S1")); + assertTrue(cvf.getStorages().contains("S2")); + assertTrue(cvf.getUpdatePeriods().get("S1").contains(HOURLY)); + assertTrue(cvf.getUpdatePeriods().get("S2").contains(DAILY)); + + // drop the virtual fact table + result = target().path("metastore").path("virtualfacts").path(table) + .queryParam("sessionid", lensSessionId).request(mediaType) + .delete(APIResult.class); + + assertSuccess(result); + + // drop the source fact table + result = target().path("metastore").path("facts").path(sourceTable) + .queryParam("cascade", "true") + .queryParam("sessionid", lensSessionId).request(mediaType) + .delete(APIResult.class); + + assertSuccess(result); + + // Drop again, this time it should give a 404 + try { + target().path("metastore").path("virtualfacts").path(table) + .queryParam("sessionid", lensSessionId).request(mediaType) + .delete(APIResult.class); + fail("Expected 404"); + } catch (NotFoundException nfe) { + // PASS + } + } finally { + setCurrentDatabase(prevDb, mediaType); + dropDatabase(DB, mediaType); + } + } + + @Test(dataProvider = "mediaTypeData") + public void testUpdateVirtualFactTable(MediaType mediaType) throws Exception { + final String table = "testUpdateVirtualFactTable"; + final String sourceTable = "testCreateVirtualFactSourceTable"; + final String DB = dbPFX + "testUpdateFactTable_DB" + mediaType.getSubtype(); + String prevDb = getCurrentDatabase(mediaType); + createDatabase(DB, mediaType); + setCurrentDatabase(DB, mediaType); + createStorage("S1", mediaType); + createStorage("S2", mediaType); + createStorage("S3", mediaType); + try { + + //first create source fact table + XFactTable source = createFactTable(sourceTable); + source.getStorageTables().getStorageTable().add(createStorageTblElement("S1", table, "HOURLY")); + source.getStorageTables().getStorageTable().add(createStorageTblElement("S2", table, "DAILY")); + + APIResult result = target() + .path("metastore") + .path("facts").queryParam("sessionid", lensSessionId) + .request(mediaType) + .post(Entity.entity( + new GenericEntity>(cubeObjectFactory.createXFactTable(source)) { + }, mediaType), + APIResult.class); + assertSuccess(result); + + //now create virtual fact table + XVirtualFactTable f = createVirtualFactTable(table, sourceTable); + + result = target() + .path("metastore") + .path("virtualfacts").queryParam("sessionid", lensSessionId) + .request(mediaType) + .post(Entity.entity( + new GenericEntity>(cubeObjectFactory.createXVirtualFactTable(f)) { + }, mediaType), + APIResult.class); + assertSuccess(result); + + // Get all virtual fact names, this should contain the virtual fact table + StringList virtualFactNames = target().path("metastore/virtualfacts") + .queryParam("sessionid", lensSessionId).request(mediaType).get(StringList.class); + assertTrue(virtualFactNames.getElements().contains(table.toLowerCase())); + + // Get the created table + JAXBElement gotFactElement = target().path("metastore/virtualfacts").path(table) + .queryParam("sessionid", lensSessionId).request(mediaType) + .get(new GenericType>() { + }); + XVirtualFactTable gotFact = gotFactElement.getValue(); + assertTrue(gotFact.getName().equalsIgnoreCase(table)); + assertEquals(gotFact.getWeight(), 10.0); + CubeVirtualFactTable cvf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, JAXBUtils.cubeFactFromFactTable(source)); + + // Do some changes to test update + cvf.alterWeight(20.0); + + XVirtualFactTable update = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cvf); + // Update + result = target().path("metastore").path("virtualfacts").path(table) + .queryParam("sessionid", lensSessionId).request(mediaType) + .put(Entity.entity(new GenericEntity>(cubeObjectFactory.createXVirtualFactTable(update)) {}, + mediaType), APIResult.class); + assertSuccess(result); + + // Get the updated table + gotFactElement = target().path("metastore/virtualfacts").path(table) + .queryParam("sessionid", lensSessionId).request(mediaType) + .get(new GenericType>() { + }); + gotFact = gotFactElement.getValue(); + CubeVirtualFactTable ucf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, JAXBUtils.cubeFactFromFactTable(source)); + + assertEquals(ucf.weight(), 20.0); + + // drop the virtual fact table + result = target().path("metastore").path("virtualfacts").path(table) + .queryParam("sessionid", lensSessionId).request(mediaType) + .delete(APIResult.class); + + assertSuccess(result); + + // drop the source fact table + result = target().path("metastore").path("facts").path(sourceTable) + .queryParam("cascade", "true") + .queryParam("sessionid", lensSessionId).request(mediaType) + .delete(APIResult.class); + + assertSuccess(result); + + // Drop again, this time it should give a 404 + try { + target().path("metastore").path("virtualfacts").path(table) + .queryParam("sessionid", lensSessionId).request(mediaType) + .delete(APIResult.class); + fail("Expected 404"); + } catch (NotFoundException nfe) { + // PASS + } + } finally { + setCurrentDatabase(prevDb, mediaType); + dropDatabase(DB, mediaType); + } + } + + @Test(dataProvider = "mediaTypeData") public void testFactStorages(MediaType mediaType) throws Exception { final String table = "testFactStorages"; From 93793d104d0a882f65d404df504c0fee91a06c53 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 27 Apr 2017 18:35:21 +0530 Subject: [PATCH 03/71] Removing unused import --- .../src/main/java/org/apache/lens/server/api/util/LensUtil.java | 1 - 1 file changed, 1 deletion(-) diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java index 93bed16a2..9d732c1c2 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java @@ -21,7 +21,6 @@ import java.net.SocketException; import java.net.SocketTimeoutException; import java.util.HashMap; -import java.util.Map; import java.util.Set; import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory; From f764fbe8715dc7f512d53ed7a35c59b6b14a9918 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 27 Apr 2017 19:04:04 +0530 Subject: [PATCH 04/71] Checkstyle correction --- .../lens/cube/metadata/CubeMetastoreClient.java | 4 ++-- .../server/metastore/TestMetastoreService.java | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 4de33019c..b40b0910b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -2744,8 +2744,8 @@ public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveE /** * Alter a virtual cube fact with new definition * - * @param factTableName fact table name - * @param cubeVirtualFactTable cube virtual fact table + * @param factTableName virtual fact table name + * @param cubeVirtualFactTable cube virtual fact table * @throws HiveException */ public void alterVirtualCubeFactTable(String factTableName, CubeVirtualFactTable cubeVirtualFactTable) diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index 4a20397a9..a512e0e3a 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -2293,7 +2293,8 @@ public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { XVirtualFactTable gotFact = gotFactElement.getValue(); assertTrue(gotFact.getName().equalsIgnoreCase(table)); assertEquals(gotFact.getWeight(), 10.0); - CubeVirtualFactTable cvf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, JAXBUtils.cubeFactFromFactTable(source)); + CubeVirtualFactTable cvf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, + JAXBUtils.cubeFactFromFactTable(source)); // Check for a column boolean foundC1 = false; @@ -2400,7 +2401,8 @@ public void testUpdateVirtualFactTable(MediaType mediaType) throws Exception { XVirtualFactTable gotFact = gotFactElement.getValue(); assertTrue(gotFact.getName().equalsIgnoreCase(table)); assertEquals(gotFact.getWeight(), 10.0); - CubeVirtualFactTable cvf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, JAXBUtils.cubeFactFromFactTable(source)); + CubeVirtualFactTable cvf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, + JAXBUtils.cubeFactFromFactTable(source)); // Do some changes to test update cvf.alterWeight(20.0); @@ -2409,17 +2411,17 @@ public void testUpdateVirtualFactTable(MediaType mediaType) throws Exception { // Update result = target().path("metastore").path("virtualfacts").path(table) .queryParam("sessionid", lensSessionId).request(mediaType) - .put(Entity.entity(new GenericEntity>(cubeObjectFactory.createXVirtualFactTable(update)) {}, - mediaType), APIResult.class); + .put(Entity.entity(new GenericEntity>(cubeObjectFactory + .createXVirtualFactTable(update)) {}, mediaType), APIResult.class); assertSuccess(result); // Get the updated table gotFactElement = target().path("metastore/virtualfacts").path(table) .queryParam("sessionid", lensSessionId).request(mediaType) - .get(new GenericType>() { - }); + .get(new GenericType>() {}); gotFact = gotFactElement.getValue(); - CubeVirtualFactTable ucf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, JAXBUtils.cubeFactFromFactTable(source)); + CubeVirtualFactTable ucf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, + JAXBUtils.cubeFactFromFactTable(source)); assertEquals(ucf.weight(), 20.0); From e8a4a77d001035225c2384d46c70a597866f03ef Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 27 Apr 2017 19:05:43 +0530 Subject: [PATCH 05/71] checkstyle fix --- .../org/apache/lens/cube/metadata/TestCubeMetastoreClient.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index ab8380c33..edf92bca7 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -135,8 +135,7 @@ private static HashMap getTimePartitionByOffsets(Object... args) { } @BeforeClass - public static void setup() throws HiveException, AlreadyExistsException, LensException, Exception { - Hive.get().dropDatabase(TestCubeMetastoreClient.class.getSimpleName(), true, true, true); + public static void setup() throws HiveException, AlreadyExistsException, LensException { SessionState.start(conf); Database database = new Database(); From 2142f787dd5ac5873b89c5a93156cf52bb940c7a Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 3 May 2017 18:50:26 +0530 Subject: [PATCH 06/71] Changes post comments --- .../lens/cube/metadata/AbstractCubeTable.java | 9 ++ .../lens/cube/metadata/CubeFactTable.java | 12 +- .../cube/metadata/CubeMetastoreClient.java | 127 +++++++++--------- .../cube/metadata/CubeVirtualFactTable.java | 44 +++++- .../cube/metadata/FactTableInterface.java | 38 ++++++ .../cube/metadata/MetastoreConstants.java | 1 + .../lens/cube/metadata/Segmentation.java | 1 + .../lens/cube/parse/CandidateTable.java | 1 + .../cube/parse/CandidateTableResolver.java | 17 ++- .../apache/lens/cube/parse/CandidateUtil.java | 2 +- .../lens/cube/parse/CubeQueryContext.java | 2 + .../lens/cube/parse/QueriedPhraseContext.java | 15 ++- .../lens/cube/parse/StorageCandidate.java | 22 ++- .../metadata/TestCubeMetastoreClient.java | 12 +- .../lens/cube/parse/TestCubeRewriter.java | 28 ++++ .../resources/schema/facts/virtualfact.xml | 27 ++++ .../api/metastore/CubeMetastoreService.java | 12 +- .../metastore/CubeMetastoreServiceImpl.java | 26 ++-- .../server/metastore/MetastoreResource.java | 16 +-- 19 files changed, 275 insertions(+), 137 deletions(-) create mode 100644 lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTableInterface.java create mode 100644 lens-cube/src/test/resources/schema/facts/virtualfact.xml diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java index 67aaff80c..f2a322d16 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java @@ -20,6 +20,7 @@ import java.util.*; +import org.apache.lens.cube.parse.StorageCandidate; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; @@ -225,4 +226,12 @@ public Date now() { return new Date(); } + + public static String getFactCubeName(String factName, Map props) { + return props.get(MetastoreUtil.getFactCubeNameKey(factName)); + } + + protected static void addCubeNames(String factName, Map props, String cubeName) { + props.put(MetastoreUtil.getFactCubeNameKey(factName), cubeName); + } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index e00122db0..8640cfa35 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -33,7 +33,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class CubeFactTable extends AbstractCubeTable { +public class CubeFactTable extends AbstractCubeTable implements FactTableInterface{ @Getter // Map> private final Map> storagePrefixUpdatePeriodMap; @@ -43,7 +43,7 @@ public class CubeFactTable extends AbstractCubeTable { public CubeFactTable(Table hiveTable) { super(hiveTable); this.storageUpdatePeriods = getUpdatePeriods(getName(), getProperties()); - this.cubeName = getCubeName(getName(), getProperties()); + this.cubeName = getFactCubeName(getName(), getProperties()); this.storagePrefixUpdatePeriodMap = getUpdatePeriodMap(getName(), getProperties()); } @@ -105,10 +105,6 @@ private static void addUpdatePeriodProperies(String name, Map pr } } - private static void addCubeNames(String factName, Map props, String cubeName) { - props.put(MetastoreUtil.getFactCubeNameKey(factName), cubeName); - } - private Map> getUpdatePeriodMap(String factName, Map props) { Map> ret = new HashMap<>(); for (Map.Entry> entry : storageUpdatePeriods.entrySet()) { @@ -146,10 +142,6 @@ private Map> getUpdatePeriods(String name, Map props) { - return props.get(MetastoreUtil.getFactCubeNameKey(factName)); - } - public Map> getUpdatePeriods() { return storageUpdatePeriods; } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index b40b0910b..fd51219ec 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -28,17 +28,7 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; -import org.apache.lens.api.metastore.XCube; -import org.apache.lens.api.metastore.XDerivedCube; -import org.apache.lens.api.metastore.XDimension; -import org.apache.lens.api.metastore.XDimensionTable; -import org.apache.lens.api.metastore.XFactTable; -import org.apache.lens.api.metastore.XSegmentation; -import org.apache.lens.api.metastore.XStorage; -import org.apache.lens.api.metastore.XStorageTableElement; -import org.apache.lens.api.metastore.XUpdatePeriod; -import org.apache.lens.api.metastore.XUpdatePeriodTableDescriptor; -import org.apache.lens.api.metastore.XUpdatePeriods; +import org.apache.lens.api.metastore.*; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.Storage.LatestInfo; import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo; @@ -138,7 +128,7 @@ public boolean isDataCompletenessCheckEnabled() { } /** extract storage name from fact and storage table name. String operation */ - private String extractStorageName(CubeFactTable fact, String storageTableName) throws LensException { + private String extractStorageName(FactTableInterface fact, String storageTableName) throws LensException { int ind = storageTableName.lastIndexOf(fact.getName()); if (ind <= 0) { throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); @@ -169,7 +159,7 @@ public Date getLatestDateOfCube(Cube cube, String timeDimension) throws HiveExce String partCol = cube.getPartitionColumnOfTimeDim(timeDimension); Date max = new Date(Long.MIN_VALUE); boolean updated = false; - for (CubeFactTable fact : getAllFacts(cube)) { + for (FactTableInterface fact : getAllFacts(cube)) { for (String storage : fact.getStorages()) { for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) { PartitionTimeline timeline = partitionTimelineCache.get(fact.getName(), storage, updatePeriod, partCol); @@ -337,6 +327,8 @@ public void createEntity(T entity) thro createCubeDimensionTable((XDimensionTable) entity); } else if (entity instanceof XSegmentation) { createSegmentation((XSegmentation) entity); + }else if (entity instanceof XVirtualFactTable) { + createVirtualFactTable((XVirtualFactTable) entity); } else { throw new LensException("Unable to create entity " + entity + " as it's unrecognizable: "+ entity.getClass()); } @@ -356,6 +348,8 @@ public void updateEntity(String name, T alterCubeDimensionTable((XDimensionTable) entity); } else if (entity instanceof XSegmentation) { alterSegmentation((XSegmentation) entity); + } else if (entity instanceof XVirtualFactTable) { + alterVirtualCubeFactTable((XVirtualFactTable) entity); } else { throw new LensException("Unable to alter entity " + entity + " as it's unrecognizable: " + entity.getClass()); } @@ -379,17 +373,20 @@ public void createCubeFactTable(XFactTable fact) throws LensException { JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables())); } - public void createVirtualFactTable(String cubeName, String factName, String sourceFactName, Double weight, + public void createVirtualFactTable(XVirtualFactTable virtualFact) throws LensException { + createVirtualFactTable(virtualFact.getCubeName(), virtualFact.getName(), virtualFact.getSourceFactName(), + virtualFact.getWeight(), JAXBUtils.mapFromXProperties(virtualFact.getProperties())); + } + + public void createVirtualFactTable(String cubeName, String virtualFactName, String sourceFactName, Double weight, Map properties) throws LensException { CubeFactTable sourceFact = getCubeFact(sourceFactName); - Map allProperties = sourceFact.getProperties(); - allProperties.putAll(properties); - CubeVirtualFactTable factTable = new CubeVirtualFactTable(cubeName, factName, - (weight != null ? weight : sourceFact.weight()), allProperties, getCubeFact(sourceFactName)); + CubeVirtualFactTable factTable = new CubeVirtualFactTable(cubeName, virtualFactName, + (weight != null ? weight : sourceFact.weight()), properties, getCubeFact(sourceFactName)); createCubeTable(factTable, null); // do a get to update cache - getVirtualCubeFact(factName, sourceFactName); + getVirtualCubeFact(virtualFactName, sourceFactName); } @@ -1426,7 +1423,7 @@ public boolean tableExists(String tblName) throws HiveException { } /** extract storage name and check in timeline cache for existance */ - public boolean factPartitionExists(CubeFactTable fact, FactPartition part, String storageTableName) + public boolean factPartitionExists(FactTableInterface fact, FactPartition part, String storageTableName) throws HiveException, LensException { String storage = extractStorageName(fact, storageTableName); return partitionTimelineCache.partitionTimeExists(fact.getName(), storage, part.getPeriod(), part.getPartCol(), @@ -1661,19 +1658,19 @@ boolean isFactTable(Table tbl) { boolean isFactTableForCube(Table tbl, String cube) { - return isFactTable(tbl) && CubeFactTable.getCubeName(tbl.getTableName(), tbl.getParameters()) + return isFactTable(tbl) && CubeFactTable.getFactCubeName(tbl.getTableName(), tbl.getParameters()) .equalsIgnoreCase(cube.toLowerCase()); } /** * Is the table name passed a virtual fact table? * - * @param tableName table name + * @param virtualTableName table name * @return true if it is cube fact, false otherwise * @throws HiveException */ - public boolean isVirtualFactTable(String tableName) throws LensException { - Table tbl = getTable(tableName); + public boolean isVirtualFactTable(String virtualTableName) throws LensException { + Table tbl = getTable(virtualTableName); return isVirtualFactTable(tbl); } @@ -1684,7 +1681,7 @@ boolean isVirtualFactTable(Table tbl) { boolean isVirtualFactTableForCube(Table tbl, String cube) { - return isVirtualFactTable(tbl) && CubeFactTable.getCubeName(tbl.getTableName(), tbl.getParameters()) + return isVirtualFactTable(tbl) && CubeFactTable.getFactCubeName(tbl.getTableName(), tbl.getParameters()) .equalsIgnoreCase(cube.toLowerCase()); } @@ -1822,14 +1819,14 @@ public CubeFactTable getFactTable(String tableName) throws LensException { /** * Get {@link CubeVirtualFactTable} object corresponding to the name * - * @param tableName The virtual fact name + * @param virtualTableName The virtual fact name * @return Returns CubeVirtualFactTable if table name passed is a virtual fact table, null otherwise * @throws LensException */ - public CubeVirtualFactTable getVirtualFactTable(String tableName) throws LensException { - return new CubeVirtualFactTable(getTableWithTypeFailFast(tableName, CubeTableType.VIRTUAL_FACT), - getTableWithTypeFailFast(allVirtualFactSourceMappings.get(tableName.trim().toLowerCase()), CubeTableType.FACT)); + public CubeVirtualFactTable getVirtualFactTable(String virtualTableName) throws LensException { + return new CubeVirtualFactTable(getTableWithTypeFailFast(virtualTableName, CubeTableType.VIRTUAL_FACT), + getTableWithTypeFailFast(allVirtualFactSourceMappings.get(virtualTableName.trim().toLowerCase()), CubeTableType.FACT)); } public Segmentation getSegmentationTable(String tableName) throws HiveException, LensException { @@ -2035,30 +2032,30 @@ private CubeFactTable getCubeFact(String tableName, boolean throwException) thro /** * Get {@link Cube} object corresponding to the name * - * @param tableName The virtual fact name + * @param virtualTableName The virtual fact name * @param sourceFactName The source fact name * @return Returns the corresponding virtual fact table object * @throws LensException if there is no fact by the name */ - public CubeVirtualFactTable getVirtualCubeFact(String tableName, String sourceFactName) throws LensException { - return getVirtualCubeFact(tableName, sourceFactName, true); + public CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, String sourceFactName) throws LensException { + return getVirtualCubeFact(virtualTableName, sourceFactName, true); } - private CubeVirtualFactTable getVirtualCubeFact(String tableName, String sourceFactName, boolean throwException) + private CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, String sourceFactName, boolean throwException) throws LensException { - tableName = tableName.trim().toLowerCase(); - CubeVirtualFactTable fact = allVirtualFactTables.get(tableName); + virtualTableName = virtualTableName.trim().toLowerCase(); + CubeVirtualFactTable fact = allVirtualFactTables.get(virtualTableName); if (fact == null) { synchronized (allVirtualFactTables) { - if (!allVirtualFactTables.containsKey(tableName)) { - Table tbl = getTableWithType(tableName, CubeTableType.VIRTUAL_FACT, throwException); + if (!allVirtualFactTables.containsKey(virtualTableName)) { + Table tbl = getTableWithType(virtualTableName, CubeTableType.VIRTUAL_FACT, throwException); Table sourceTbl = getTableWithType(sourceFactName, CubeTableType.FACT, throwException); fact = (tbl == null || sourceTbl == null) ? null : new CubeVirtualFactTable(tbl, sourceTbl); if (enableCaching && fact != null) { - allVirtualFactTables.put(tableName, fact); - allVirtualFactSourceMappings.put(tableName, sourceFactName); + allVirtualFactTables.put(virtualTableName, fact); + allVirtualFactSourceMappings.put(virtualTableName, sourceFactName); } } else { - fact = allVirtualFactTables.get(tableName); + fact = allVirtualFactTables.get(virtualTableName); } } } @@ -2311,7 +2308,7 @@ private Collection getAllHiveTableNames() throws HiveException, LensExce * @return List of fact tables * @throws LensException */ - public List getAllFacts(CubeInterface cube) throws LensException { + public List getAllFacts(CubeInterface cube) throws LensException { String cubeName = null; if (cube != null) { if (cube instanceof DerivedCube) { @@ -2319,8 +2316,8 @@ public List getAllFacts(CubeInterface cube) throws LensException } cubeName = cube.getName(); } - List cubeFacts = new ArrayList<>(); - for (CubeFactTable fact : getAllFacts()) { + List cubeFacts = new ArrayList<>(); + for (FactTableInterface fact : getAllFacts()) { if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { cubeFacts.add(fact); } @@ -2336,7 +2333,7 @@ public List getAllFacts(CubeInterface cube) throws LensException * @return List of virtual fact tables * @throws LensException */ - public List getAllVirtualFacts(CubeInterface cube) throws LensException { + public List getAllVirtualFacts(CubeInterface cube) throws LensException { String cubeName = null; if (cube != null) { if (cube instanceof DerivedCube) { @@ -2344,13 +2341,13 @@ public List getAllVirtualFacts(CubeInterface cube) throws } cubeName = cube.getName(); } - List cubeFacts = new ArrayList<>(); - for (CubeVirtualFactTable fact : getAllVirtualFacts()) { + List cubeVirtualFacts = new ArrayList<>(); + for (FactTableInterface fact : getAllVirtualFacts()) { if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { - cubeFacts.add(fact); + cubeVirtualFacts.add(fact); } } - return cubeFacts; + return cubeVirtualFacts; } public List getAllSegmentations(CubeInterface cube) throws LensException { @@ -2461,18 +2458,18 @@ private boolean alterCubeTable(String table, Table hiveTable, AbstractCubeTable /** * - * @param table table name + * @param virtualTable table name * @param hiveTable hive table * @param cubeTable lens cube table * @throws LensException */ - private void alterCubeVirtualFactTable(String table, Table hiveTable, AbstractCubeTable cubeTable) throws + private void alterCubeVirtualFactTable(String virtualTable, Table hiveTable, AbstractCubeTable cubeTable) throws LensException { hiveTable.getParameters().putAll(cubeTable.getProperties()); hiveTable.getTTable().getParameters().putAll(cubeTable.getProperties()); try { - getClient().alterTable(table, hiveTable, null); + getClient().alterTable(virtualTable, hiveTable, null); } catch (Exception e) { throw new LensException(e); } @@ -2607,14 +2604,14 @@ public void dropFact(String factName, boolean cascade) throws LensException { /** * Drop a virtual fact * - * @param factName virtual fact name + * @param virtualFactName virtual fact name * @throws LensException */ - public void dropVirtualFact(String factName) throws LensException { - getTableWithTypeFailFast(factName, CubeTableType.VIRTUAL_FACT); - dropHiveTable(factName); - allVirtualFactTables.remove(factName.trim().toLowerCase()); - allVirtualFactSourceMappings.remove(factName); + public void dropVirtualFact(String virtualFactName) throws LensException { + getTableWithTypeFailFast(virtualFactName, CubeTableType.VIRTUAL_FACT); + dropHiveTable(virtualFactName); + allVirtualFactTables.remove(virtualFactName.trim().toLowerCase()); + allVirtualFactSourceMappings.remove(virtualFactName); } public void dropSegmentation(String segName) throws LensException { @@ -2710,6 +2707,10 @@ public void alterCubeFactTable(XFactTable fact) throws LensException, HiveExcept JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); } + + public void alterVirtualCubeFactTable(XVirtualFactTable virtualFact) throws LensException, HiveException { + alterVirtualCubeFactTable(JAXBUtils.cubeVirtualFactFromFactTable(virtualFact , getCubeFact(virtualFact.getSourceFactName()))); + } /** * Alter a cubefact with new definition and alter underlying storage tables as well. * @@ -2743,17 +2744,17 @@ public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveE /** * Alter a virtual cube fact with new definition - * - * @param factTableName virtual fact table name * @param cubeVirtualFactTable cube virtual fact table * @throws HiveException */ - public void alterVirtualCubeFactTable(String factTableName, CubeVirtualFactTable cubeVirtualFactTable) + public void alterVirtualCubeFactTable(CubeVirtualFactTable cubeVirtualFactTable) throws HiveException, LensException { - Table factTbl = getTableWithTypeFailFast(factTableName, CubeTableType.VIRTUAL_FACT); - alterCubeVirtualFactTable(factTableName, factTbl, cubeVirtualFactTable); - updateVirtualFactCache(factTableName); + String virtualFactTableName = cubeVirtualFactTable.getName(); + Table factTbl = getTableWithTypeFailFast(virtualFactTableName, CubeTableType.VIRTUAL_FACT); + + alterCubeVirtualFactTable(virtualFactTableName, factTbl, cubeVirtualFactTable); + updateVirtualFactCache(virtualFactTableName); } public void alterSegmentation(String segName, Segmentation seg) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index 8df40fc47..a0a8868e8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -20,43 +20,63 @@ import java.util.*; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; import lombok.Getter; -public class CubeVirtualFactTable extends CubeFactTable { +public class CubeVirtualFactTable extends AbstractCubeTable implements FactTableInterface{ @Getter private CubeFactTable sourceCubeFactTable; + private String cubeName; public CubeVirtualFactTable(Table hiveTable, Table sourceHiveTable) { super(hiveTable); + this.cubeName = getFactCubeName(getName(), getProperties()); this.sourceCubeFactTable = new CubeFactTable(sourceHiveTable); } - public CubeVirtualFactTable(String cubeName, String factName, double weight, Map properties, + public CubeVirtualFactTable(String cubeName, String virtualFactName, double weight, Map properties, CubeFactTable sourceFact) { - super(cubeName, factName, sourceFact.getColumns(), sourceFact.getUpdatePeriods(), weight, properties); + this(cubeName, virtualFactName, sourceFact.getColumns(), weight, properties); this.sourceCubeFactTable = sourceFact; } - public CubeVirtualFactTable(String cubeName, String factName, Map properties, + public CubeVirtualFactTable(String cubeName, String virtualFactName, Map properties, CubeFactTable sourceFact) { - super(cubeName, factName, sourceFact.getColumns(), sourceFact.getUpdatePeriods(), sourceFact.weight(), properties); + this(cubeName, virtualFactName, sourceFact.getColumns(), sourceFact.weight(), properties); this.sourceCubeFactTable = sourceFact; } - public CubeVirtualFactTable(String cubeName, String factName, CubeFactTable sourceFact) { - super(cubeName, factName, sourceFact.getColumns(), sourceFact.getUpdatePeriods(), sourceFact.weight(), + public CubeVirtualFactTable(String cubeName, String virtualFactName, CubeFactTable sourceFact) { + this(cubeName, virtualFactName, sourceFact.getColumns(), sourceFact.weight(), new HashMap()); this.sourceCubeFactTable = sourceFact; } + public CubeVirtualFactTable(String cubeName, String virtualFactName, List columns, + double weight, Map properties) { + super(virtualFactName, columns, properties, weight); + this.cubeName = cubeName; + addProperties(); + } + + @Override + protected void addProperties() { + super.addProperties(); + addCubeNames(getName(), getProperties(), cubeName); + } @Override public CubeTableType getTableType() { return CubeTableType.VIRTUAL_FACT; } + @Override + public List getValidColumns() { + return this.sourceCubeFactTable.getValidColumns(); + } + @Override public Set getStorages() { return this.sourceCubeFactTable.getStorages(); @@ -67,9 +87,19 @@ public Map> getUpdatePeriods() { return this.sourceCubeFactTable.getUpdatePeriods(); } + @Override + public String getCubeName() { + return this.cubeName; + } + @Override public String getDataCompletenessTag() { return this.sourceCubeFactTable.getDataCompletenessTag(); } + @Override + public boolean isAggregated() { + return this.sourceCubeFactTable.isAggregated(); + } + } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTableInterface.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTableInterface.java new file mode 100644 index 000000000..8fcabcb75 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTableInterface.java @@ -0,0 +1,38 @@ +package org.apache.lens.cube.metadata; + +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +/** + * Created by rajithar on 3/5/17. + */ +public interface FactTableInterface extends Named{ + + public Map> getUpdatePeriods(); + + public String getCubeName(); + + public Set getStorages(); + + public CubeTableType getTableType(); + + public Map getProperties(); + + public List getValidColumns(); + + public double weight(); + + public Set getAllFieldNames(); + + public String getDataCompletenessTag(); + + public List getColumns(); + + public boolean isAggregated(); + + public Date getDateFromProperty(String propKey, boolean relative, boolean start); +} diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java index 88500fdc8..c2e9fc9e1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java @@ -58,6 +58,7 @@ private MetastoreConstants() { public static final String FACT_COL_START_TIME_PFX = "cube.fact.col.start.time."; public static final String FACT_COL_END_TIME_PFX = "cube.fact.col.end.time."; public static final String FACT_DATA_COMPLETENESS_TAG = "cube.fact.datacompleteness.tag"; + public static final String VIRTUAL_FACT_FILTER = " cube.fact.query.where.filter"; // Segmentation constants public static final String SEGMENTATION_KEY_PFX = "cube.segmentation.internal."; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java index 1e3cef39b..57be74662 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java @@ -182,6 +182,7 @@ public Date getRelativeEndTime() { public Date getEndTime() { return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime())); } + static String getCubeName(String segName, Map props) { return props.get(MetastoreUtil.getSegmentationCubeNameKey(segName)); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java index c909545cc..54094b741 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java @@ -22,6 +22,7 @@ import java.util.Set; import org.apache.lens.cube.metadata.AbstractCubeTable; +import org.apache.lens.cube.metadata.FactTableInterface; /** * Candidate table interface diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index 6d61f1fc1..c9b83cd69 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -84,12 +84,13 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { private void populateCandidateTables(CubeQueryContext cubeql) throws LensException { if (cubeql.getCube() != null) { - List factTables = cubeql.getMetastoreClient().getAllFacts(cubeql.getCube()); + List factTables = cubeql.getMetastoreClient().getAllFacts(cubeql.getCube()); + factTables.addAll(cubeql.getMetastoreClient().getAllVirtualFacts(cubeql.getCube())); if (factTables.isEmpty()) { throw new LensException(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), cubeql.getCube().getName() + " does not have any facts"); } - for (CubeFactTable fact : factTables) { + for (FactTableInterface fact : factTables) { if (fact.getUpdatePeriods().isEmpty()) { log.info("Not considering fact: {} as it has no update periods", fact.getName()); } else { @@ -99,6 +100,18 @@ private void populateCandidateTables(CubeQueryContext cubeql) throws LensExcepti } } } + +// for (CubeVirtualFactTable vfact : vfactTables) { +// if (vfact.getUpdatePeriods().isEmpty()) { +// log.info("Not considering virtual fact: {} as it has no update periods", vfact.getName()); +// } else { +// for (String s : vfact.getStorages()) { +// StorageCandidate sc = new StorageCandidate(cubeql.getCube(), vfact, s, cubeql); +// cubeql.getCandidates().add(sc); +// } +// } +// } + log.info("Populated storage candidates: {}", cubeql.getCandidates()); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java index b9ff0eff2..b393f9638 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java @@ -213,7 +213,7 @@ public static StorageCandidate cloneStorageCandidate(StorageCandidate sc) throws return new StorageCandidate(sc); } - public static boolean factHasColumn(CubeFactTable fact, String column) { + public static boolean factHasColumn(FactTableInterface fact, String column) { for (FieldSchema factField : fact.getColumns()) { if (factField.getName().equals(column)) { return true; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index c63c4c0f2..8ed97db9e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -1024,7 +1024,9 @@ public String toHQL() throws LensException { String where = getWhere(sc, autoJoinCtx, sc.getQueryAst().getWhereAST(), getAliasForTableName(sc.getBaseTable().getName()), shouldReplaceDimFilterWithFactFilter(), storageTable, dimsToQuery); + sc.setWhereString(where); + } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java index 310a655cd..be21b5143 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -20,6 +20,7 @@ import java.util.*; +import org.apache.lens.cube.metadata.FactTableInterface; import org.apache.lens.cube.metadata.MetastoreConstants; import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.server.api.error.LensException; @@ -151,33 +152,33 @@ private static boolean isColumnAvailableTill(@NonNull final Date date, Date endT public static boolean isFactColumnValidForRange(CubeQueryContext cubeql, StorageCandidate sc, String col) { for (TimeRange range : cubeql.getTimeRanges()) { - if (!isColumnAvailableInRange(range, getFactColumnStartTime(sc, col), getFactColumnEndTime(sc, col))) { + if (!isColumnAvailableInRange(range, getFactColumnStartTime(sc.getFact(), col), getFactColumnEndTime(sc.getFact(), col))) { return false; } } return true; } - public static Date getFactColumnStartTime(StorageCandidate sc, String factCol) { + public static Date getFactColumnStartTime(FactTableInterface ft, String factCol) { Date startTime = null; - for (String key : sc.getTable().getProperties().keySet()) { + for (String key : ft.getProperties().keySet()) { if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_START_TIME_PFX); if (factCol.equals(propCol)) { - startTime = sc.getTable().getDateFromProperty(key, false, true); + startTime = ft.getDateFromProperty(key, false, true); } } } return startTime; } - public static Date getFactColumnEndTime(StorageCandidate sc, String factCol) { + public static Date getFactColumnEndTime(FactTableInterface ft, String factCol) { Date endTime = null; - for (String key : sc.getTable().getProperties().keySet()) { + for (String key : ft.getProperties().keySet()) { if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_END_TIME_PFX); if (factCol.equals(propCol)) { - endTime = sc.getTable().getDateFromProperty(key, false, true); + endTime = ft.getDateFromProperty(key, false, true); } } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 95e3c9573..01904e1f8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -18,6 +18,7 @@ */ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.metadata.MetastoreConstants.VIRTUAL_FACT_FILTER; import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipUpdatePeriodCode; import static org.apache.lens.cube.parse.CandidateTablePruneCause.timeDimNotSupported; @@ -30,16 +31,7 @@ import java.text.SimpleDateFormat; import java.util.*; -import org.apache.lens.cube.metadata.AbstractCubeTable; -import org.apache.lens.cube.metadata.CubeFactTable; -import org.apache.lens.cube.metadata.CubeInterface; -import org.apache.lens.cube.metadata.CubeMetastoreClient; -import org.apache.lens.cube.metadata.DateUtil; -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.cube.metadata.FactPartition; -import org.apache.lens.cube.metadata.MetastoreUtil; -import org.apache.lens.cube.metadata.TimeRange; -import org.apache.lens.cube.metadata.UpdatePeriod; +import org.apache.lens.cube.metadata.*; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.DataCompletenessChecker; @@ -114,7 +106,7 @@ public class StorageCandidate implements Candidate, CandidateTable { * Participating fact, storage and dimensions for this StorageCandidate */ @Getter - private CubeFactTable fact; + private FactTableInterface fact; @Getter private String storageName; @Getter @@ -182,7 +174,7 @@ public StorageCandidate(StorageCandidate sc) throws LensException { this.answerableMeasurePhraseIndices = sc.answerableMeasurePhraseIndices; } - public StorageCandidate(CubeInterface cube, CubeFactTable fact, String storageName, CubeQueryContext cubeql) + public StorageCandidate(CubeInterface cube, FactTableInterface fact, String storageName, CubeQueryContext cubeql) throws LensException { if ((cube == null) || (fact == null) || (storageName == null)) { throw new IllegalArgumentException("Cube,fact and storageName should be non null"); @@ -281,7 +273,9 @@ private void setMissingExpressions(Set queriedDims) throws LensExcept setWhereString(joinWithAnd( genWhereClauseWithDimPartitions(whereString, queriedDims), cubeql.getConf().getBoolean( CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, CubeQueryConfUtil.DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL) - ? getPostSelectionWhereClause() : null)); + ? getPostSelectionWhereClause() : null, + this.fact.getTableType().equals(CubeTableType.VIRTUAL_FACT) + ? this.fact.getProperties().get(VIRTUAL_FACT_FILTER) :null)); if (cubeql.getHavingAST() != null) { queryAst.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST())); } @@ -383,7 +377,7 @@ public String getStorageString(String alias) { @Override public AbstractCubeTable getTable() { - return fact; + return (AbstractCubeTable) fact; } @Override diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index edf92bca7..ab9e051e8 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -1277,7 +1277,7 @@ public void testVirtualCubeFact() throws Exception { assertEquals(client.getAllVirtualFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(), virtualFactName.toLowerCase()); - CubeVirtualFactTable cubeFact2 = new CubeVirtualFactTable(cubeTbl, client.getHiveTable(sourceFactName)); + CubeVirtualFactTable cubeFact2 = client.getVirtualFactTable(virtualFactName);//new CubeVirtualFactTable(cubeTbl, client.getHiveTable(sourceFactName)); assertTrue(cubeVirtualFact.equals(cubeFact2)); //alter virtual fact @@ -1285,7 +1285,7 @@ public void testVirtualCubeFact() throws Exception { Map alterVirtualFactAllPropertiesMap = cubeVirtualFact.getProperties(); alterVirtualFactAllPropertiesMap.putAll(alterVirtualFactPropertiesMap); cubeVirtualFact = new CubeVirtualFactTable(CUBE_NAME, virtualFactName, alterVirtualFactPropertiesMap, cubeFact); - client.alterVirtualCubeFactTable(virtualFactName, cubeVirtualFact); + client.alterVirtualCubeFactTable(cubeVirtualFact); cubeFact2 = new CubeVirtualFactTable(cubeTbl, client.getHiveTable(sourceFactName)); assertEquals(cubeFact2.getProperties().get("name1"), "newvalue2"); assertEquals(cubeFact2.getProperties().get("name3"), "value3"); @@ -1481,15 +1481,15 @@ public void testAlterCubeFact() throws Exception { client.dropStorageFromFact(factName, c2); storageTableName = getFactOrDimtableStorageTableName(factName, c2); assertFalse(client.tableExists(storageTableName)); - List cubeFacts = client.getAllFacts(client.getCube(CUBE_NAME)); + List cubeFacts = client.getAllFacts(client.getCube(CUBE_NAME)); List cubeFactNames = new ArrayList<>(); - for (CubeFactTable cfact : cubeFacts) { + for (FactTableInterface cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } assertTrue(cubeFactNames.contains(factName.toLowerCase())); cubeFacts = client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)); cubeFactNames = new ArrayList<>(); - for (CubeFactTable cfact : cubeFacts) { + for (FactTableInterface cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } assertTrue(cubeFactNames.contains(factName.toLowerCase())); @@ -1499,7 +1499,7 @@ public void testAlterCubeFact() throws Exception { assertFalse(client.tableExists(factName)); cubeFacts = client.getAllFacts(cube); cubeFactNames = new ArrayList<>(); - for (CubeFactTable cfact : cubeFacts) { + for (FactTableInterface cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } assertFalse(cubeFactNames.contains(factName.toLowerCase())); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index bab1080f0..bda38508b 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -144,6 +144,21 @@ public void testCubeQuery() throws Exception { // assertNotNull(rewrittenQuery.getNonExistingParts()); } + +// @Test +// public void testCubeQuery2() throws Exception { +// CubeQueryContext rewrittenQuery = +// rewriteCtx("select SUM(msr2) from virtual_fact_cube where " + TWO_DAYS_RANGE, getConfWithStorages("C2")); +// String expected = +// getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, +// getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); +// String hql = rewrittenQuery.toHQL(); +// compareQueries(hql, expected); +// System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); +//// assertNotNull(rewrittenQuery.getNonExistingParts()); +// } + + @Test public void testMaxCoveringFact() throws Exception { Configuration conf = getConf(); @@ -1600,6 +1615,19 @@ public void testAliasNameSameAsColumnName() throws Exception { } } + @Test + public void testVirtualFact() throws Exception { + String query = "SELECT msr2 as msr2 from testCube WHERE " + TWO_DAYS_RANGE; + try { + String hql = rewrite(query, getConf()); + assertNotNull(hql); + System.out.println("@@HQL " + hql); + } catch (NullPointerException npe) { + fail(npe.getMessage()); + log.error("Not expecting null pointer exception", npe); + } + } + @Test public void testDimAttributeQueryWithFact() throws Exception { String query = "select count (distinct dim1) from testCube where " + TWO_DAYS_RANGE; diff --git a/lens-cube/src/test/resources/schema/facts/virtualfact.xml b/lens-cube/src/test/resources/schema/facts/virtualfact.xml new file mode 100644 index 000000000..48c4a4676 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/virtualfact.xml @@ -0,0 +1,27 @@ + + + + + + + diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java index 4bc1bb837..c53b51b62 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java @@ -425,11 +425,11 @@ int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, St * Get virtual fact table given by name * * @param sessionid The sessionid - * @param fact The virtual fact table name + * @param virtualFact The virtual fact table name * @return {@link XFactTable} * @throws LensException */ - XVirtualFactTable getVirtualFactTable(LensSessionHandle sessionid, String fact) throws LensException; + XVirtualFactTable getVirtualFactTable(LensSessionHandle sessionid, String virtualFact) throws LensException; /** * Create virtual fact table @@ -444,19 +444,19 @@ int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, St * Update/Alter virtual fact table * * @param sessionid The sessionid - * @param fact The virtual fact table's new definition + * @param virtualFact The virtual fact table's new definition * @throws LensException */ - void updateVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable fact) throws LensException; + void updateVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable virtualFact) throws LensException; /** * Drop virtual fact table. * * @param sessionid The sessionid - * @param fact The virtual fact table name + * @param virtualFact The virtual fact table name * @throws LensException */ - void dropVirtualFactTable(LensSessionHandle sessionid, String fact) throws LensException; + void dropVirtualFactTable(LensSessionHandle sessionid, String virtualFact) throws LensException; /** * Get all virtual fact names diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index 47b795c17..f713a3233 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -442,10 +442,10 @@ public List getAllFactNames(LensSessionHandle sessionid, String cubeName } @Override - public XVirtualFactTable getVirtualFactTable(LensSessionHandle sessionid, String fact) throws LensException { + public XVirtualFactTable getVirtualFactTable(LensSessionHandle sessionid, String virtualFact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); - CubeVirtualFactTable cft = msClient.getVirtualFactTable(fact); + CubeVirtualFactTable cft = msClient.getVirtualFactTable(virtualFact); XVirtualFactTable factTable = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cft); factTable.setSourceFactName(cft.getSourceCubeFactTable().getName()); return factTable; @@ -465,13 +465,13 @@ public void createVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTabl } @Override - public void updateVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable fact) throws LensException { + public void updateVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable virtualFact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { - CubeVirtualFactTable cubeVirtualFactTable = cubeVirtualFactFromFactTable(fact, - getClient(sessionid).getCubeFact(fact.getSourceFactName())); + CubeVirtualFactTable cubeVirtualFactTable = cubeVirtualFactFromFactTable(virtualFact, + getClient(sessionid).getCubeFact(virtualFact.getSourceFactName())); - getClient(sessionid).alterVirtualCubeFactTable(fact.getName(), cubeVirtualFactTable); - log.info("Updated virtual fact table " + fact.getName()); + getClient(sessionid).alterVirtualCubeFactTable(cubeVirtualFactTable); + log.info("Updated virtual fact table " + virtualFact.getName()); } catch (HiveException e) { throw new LensException(e); } @@ -479,10 +479,10 @@ public void updateVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTabl @Override - public void dropVirtualFactTable(LensSessionHandle sessionid, String fact) throws LensException { + public void dropVirtualFactTable(LensSessionHandle sessionid, String virtualFact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { - getClient(sessionid).dropVirtualFact(fact); - log.info("Dropped virtual fact table " + fact); + getClient(sessionid).dropVirtualFact(virtualFact); + log.info("Dropped virtual fact table " + virtualFact); } } @@ -494,9 +494,9 @@ public List getAllVirtualFactNames(LensSessionHandle sessionid, String c if (cubeName != null && fact == null) { throw new LensException("Could not get table: " + cubeName + " as a cube"); } - Collection facts = client.getAllVirtualFacts(fact); - List factNames = new ArrayList<>(facts.size()); - for (CubeVirtualFactTable cft : facts) { + Collection virtualFacts = client.getAllVirtualFacts(fact); + List factNames = new ArrayList<>(virtualFacts.size()); + for (CubeVirtualFactTable cft : virtualFacts) { factNames.add(cft.getName()); } return factNames; diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index 8dbf472d9..fa876d5dd 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -919,21 +919,21 @@ public APIResult createSegmentation(@QueryParam("sessionid") LensSessionHandle s /** - * Update fact table definition + * Update virtualFact table definition * * @param sessionid The sessionid in which user is working - * @param factName name of the virtual fact table - * @param fact The {@link XFactTable} representation of the updated virtual fact table definition + * @param factName name of the virtual virtualFact table + * @param virtualFact The {@link XFactTable} representation of the updated virtual virtualFact table definition * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful. {@link APIResult} with * state {@link Status#FAILED}, if update has failed */ @PUT @Path("/virtualfacts/{virtualFactName}") public APIResult updateVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, - @PathParam("factName") String factName, XVirtualFactTable fact) + @PathParam("factName") String factName, XVirtualFactTable virtualFact) throws LensException { checkSessionId(sessionid); - getSvc().updateVirtualFactTable(sessionid, fact); + getSvc().updateVirtualFactTable(sessionid, virtualFact); return success(); } @@ -997,16 +997,16 @@ public APIResult dropFactTable(@QueryParam("sessionid") LensSessionHandle sessio * Drop the virtual fact table, specified by name * * @param sessionid The sessionid in which user is working - * @param factName The virtual fact table name + * @param virtualFactName The virtual fact table name * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if drop was successful. {@link APIResult} with state * {@link Status#FAILED}, if drop has failed */ @DELETE @Path("/virtualfacts/{virtualFactName}") public APIResult dropVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, - @PathParam("virtualFactName") String factName) + @PathParam("virtualFactName") String virtualFactName) throws LensException { - return Entity.VIRTUALFACT.delete(sessionid, factName); + return Entity.VIRTUALFACT.delete(sessionid, virtualFactName); } /** From 65f6e2e61273158e70c570d98eb98d3b2e588af4 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 3 May 2017 19:06:39 +0530 Subject: [PATCH 07/71] Changes post comments --- .../lens/cube/metadata/FactTableInterface.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTableInterface.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTableInterface.java index 8fcabcb75..e9229209f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTableInterface.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTableInterface.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.lens.cube.metadata; import java.util.Date; From 5d559b234916385da2a33b89b1776e4b4f322c49 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 3 May 2017 20:16:29 +0530 Subject: [PATCH 08/71] checkstyle fixes --- .../apache/lens/cube/metadata/AbstractCubeTable.java | 1 - .../apache/lens/cube/metadata/CubeMetastoreClient.java | 10 ++++++---- .../org/apache/lens/cube/parse/CandidateTable.java | 1 - .../apache/lens/cube/parse/QueriedPhraseContext.java | 3 ++- .../lens/cube/metadata/TestCubeMetastoreClient.java | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java index f2a322d16..815b9ea1e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java @@ -20,7 +20,6 @@ import java.util.*; -import org.apache.lens.cube.parse.StorageCandidate; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index fd51219ec..26a1ec8e6 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -1826,7 +1826,8 @@ public CubeFactTable getFactTable(String tableName) throws LensException { public CubeVirtualFactTable getVirtualFactTable(String virtualTableName) throws LensException { return new CubeVirtualFactTable(getTableWithTypeFailFast(virtualTableName, CubeTableType.VIRTUAL_FACT), - getTableWithTypeFailFast(allVirtualFactSourceMappings.get(virtualTableName.trim().toLowerCase()), CubeTableType.FACT)); + getTableWithTypeFailFast(allVirtualFactSourceMappings.get(virtualTableName.trim().toLowerCase()), + CubeTableType.FACT)); } public Segmentation getSegmentationTable(String tableName) throws HiveException, LensException { @@ -2040,8 +2041,8 @@ private CubeFactTable getCubeFact(String tableName, boolean throwException) thro public CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, String sourceFactName) throws LensException { return getVirtualCubeFact(virtualTableName, sourceFactName, true); } - private CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, String sourceFactName, boolean throwException) - throws LensException { + private CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, String sourceFactName, + boolean throwException) throws LensException { virtualTableName = virtualTableName.trim().toLowerCase(); CubeVirtualFactTable fact = allVirtualFactTables.get(virtualTableName); if (fact == null) { @@ -2709,7 +2710,8 @@ public void alterCubeFactTable(XFactTable fact) throws LensException, HiveExcept } public void alterVirtualCubeFactTable(XVirtualFactTable virtualFact) throws LensException, HiveException { - alterVirtualCubeFactTable(JAXBUtils.cubeVirtualFactFromFactTable(virtualFact , getCubeFact(virtualFact.getSourceFactName()))); + alterVirtualCubeFactTable(JAXBUtils.cubeVirtualFactFromFactTable(virtualFact , + getCubeFact(virtualFact.getSourceFactName()))); } /** * Alter a cubefact with new definition and alter underlying storage tables as well. diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java index 54094b741..c909545cc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java @@ -22,7 +22,6 @@ import java.util.Set; import org.apache.lens.cube.metadata.AbstractCubeTable; -import org.apache.lens.cube.metadata.FactTableInterface; /** * Candidate table interface diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java index be21b5143..c14437efa 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -152,7 +152,8 @@ private static boolean isColumnAvailableTill(@NonNull final Date date, Date endT public static boolean isFactColumnValidForRange(CubeQueryContext cubeql, StorageCandidate sc, String col) { for (TimeRange range : cubeql.getTimeRanges()) { - if (!isColumnAvailableInRange(range, getFactColumnStartTime(sc.getFact(), col), getFactColumnEndTime(sc.getFact(), col))) { + if (!isColumnAvailableInRange(range, getFactColumnStartTime(sc.getFact(), col), + getFactColumnEndTime(sc.getFact(), col))) { return false; } } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index ab9e051e8..649abef69 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -1277,7 +1277,7 @@ public void testVirtualCubeFact() throws Exception { assertEquals(client.getAllVirtualFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(), virtualFactName.toLowerCase()); - CubeVirtualFactTable cubeFact2 = client.getVirtualFactTable(virtualFactName);//new CubeVirtualFactTable(cubeTbl, client.getHiveTable(sourceFactName)); + CubeVirtualFactTable cubeFact2 = client.getVirtualFactTable(virtualFactName); assertTrue(cubeVirtualFact.equals(cubeFact2)); //alter virtual fact From 4e30adf46cdc6317a7c6917dd3c19b752636f38a Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 3 May 2017 23:11:35 +0530 Subject: [PATCH 09/71] checkstyle fix --- .../java/org/apache/lens/cube/metadata/CubeMetastoreClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 26a1ec8e6..c6f769598 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -2710,7 +2710,7 @@ public void alterCubeFactTable(XFactTable fact) throws LensException, HiveExcept } public void alterVirtualCubeFactTable(XVirtualFactTable virtualFact) throws LensException, HiveException { - alterVirtualCubeFactTable(JAXBUtils.cubeVirtualFactFromFactTable(virtualFact , + alterVirtualCubeFactTable(JAXBUtils.cubeVirtualFactFromFactTable(virtualFact, getCubeFact(virtualFact.getSourceFactName()))); } /** From 4deb9dd56277517fa9aa81e7b1daf66554130a4b Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 3 May 2017 23:30:17 +0530 Subject: [PATCH 10/71] checkstyle fix --- .../lens/server/metastore/CubeMetastoreServiceImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index f713a3233..48d458522 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -432,9 +432,9 @@ public List getAllFactNames(LensSessionHandle sessionid, String cubeName if (cubeName != null && fact == null) { throw new LensException("Could not get table: " + cubeName + " as a cube"); } - Collection facts = client.getAllFacts(fact); + Collection facts = client.getAllFacts(fact); List factNames = new ArrayList<>(facts.size()); - for (CubeFactTable cft : facts) { + for (FactTableInterface cft : facts) { factNames.add(cft.getName()); } return factNames; @@ -494,9 +494,9 @@ public List getAllVirtualFactNames(LensSessionHandle sessionid, String c if (cubeName != null && fact == null) { throw new LensException("Could not get table: " + cubeName + " as a cube"); } - Collection virtualFacts = client.getAllVirtualFacts(fact); + Collection virtualFacts = client.getAllVirtualFacts(fact); List factNames = new ArrayList<>(virtualFacts.size()); - for (CubeVirtualFactTable cft : virtualFacts) { + for (FactTableInterface cft : virtualFacts) { factNames.add(cft.getName()); } return factNames; From 7710ac394687524154319e383821e22a225a82f3 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 4 May 2017 16:42:18 +0530 Subject: [PATCH 11/71] Test cases fix post refactoring --- .../cube/metadata/CubeVirtualFactTable.java | 12 +++- .../cube/metadata/MetastoreConstants.java | 1 + .../lens/cube/metadata/MetastoreUtil.java | 4 ++ .../metadata/TestCubeMetastoreClient.java | 19 +++---- .../metastore/TestMetastoreService.java | 55 +++++++++---------- 5 files changed, 49 insertions(+), 42 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index a0a8868e8..d6ab56bd6 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -31,7 +31,7 @@ public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable private CubeFactTable sourceCubeFactTable; private String cubeName; - public CubeVirtualFactTable(Table hiveTable, Table sourceHiveTable) { + public CubeVirtualFactTable(Table hiveTable , Table sourceHiveTable) { super(hiveTable); this.cubeName = getFactCubeName(getName(), getProperties()); this.sourceCubeFactTable = new CubeFactTable(sourceHiveTable); @@ -66,7 +66,17 @@ public CubeVirtualFactTable(String cubeName, String virtualFactName, List props) { + return props.get(MetastoreUtil.getSourceFactNameKey(factName)); + } + + protected static void addSourceFactName(String factName, Map props, String sourceFactName) { + props.put(MetastoreUtil.getSourceFactNameKey(factName), sourceFactName); + } + @Override public CubeTableType getTableType() { return CubeTableType.VIRTUAL_FACT; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java index c2e9fc9e1..d5126c124 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java @@ -49,6 +49,7 @@ private MetastoreConstants() { public static final String FACT_KEY_PFX = "cube.fact."; public static final String UPDATE_PERIOD_SFX = ".updateperiods"; public static final String CUBE_NAME_SFX = ".cubename"; + public static final String SOURCE_NAME_SFX = ".source"; public static final String VALID_COLUMNS_SFX = ".valid.columns"; public static final String FACT_AGGREGATED_PROPERTY = "cube.fact.is.aggregated"; public static final String FACT_ABSOLUTE_START_TIME = "cube.fact.absolute.start.time"; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index 599027fbc..0dc4088de 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -350,6 +350,10 @@ public static String getFactCubeNameKey(String name) { return getFactKeyPrefix(name) + CUBE_NAME_SFX; } + public static String getSourceFactNameKey(String name) { + return getFactKeyPrefix(name) + SOURCE_NAME_SFX; + } + public static String getValidColumnsKey(String name) { return getFactKeyPrefix(name) + VALID_COLUMNS_SFX; } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 649abef69..c56b0effd 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -1259,19 +1259,16 @@ public void testVirtualCubeFact() throws Exception { Map virtualFactPropertiesMap = getHashMap("name1", "newvalue1"); - Map virtualFactAllPropertiesMap = cubeFact.getProperties(); - virtualFactAllPropertiesMap.putAll(virtualFactPropertiesMap); CubeVirtualFactTable cubeVirtualFact = new CubeVirtualFactTable(CUBE_NAME, virtualFactName, - virtualFactAllPropertiesMap, cubeFact); + virtualFactPropertiesMap, cubeFact); // create virtual cube fact client.createVirtualFactTable(CUBE_NAME, virtualFactName, sourceFactName, null, virtualFactPropertiesMap); assertTrue(client.tableExists(virtualFactName)); - Table cubeTbl = client.getHiveTable(virtualFactName); - assertTrue(client.isVirtualFactTable(cubeTbl)); - assertTrue(client.isVirtualFactTableForCube(cubeTbl, CUBE_NAME)); - assertEquals(cubeVirtualFact.weight(), 10.0); - + Table virtualTbl = client.getHiveTable(virtualFactName); + Table sourceTbl = client.getHiveTable(sourceFactName); + assertTrue(client.isVirtualFactTable(virtualTbl)); + assertTrue(client.isVirtualFactTableForCube(virtualTbl, CUBE_NAME)); //get virtual fact assertEquals(client.getAllVirtualFacts(client.getCube(CUBE_NAME)).get(0).getName(), virtualFactName.toLowerCase()); assertEquals(client.getAllVirtualFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(), @@ -1282,11 +1279,11 @@ public void testVirtualCubeFact() throws Exception { //alter virtual fact Map alterVirtualFactPropertiesMap = getHashMap("name1", "newvalue2", "name3", "value3"); - Map alterVirtualFactAllPropertiesMap = cubeVirtualFact.getProperties(); - alterVirtualFactAllPropertiesMap.putAll(alterVirtualFactPropertiesMap); + //Map alterVirtualFactAllPropertiesMap = cubeVirtualFact.getProperties(); + // alterVirtualFactAllPropertiesMap.putAll(alterVirtualFactPropertiesMap); cubeVirtualFact = new CubeVirtualFactTable(CUBE_NAME, virtualFactName, alterVirtualFactPropertiesMap, cubeFact); client.alterVirtualCubeFactTable(cubeVirtualFact); - cubeFact2 = new CubeVirtualFactTable(cubeTbl, client.getHiveTable(sourceFactName)); + cubeFact2 = new CubeVirtualFactTable(virtualTbl, client.getHiveTable(sourceFactName)); assertEquals(cubeFact2.getProperties().get("name1"), "newvalue2"); assertEquals(cubeFact2.getProperties().get("name3"), "value3"); diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index a512e0e3a..b0f1824a4 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -1869,7 +1869,7 @@ private XVirtualFactTable createVirtualFactTable(String factName, final String c f.setCubeName(cubeName); f.setSourceFactName(sourceFactName); - Map properties = LensUtil.getHashMap("foo", "bar"); + Map properties = LensUtil.getHashMap("foo1", "bar1"); f.getProperties().getProperty().addAll(JAXBUtils.xPropertiesFromMap(properties)); return f; } @@ -2242,20 +2242,20 @@ public void testUpdateFactTable(MediaType mediaType) throws Exception { @Test(dataProvider = "mediaTypeData") public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { - final String table = "testCreateVirtualFactTable"; - final String sourceTable = "testCreateVirtualFactSourceTable"; + final String virtualFactTable = "testCreateVirtualFactTable"; + final String sourceFactTable = "testCreateVirtualFactSourceTable"; final String DB = dbPFX + "testCreateVirtualFactTable_DB" + mediaType.getSubtype(); String prevDb = getCurrentDatabase(mediaType); createDatabase(DB, mediaType); setCurrentDatabase(DB, mediaType); - createStorage("S1", mediaType); - createStorage("S2", mediaType); + createStorage("VS1", mediaType); + createStorage("VS2", mediaType); try { //first create source fact table - XFactTable source = createFactTable(sourceTable); - source.getStorageTables().getStorageTable().add(createStorageTblElement("S1", table, "HOURLY")); - source.getStorageTables().getStorageTable().add(createStorageTblElement("S2", table, "DAILY")); + XFactTable source = createFactTable(sourceFactTable); + source.getStorageTables().getStorageTable().add(createStorageTblElement("VS1", virtualFactTable, "HOURLY")); + source.getStorageTables().getStorageTable().add(createStorageTblElement("VS2", virtualFactTable, "DAILY")); APIResult result = target() .path("metastore") @@ -2268,7 +2268,7 @@ public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { assertSuccess(result); //now create virtual fact table - XVirtualFactTable f = createVirtualFactTable(table, sourceTable); + XVirtualFactTable f = createVirtualFactTable(virtualFactTable, sourceFactTable); result = target() .path("metastore") @@ -2283,15 +2283,15 @@ public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { // Get all virtual fact names, this should contain the virtual fact table StringList virtualFactNames = target().path("metastore/virtualfacts") .queryParam("sessionid", lensSessionId).request(mediaType).get(StringList.class); - assertTrue(virtualFactNames.getElements().contains(table.toLowerCase())); + assertTrue(virtualFactNames.getElements().contains(virtualFactTable.toLowerCase())); // Get the created table - JAXBElement gotFactElement = target().path("metastore/virtualfacts").path(table) + JAXBElement gotFactElement = target().path("metastore/virtualfacts").path(virtualFactTable) .queryParam("sessionid", lensSessionId).request(mediaType) .get(new GenericType>() { }); XVirtualFactTable gotFact = gotFactElement.getValue(); - assertTrue(gotFact.getName().equalsIgnoreCase(table)); + assertTrue(gotFact.getName().equalsIgnoreCase(virtualFactTable)); assertEquals(gotFact.getWeight(), 10.0); CubeVirtualFactTable cvf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, JAXBUtils.cubeFactFromFactTable(source)); @@ -2305,27 +2305,22 @@ public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { } } - //Check for column with start time - Map props = JAXBUtils.mapFromXProperties(gotFact.getProperties()); - assertEquals(props.get(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("c3")), "2016-01-01"); - assertEquals(props.get(MetastoreConstants.FACT_COL_END_TIME_PFX.concat("c3")), "2017-01-01"); - assertTrue(foundC1); - assertEquals(cvf.getProperties().get("foo"), "bar"); - assertTrue(cvf.getStorages().contains("S1")); - assertTrue(cvf.getStorages().contains("S2")); - assertTrue(cvf.getUpdatePeriods().get("S1").contains(HOURLY)); - assertTrue(cvf.getUpdatePeriods().get("S2").contains(DAILY)); + assertEquals(cvf.getProperties().get("foo1"), "bar1"); + assertTrue(cvf.getStorages().contains("VS1")); + assertTrue(cvf.getStorages().contains("VS2")); + assertTrue(cvf.getUpdatePeriods().get("VS1").contains(HOURLY)); + assertTrue(cvf.getUpdatePeriods().get("VS2").contains(DAILY)); // drop the virtual fact table - result = target().path("metastore").path("virtualfacts").path(table) + result = target().path("metastore").path("virtualfacts").path(virtualFactTable) .queryParam("sessionid", lensSessionId).request(mediaType) .delete(APIResult.class); assertSuccess(result); // drop the source fact table - result = target().path("metastore").path("facts").path(sourceTable) + result = target().path("metastore").path("facts").path(sourceFactTable) .queryParam("cascade", "true") .queryParam("sessionid", lensSessionId).request(mediaType) .delete(APIResult.class); @@ -2334,7 +2329,7 @@ public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { // Drop again, this time it should give a 404 try { - target().path("metastore").path("virtualfacts").path(table) + target().path("metastore").path("virtualfacts").path(virtualFactTable) .queryParam("sessionid", lensSessionId).request(mediaType) .delete(APIResult.class); fail("Expected 404"); @@ -2355,15 +2350,15 @@ public void testUpdateVirtualFactTable(MediaType mediaType) throws Exception { String prevDb = getCurrentDatabase(mediaType); createDatabase(DB, mediaType); setCurrentDatabase(DB, mediaType); - createStorage("S1", mediaType); - createStorage("S2", mediaType); - createStorage("S3", mediaType); + createStorage("VS1", mediaType); + createStorage("VS2", mediaType); + try { //first create source fact table XFactTable source = createFactTable(sourceTable); - source.getStorageTables().getStorageTable().add(createStorageTblElement("S1", table, "HOURLY")); - source.getStorageTables().getStorageTable().add(createStorageTblElement("S2", table, "DAILY")); + source.getStorageTables().getStorageTable().add(createStorageTblElement("VS1", table, "HOURLY")); + source.getStorageTables().getStorageTable().add(createStorageTblElement("VS2", table, "DAILY")); APIResult result = target() .path("metastore") From fd0eafab1479fe7e5a1c9c2da04f020ce41dec4e Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 4 May 2017 17:21:28 +0530 Subject: [PATCH 12/71] Test cases fix post refactoring --- .../org/apache/lens/cube/metadata/CubeVirtualFactTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index d6ab56bd6..47697f5a5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -31,7 +31,7 @@ public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable private CubeFactTable sourceCubeFactTable; private String cubeName; - public CubeVirtualFactTable(Table hiveTable , Table sourceHiveTable) { + public CubeVirtualFactTable(Table hiveTable, Table sourceHiveTable) { super(hiveTable); this.cubeName = getFactCubeName(getName(), getProperties()); this.sourceCubeFactTable = new CubeFactTable(sourceHiveTable); From bb3796a8ade26dd35356016ffbf435e10baa946e Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Fri, 5 May 2017 16:55:42 +0530 Subject: [PATCH 13/71] changes for review comments --- .../lens/cube/metadata/CubeFactTable.java | 2 +- .../cube/metadata/CubeMetastoreClient.java | 66 ++++++++++++++++--- .../cube/metadata/CubeVirtualFactTable.java | 57 ++++++++-------- ...FactTableInterface.java => FactTable.java} | 2 +- .../apache/lens/cube/metadata/JAXBUtils.java | 5 +- .../cube/parse/CandidateTableResolver.java | 15 +---- .../apache/lens/cube/parse/CandidateUtil.java | 2 +- .../lens/cube/parse/QueriedPhraseContext.java | 6 +- .../lens/cube/parse/StorageCandidate.java | 5 +- .../metadata/TestCubeMetastoreClient.java | 54 ++++++++++----- .../lens/cube/parse/TestCubeRewriter.java | 24 +++---- .../metastore/CubeMetastoreServiceImpl.java | 8 +-- 12 files changed, 149 insertions(+), 97 deletions(-) rename lens-cube/src/main/java/org/apache/lens/cube/metadata/{FactTableInterface.java => FactTable.java} (96%) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index 8640cfa35..e796a0534 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -33,7 +33,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class CubeFactTable extends AbstractCubeTable implements FactTableInterface{ +public class CubeFactTable extends AbstractCubeTable implements FactTable { @Getter // Map> private final Map> storagePrefixUpdatePeriodMap; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index c6f769598..988b8d11e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -26,6 +26,7 @@ import java.text.ParseException; import java.util.*; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import org.apache.lens.api.metastore.*; @@ -57,6 +58,7 @@ import org.jvnet.jaxb2_commons.lang.HashCode; import org.jvnet.jaxb2_commons.lang.ToString; +import com.google.common.base.*; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -89,6 +91,8 @@ private CubeMetastoreClient(HiveConf conf) { private volatile boolean allDimTablesPopulated = false; // map from fact name to fact table private final Map allFactTables = Maps.newConcurrentMap(); + // map from fact name to all virtual fact tables + private final Map> factToVirtualFactTables = Maps.newConcurrentMap(); // map from virtual fact name to virtual fact table private final Map allVirtualFactTables = Maps.newConcurrentMap(); // map from virtual fact name to source fact table @@ -128,7 +132,7 @@ public boolean isDataCompletenessCheckEnabled() { } /** extract storage name from fact and storage table name. String operation */ - private String extractStorageName(FactTableInterface fact, String storageTableName) throws LensException { + private String extractStorageName(FactTable fact, String storageTableName) throws LensException { int ind = storageTableName.lastIndexOf(fact.getName()); if (ind <= 0) { throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); @@ -159,7 +163,7 @@ public Date getLatestDateOfCube(Cube cube, String timeDimension) throws HiveExce String partCol = cube.getPartitionColumnOfTimeDim(timeDimension); Date max = new Date(Long.MIN_VALUE); boolean updated = false; - for (FactTableInterface fact : getAllFacts(cube)) { + for (FactTable fact : getAllFacts(cube)) { for (String storage : fact.getStorages()) { for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) { PartitionTimeline timeline = partitionTimelineCache.get(fact.getName(), storage, updatePeriod, partCol); @@ -382,8 +386,10 @@ public void createVirtualFactTable(String cubeName, String virtualFactName, Stri Map properties) throws LensException { CubeFactTable sourceFact = getCubeFact(sourceFactName); + com.google.common.base.Optional optionalWeight = com.google.common.base.Optional.fromNullable(weight); + CubeVirtualFactTable factTable = new CubeVirtualFactTable(cubeName, virtualFactName, - (weight != null ? weight : sourceFact.weight()), properties, getCubeFact(sourceFactName)); + optionalWeight, properties, sourceFact); createCubeTable(factTable, null); // do a get to update cache getVirtualCubeFact(virtualFactName, sourceFactName); @@ -1423,7 +1429,7 @@ public boolean tableExists(String tblName) throws HiveException { } /** extract storage name and check in timeline cache for existance */ - public boolean factPartitionExists(FactTableInterface fact, FactPartition part, String storageTableName) + public boolean factPartitionExists(FactTable fact, FactPartition part, String storageTableName) throws HiveException, LensException { String storage = extractStorageName(fact, storageTableName); return partitionTimelineCache.partitionTimeExists(fact.getName(), storage, part.getPeriod(), part.getPartCol(), @@ -2044,6 +2050,7 @@ public CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, String s private CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, String sourceFactName, boolean throwException) throws LensException { virtualTableName = virtualTableName.trim().toLowerCase(); + sourceFactName = sourceFactName.trim().toLowerCase(); CubeVirtualFactTable fact = allVirtualFactTables.get(virtualTableName); if (fact == null) { synchronized (allVirtualFactTables) { @@ -2054,6 +2061,14 @@ private CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, String if (enableCaching && fact != null) { allVirtualFactTables.put(virtualTableName, fact); allVirtualFactSourceMappings.put(virtualTableName, sourceFactName); + if(factToVirtualFactTables.get(sourceFactName) != null){ + List prevList = factToVirtualFactTables.get(sourceFactName); + prevList.add(virtualTableName); + }else{ + List newList = new ArrayList<>(); + newList.add(virtualTableName); + factToVirtualFactTables.put(sourceFactName, newList); + } } } else { fact = allVirtualFactTables.get(virtualTableName); @@ -2309,7 +2324,7 @@ private Collection getAllHiveTableNames() throws HiveException, LensExce * @return List of fact tables * @throws LensException */ - public List getAllFacts(CubeInterface cube) throws LensException { + public List getAllFacts(CubeInterface cube) throws LensException { String cubeName = null; if (cube != null) { if (cube instanceof DerivedCube) { @@ -2317,8 +2332,8 @@ public List getAllFacts(CubeInterface cube) throws LensExcep } cubeName = cube.getName(); } - List cubeFacts = new ArrayList<>(); - for (FactTableInterface fact : getAllFacts()) { + List cubeFacts = new ArrayList<>(); + for (FactTable fact : getAllFacts()) { if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { cubeFacts.add(fact); } @@ -2334,7 +2349,7 @@ public List getAllFacts(CubeInterface cube) throws LensExcep * @return List of virtual fact tables * @throws LensException */ - public List getAllVirtualFacts(CubeInterface cube) throws LensException { + public List getAllVirtualFacts(CubeInterface cube) throws LensException { String cubeName = null; if (cube != null) { if (cube instanceof DerivedCube) { @@ -2342,8 +2357,8 @@ public List getAllVirtualFacts(CubeInterface cube) throws Le } cubeName = cube.getName(); } - List cubeVirtualFacts = new ArrayList<>(); - for (FactTableInterface fact : getAllVirtualFacts()) { + List cubeVirtualFacts = new ArrayList<>(); + for (FactTable fact : getAllVirtualFacts()) { if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { cubeVirtualFacts.add(fact); } @@ -2600,6 +2615,21 @@ public void dropFact(String factName, boolean cascade) throws LensException { } dropHiveTable(factName); allFactTables.remove(factName.trim().toLowerCase()); + dropAllVirtualFactTablesFromCache(factName); + } + + private void dropAllVirtualFactTablesFromCache(String cubeFactTableName) throws LensException { + if (enableCaching) { + cubeFactTableName = cubeFactTableName.trim().toLowerCase(); + if (factToVirtualFactTables.get(cubeFactTableName) != null) { + synchronized (allVirtualFactTables) { + List virtualFactTableNames = factToVirtualFactTables.get(cubeFactTableName); + for (String vf : virtualFactTableNames) { + dropVirtualFact(vf.trim().toLowerCase()); + } + } + } + } } /** @@ -2738,6 +2768,7 @@ public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable } } updateFactCache(factTableName); + updateAllVirtualFacts(getCubeFact(factTableName)); } public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException { @@ -2781,6 +2812,21 @@ private void updateFactCache(String factTableName) throws LensException { } } + private void updateAllVirtualFacts(CubeFactTable cubeFactTable) throws LensException { + if (enableCaching) { + String cubeFactTableName = cubeFactTable.getName().trim().toLowerCase(); + if (factToVirtualFactTables.get(cubeFactTableName) != null) { + synchronized (allVirtualFactTables) { + List virtualFactTableNames = factToVirtualFactTables.get(cubeFactTableName); + for (String vf : virtualFactTableNames) { + CubeVirtualFactTable cvf = getVirtualCubeFact(vf, allVirtualFactSourceMappings.get(vf)); + cvf.setSourceCubeFactTable(cubeFactTable); + allVirtualFactTables.put(vf.trim().toLowerCase(), cvf); + } + } + } + } + } private void updateVirtualFactCache(String factTableName) throws LensException { if (enableCaching) { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index 47697f5a5..b28f0061f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -19,46 +19,41 @@ package org.apache.lens.cube.metadata; import java.util.*; +import com.google.common.base.Optional; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; import lombok.Getter; +import lombok.Setter; -public class CubeVirtualFactTable extends AbstractCubeTable implements FactTableInterface{ +public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable { @Getter + @Setter private CubeFactTable sourceCubeFactTable; private String cubeName; + private static final List COLUMNS = new ArrayList(); + @Getter + private Optional virtualFactWeight = Optional.absent(); + + static { + COLUMNS.add(new FieldSchema("dummy", "string", "dummy column")); + } public CubeVirtualFactTable(Table hiveTable, Table sourceHiveTable) { super(hiveTable); this.cubeName = getFactCubeName(getName(), getProperties()); +// this.virtualFactWeight = hiveTable.we this.sourceCubeFactTable = new CubeFactTable(sourceHiveTable); } - public CubeVirtualFactTable(String cubeName, String virtualFactName, double weight, Map properties, - CubeFactTable sourceFact) { - this(cubeName, virtualFactName, sourceFact.getColumns(), weight, properties); - this.sourceCubeFactTable = sourceFact; - } - - public CubeVirtualFactTable(String cubeName, String virtualFactName, Map properties, - CubeFactTable sourceFact) { - this(cubeName, virtualFactName, sourceFact.getColumns(), sourceFact.weight(), properties); - this.sourceCubeFactTable = sourceFact; - } - - public CubeVirtualFactTable(String cubeName, String virtualFactName, CubeFactTable sourceFact) { - this(cubeName, virtualFactName, sourceFact.getColumns(), sourceFact.weight(), - new HashMap()); - this.sourceCubeFactTable = sourceFact; - } - - public CubeVirtualFactTable(String cubeName, String virtualFactName, List columns, - double weight, Map properties) { - super(virtualFactName, columns, properties, weight); + public CubeVirtualFactTable(String cubeName, String virtualFactName, Optional weight, + Map properties, CubeFactTable sourceFact) { + super(virtualFactName, COLUMNS, properties, weight.isPresent() ? weight.get() : sourceFact.weight()); this.cubeName = cubeName; + this.virtualFactWeight = weight; + this.sourceCubeFactTable = sourceFact; addProperties(); } @@ -66,15 +61,6 @@ public CubeVirtualFactTable(String cubeName, String virtualFactName, List props) { - return props.get(MetastoreUtil.getSourceFactNameKey(factName)); - } - - protected static void addSourceFactName(String factName, Map props, String sourceFactName) { - props.put(MetastoreUtil.getSourceFactNameKey(factName), sourceFactName); } @Override @@ -112,4 +98,13 @@ public boolean isAggregated() { return this.sourceCubeFactTable.isAggregated(); } + @Override + public List getColumns() { + return this.sourceCubeFactTable.getColumns(); + } + + @Override + public double weight() { + return virtualFactWeight.isPresent() ? virtualFactWeight.get() : sourceCubeFactTable.weight(); + } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTableInterface.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java similarity index 96% rename from lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTableInterface.java rename to lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java index e9229209f..6fdcaf2cb 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTableInterface.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java @@ -28,7 +28,7 @@ /** * Created by rajithar on 3/5/17. */ -public interface FactTableInterface extends Named{ +public interface FactTable extends Named { public Map> getUpdatePeriods(); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java index c3a5e186a..b87d83a5f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java @@ -723,8 +723,11 @@ public static CubeVirtualFactTable cubeVirtualFactFromFactTable(XVirtualFactTabl //Update properties from source fact Map allProperties = sourceFactTable.getProperties(); allProperties.putAll(mapFromXProperties(fact.getProperties())); + + com.google.common.base.Optional optionalWeight = com.google.common.base.Optional.fromNullable(fact.getWeight()); + return new CubeVirtualFactTable(fact.getCubeName(), fact.getName(), - fact.getWeight() != null ? fact.getWeight() : sourceFactTable.weight(), allProperties, sourceFactTable); + optionalWeight, allProperties, sourceFactTable); } public static Segmentation segmentationFromXSegmentation(XSegmentation seg) throws LensException { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index c9b83cd69..d795c1cbf 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -84,13 +84,13 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { private void populateCandidateTables(CubeQueryContext cubeql) throws LensException { if (cubeql.getCube() != null) { - List factTables = cubeql.getMetastoreClient().getAllFacts(cubeql.getCube()); + List factTables = cubeql.getMetastoreClient().getAllFacts(cubeql.getCube()); factTables.addAll(cubeql.getMetastoreClient().getAllVirtualFacts(cubeql.getCube())); if (factTables.isEmpty()) { throw new LensException(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), cubeql.getCube().getName() + " does not have any facts"); } - for (FactTableInterface fact : factTables) { + for (FactTable fact : factTables) { if (fact.getUpdatePeriods().isEmpty()) { log.info("Not considering fact: {} as it has no update periods", fact.getName()); } else { @@ -101,17 +101,6 @@ private void populateCandidateTables(CubeQueryContext cubeql) throws LensExcepti } } -// for (CubeVirtualFactTable vfact : vfactTables) { -// if (vfact.getUpdatePeriods().isEmpty()) { -// log.info("Not considering virtual fact: {} as it has no update periods", vfact.getName()); -// } else { -// for (String s : vfact.getStorages()) { -// StorageCandidate sc = new StorageCandidate(cubeql.getCube(), vfact, s, cubeql); -// cubeql.getCandidates().add(sc); -// } -// } -// } - log.info("Populated storage candidates: {}", cubeql.getCandidates()); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java index b393f9638..aea240e8c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java @@ -213,7 +213,7 @@ public static StorageCandidate cloneStorageCandidate(StorageCandidate sc) throws return new StorageCandidate(sc); } - public static boolean factHasColumn(FactTableInterface fact, String column) { + public static boolean factHasColumn(FactTable fact, String column) { for (FieldSchema factField : fact.getColumns()) { if (factField.getName().equals(column)) { return true; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java index c14437efa..1c2cfee3d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -20,7 +20,7 @@ import java.util.*; -import org.apache.lens.cube.metadata.FactTableInterface; +import org.apache.lens.cube.metadata.FactTable; import org.apache.lens.cube.metadata.MetastoreConstants; import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.server.api.error.LensException; @@ -160,7 +160,7 @@ public static boolean isFactColumnValidForRange(CubeQueryContext cubeql, Storage return true; } - public static Date getFactColumnStartTime(FactTableInterface ft, String factCol) { + public static Date getFactColumnStartTime(FactTable ft, String factCol) { Date startTime = null; for (String key : ft.getProperties().keySet()) { if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) { @@ -173,7 +173,7 @@ public static Date getFactColumnStartTime(FactTableInterface ft, String factCol) return startTime; } - public static Date getFactColumnEndTime(FactTableInterface ft, String factCol) { + public static Date getFactColumnEndTime(FactTable ft, String factCol) { Date endTime = null; for (String key : ft.getProperties().keySet()) { if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 01904e1f8..bc8223215 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -106,7 +106,7 @@ public class StorageCandidate implements Candidate, CandidateTable { * Participating fact, storage and dimensions for this StorageCandidate */ @Getter - private FactTableInterface fact; + private FactTable fact; @Getter private String storageName; @Getter @@ -174,7 +174,7 @@ public StorageCandidate(StorageCandidate sc) throws LensException { this.answerableMeasurePhraseIndices = sc.answerableMeasurePhraseIndices; } - public StorageCandidate(CubeInterface cube, FactTableInterface fact, String storageName, CubeQueryContext cubeql) + public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, CubeQueryContext cubeql) throws LensException { if ((cube == null) || (fact == null) || (storageName == null)) { throw new IllegalArgumentException("Cube,fact and storageName should be non null"); @@ -275,6 +275,7 @@ private void setMissingExpressions(Set queriedDims) throws LensExcept CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, CubeQueryConfUtil.DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL) ? getPostSelectionWhereClause() : null, this.fact.getTableType().equals(CubeTableType.VIRTUAL_FACT) + && this.fact.getProperties().get(VIRTUAL_FACT_FILTER) != null ? this.fact.getProperties().get(VIRTUAL_FACT_FILTER) :null)); if (cubeql.getHavingAST() != null) { queryAst.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST())); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index c56b0effd..8e18f2ea1 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -31,6 +31,8 @@ import java.util.*; +import javax.ws.rs.NotFoundException; + import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol; @@ -1225,7 +1227,6 @@ public void testCubeFact() throws Exception { assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); } - @Test(priority = 2) public void testVirtualCubeFact() throws Exception { @@ -1248,7 +1249,7 @@ public void testVirtualCubeFact() throws Exception { Map storageTables = getHashMap(c1, s1); Map sourceFactPropertiesMap = getHashMap("name1", "value1", "name2", "value2"); - CubeFactTable cubeFact = new CubeFactTable(CUBE_NAME, sourceFactName, factColumns, updatePeriods, 10.0, + CubeFactTable sourceFact = new CubeFactTable(CUBE_NAME, sourceFactName, factColumns, updatePeriods, 10.0, sourceFactPropertiesMap); // create cube fact @@ -1260,37 +1261,54 @@ public void testVirtualCubeFact() throws Exception { Map virtualFactPropertiesMap = getHashMap("name1", "newvalue1"); CubeVirtualFactTable cubeVirtualFact = new CubeVirtualFactTable(CUBE_NAME, virtualFactName, - virtualFactPropertiesMap, cubeFact); + com.google.common.base.Optional.fromNullable(null), virtualFactPropertiesMap, sourceFact); // create virtual cube fact client.createVirtualFactTable(CUBE_NAME, virtualFactName, sourceFactName, null, virtualFactPropertiesMap); assertTrue(client.tableExists(virtualFactName)); Table virtualTbl = client.getHiveTable(virtualFactName); - Table sourceTbl = client.getHiveTable(sourceFactName); assertTrue(client.isVirtualFactTable(virtualTbl)); assertTrue(client.isVirtualFactTableForCube(virtualTbl, CUBE_NAME)); + //get virtual fact assertEquals(client.getAllVirtualFacts(client.getCube(CUBE_NAME)).get(0).getName(), virtualFactName.toLowerCase()); assertEquals(client.getAllVirtualFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(), virtualFactName.toLowerCase()); - CubeVirtualFactTable cubeFact2 = client.getVirtualFactTable(virtualFactName); - assertTrue(cubeVirtualFact.equals(cubeFact2)); + CubeVirtualFactTable actualcubeVirtualFact = client.getVirtualFactTable(virtualFactName); + assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); //alter virtual fact Map alterVirtualFactPropertiesMap = getHashMap("name1", "newvalue2", "name3", "value3"); - //Map alterVirtualFactAllPropertiesMap = cubeVirtualFact.getProperties(); - // alterVirtualFactAllPropertiesMap.putAll(alterVirtualFactPropertiesMap); - cubeVirtualFact = new CubeVirtualFactTable(CUBE_NAME, virtualFactName, alterVirtualFactPropertiesMap, cubeFact); + cubeVirtualFact = new CubeVirtualFactTable(CUBE_NAME, virtualFactName, + com.google.common.base.Optional.fromNullable(null), alterVirtualFactPropertiesMap, sourceFact); client.alterVirtualCubeFactTable(cubeVirtualFact); - cubeFact2 = new CubeVirtualFactTable(virtualTbl, client.getHiveTable(sourceFactName)); - assertEquals(cubeFact2.getProperties().get("name1"), "newvalue2"); - assertEquals(cubeFact2.getProperties().get("name3"), "value3"); + actualcubeVirtualFact = client.getVirtualFactTable(virtualFactName); + assertEquals(actualcubeVirtualFact.getProperties().get("name1"), "newvalue2"); + assertEquals(actualcubeVirtualFact.getProperties().get("name3"), "value3"); + assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); - //drop virtual fact - client.dropVirtualFact(virtualFactName); + //alter source fact + FieldSchema newcol = new FieldSchema("newcol", "int", "new col for part"); + sourceFact.alterColumn(newcol); + sourceFact.alterWeight(100); + client.alterCubeFactTable(sourceFact.getName(), sourceFact, storageTables, new HashMap()); + actualcubeVirtualFact = client.getVirtualFactTable(virtualFactName); + assertTrue(actualcubeVirtualFact.getColumns().contains(newcol)); + assertEquals(actualcubeVirtualFact.weight(),100.0); + assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); + + //drop source fact client.dropFact(sourceFactName, true); + //drop virtual fact + try{ + client.dropVirtualFact(virtualFactName); + fail("Expected 404"); + } catch (LensException nfe) { + // PASS since virtual fact is already dropped when source fact was dropped + } + assertFalse(client.tableExists(virtualFactName)); } @@ -1478,15 +1496,15 @@ public void testAlterCubeFact() throws Exception { client.dropStorageFromFact(factName, c2); storageTableName = getFactOrDimtableStorageTableName(factName, c2); assertFalse(client.tableExists(storageTableName)); - List cubeFacts = client.getAllFacts(client.getCube(CUBE_NAME)); + List cubeFacts = client.getAllFacts(client.getCube(CUBE_NAME)); List cubeFactNames = new ArrayList<>(); - for (FactTableInterface cfact : cubeFacts) { + for (FactTable cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } assertTrue(cubeFactNames.contains(factName.toLowerCase())); cubeFacts = client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)); cubeFactNames = new ArrayList<>(); - for (FactTableInterface cfact : cubeFacts) { + for (FactTable cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } assertTrue(cubeFactNames.contains(factName.toLowerCase())); @@ -1496,7 +1514,7 @@ public void testAlterCubeFact() throws Exception { assertFalse(client.tableExists(factName)); cubeFacts = client.getAllFacts(cube); cubeFactNames = new ArrayList<>(); - for (FactTableInterface cfact : cubeFacts) { + for (FactTable cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } assertFalse(cubeFactNames.contains(factName.toLowerCase())); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index bda38508b..960f32b3a 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -145,18 +145,18 @@ public void testCubeQuery() throws Exception { } -// @Test -// public void testCubeQuery2() throws Exception { -// CubeQueryContext rewrittenQuery = -// rewriteCtx("select SUM(msr2) from virtual_fact_cube where " + TWO_DAYS_RANGE, getConfWithStorages("C2")); -// String expected = -// getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, -// getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); -// String hql = rewrittenQuery.toHQL(); -// compareQueries(hql, expected); -// System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); -//// assertNotNull(rewrittenQuery.getNonExistingParts()); -// } + @Test + public void testCubeQuery2() throws Exception { + CubeQueryContext rewrittenQuery = + rewriteCtx("select SUM(msr2) from virtual_fact_cube where " + TWO_DAYS_RANGE, getConfWithStorages("C2")); + String expected = + getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, + getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); + String hql = rewrittenQuery.toHQL(); + compareQueries(hql, expected); + System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); +// assertNotNull(rewrittenQuery.getNonExistingParts()); + } @Test diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index 48d458522..f876229ab 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -432,9 +432,9 @@ public List getAllFactNames(LensSessionHandle sessionid, String cubeName if (cubeName != null && fact == null) { throw new LensException("Could not get table: " + cubeName + " as a cube"); } - Collection facts = client.getAllFacts(fact); + Collection facts = client.getAllFacts(fact); List factNames = new ArrayList<>(facts.size()); - for (FactTableInterface cft : facts) { + for (FactTable cft : facts) { factNames.add(cft.getName()); } return factNames; @@ -494,9 +494,9 @@ public List getAllVirtualFactNames(LensSessionHandle sessionid, String c if (cubeName != null && fact == null) { throw new LensException("Could not get table: " + cubeName + " as a cube"); } - Collection virtualFacts = client.getAllVirtualFacts(fact); + Collection virtualFacts = client.getAllVirtualFacts(fact); List factNames = new ArrayList<>(virtualFacts.size()); - for (FactTableInterface cft : virtualFacts) { + for (FactTable cft : virtualFacts) { factNames.add(cft.getName()); } return factNames; From d03af69c97158c9eb8cb2ca9515ed75a1448363b Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Fri, 5 May 2017 17:11:08 +0530 Subject: [PATCH 14/71] changes for review comments --- .../lens/cube/parse/TestCubeRewriter.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 960f32b3a..721008ea2 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -144,19 +144,19 @@ public void testCubeQuery() throws Exception { // assertNotNull(rewrittenQuery.getNonExistingParts()); } - - @Test - public void testCubeQuery2() throws Exception { - CubeQueryContext rewrittenQuery = - rewriteCtx("select SUM(msr2) from virtual_fact_cube where " + TWO_DAYS_RANGE, getConfWithStorages("C2")); - String expected = - getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, - getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); - String hql = rewrittenQuery.toHQL(); - compareQueries(hql, expected); - System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); -// assertNotNull(rewrittenQuery.getNonExistingParts()); - } +// +// @Test +// public void testCubeQuery2() throws Exception { +// CubeQueryContext rewrittenQuery = +// rewriteCtx("select SUM(msr2) from virtual_fact_cube where " + TWO_DAYS_RANGE, getConfWithStorages("C2")); +// String expected = +// getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, +// getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); +// String hql = rewrittenQuery.toHQL(); +// compareQueries(hql, expected); +// System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); +//// assertNotNull(rewrittenQuery.getNonExistingParts()); +// } @Test From 4c156d4058299389726ec112d50ad1438c4ca991 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Sun, 7 May 2017 23:15:19 +0530 Subject: [PATCH 15/71] Test case fix --- .../lens/cube/metadata/AbstractCubeTable.java | 2 +- .../cube/metadata/CubeMetastoreClient.java | 43 +++++++++++-------- .../cube/metadata/CubeVirtualFactTable.java | 16 ++++++- .../apache/lens/cube/metadata/JAXBUtils.java | 2 +- .../cube/metadata/MetastoreConstants.java | 2 +- .../lens/cube/parse/StorageCandidate.java | 27 +++++++----- .../lens/cube/parse/StorageTableResolver.java | 4 +- .../metadata/TestCubeMetastoreClient.java | 9 ++-- .../apache/lens/cube/parse/CubeTestSetup.java | 1 + .../lens/cube/parse/TestCubeRewriter.java | 23 +++++----- .../schema/cubes/base/virtualcube.xml | 35 +++++++++++++++ .../resources/schema/facts/virtualfact.xml | 2 +- 12 files changed, 110 insertions(+), 56 deletions(-) create mode 100644 lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java index 815b9ea1e..4d9217476 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java @@ -67,7 +67,7 @@ public static double getWeight(Map properties, String name) { protected void addProperties() { properties.put(MetastoreConstants.TABLE_TYPE_KEY, getTableType().name()); - properties.put(MetastoreUtil.getCubeTableWeightKey(name), String.valueOf(weight)); + properties.put(MetastoreUtil.getCubeTableWeightKey(name), String.valueOf(weight())); } public String getName() { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 988b8d11e..8f1b1dd9a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -26,7 +26,6 @@ import java.text.ParseException; import java.util.*; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import org.apache.lens.api.metastore.*; @@ -35,6 +34,7 @@ import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo; import org.apache.lens.cube.metadata.timeline.PartitionTimeline; import org.apache.lens.cube.metadata.timeline.PartitionTimelineFactory; +import org.apache.lens.cube.parse.StorageCandidate; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.DataCompletenessChecker; @@ -58,7 +58,6 @@ import org.jvnet.jaxb2_commons.lang.HashCode; import org.jvnet.jaxb2_commons.lang.ToString; -import com.google.common.base.*; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -133,7 +132,7 @@ public boolean isDataCompletenessCheckEnabled() { /** extract storage name from fact and storage table name. String operation */ private String extractStorageName(FactTable fact, String storageTableName) throws LensException { - int ind = storageTableName.lastIndexOf(fact.getName()); + int ind = storageTableName.lastIndexOf(StorageCandidate.getStorageFactName(fact)); if (ind <= 0) { throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); } @@ -1432,8 +1431,8 @@ public boolean tableExists(String tblName) throws HiveException { public boolean factPartitionExists(FactTable fact, FactPartition part, String storageTableName) throws HiveException, LensException { String storage = extractStorageName(fact, storageTableName); - return partitionTimelineCache.partitionTimeExists(fact.getName(), storage, part.getPeriod(), part.getPartCol(), - part.getPartSpec()); + return partitionTimelineCache.partitionTimeExists(StorageCandidate.getStorageFactName(fact), storage, + part.getPeriod(), part.getPartCol(), part.getPartSpec()); } public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod, @@ -2061,7 +2060,7 @@ private CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, String if (enableCaching && fact != null) { allVirtualFactTables.put(virtualTableName, fact); allVirtualFactSourceMappings.put(virtualTableName, sourceFactName); - if(factToVirtualFactTables.get(sourceFactName) != null){ + if (factToVirtualFactTables.get(sourceFactName) != null) { List prevList = factToVirtualFactTables.get(sourceFactName); prevList.add(virtualTableName); }else{ @@ -2419,8 +2418,8 @@ public List getAllDimensionTables(Dimension dim) throws Lens return dimTables; } - public boolean partColExists(String fact, String storage, String partCol) throws LensException { - for (String storageTable : getStorageTables(fact, storage)) { + public boolean partColExists(FactTable factTable, String storage, String partCol) throws LensException { + for (String storageTable : getStorageTables(factTable, storage)) { for (FieldSchema f : getTable(storageTable).getPartCols()) { if (f.getName().equalsIgnoreCase(partCol)) { return true; @@ -2435,20 +2434,21 @@ public boolean partColExists(String fact, String storage, String partCol) throws * Note: If each update period in the storage has a different storage table, this method will return N Storage Tables * where N is the number of update periods in the storage (LENS-1386) * - * @param fact + * @param factTable * @param storage * @return * @throws LensException */ - public Set getStorageTables(String fact, String storage) throws LensException { + public Set getStorageTables(FactTable factTable, String storage) throws LensException { Set uniqueStorageTables = new HashSet<>(); - for (UpdatePeriod updatePeriod : getFactTable(fact).getUpdatePeriods().get(storage)) { - uniqueStorageTables.add(getStorageTableName(fact, storage, updatePeriod)); + + for (UpdatePeriod updatePeriod : factTable.getUpdatePeriods().get(storage)) { + String factName = StorageCandidate.getStorageFactName(factTable); + uniqueStorageTables.add(getStorageTableName(factName, storage, updatePeriod)); } return uniqueStorageTables; } - /** * * @param table table name @@ -2622,11 +2622,10 @@ private void dropAllVirtualFactTablesFromCache(String cubeFactTableName) throws if (enableCaching) { cubeFactTableName = cubeFactTableName.trim().toLowerCase(); if (factToVirtualFactTables.get(cubeFactTableName) != null) { - synchronized (allVirtualFactTables) { - List virtualFactTableNames = factToVirtualFactTables.get(cubeFactTableName); - for (String vf : virtualFactTableNames) { - dropVirtualFact(vf.trim().toLowerCase()); - } + List virtualFactTableNames = factToVirtualFactTables.get(cubeFactTableName); + factToVirtualFactTables.remove(cubeFactTableName); + for (String vf : virtualFactTableNames) { + dropVirtualFact(vf.trim().toLowerCase()); } } } @@ -2639,10 +2638,16 @@ private void dropAllVirtualFactTablesFromCache(String cubeFactTableName) throws * @throws LensException */ public void dropVirtualFact(String virtualFactName) throws LensException { + virtualFactName = virtualFactName.trim().toLowerCase(); getTableWithTypeFailFast(virtualFactName, CubeTableType.VIRTUAL_FACT); + String sourceFactTable = allVirtualFactSourceMappings.get(virtualFactName); + if (factToVirtualFactTables.get(sourceFactTable) != null + && factToVirtualFactTables.get(sourceFactTable).contains(virtualFactName)) { + factToVirtualFactTables.get(sourceFactTable).remove(virtualFactName); + } dropHiveTable(virtualFactName); allVirtualFactTables.remove(virtualFactName.trim().toLowerCase()); - allVirtualFactSourceMappings.remove(virtualFactName); + allVirtualFactSourceMappings.remove(virtualFactName.trim().toLowerCase()); } public void dropSegmentation(String segName) throws LensException { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index b28f0061f..a88ce53cc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -19,11 +19,12 @@ package org.apache.lens.cube.metadata; import java.util.*; -import com.google.common.base.Optional; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; +import com.google.common.base.Optional; + import lombok.Getter; import lombok.Setter; @@ -44,7 +45,7 @@ public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable public CubeVirtualFactTable(Table hiveTable, Table sourceHiveTable) { super(hiveTable); this.cubeName = getFactCubeName(getName(), getProperties()); -// this.virtualFactWeight = hiveTable.we + this.virtualFactWeight = Optional.of(getWeight(getProperties(), getName())); this.sourceCubeFactTable = new CubeFactTable(sourceHiveTable); } @@ -57,6 +58,17 @@ public CubeVirtualFactTable(String cubeName, String virtualFactName, Optional allProperties = sourceFactTable.getProperties(); allProperties.putAll(mapFromXProperties(fact.getProperties())); - com.google.common.base.Optional optionalWeight = com.google.common.base.Optional.fromNullable(fact.getWeight()); + Optional optionalWeight = Optional.fromNullable(fact.getWeight()); return new CubeVirtualFactTable(fact.getCubeName(), fact.getName(), optionalWeight, allProperties, sourceFactTable); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java index d5126c124..945fe26bb 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java @@ -59,7 +59,7 @@ private MetastoreConstants() { public static final String FACT_COL_START_TIME_PFX = "cube.fact.col.start.time."; public static final String FACT_COL_END_TIME_PFX = "cube.fact.col.end.time."; public static final String FACT_DATA_COMPLETENESS_TAG = "cube.fact.datacompleteness.tag"; - public static final String VIRTUAL_FACT_FILTER = " cube.fact.query.where.filter"; + public static final String VIRTUAL_FACT_FILTER = "cube.fact.query.where.filter"; // Segmentation constants public static final String SEGMENTATION_KEY_PFX = "cube.segmentation.internal."; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index bc8223215..d3a4640e9 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -183,7 +183,7 @@ public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, this.fact = fact; this.cubeql = cubeql; this.storageName = storageName; - this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), storageName); + this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(getStorageFactName(fact), storageName); this.conf = cubeql.getConf(); this.name = fact.getName(); this.processTimePartCol = conf.get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); @@ -195,7 +195,7 @@ public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, completenessThreshold = conf .getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); client = cubeql.getMetastoreClient(); - Set storageTblNames = client.getStorageTables(fact.getName(), storageName); + Set storageTblNames = client.getStorageTables(fact, storageName); if (storageTblNames.size() > 1) { isStorageTblsAtUpdatePeriodLevel = true; } else { @@ -205,6 +205,11 @@ public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, setStorageStartAndEndDate(); } + public static String getStorageFactName(FactTable fact) { + return fact.getTableType().equals(CubeTableType.VIRTUAL_FACT) + ? ((CubeVirtualFactTable) fact).getSourceCubeFactTable().getName() : fact.getName(); + } + /** * Sets Storage candidates start and end time based on underlying storage-tables * @@ -247,8 +252,8 @@ public void setStorageStartAndEndDate() throws LensException { List startDates = new ArrayList<>(); List endDates = new ArrayList<>(); for (String storageTablePrefix : getValidStorageTableNames()) { - startDates.add(client.getStorageTableStartDate(storageTablePrefix, fact.getName())); - endDates.add(client.getStorageTableEndDate(storageTablePrefix, fact.getName())); + startDates.add(client.getStorageTableStartDate(storageTablePrefix, getStorageFactName(fact))); + endDates.add(client.getStorageTableEndDate(storageTablePrefix, getStorageFactName(fact))); } this.startTime = Collections.min(startDates); this.endTime = Collections.max(endDates); @@ -264,19 +269,19 @@ private Set getValidStorageTableNames() throws LensException { return uniqueStorageTables; } else { //Get all storage tables. - return client.getStorageTables(fact.getName(), storageName); + return client.getStorageTables(fact, storageName); } } private void setMissingExpressions(Set queriedDims) throws LensException { setFromString(String.format("%s", getFromTable())); - setWhereString(joinWithAnd( + setWhereString(joinWithAnd(this.fact.getTableType().equals(CubeTableType.VIRTUAL_FACT) + && this.fact.getProperties().get(VIRTUAL_FACT_FILTER) != null + ? this.fact.getProperties().get(VIRTUAL_FACT_FILTER) :null, genWhereClauseWithDimPartitions(whereString, queriedDims), cubeql.getConf().getBoolean( CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, CubeQueryConfUtil.DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL) - ? getPostSelectionWhereClause() : null, - this.fact.getTableType().equals(CubeTableType.VIRTUAL_FACT) - && this.fact.getProperties().get(VIRTUAL_FACT_FILTER) != null - ? this.fact.getProperties().get(VIRTUAL_FACT_FILTER) :null)); + ? getPostSelectionWhereClause() : null + )); if (cubeql.getHavingAST() != null) { queryAst.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST())); } @@ -474,7 +479,7 @@ private boolean getPartitions(Date fromDate, Date toDate, String partCol, Set alterVirtualFactPropertiesMap = getHashMap("name1", "newvalue2", "name3", "value3"); cubeVirtualFact = new CubeVirtualFactTable(CUBE_NAME, virtualFactName, - com.google.common.base.Optional.fromNullable(null), alterVirtualFactPropertiesMap, sourceFact); + com.google.common.base.Optional.fromNullable(null), alterVirtualFactPropertiesMap, + sourceFact); client.alterVirtualCubeFactTable(cubeVirtualFact); actualcubeVirtualFact = client.getVirtualFactTable(virtualFactName); assertEquals(actualcubeVirtualFact.getProperties().get("name1"), "newvalue2"); @@ -1295,14 +1294,14 @@ public void testVirtualCubeFact() throws Exception { client.alterCubeFactTable(sourceFact.getName(), sourceFact, storageTables, new HashMap()); actualcubeVirtualFact = client.getVirtualFactTable(virtualFactName); assertTrue(actualcubeVirtualFact.getColumns().contains(newcol)); - assertEquals(actualcubeVirtualFact.weight(),100.0); + assertEquals(actualcubeVirtualFact.weight(), 100.0); assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); //drop source fact client.dropFact(sourceFactName, true); //drop virtual fact - try{ + try { client.dropVirtualFact(virtualFactName); fail("Expected 404"); } catch (LensException nfe) { diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 033264c81..0658ac340 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -99,6 +99,7 @@ public class CubeTestSetup { public static final String TEST_CUBE_NAME = "testCube"; public static final String DERIVED_CUBE_NAME = "derivedCube"; public static final String BASE_CUBE_NAME = "baseCube"; + public static final String VIRTUAL_CUBE_NAME = "virtualCube"; private static String c0 = "C0"; private static String c1 = "C1"; diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 721008ea2..e8c17a28a 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -144,20 +144,17 @@ public void testCubeQuery() throws Exception { // assertNotNull(rewrittenQuery.getNonExistingParts()); } -// -// @Test -// public void testCubeQuery2() throws Exception { -// CubeQueryContext rewrittenQuery = -// rewriteCtx("select SUM(msr2) from virtual_fact_cube where " + TWO_DAYS_RANGE, getConfWithStorages("C2")); -// String expected = -// getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, -// getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); -// String hql = rewrittenQuery.toHQL(); -// compareQueries(hql, expected); -// System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); -//// assertNotNull(rewrittenQuery.getNonExistingParts()); -// } + @Test + public void testVirtualFactCubeQuery() throws Exception { + CubeQueryContext rewrittenQuery = + rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); + String expected = + getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", "( dim1 = 10 )", null, + getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, "C1_summary1")); + String hql = rewrittenQuery.toHQL(); + compareQueries(hql, expected); + } @Test public void testMaxCoveringFact() throws Exception { diff --git a/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml new file mode 100644 index 000000000..f614b8cd1 --- /dev/null +++ b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + diff --git a/lens-cube/src/test/resources/schema/facts/virtualfact.xml b/lens-cube/src/test/resources/schema/facts/virtualfact.xml index 48c4a4676..860965f11 100644 --- a/lens-cube/src/test/resources/schema/facts/virtualfact.xml +++ b/lens-cube/src/test/resources/schema/facts/virtualfact.xml @@ -19,7 +19,7 @@ under the License. --> - From 838802bd50c9cf9a7c2f651bc860e7cf950f28cb Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Mon, 8 May 2017 00:04:36 +0530 Subject: [PATCH 16/71] Test case fix --- .../lens/cube/metadata/CubeVirtualFactTable.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index a88ce53cc..a51328204 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -45,8 +45,13 @@ public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable public CubeVirtualFactTable(Table hiveTable, Table sourceHiveTable) { super(hiveTable); this.cubeName = getFactCubeName(getName(), getProperties()); - this.virtualFactWeight = Optional.of(getWeight(getProperties(), getName())); this.sourceCubeFactTable = new CubeFactTable(sourceHiveTable); + + this.virtualFactWeight = Optional.absent(); + String wtStr = getProperties().get(MetastoreUtil.getCubeTableWeightKey(getName())); + if(wtStr != null ) { + this.virtualFactWeight = Optional.of(Double.parseDouble(wtStr)); + } } public CubeVirtualFactTable(String cubeName, String virtualFactName, Optional weight, @@ -71,7 +76,10 @@ public void alterWeight(double weight) { @Override protected void addProperties() { - super.addProperties(); + getProperties().put(MetastoreConstants.TABLE_TYPE_KEY, getTableType().name()); + if(virtualFactWeight.isPresent()) { + getProperties().put(MetastoreUtil.getCubeTableWeightKey(this.getName()), String.valueOf(virtualFactWeight.get())); + } addCubeNames(getName(), getProperties(), cubeName); } From e5a49a082ab4ffdbdf8f5142e76bdd59afea5312 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Mon, 8 May 2017 10:53:53 +0530 Subject: [PATCH 17/71] checkstyle fix --- .../cube/metadata/CubeVirtualFactTable.java | 4 +- .../lens/cube/parse/StorageCandidate.java | 3 +- .../lens/cube/parse/TestCubeRewriter.java | 38 +++++++------------ .../schema/cubes/base/virtualcube.xml | 1 + 4 files changed, 18 insertions(+), 28 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index a51328204..11dc00e81 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -49,7 +49,7 @@ public CubeVirtualFactTable(Table hiveTable, Table sourceHiveTable) { this.virtualFactWeight = Optional.absent(); String wtStr = getProperties().get(MetastoreUtil.getCubeTableWeightKey(getName())); - if(wtStr != null ) { + if (wtStr != null) { this.virtualFactWeight = Optional.of(Double.parseDouble(wtStr)); } } @@ -77,7 +77,7 @@ public void alterWeight(double weight) { @Override protected void addProperties() { getProperties().put(MetastoreConstants.TABLE_TYPE_KEY, getTableType().name()); - if(virtualFactWeight.isPresent()) { + if (virtualFactWeight.isPresent()) { getProperties().put(MetastoreUtil.getCubeTableWeightKey(this.getName()), String.valueOf(virtualFactWeight.get())); } addCubeNames(getName(), getProperties(), cubeName); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index d3a4640e9..d6a2499cb 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -280,8 +280,7 @@ private void setMissingExpressions(Set queriedDims) throws LensExcept ? this.fact.getProperties().get(VIRTUAL_FACT_FILTER) :null, genWhereClauseWithDimPartitions(whereString, queriedDims), cubeql.getConf().getBoolean( CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, CubeQueryConfUtil.DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL) - ? getPostSelectionWhereClause() : null - )); + ? getPostSelectionWhereClause() : null)); if (cubeql.getHavingAST() != null) { queryAst.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST())); } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index e8c17a28a..96c6adc4d 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -144,17 +144,20 @@ public void testCubeQuery() throws Exception { // assertNotNull(rewrittenQuery.getNonExistingParts()); } - - @Test - public void testVirtualFactCubeQuery() throws Exception { - CubeQueryContext rewrittenQuery = - rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); - String expected = - getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", "( dim1 = 10 )", null, - getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, "C1_summary1")); - String hql = rewrittenQuery.toHQL(); - compareQueries(hql, expected); - } +// +// @Test +// public void testVirtualFactCubeQuery() throws Exception { +// Configuration conf = getConf(); +// conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); +// CubeQueryContext rewrittenQuery = +// rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); +// String expected = +// getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", "( dim1 = 10 )", null, +// getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, "C1_summary1")); +// String hql = rewrittenQuery.toHQL(); +// compareQueries(hql, expected); +// System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); +// } @Test public void testMaxCoveringFact() throws Exception { @@ -1612,19 +1615,6 @@ public void testAliasNameSameAsColumnName() throws Exception { } } - @Test - public void testVirtualFact() throws Exception { - String query = "SELECT msr2 as msr2 from testCube WHERE " + TWO_DAYS_RANGE; - try { - String hql = rewrite(query, getConf()); - assertNotNull(hql); - System.out.println("@@HQL " + hql); - } catch (NullPointerException npe) { - fail(npe.getMessage()); - log.error("Not expecting null pointer exception", npe); - } - } - @Test public void testDimAttributeQueryWithFact() throws Exception { String query = "select count (distinct dim1) from testCube where " + TWO_DAYS_RANGE; diff --git a/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml index f614b8cd1..2a5e0b0c5 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml @@ -27,6 +27,7 @@ + From 557483588c4747360420cbce4e4755eb733f6687 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Mon, 8 May 2017 10:55:53 +0530 Subject: [PATCH 18/71] checkstyle fix --- .../lens/cube/parse/TestCubeRewriter.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 96c6adc4d..6b1fe8024 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -144,20 +144,20 @@ public void testCubeQuery() throws Exception { // assertNotNull(rewrittenQuery.getNonExistingParts()); } -// -// @Test -// public void testVirtualFactCubeQuery() throws Exception { -// Configuration conf = getConf(); -// conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); -// CubeQueryContext rewrittenQuery = -// rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); -// String expected = -// getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", "( dim1 = 10 )", null, -// getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, "C1_summary1")); -// String hql = rewrittenQuery.toHQL(); -// compareQueries(hql, expected); -// System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); -// } + + @Test + public void testVirtualFactCubeQuery() throws Exception { + Configuration conf = getConf(); + conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); + String expected = + getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", "( dim1 = 10 )", null, + getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, "C1_summary1")); + String hql = rewrittenQuery.toHQL(); + compareQueries(hql, expected); + System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); + } @Test public void testMaxCoveringFact() throws Exception { From 35e1582e5a0e29c6b15d26b3fc523abadde9ad9f Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Mon, 8 May 2017 10:57:52 +0530 Subject: [PATCH 19/71] checkstyle fix --- .../java/org/apache/lens/cube/parse/TestCubeRewriter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 6b1fe8024..4e3632f2e 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -152,8 +152,9 @@ public void testVirtualFactCubeQuery() throws Exception { CubeQueryContext rewrittenQuery = rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); String expected = - getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", "( dim1 = 10 )", null, - getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, "C1_summary1")); + getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", + "( dim1 = 10 )", null, getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, + "C1_summary1")); String hql = rewrittenQuery.toHQL(); compareQueries(hql, expected); System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); From a49eec9622b336b27c4637349bd1cb8f9b35f1e1 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Mon, 8 May 2017 10:58:11 +0530 Subject: [PATCH 20/71] checkstyle fix --- .../test/java/org/apache/lens/cube/parse/TestCubeRewriter.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 4e3632f2e..571ea3a97 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -151,8 +151,7 @@ public void testVirtualFactCubeQuery() throws Exception { conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); CubeQueryContext rewrittenQuery = rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); - String expected = - getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", + String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", "( dim1 = 10 )", null, getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, "C1_summary1")); String hql = rewrittenQuery.toHQL(); From d522a4c1556fc74529c31c7c33b7dc50167aa765 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 11 May 2017 16:01:07 +0530 Subject: [PATCH 21/71] More changes --- .../lens/cube/metadata/AbstractCubeTable.java | 34 +- .../lens/cube/metadata/CubeFactTable.java | 29 +- .../cube/metadata/CubeMetastoreClient.java | 379 +++++++++--------- .../lens/cube/metadata/CubeTableType.java | 2 +- .../cube/metadata/CubeVirtualFactTable.java | 50 ++- .../apache/lens/cube/metadata/FactTable.java | 27 +- .../apache/lens/cube/metadata/JAXBUtils.java | 9 +- .../lens/cube/metadata/MetastoreUtil.java | 20 +- .../lens/cube/metadata/Segmentation.java | 12 +- .../cube/parse/CandidateTableResolver.java | 7 +- .../lens/cube/parse/QueriedPhraseContext.java | 5 +- .../lens/cube/parse/StorageCandidate.java | 5 +- .../lens/cube/metadata/CubeFactTableTest.java | 2 +- .../metadata/TestCubeMetastoreClient.java | 14 +- .../apache/lens/cube/parse/CubeTestSetup.java | 21 +- .../api/metastore/CubeMetastoreService.java | 18 +- .../metastore/CubeMetastoreServiceImpl.java | 48 +-- .../server/metastore/MetastoreResource.java | 18 +- 18 files changed, 386 insertions(+), 314 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java index 4d9217476..8db03d1ec 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; +import com.google.common.collect.Lists; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -181,22 +182,22 @@ public boolean equals(Object obj) { } return true; } - - public Date getDateFromProperty(String propKey, boolean relative, boolean start) { - String prop = getProperties().get(propKey); - try { - if (StringUtils.isNotBlank(prop)) { - if (relative) { - return DateUtil.resolveRelativeDate(prop, now()); - } else { - return DateUtil.resolveAbsoluteDate(prop); - } - } - } catch (LensException e) { - log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop); - } - return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE; - } +// +// public Date getDateFromProperty(String propKey, boolean relative, boolean start) { +// String prop = getProperties().get(propKey); +// try { +// if (StringUtils.isNotBlank(prop)) { +// if (relative) { +// return DateUtil.resolveRelativeDate(prop, now()); +// } else { +// return DateUtil.resolveAbsoluteDate(prop); +// } +// } +// } catch (LensException e) { +// log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop); +// } +// return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE; +// } @Override @@ -233,4 +234,5 @@ public static String getFactCubeName(String factName, Map props) protected static void addCubeNames(String factName, Map props, String cubeName) { props.put(MetastoreUtil.getFactCubeNameKey(factName), cubeName); } + } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index e796a0534..e680b6d7d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -372,12 +372,28 @@ public void setAggregated(boolean isAggregated) { getProperties().put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, Boolean.toString(isAggregated)); } + @Override + public boolean isVirtualFact() { + return false; + } + + @Override + public String getStorageFactName() { + return this.getName(); + } + + public String getTablePrefix(String storage, UpdatePeriod updatePeriod) { + return storagePrefixUpdatePeriodMap.get(storage).get(updatePeriod); + } + public Date getAbsoluteStartTime() { - return getDateFromProperty(MetastoreConstants.FACT_ABSOLUTE_START_TIME, false, true); + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME), + false, true); } public Date getRelativeStartTime() { - return getDateFromProperty(MetastoreConstants.FACT_RELATIVE_START_TIME, true, true); + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_RELATIVE_START_TIME), + true, true); } public Date getStartTime() { @@ -385,18 +401,17 @@ public Date getStartTime() { } public Date getAbsoluteEndTime() { - return getDateFromProperty(MetastoreConstants.FACT_ABSOLUTE_END_TIME, false, false); + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_END_TIME), + false, false); } public Date getRelativeEndTime() { - return getDateFromProperty(MetastoreConstants.FACT_RELATIVE_END_TIME, true, false); + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_RELATIVE_END_TIME), + true, false); } public Date getEndTime() { return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime())); } - public String getTablePrefix(String storage, UpdatePeriod updatePeriod) { - return storagePrefixUpdatePeriodMap.get(storage).get(updatePeriod); - } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 8f1b1dd9a..edfb769e8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -34,7 +34,6 @@ import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo; import org.apache.lens.cube.metadata.timeline.PartitionTimeline; import org.apache.lens.cube.metadata.timeline.PartitionTimelineFactory; -import org.apache.lens.cube.parse.StorageCandidate; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.DataCompletenessChecker; @@ -89,13 +88,11 @@ private CubeMetastoreClient(HiveConf conf) { private final Map allDimTables = Maps.newConcurrentMap(); private volatile boolean allDimTablesPopulated = false; // map from fact name to fact table - private final Map allFactTables = Maps.newConcurrentMap(); + private final Map allFactTables = Maps.newConcurrentMap(); // map from fact name to all virtual fact tables private final Map> factToVirtualFactTables = Maps.newConcurrentMap(); - // map from virtual fact name to virtual fact table - private final Map allVirtualFactTables = Maps.newConcurrentMap(); - // map from virtual fact name to source fact table - private final Map allVirtualFactSourceMappings = Maps.newConcurrentMap(); +// // map from virtual fact name to virtual fact table +// private final Map allVirtualFactTables = Maps.newConcurrentMap(); private volatile boolean allFactTablesPopulated = false; private volatile boolean allVirtualFactTablesPopulated = false; //map from segmentation name to segmentation @@ -132,7 +129,7 @@ public boolean isDataCompletenessCheckEnabled() { /** extract storage name from fact and storage table name. String operation */ private String extractStorageName(FactTable fact, String storageTableName) throws LensException { - int ind = storageTableName.lastIndexOf(StorageCandidate.getStorageFactName(fact)); + int ind = storageTableName.lastIndexOf(fact.getStorageFactName()); if (ind <= 0) { throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); } @@ -189,7 +186,7 @@ public List getTimelines(String factName, String storage, Str throws LensException, HiveException { UpdatePeriod updatePeriod = updatePeriodStr == null ? null : UpdatePeriod.valueOf(updatePeriodStr.toUpperCase()); List ret = Lists.newArrayList(); - CubeFactTable fact = getCubeFact(factName); + CubeFactTable fact = (CubeFactTable) getCubeFact(factName); List storageList = Lists.newArrayList(); if (storage != null) { storageList.add(storage); @@ -383,7 +380,7 @@ public void createVirtualFactTable(XVirtualFactTable virtualFact) throws LensExc public void createVirtualFactTable(String cubeName, String virtualFactName, String sourceFactName, Double weight, Map properties) throws LensException { - CubeFactTable sourceFact = getCubeFact(sourceFactName); + FactTable sourceFact = getCubeFact(sourceFactName); com.google.common.base.Optional optionalWeight = com.google.common.base.Optional.fromNullable(weight); @@ -391,7 +388,7 @@ public void createVirtualFactTable(String cubeName, String virtualFactName, Stri optionalWeight, properties, sourceFact); createCubeTable(factTable, null); // do a get to update cache - getVirtualCubeFact(virtualFactName, sourceFactName); + getCubeFact(virtualFactName); } @@ -1102,14 +1099,14 @@ private List addPartitions(String factOrDimTable, String storageName, public Date getStorageTableStartDate(String storageTable, String factTableName) throws LensException { List startDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableStartTimesKey()); - startDates.add(getFactTable(factTableName).getStartTime()); + startDates.add(getCubeFact(factTableName).getStartTime()); return Collections.max(startDates); } public Date getStorageTableEndDate(String storageTable, String factTableName) throws LensException { List endDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableEndTimesKey()); - endDates.add(getFactTable(factTableName).getEndTime()); + endDates.add((getCubeFact(factTableName)).getEndTime()); return Collections.min(endDates); } @@ -1431,7 +1428,7 @@ public boolean tableExists(String tblName) throws HiveException { public boolean factPartitionExists(FactTable fact, FactPartition part, String storageTableName) throws HiveException, LensException { String storage = extractStorageName(fact, storageTableName); - return partitionTimelineCache.partitionTimeExists(StorageCandidate.getStorageFactName(fact), storage, + return partitionTimelineCache.partitionTimeExists(fact.getStorageFactName(), storage, part.getPeriod(), part.getPartCol(), part.getPartSpec()); } @@ -1680,8 +1677,8 @@ public boolean isVirtualFactTable(String virtualTableName) throws LensException } boolean isVirtualFactTable(Table tbl) { - String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY); - return CubeTableType.VIRTUAL_FACT.name().equals(tableType); + String sourceFactName = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); + return sourceFactName != null ? true : false ; } @@ -1768,7 +1765,7 @@ boolean isDimension(Table tbl) { } public XFactTable getXFactTable(String tableName) throws LensException { - return getXFactTable(getFactTable(tableName)); + return getXFactTable((CubeFactTable) getCubeFact(tableName)); } public XFactTable getXFactTable(CubeFactTable cft) throws LensException { @@ -1816,24 +1813,24 @@ public XFactTable getXFactTable(CubeFactTable cft) throws LensException { * @return Returns CubeFactTable if table name passed is a fact table, null otherwise * @throws LensException */ - - public CubeFactTable getFactTable(String tableName) throws LensException { - return new CubeFactTable(getTableWithTypeFailFast(tableName, CubeTableType.FACT)); - } - - /** - * Get {@link CubeVirtualFactTable} object corresponding to the name - * - * @param virtualTableName The virtual fact name - * @return Returns CubeVirtualFactTable if table name passed is a virtual fact table, null otherwise - * @throws LensException - */ - - public CubeVirtualFactTable getVirtualFactTable(String virtualTableName) throws LensException { - return new CubeVirtualFactTable(getTableWithTypeFailFast(virtualTableName, CubeTableType.VIRTUAL_FACT), - getTableWithTypeFailFast(allVirtualFactSourceMappings.get(virtualTableName.trim().toLowerCase()), - CubeTableType.FACT)); - } +// +// public FactTable getFactTable(String tableName) throws LensException { +// return new CubeFactTable(getTableWithTypeFailFast(tableName, CubeTableType.FACT)); +// } +// +// /** +// * Get {@link CubeVirtualFactTable} object corresponding to the name +// * +// * @param virtualTableName The virtual fact name +// * @return Returns CubeVirtualFactTable if table name passed is a virtual fact table, null otherwise +// * @throws LensException +// */ +// +// public CubeVirtualFactTable getVirtualFactTable(String virtualTableName) throws LensException { +// return new CubeVirtualFactTable(getTableWithTypeFailFast(virtualTableName, CubeTableType.VIRTUAL_FACT), +// getTableWithTypeFailFast(allVirtualFactSourceMappings.get(virtualTableName.trim().toLowerCase()), +// CubeTableType.FACT)); +// } public Segmentation getSegmentationTable(String tableName) throws HiveException, LensException { return new Segmentation(getTableWithTypeFailFast(tableName, CubeTableType.SEGMENTATION)); @@ -2012,17 +2009,17 @@ private Dimension getDimension(String tableName, boolean throwException) throws * @return Returns cube is table name passed is a cube * @throws LensException if there is no cube by the name */ - public CubeFactTable getCubeFact(String tableName) throws LensException { + public FactTable getCubeFact(String tableName) throws LensException { return getCubeFact(tableName, true); } - private CubeFactTable getCubeFact(String tableName, boolean throwException) throws LensException { + private FactTable getCubeFact(String tableName, boolean throwException) throws LensException { tableName = tableName.trim().toLowerCase(); - CubeFactTable fact = allFactTables.get(tableName); + FactTable fact = allFactTables.get(tableName); if (fact == null) { synchronized (allFactTables) { if (!allFactTables.containsKey(tableName)) { Table tbl = getTableWithType(tableName, CubeTableType.FACT, throwException); - fact = tbl == null ? null : new CubeFactTable(tbl); + fact = tbl == null ? null : getFactTable(tbl); if (enableCaching && fact != null) { allFactTables.put(tableName, fact); } @@ -2034,48 +2031,54 @@ private CubeFactTable getCubeFact(String tableName, boolean throwException) thro return fact; } - - /** - * Get {@link Cube} object corresponding to the name - * - * @param virtualTableName The virtual fact name - * @param sourceFactName The source fact name - * @return Returns the corresponding virtual fact table object - * @throws LensException if there is no fact by the name - */ - public CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, String sourceFactName) throws LensException { - return getVirtualCubeFact(virtualTableName, sourceFactName, true); - } - private CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, String sourceFactName, - boolean throwException) throws LensException { - virtualTableName = virtualTableName.trim().toLowerCase(); - sourceFactName = sourceFactName.trim().toLowerCase(); - CubeVirtualFactTable fact = allVirtualFactTables.get(virtualTableName); - if (fact == null) { - synchronized (allVirtualFactTables) { - if (!allVirtualFactTables.containsKey(virtualTableName)) { - Table tbl = getTableWithType(virtualTableName, CubeTableType.VIRTUAL_FACT, throwException); - Table sourceTbl = getTableWithType(sourceFactName, CubeTableType.FACT, throwException); - fact = (tbl == null || sourceTbl == null) ? null : new CubeVirtualFactTable(tbl, sourceTbl); - if (enableCaching && fact != null) { - allVirtualFactTables.put(virtualTableName, fact); - allVirtualFactSourceMappings.put(virtualTableName, sourceFactName); - if (factToVirtualFactTables.get(sourceFactName) != null) { - List prevList = factToVirtualFactTables.get(sourceFactName); - prevList.add(virtualTableName); - }else{ - List newList = new ArrayList<>(); - newList.add(virtualTableName); - factToVirtualFactTables.put(sourceFactName, newList); - } - } - } else { - fact = allVirtualFactTables.get(virtualTableName); - } - } - } - return fact; - } + private FactTable getFactTable(Table tbl) throws LensException { + String sourceFact = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); + if (sourceFact != null) { + return new CubeVirtualFactTable(tbl, (CubeFactTable) getCubeFact(sourceFact)); + } else { + return new CubeFactTable(tbl); + } + } + +// +// /** +// * Get {@link Cube} object corresponding to the name +// * +// * @param virtualTableName The virtual fact name +// * @return Returns the corresponding virtual fact table object +// * @throws LensException if there is no fact by the name +// */ +// public CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, CubeFactTable sourceFact) throws LensException { +// return getVirtualCubeFact(virtualTableName, sourceFact, true); +// } +// private CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, CubeFactTable sourceFact, +// boolean throwException) throws LensException { +// virtualTableName = virtualTableName.trim().toLowerCase(); +// +// CubeVirtualFactTable fact = allFactTables.get(virtualTableName); +// if (fact == null) { +// synchronized (allVirtualFactTables) { +// if (!allVirtualFactTables.containsKey(virtualTableName)) { +// Table virtualTbl = getTableWithType(virtualTableName, CubeTableType.FACT, throwException); +// fact = (virtualTbl == null) ? null : new CubeVirtualFactTable(virtualTbl, sourceFact); +// if (enableCaching && fact != null) { +// allVirtualFactTables.put(virtualTableName, fact); +// if (factToVirtualFactTables.get(sourceFact.getName()) != null) { +// List prevList = factToVirtualFactTables.get(sourceFact.getName()); +// prevList.add(virtualTableName); +// }else{ +// List newList = new ArrayList<>(); +// newList.add(virtualTableName); +// factToVirtualFactTables.put(sourceFact.getName(), newList); +// } +// } +// } else { +// fact = allVirtualFactTables.get(virtualTableName); +// } +// } +// } +// return fact; +// } public Segmentation getSegmentation(String segName) throws LensException { return getSegmentation(segName, true); @@ -2223,12 +2226,12 @@ public Collection getAllDimensions() throws LensException { * @return List of Cube Fact Table objects * @throws LensException */ - public Collection getAllFacts() throws LensException { + public Collection getAllFacts() throws LensException { if (!allFactTablesPopulated) { - List facts = new ArrayList<>(); + List facts = new ArrayList<>(); try { for (String table : getAllHiveTableNames()) { - CubeFactTable fact = getCubeFact(table, false); + FactTable fact = getCubeFact(table, false); if (fact != null) { facts.add(fact); } @@ -2243,36 +2246,35 @@ public Collection getAllFacts() throws LensException { } } - - /** - * Get all virtual facts in metastore - * - * @return List of Virtual Cube Fact Table objects - * @throws LensException - */ - public Collection getAllVirtualFacts() throws LensException { - if (!allVirtualFactTablesPopulated) { - List facts = new ArrayList<>(); - try { - for (String table : getAllHiveTableNames()) { - String sourceFactName = allVirtualFactSourceMappings.get(table); - CubeVirtualFactTable fact = null; - if (null != table && null != sourceFactName) { - fact = getVirtualCubeFact(table, allVirtualFactSourceMappings.get(table), false); - } - if (fact != null) { - facts.add(fact); - } - } - } catch (HiveException e) { - throw new LensException("Could not get all virtual fact tables", e); - } - allVirtualFactTablesPopulated = enableCaching; - return facts; - } else { - return allVirtualFactTables.values(); - } - } +// +// /** +// * Get all virtual facts in metastore +// * +// * @return List of Virtual Cube Fact Table objects +// * @throws LensException +// */ +// public Collection getAllVirtualFacts() throws LensException { +// if (!allVirtualFactTablesPopulated) { +// List facts = new ArrayList<>(); +// try { +// for (String table : getAllHiveTableNames()) { +// CubeVirtualFactTable fact = null; +// if (null != table) { +// fact = getVirtualCubeFact(table, false); +// } +// if (fact != null) { +// facts.add(fact); +// } +// } +// } catch (HiveException e) { +// throw new LensException("Could not get all virtual fact tables", e); +// } +// allVirtualFactTablesPopulated = enableCaching; +// return facts; +// } else { +// return allVirtualFactTables.values(); +// } +// } /** * Get all segmentations in metastore @@ -2340,30 +2342,30 @@ public List getAllFacts(CubeInterface cube) throws LensException { return cubeFacts; } - - /** - * Get all virtual fact tables of the cube. - * - * @param cube Cube object - * @return List of virtual fact tables - * @throws LensException - */ - public List getAllVirtualFacts(CubeInterface cube) throws LensException { - String cubeName = null; - if (cube != null) { - if (cube instanceof DerivedCube) { - cube = ((DerivedCube) cube).getParent(); - } - cubeName = cube.getName(); - } - List cubeVirtualFacts = new ArrayList<>(); - for (FactTable fact : getAllVirtualFacts()) { - if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { - cubeVirtualFacts.add(fact); - } - } - return cubeVirtualFacts; - } +// +// /** +// * Get all virtual fact tables of the cube. +// * +// * @param cube Cube object +// * @return List of virtual fact tables +// * @throws LensException +// */ +// public List getAllVirtualFacts(CubeInterface cube) throws LensException { +// String cubeName = null; +// if (cube != null) { +// if (cube instanceof DerivedCube) { +// cube = ((DerivedCube) cube).getParent(); +// } +// cubeName = cube.getName(); +// } +// List cubeVirtualFacts = new ArrayList<>(); +// for (FactTable fact : getAllVirtualFacts()) { +// if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { +// cubeVirtualFacts.add(fact); +// } +// } +// return cubeVirtualFacts; +// } public List getAllSegmentations(CubeInterface cube) throws LensException { String cubeName = null; @@ -2443,7 +2445,7 @@ public Set getStorageTables(FactTable factTable, String storage) throws Set uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : factTable.getUpdatePeriods().get(storage)) { - String factName = StorageCandidate.getStorageFactName(factTable); + String factName = factTable.getStorageFactName(); uniqueStorageTables.add(getStorageTableName(factName, storage, updatePeriod)); } return uniqueStorageTables; @@ -2471,25 +2473,25 @@ private boolean alterCubeTable(String table, Table hiveTable, AbstractCubeTable } return columnsChanged; } - - /** - * - * @param virtualTable table name - * @param hiveTable hive table - * @param cubeTable lens cube table - * @throws LensException - */ - private void alterCubeVirtualFactTable(String virtualTable, Table hiveTable, AbstractCubeTable cubeTable) throws - LensException { - - hiveTable.getParameters().putAll(cubeTable.getProperties()); - hiveTable.getTTable().getParameters().putAll(cubeTable.getProperties()); - try { - getClient().alterTable(virtualTable, hiveTable, null); - } catch (Exception e) { - throw new LensException(e); - } - } +// +// /** +// * +// * @param virtualTable table name +// * @param hiveTable hive table +// * @param cubeTable lens cube table +// * @throws LensException +// */ +// private void alterCubeVirtualFactTable(String virtualTable, Table hiveTable, AbstractCubeTable cubeTable) throws +// LensException { +// +// hiveTable.getParameters().putAll(cubeTable.getProperties()); +// hiveTable.getTTable().getParameters().putAll(cubeTable.getProperties()); +// try { +// getClient().alterTable(virtualTable, hiveTable, null); +// } catch (Exception e) { +// throw new LensException(e); +// } +// } public void pushHiveTable(Table hiveTable) throws HiveException, LensException { alterHiveTable(hiveTable.getTableName(), hiveTable); @@ -2607,7 +2609,7 @@ public void dropDimension(String dimName) throws LensException { */ public void dropFact(String factName, boolean cascade) throws LensException { getTableWithTypeFailFast(factName, CubeTableType.FACT); - CubeFactTable fact = getFactTable(factName); + FactTable fact = getCubeFact(factName); if (cascade) { for (String storage : fact.getStorages()) { dropStorageFromFact(factName, storage, false); @@ -2639,15 +2641,15 @@ private void dropAllVirtualFactTablesFromCache(String cubeFactTableName) throws */ public void dropVirtualFact(String virtualFactName) throws LensException { virtualFactName = virtualFactName.trim().toLowerCase(); - getTableWithTypeFailFast(virtualFactName, CubeTableType.VIRTUAL_FACT); - String sourceFactTable = allVirtualFactSourceMappings.get(virtualFactName); + Table virtualTbl = getTableWithTypeFailFast(virtualFactName, CubeTableType.FACT); + String sourceFactTable = virtualTbl.getParameters().get(getSourceFactNameKey(virtualTbl.getTableName())); if (factToVirtualFactTables.get(sourceFactTable) != null && factToVirtualFactTables.get(sourceFactTable).contains(virtualFactName)) { factToVirtualFactTables.get(sourceFactTable).remove(virtualFactName); } dropHiveTable(virtualFactName); - allVirtualFactTables.remove(virtualFactName.trim().toLowerCase()); - allVirtualFactSourceMappings.remove(virtualFactName.trim().toLowerCase()); + allFactTables.remove(virtualFactName.trim().toLowerCase()); + } public void dropSegmentation(String segName) throws LensException { @@ -2664,7 +2666,7 @@ public void dropSegmentation(String segName) throws LensException { * @throws LensException */ public void dropStorageFromFact(String factName, String storage) throws LensException { - CubeFactTable cft = getFactTable(factName); + CubeFactTable cft = (CubeFactTable) getCubeFact(factName); dropHiveTablesForStorage(factName, storage); cft.dropStorage(storage); alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft); @@ -2672,7 +2674,7 @@ public void dropStorageFromFact(String factName, String storage) throws LensExce } private void dropHiveTablesForStorage(String factName, String storage) throws LensException{ - CubeFactTable cft = getFactTable(factName); + CubeFactTable cft = (CubeFactTable) getCubeFact(factName); Set droppedTables = new HashSet<>(); for (Map.Entry updatePeriodEntry : cft.getStoragePrefixUpdatePeriodMap().get(storage).entrySet()) { UpdatePeriod updatePeriod = (UpdatePeriod) updatePeriodEntry.getKey(); @@ -2688,7 +2690,7 @@ private void dropStorageFromFact(String factName, String storage, boolean update throws LensException { dropHiveTablesForStorage(factName, storage); if (updateFact) { - CubeFactTable cft = getFactTable(factName); + CubeFactTable cft = (CubeFactTable) getCubeFact(factName); cft.dropStorage(storage); alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft); updateFactCache(factName); @@ -2745,8 +2747,8 @@ public void alterCubeFactTable(XFactTable fact) throws LensException, HiveExcept } public void alterVirtualCubeFactTable(XVirtualFactTable virtualFact) throws LensException, HiveException { - alterVirtualCubeFactTable(JAXBUtils.cubeVirtualFactFromFactTable(virtualFact, - getCubeFact(virtualFact.getSourceFactName()))); + alterCubeFactTable(virtualFact.getName(), JAXBUtils.cubeVirtualFactFromFactTable(virtualFact, + getCubeFact(virtualFact.getSourceFactName())), null, new HashMap<>()); } /** * Alter a cubefact with new definition and alter underlying storage tables as well. @@ -2757,7 +2759,7 @@ public void alterVirtualCubeFactTable(XVirtualFactTable virtualFact) throws Lens * * @throws HiveException */ - public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable, + public void alterCubeFactTable(String factTableName, FactTable cubeFactTable, Map storageTableDescs, Map props) throws HiveException, LensException { @@ -2765,7 +2767,7 @@ public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable if (!props.isEmpty()) { cubeFactTable.getProperties().putAll(props); } - alterCubeTable(factTableName, factTbl, cubeFactTable); + alterCubeTable(factTableName, factTbl, (AbstractCubeTable) cubeFactTable); if (storageTableDescs != null) { // create/alter tables for each storage for (Map.Entry entry : storageTableDescs.entrySet()) { @@ -2773,7 +2775,10 @@ public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable } } updateFactCache(factTableName); - updateAllVirtualFacts(getCubeFact(factTableName)); + + if(cubeFactTable.getTableType().equals(CubeTableType.FACT)) { + updateAllVirtualFacts(getCubeFact(factTableName)); + } } public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException { @@ -2787,12 +2792,7 @@ public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveE */ public void alterVirtualCubeFactTable(CubeVirtualFactTable cubeVirtualFactTable) throws HiveException, LensException { - - String virtualFactTableName = cubeVirtualFactTable.getName(); - Table factTbl = getTableWithTypeFailFast(virtualFactTableName, CubeTableType.VIRTUAL_FACT); - - alterCubeVirtualFactTable(virtualFactTableName, factTbl, cubeVirtualFactTable); - updateVirtualFactCache(virtualFactTableName); + alterCubeFactTable(cubeVirtualFactTable.getName(), cubeVirtualFactTable, null, new HashMap<>()); } public void alterSegmentation(String segName, Segmentation seg) @@ -2813,33 +2813,32 @@ private void updateSegmentationCache(String segmentName) throws HiveException, L private void updateFactCache(String factTableName) throws LensException { if (enableCaching) { - allFactTables.put(factTableName.trim().toLowerCase(), new CubeFactTable(refreshTable(factTableName))); + allFactTables.put(factTableName.trim().toLowerCase(), getCubeFact(factTableName)); } } - private void updateAllVirtualFacts(CubeFactTable cubeFactTable) throws LensException { + private void updateAllVirtualFacts(FactTable cubeFactTable) throws LensException { if (enableCaching) { String cubeFactTableName = cubeFactTable.getName().trim().toLowerCase(); if (factToVirtualFactTables.get(cubeFactTableName) != null) { - synchronized (allVirtualFactTables) { + synchronized (allFactTables) { List virtualFactTableNames = factToVirtualFactTables.get(cubeFactTableName); for (String vf : virtualFactTableNames) { - CubeVirtualFactTable cvf = getVirtualCubeFact(vf, allVirtualFactSourceMappings.get(vf)); + CubeVirtualFactTable cvf = (CubeVirtualFactTable) getCubeFact(vf); cvf.setSourceCubeFactTable(cubeFactTable); - allVirtualFactTables.put(vf.trim().toLowerCase(), cvf); + allFactTables.put(vf.trim().toLowerCase(), cvf); } } } } } - - private void updateVirtualFactCache(String factTableName) throws LensException { - if (enableCaching) { - allVirtualFactTables.put(factTableName.trim().toLowerCase(), - new CubeVirtualFactTable(refreshTable(factTableName), - getHiveTable(allVirtualFactSourceMappings.get(factTableName.trim().toLowerCase())))); - } - } +// +// private void updateVirtualFactCache(String factTableName, CubeFactTable cubeFactTable) throws LensException { +// if (enableCaching) { +// allVirtualFactTables.put(factTableName.trim().toLowerCase(), +// new CubeVirtualFactTable(refreshTable(factTableName), cubeFactTable )); +// } +// } private void updateDimCache(String dimTblName) throws LensException { if (enableCaching) { @@ -2948,7 +2947,7 @@ private String getStorageTablePrefixFromStorage(String factOrDimTableName, Strin return storage; } if (isFactTable(factOrDimTableName)) { - return getFactTable(factOrDimTableName).getTablePrefix(storage, updatePeriod); + return ((CubeFactTable)getCubeFact(factOrDimTableName)).getTablePrefix(storage, updatePeriod); } else { return storage; } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java index 85075fba4..6dd35b318 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java @@ -20,5 +20,5 @@ package org.apache.lens.cube.metadata; public enum CubeTableType { - CUBE, DIMENSION, FACT, DIM_TABLE, STORAGE, SEGMENTATION, VIRTUAL_FACT + CUBE, DIMENSION, FACT, DIM_TABLE, STORAGE, SEGMENTATION } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index 11dc00e81..0fc7c36be 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -25,6 +25,7 @@ import com.google.common.base.Optional; +import com.google.common.collect.Lists; import lombok.Getter; import lombok.Setter; @@ -32,7 +33,7 @@ public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable @Getter @Setter - private CubeFactTable sourceCubeFactTable; + private FactTable sourceCubeFactTable; private String cubeName; private static final List COLUMNS = new ArrayList(); @Getter @@ -42,10 +43,10 @@ public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable COLUMNS.add(new FieldSchema("dummy", "string", "dummy column")); } - public CubeVirtualFactTable(Table hiveTable, Table sourceHiveTable) { + public CubeVirtualFactTable(Table hiveTable, CubeFactTable sourceCubeFactTable) { super(hiveTable); this.cubeName = getFactCubeName(getName(), getProperties()); - this.sourceCubeFactTable = new CubeFactTable(sourceHiveTable); + this.sourceCubeFactTable = sourceCubeFactTable; this.virtualFactWeight = Optional.absent(); String wtStr = getProperties().get(MetastoreUtil.getCubeTableWeightKey(getName())); @@ -55,7 +56,7 @@ public CubeVirtualFactTable(Table hiveTable, Table sourceHiveTable) { } public CubeVirtualFactTable(String cubeName, String virtualFactName, Optional weight, - Map properties, CubeFactTable sourceFact) { + Map properties, FactTable sourceFact) { super(virtualFactName, COLUMNS, properties, weight.isPresent() ? weight.get() : sourceFact.weight()); this.cubeName = cubeName; this.virtualFactWeight = weight; @@ -85,7 +86,7 @@ protected void addProperties() { @Override public CubeTableType getTableType() { - return CubeTableType.VIRTUAL_FACT; + return CubeTableType.FACT; } @Override @@ -127,4 +128,43 @@ public List getColumns() { public double weight() { return virtualFactWeight.isPresent() ? virtualFactWeight.get() : sourceCubeFactTable.weight(); } + + public Date getAbsoluteStartTime() { + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME), + false, true); + } + + public Date getRelativeStartTime() { + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_RELATIVE_START_TIME), + true, true); + } + + public Date getStartTime() { + return Collections.max(Lists.newArrayList(getRelativeStartTime(), getAbsoluteStartTime())); + } + + public Date getAbsoluteEndTime() { + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_END_TIME), + false, false); + } + + public Date getRelativeEndTime() { + return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_RELATIVE_END_TIME), + true, false); + } + + public Date getEndTime() { + return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime())); + } + + + @Override + public boolean isVirtualFact() { + return true; + } + + @Override + public String getStorageFactName() { + return this.sourceCubeFactTable.getName(); + } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java index 6fdcaf2cb..422d90b72 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java @@ -18,16 +18,12 @@ */ package org.apache.lens.cube.metadata; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.apache.hadoop.hive.metastore.api.FieldSchema; -/** - * Created by rajithar on 3/5/17. - */ +import com.google.common.collect.Lists; + public interface FactTable extends Named { public Map> getUpdatePeriods(); @@ -52,5 +48,20 @@ public interface FactTable extends Named { public boolean isAggregated(); - public Date getDateFromProperty(String propKey, boolean relative, boolean start); + public Date getAbsoluteStartTime(); + + public Date getRelativeStartTime(); + + public Date getStartTime(); + + public Date getAbsoluteEndTime(); + + public Date getRelativeEndTime(); + + public Date getEndTime(); + + public boolean isVirtualFact(); + + public String getStorageFactName(); + } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java index 632ff05e8..7dfb16683 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java @@ -718,16 +718,13 @@ public static CubeFactTable cubeFactFromFactTable(XFactTable fact) throws LensEx mapFromXProperties(fact.getProperties()), storageTablePrefixMap); } - public static CubeVirtualFactTable cubeVirtualFactFromFactTable(XVirtualFactTable fact, CubeFactTable sourceFactTable) + public static CubeVirtualFactTable cubeVirtualFactFromFactTable(XVirtualFactTable fact, FactTable sourceFactTable) throws LensException { - //Update properties from source fact - Map allProperties = sourceFactTable.getProperties(); - allProperties.putAll(mapFromXProperties(fact.getProperties())); Optional optionalWeight = Optional.fromNullable(fact.getWeight()); return new CubeVirtualFactTable(fact.getCubeName(), fact.getName(), - optionalWeight, allProperties, sourceFactTable); + optionalWeight, mapFromXProperties(fact.getProperties()), sourceFactTable); } public static Segmentation segmentationFromXSegmentation(XSegmentation seg) throws LensException { @@ -748,7 +745,7 @@ public static Segmentation segmentationFromXSegmentation(XSegmentation seg) thro } - public static XFactTable factTableFromCubeFactTable(CubeFactTable cFact) { + public static XFactTable factTableFromCubeFactTable(FactTable cFact) { XFactTable fact = XCF.createXFactTable(); fact.setName(cFact.getName()); fact.setColumns(new XColumns()); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index 0dc4088de..fb3a7b2bc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -332,7 +332,7 @@ public static String getCubeTableKeyPrefix(String tableName) { } // ////////////////////////// - // Fact propertes /// + // Fact properties /// // ///////////////////////// public static String getFactStorageListKey(String name) { return getFactKeyPrefix(name) + STORAGE_LIST_SFX; @@ -389,6 +389,24 @@ public static String getSegmentPropertyKey(String segName) { // ////////////////////////// // Utils /// // ///////////////////////// + + public static Date getDateFromProperty(String prop, boolean relative, boolean start) { + // String prop = getProperties().get(propKey); + try { + if (StringUtils.isNotBlank(prop)) { + if (relative) { + return DateUtil.resolveRelativeDate(prop, new Date()); + } else { + return DateUtil.resolveAbsoluteDate(prop); + } + } + } catch (LensException e) { + // log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop); + } + return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE; + } + + public static String getNamedStr(Collection set) { if (set == null) { return ""; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java index 57be74662..54059d113 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java @@ -160,11 +160,13 @@ public Set getStorages() { public Date getAbsoluteStartTime() { - return getDateFromProperty(MetastoreConstants.SEGMENTATION_ABSOLUTE_START_TIME, false, true); + return MetastoreUtil.getDateFromProperty(this.getProperties() + .get(MetastoreConstants.SEGMENTATION_ABSOLUTE_START_TIME), false, true); } public Date getRelativeStartTime() { - return getDateFromProperty(MetastoreConstants.SEGMENTATION_RELATIVE_START_TIME, true, true); + return MetastoreUtil.getDateFromProperty(this.getProperties() + .get(MetastoreConstants.SEGMENTATION_RELATIVE_START_TIME), true, true); } public Date getStartTime() { @@ -172,11 +174,13 @@ public Date getStartTime() { } public Date getAbsoluteEndTime() { - return getDateFromProperty(MetastoreConstants.SEGMENTATION_ABSOLUTE_END_TIME, false, false); + return MetastoreUtil.getDateFromProperty(this.getProperties() + .get(MetastoreConstants.SEGMENTATION_ABSOLUTE_END_TIME), false, false); } public Date getRelativeEndTime() { - return getDateFromProperty(MetastoreConstants.SEGMENTATION_RELATIVE_END_TIME, true, false); + return MetastoreUtil.getDateFromProperty(this.getProperties() + .get(MetastoreConstants.SEGMENTATION_RELATIVE_END_TIME), true, false); } public Date getEndTime() { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index d795c1cbf..1ffdca7ed 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -85,7 +85,6 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { private void populateCandidateTables(CubeQueryContext cubeql) throws LensException { if (cubeql.getCube() != null) { List factTables = cubeql.getMetastoreClient().getAllFacts(cubeql.getCube()); - factTables.addAll(cubeql.getMetastoreClient().getAllVirtualFacts(cubeql.getCube())); if (factTables.isEmpty()) { throw new LensException(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), cubeql.getCube().getName() + " does not have any facts"); @@ -208,7 +207,8 @@ private static Date getFactColumnStartTime(CandidateTable table, String factCol) if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_START_TIME_PFX); if (factCol.equals(propCol)) { - startTime = ((StorageCandidate) table).getFact().getDateFromProperty(key, false, true); + startTime = MetastoreUtil.getDateFromProperty(((StorageCandidate) table).getFact().getProperties().get(key), + false, true); } } } @@ -223,7 +223,8 @@ private static Date getFactColumnEndTime(CandidateTable table, String factCol) { if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_END_TIME_PFX); if (factCol.equals(propCol)) { - endTime = ((StorageCandidate) table).getFact().getDateFromProperty(key, false, true); + endTime = MetastoreUtil.getDateFromProperty(((StorageCandidate) table).getFact().getProperties().get(key), + false, true); } } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java index 1c2cfee3d..929f5cade 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -22,6 +22,7 @@ import org.apache.lens.cube.metadata.FactTable; import org.apache.lens.cube.metadata.MetastoreConstants; +import org.apache.lens.cube.metadata.MetastoreUtil; import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.server.api.error.LensException; @@ -166,7 +167,7 @@ public static Date getFactColumnStartTime(FactTable ft, String factCol) { if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_START_TIME_PFX); if (factCol.equals(propCol)) { - startTime = ft.getDateFromProperty(key, false, true); + startTime = MetastoreUtil.getDateFromProperty(ft.getProperties().get(key), false, true); } } } @@ -179,7 +180,7 @@ public static Date getFactColumnEndTime(FactTable ft, String factCol) { if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_END_TIME_PFX); if (factCol.equals(propCol)) { - endTime = ft.getDateFromProperty(key, false, true); + endTime = MetastoreUtil.getDateFromProperty(ft.getProperties().get(key), false, true); } } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index d6a2499cb..f442edea7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -206,7 +206,7 @@ public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, } public static String getStorageFactName(FactTable fact) { - return fact.getTableType().equals(CubeTableType.VIRTUAL_FACT) + return fact.getProperties().get(MetastoreUtil.getSourceFactNameKey(fact.getName()))!= null ? ((CubeVirtualFactTable) fact).getSourceCubeFactTable().getName() : fact.getName(); } @@ -275,8 +275,7 @@ private Set getValidStorageTableNames() throws LensException { private void setMissingExpressions(Set queriedDims) throws LensException { setFromString(String.format("%s", getFromTable())); - setWhereString(joinWithAnd(this.fact.getTableType().equals(CubeTableType.VIRTUAL_FACT) - && this.fact.getProperties().get(VIRTUAL_FACT_FILTER) != null + setWhereString(joinWithAnd(this.fact.getProperties().get(VIRTUAL_FACT_FILTER) != null ? this.fact.getProperties().get(VIRTUAL_FACT_FILTER) :null, genWhereClauseWithDimPartitions(whereString, queriedDims), cubeql.getConf().getBoolean( CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, CubeQueryConfUtil.DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL) diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java index 42e32cb8e..8ceadc718 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java @@ -69,7 +69,7 @@ private CubeFactTable getMockCubeFactTable(Map properties) { when(cubeFactTable.getProperties()).thenReturn(properties); - when(cubeFactTable.getDateFromProperty(anyString(), anyBoolean(), anyBoolean())).thenCallRealMethod(); + when(MetastoreUtil.getDateFromProperty(properties.get(anyString()), anyBoolean(), anyBoolean())).thenCallRealMethod(); when(cubeFactTable.getRelativeStartTime()).thenCallRealMethod(); when(cubeFactTable.getAbsoluteStartTime()).thenCallRealMethod(); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 9b9d49ce0..221e24950 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -1269,11 +1269,11 @@ public void testVirtualCubeFact() throws Exception { assertTrue(client.isVirtualFactTableForCube(virtualTbl, CUBE_NAME)); //get virtual fact - assertEquals(client.getAllVirtualFacts(client.getCube(CUBE_NAME)).get(0).getName(), virtualFactName.toLowerCase()); - assertEquals(client.getAllVirtualFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(), + assertEquals(client.getAllFacts(client.getCube(CUBE_NAME)).get(0).getName(), virtualFactName.toLowerCase()); + assertEquals(client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(), virtualFactName.toLowerCase()); - CubeVirtualFactTable actualcubeVirtualFact = client.getVirtualFactTable(virtualFactName); + CubeVirtualFactTable actualcubeVirtualFact = (CubeVirtualFactTable) (client.getCubeFact(virtualFactName)); assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); //alter virtual fact @@ -1282,7 +1282,7 @@ public void testVirtualCubeFact() throws Exception { com.google.common.base.Optional.fromNullable(null), alterVirtualFactPropertiesMap, sourceFact); client.alterVirtualCubeFactTable(cubeVirtualFact); - actualcubeVirtualFact = client.getVirtualFactTable(virtualFactName); + actualcubeVirtualFact = (CubeVirtualFactTable) client.getCubeFact(virtualFactName); assertEquals(actualcubeVirtualFact.getProperties().get("name1"), "newvalue2"); assertEquals(actualcubeVirtualFact.getProperties().get("name3"), "value3"); assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); @@ -1292,7 +1292,7 @@ public void testVirtualCubeFact() throws Exception { sourceFact.alterColumn(newcol); sourceFact.alterWeight(100); client.alterCubeFactTable(sourceFact.getName(), sourceFact, storageTables, new HashMap()); - actualcubeVirtualFact = client.getVirtualFactTable(virtualFactName); + actualcubeVirtualFact = (CubeVirtualFactTable) client.getCubeFact(virtualFactName); assertTrue(actualcubeVirtualFact.getColumns().contains(newcol)); assertEquals(actualcubeVirtualFact.weight(), 100.0); assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); @@ -1460,7 +1460,7 @@ public void testAlterCubeFact() throws Exception { updatePeriodStoragePrefix.put(DAILY, c4); factTable.addStorage(c4, hourlyAndDaily, updatePeriodStoragePrefix); client.alterCubeFactTable(factName, factTable, storageTables, new HashMap()); - CubeFactTable altered2 = client.getCubeFact(factName); + CubeFactTable altered2 = (CubeFactTable) client.getCubeFact(factName); assertTrue(client.tableExists(c1TableName)); Table alteredC1Table = client.getTable(c1TableName); assertEquals(alteredC1Table.getInputFormatClass(), SequenceFileInputFormat.class); @@ -1487,7 +1487,7 @@ public void testAlterCubeFact() throws Exception { Map storageTableDescMap = new HashMap<>(); storageTableDescMap.put(c3, s1); client.addStorage(altered2, c3, hourlyAndDaily, storageTableDescMap, updatePeriodStoragePrefix); - CubeFactTable altered3 = client.getCubeFact(factName); + CubeFactTable altered3 = (CubeFactTable) client.getCubeFact(factName); assertTrue(altered3.getStorages().contains("C3")); assertTrue(altered3.getUpdatePeriods().get("C3").equals(hourlyAndDaily)); String storageTableName = getFactOrDimtableStorageTableName(factName, c3); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 0658ac340..7630ca494 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -534,7 +534,7 @@ private void assertTestFactTimelineClass(CubeMetastoreClient client) throws Exce client.clearHiveTableCache(); - CubeFactTable fact = client.getFactTable(factName); + CubeFactTable fact = (CubeFactTable) client.getCubeFact(factName); Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c1))); assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); for (UpdatePeriod period : Lists.newArrayList(MINUTELY, HOURLY, DAILY, MONTHLY, YEARLY, QUARTERLY)) { @@ -583,7 +583,7 @@ private void assertTimeline(CubeMetastoreClient client, String factName, String private void createCubeCheapFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "cheapFact"; - CubeFactTable fact = client.getFactTable(factName); + CubeFactTable fact = (CubeFactTable) client.getCubeFact(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); @@ -615,7 +615,7 @@ private void createCubeCheapFactPartitions(CubeMetastoreClient client) throws Hi private void createTestFact2Partitions(CubeMetastoreClient client) throws Exception { String factName = "testFact2"; - CubeFactTable fact = client.getFactTable(factName); + CubeFactTable fact = (CubeFactTable) client.getCubeFact(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); @@ -703,7 +703,7 @@ private void createTestFact2Partitions(CubeMetastoreClient client) throws Except private void createTestFact2RawPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "testFact2_raw"; - CubeFactTable fact2 = client.getFactTable(factName); + CubeFactTable fact2 = (CubeFactTable) client.getCubeFact(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); @@ -785,7 +785,8 @@ private void dump(CubeMetastoreClient client) throws LensException, IOException // bw.write(ToXMLString.toString(JAXBUtils.xCubeFromHiveCube(cubeInterface))); // } // } - for (CubeFactTable cubeFactTable : client.getAllFacts()) { + for (FactTable factTable : client.getAllFacts()) { + CubeFactTable cubeFactTable = (CubeFactTable) factTable; try(BufferedWriter bw = new BufferedWriter(new FileWriter(getClass() .getResource("/schema/facts").getPath()+"/"+cubeFactTable.getName()+".xml"))) { bw.write(ToXMLString.toString(client.getXFactTable(cubeFactTable))); @@ -818,25 +819,25 @@ public void dropSources(HiveConf conf, String dbName) throws Exception { private void createSummaryPartitions(CubeMetastoreClient client) throws Exception { String factName = "summary1"; - CubeFactTable fact1 = client.getFactTable(factName); + CubeFactTable fact1 = (CubeFactTable) client.getCubeFact(factName); createPIEParts(client, fact1, c2); factName = "summary2"; - CubeFactTable fact2 = client.getFactTable(factName); + CubeFactTable fact2 = (CubeFactTable) client.getCubeFact(factName); createPIEParts(client, fact2, c2); factName = "summary3"; - CubeFactTable fact3 = client.getFactTable(factName); + CubeFactTable fact3 = (CubeFactTable) client.getCubeFact(factName); createPIEParts(client, fact3, c2); factName = "summary4"; - CubeFactTable fact4 = client.getFactTable(factName); + CubeFactTable fact4 = (CubeFactTable) client.getCubeFact(factName); createPIEParts(client, fact4, c2); } private void createBaseCubeFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "testFact5_RAW_BASE"; - CubeFactTable fact = client.getFactTable(factName); + CubeFactTable fact = (CubeFactTable) client.getCubeFact(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java index c53b51b62..729041321 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java @@ -458,15 +458,15 @@ int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, St */ void dropVirtualFactTable(LensSessionHandle sessionid, String virtualFact) throws LensException; - /** - * Get all virtual fact names - * - * @param sessionid The sessionid - * @param cubeName optional filter filter facts by cube name. - * @return List of virtual fact table names - * @throws LensException - */ - List getAllVirtualFactNames(LensSessionHandle sessionid, String cubeName) throws LensException; +// /** +// * Get all virtual fact names +// * +// * @param sessionid The sessionid +// * @param cubeName optional filter filter facts by cube name. +// * @return List of virtual fact table names +// * @throws LensException +// */ +// List getAllVirtualFactNames(LensSessionHandle sessionid, String cubeName) throws LensException; /** * Get all storages of fact diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index f876229ab..79b964906 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -323,7 +323,7 @@ public List getAllDimTableNames(LensSessionHandle sessionid, String dime public void dropAllStoragesOfFact(LensSessionHandle sessionid, String factName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); - CubeFactTable tab = msClient.getFactTable(factName); + CubeFactTable tab = (CubeFactTable) msClient.getCubeFact(factName); int total = tab.getStorages().size(); int i = 0; List storageNames = new ArrayList<>(tab.getStorages()); @@ -445,7 +445,7 @@ public List getAllFactNames(LensSessionHandle sessionid, String cubeName public XVirtualFactTable getVirtualFactTable(LensSessionHandle sessionid, String virtualFact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); - CubeVirtualFactTable cft = msClient.getVirtualFactTable(virtualFact); + CubeVirtualFactTable cft = (CubeVirtualFactTable) msClient.getCubeFact(virtualFact); XVirtualFactTable factTable = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cft); factTable.setSourceFactName(cft.getSourceCubeFactTable().getName()); return factTable; @@ -485,23 +485,23 @@ public void dropVirtualFactTable(LensSessionHandle sessionid, String virtualFact log.info("Dropped virtual fact table " + virtualFact); } } - - @Override - public List getAllVirtualFactNames(LensSessionHandle sessionid, String cubeName) throws LensException { - try (SessionContext ignored = new SessionContext(sessionid)){ - CubeMetastoreClient client = getClient(sessionid); - CubeInterface fact = client.getCube(cubeName); - if (cubeName != null && fact == null) { - throw new LensException("Could not get table: " + cubeName + " as a cube"); - } - Collection virtualFacts = client.getAllVirtualFacts(fact); - List factNames = new ArrayList<>(virtualFacts.size()); - for (FactTable cft : virtualFacts) { - factNames.add(cft.getName()); - } - return factNames; - } - } +// +// @Override +// public List getAllVirtualFactNames(LensSessionHandle sessionid, String cubeName) throws LensException { +// try (SessionContext ignored = new SessionContext(sessionid)){ +// CubeMetastoreClient client = getClient(sessionid); +// CubeInterface fact = client.getCube(cubeName); +// if (cubeName != null && fact == null) { +// throw new LensException("Could not get table: " + cubeName + " as a cube"); +// } +// Collection virtualFacts = client.getAllFacts(fact); +// List factNames = new ArrayList<>(virtualFacts.size()); +// for (FactTable cft : virtualFacts) { +// factNames.add(cft.getName()); +// } +// return factNames; +// } +// } @Override public List getAllSegmentations(LensSessionHandle sessionid, String cubeName) throws LensException { @@ -530,7 +530,7 @@ public List getStoragesOfFact(LensSessionHandle sessionid, String fact) throw new NotFoundException("Not a fact table " + fact); } - CubeFactTable cft = msClient.getFactTable(fact); + CubeFactTable cft = (CubeFactTable) msClient.getCubeFact(fact); if (cft != null) { return new ArrayList<>(cft.getStorages()); } else { @@ -543,7 +543,7 @@ public XStorageTableElement getStorageOfFact(LensSessionHandle sessionid, String throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); - CubeFactTable factTable = msClient.getFactTable(fact); + FactTable factTable = msClient.getCubeFact(fact); Set updatePeriods = factTable.getUpdatePeriods().get(storageName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName))); @@ -582,7 +582,7 @@ public void addStorageToFact(LensSessionHandle sessionid, String fact, XStorageT CubeMetastoreClient msClient = getClient(sessionid); XStorageTables tables = new XStorageTables(); tables.getStorageTable().add(storageTable); - msClient.addStorage(msClient.getFactTable(fact), storageTable.getStorageName(), updatePeriods, + msClient.addStorage((CubeFactTable) msClient.getCubeFact(fact), storageTable.getStorageName(), updatePeriods, JAXBUtils.tableDescPrefixMapFromXStorageTables(tables), JAXBUtils.storageTablePrefixMapOfStorage(tables).get(storageTable.getStorageName())); log.info("Added storage " + storageTable.getStorageName() + ":" + updatePeriods + " for fact " + fact); @@ -603,7 +603,7 @@ public void dropStorageOfFact(LensSessionHandle sessionid, String fact, String s private CubeFactTable checkFactStorage(LensSessionHandle sessionid, String fact, String storage) throws HiveException, LensException { CubeMetastoreClient client = getClient(sessionid); - CubeFactTable factTable = client.getCubeFact(fact); + CubeFactTable factTable = (CubeFactTable) client.getCubeFact(fact); client.verifyStorageExists(factTable, storage); return factTable; } @@ -612,7 +612,7 @@ private Set getAllTablesForStorage(LensSessionHandle sessionHandle, Stri throws LensException { Set storageTableNames = new HashSet<>(); if (getClient(sessionHandle).isFactTable(fact)) { - CubeFactTable cft = getClient(sessionHandle).getCubeFact(fact); + CubeFactTable cft = (CubeFactTable) getClient(sessionHandle).getCubeFact(fact); Map storageMap = cft.getStoragePrefixUpdatePeriodMap().get(storageName); for (Map.Entry entry : storageMap.entrySet()) { storageTableNames.add(MetastoreUtil.getStorageTableName(fact, Storage.getPrefix((String) entry.getValue()))); diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index fa876d5dd..07c5c2237 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -183,7 +183,7 @@ public void doDelete(LensSessionHandle sessionid, String entityName, Boolean cas }, VIRTUALFACT { @Override public List doGetAll(LensSessionHandle sessionid) throws LensException { - return getSvc().getAllVirtualFactNames(sessionid, null); + return getSvc().getAllFactNames(sessionid, null); } @Override @@ -678,22 +678,6 @@ public StringList getAllDimensionTablesOfDimension( return new StringList(getSvc().getAllDimTableNames(sessionid, dimensionName)); } - /** - * Get all virtual facts that belong to a cube in the metastore - * - * @param sessionid The sessionid in which user is working - * @param cubeName name of the base cube or derived cube - * @return StringList consisting of all the virtual fact names in the given cube - */ - @GET - @Path("/cubes/{cubeName}/virtualfacts") - public StringList getAllVirtualFactsOfCube( - @QueryParam("sessionid") LensSessionHandle sessionid, @PathParam("cubeName") String cubeName) - throws LensException { - checkSessionId(sessionid); - return new StringList(getSvc().getAllVirtualFactNames(sessionid, cubeName)); - } - /** * Get all facts that belong to a cube in the metastore * From 6d48eafd1b94f7bcbf27c04e2062c04214339d1b Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 11 May 2017 17:32:34 +0530 Subject: [PATCH 22/71] More changes --- .../org/apache/lens/cube/metadata/MetastoreUtil.java | 1 - .../apache/lens/cube/metadata/CubeFactTableTest.java | 10 ++++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index fb3a7b2bc..44a460641 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -391,7 +391,6 @@ public static String getSegmentPropertyKey(String segName) { // ///////////////////////// public static Date getDateFromProperty(String prop, boolean relative, boolean start) { - // String prop = getProperties().get(propKey); try { if (StringUtils.isNotBlank(prop)) { if (relative) { diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java index 8ceadc718..258c3c0f4 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java @@ -38,10 +38,10 @@ public class CubeFactTableTest { @DataProvider(name = "properties") public Object[][] factProperties() throws LensException { - String minus1DaysRelative = "now -1 days"; - String minus2DaysRelative = "now -2 days"; - String plus1DaysRelative = "now +1 days"; - String plus2DaysRelative = "now +2 days"; + String minus1DaysRelative = "now.day -1 days"; + String minus2DaysRelative = "now.day -2 days"; + String plus1DaysRelative = "now.day +1 days"; + String plus2DaysRelative = "now.day +2 days"; String minus1DaysAbsolute = DateUtil.relativeToAbsolute(minus1DaysRelative, now); String minus2DaysAbsolute = DateUtil.relativeToAbsolute(minus2DaysRelative, now); @@ -69,8 +69,6 @@ private CubeFactTable getMockCubeFactTable(Map properties) { when(cubeFactTable.getProperties()).thenReturn(properties); - when(MetastoreUtil.getDateFromProperty(properties.get(anyString()), anyBoolean(), anyBoolean())).thenCallRealMethod(); - when(cubeFactTable.getRelativeStartTime()).thenCallRealMethod(); when(cubeFactTable.getAbsoluteStartTime()).thenCallRealMethod(); From ff70054dc8adf1552270fb123c09871f9106f0ca Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Fri, 12 May 2017 08:12:06 +0530 Subject: [PATCH 23/71] More changes for virtual fact --- .../lens/cube/metadata/AbstractCubeTable.java | 4 - .../cube/metadata/CubeMetastoreClient.java | 180 +++--------------- .../cube/metadata/CubeVirtualFactTable.java | 1 + .../apache/lens/cube/metadata/FactTable.java | 7 +- .../lens/cube/metadata/CubeFactTableTest.java | 2 - .../metadata/TestCubeMetastoreClient.java | 9 +- src/site/apt/user/cli.apt | 2 +- 7 files changed, 38 insertions(+), 167 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java index 8db03d1ec..4f7a9dec7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java @@ -20,13 +20,9 @@ import java.util.*; -import org.apache.lens.server.api.error.LensException; - -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; -import com.google.common.collect.Lists; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index edfb769e8..05cf10bc5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -1678,7 +1678,7 @@ public boolean isVirtualFactTable(String virtualTableName) throws LensException boolean isVirtualFactTable(Table tbl) { String sourceFactName = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); - return sourceFactName != null ? true : false ; + return sourceFactName != null; } @@ -1806,31 +1806,6 @@ public XFactTable getXFactTable(CubeFactTable cft) throws LensException { } return factTable; } - /** - * Get {@link CubeFactTable} object corresponding to the name - * - * @param tableName The cube fact name - * @return Returns CubeFactTable if table name passed is a fact table, null otherwise - * @throws LensException - */ -// -// public FactTable getFactTable(String tableName) throws LensException { -// return new CubeFactTable(getTableWithTypeFailFast(tableName, CubeTableType.FACT)); -// } -// -// /** -// * Get {@link CubeVirtualFactTable} object corresponding to the name -// * -// * @param virtualTableName The virtual fact name -// * @return Returns CubeVirtualFactTable if table name passed is a virtual fact table, null otherwise -// * @throws LensException -// */ -// -// public CubeVirtualFactTable getVirtualFactTable(String virtualTableName) throws LensException { -// return new CubeVirtualFactTable(getTableWithTypeFailFast(virtualTableName, CubeTableType.VIRTUAL_FACT), -// getTableWithTypeFailFast(allVirtualFactSourceMappings.get(virtualTableName.trim().toLowerCase()), -// CubeTableType.FACT)); -// } public Segmentation getSegmentationTable(String tableName) throws HiveException, LensException { return new Segmentation(getTableWithTypeFailFast(tableName, CubeTableType.SEGMENTATION)); @@ -2019,7 +1994,22 @@ private FactTable getCubeFact(String tableName, boolean throwException) throws L synchronized (allFactTables) { if (!allFactTables.containsKey(tableName)) { Table tbl = getTableWithType(tableName, CubeTableType.FACT, throwException); - fact = tbl == null ? null : getFactTable(tbl); + if (tbl != null){ + String sourceFactName = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); + if (sourceFactName != null) { + fact = new CubeVirtualFactTable(tbl, (CubeFactTable) getCubeFact(sourceFactName)); + if (factToVirtualFactTables.get(sourceFactName) != null) { + List prevList = factToVirtualFactTables.get(sourceFactName); + prevList.add(tableName); + }else{ + List newList = new ArrayList<>(); + newList.add(tableName); + factToVirtualFactTables.put(sourceFactName, newList); + } + } else { + fact = new CubeFactTable(tbl); + } + } if (enableCaching && fact != null) { allFactTables.put(tableName, fact); } @@ -2040,46 +2030,6 @@ private FactTable getFactTable(Table tbl) throws LensException { } } -// -// /** -// * Get {@link Cube} object corresponding to the name -// * -// * @param virtualTableName The virtual fact name -// * @return Returns the corresponding virtual fact table object -// * @throws LensException if there is no fact by the name -// */ -// public CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, CubeFactTable sourceFact) throws LensException { -// return getVirtualCubeFact(virtualTableName, sourceFact, true); -// } -// private CubeVirtualFactTable getVirtualCubeFact(String virtualTableName, CubeFactTable sourceFact, -// boolean throwException) throws LensException { -// virtualTableName = virtualTableName.trim().toLowerCase(); -// -// CubeVirtualFactTable fact = allFactTables.get(virtualTableName); -// if (fact == null) { -// synchronized (allVirtualFactTables) { -// if (!allVirtualFactTables.containsKey(virtualTableName)) { -// Table virtualTbl = getTableWithType(virtualTableName, CubeTableType.FACT, throwException); -// fact = (virtualTbl == null) ? null : new CubeVirtualFactTable(virtualTbl, sourceFact); -// if (enableCaching && fact != null) { -// allVirtualFactTables.put(virtualTableName, fact); -// if (factToVirtualFactTables.get(sourceFact.getName()) != null) { -// List prevList = factToVirtualFactTables.get(sourceFact.getName()); -// prevList.add(virtualTableName); -// }else{ -// List newList = new ArrayList<>(); -// newList.add(virtualTableName); -// factToVirtualFactTables.put(sourceFact.getName(), newList); -// } -// } -// } else { -// fact = allVirtualFactTables.get(virtualTableName); -// } -// } -// } -// return fact; -// } - public Segmentation getSegmentation(String segName) throws LensException { return getSegmentation(segName, true); } @@ -2246,36 +2196,6 @@ public Collection getAllFacts() throws LensException { } } -// -// /** -// * Get all virtual facts in metastore -// * -// * @return List of Virtual Cube Fact Table objects -// * @throws LensException -// */ -// public Collection getAllVirtualFacts() throws LensException { -// if (!allVirtualFactTablesPopulated) { -// List facts = new ArrayList<>(); -// try { -// for (String table : getAllHiveTableNames()) { -// CubeVirtualFactTable fact = null; -// if (null != table) { -// fact = getVirtualCubeFact(table, false); -// } -// if (fact != null) { -// facts.add(fact); -// } -// } -// } catch (HiveException e) { -// throw new LensException("Could not get all virtual fact tables", e); -// } -// allVirtualFactTablesPopulated = enableCaching; -// return facts; -// } else { -// return allVirtualFactTables.values(); -// } -// } - /** * Get all segmentations in metastore * @@ -2342,31 +2262,6 @@ public List getAllFacts(CubeInterface cube) throws LensException { return cubeFacts; } -// -// /** -// * Get all virtual fact tables of the cube. -// * -// * @param cube Cube object -// * @return List of virtual fact tables -// * @throws LensException -// */ -// public List getAllVirtualFacts(CubeInterface cube) throws LensException { -// String cubeName = null; -// if (cube != null) { -// if (cube instanceof DerivedCube) { -// cube = ((DerivedCube) cube).getParent(); -// } -// cubeName = cube.getName(); -// } -// List cubeVirtualFacts = new ArrayList<>(); -// for (FactTable fact : getAllVirtualFacts()) { -// if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { -// cubeVirtualFacts.add(fact); -// } -// } -// return cubeVirtualFacts; -// } - public List getAllSegmentations(CubeInterface cube) throws LensException { String cubeName = null; if (cube != null) { @@ -2473,25 +2368,6 @@ private boolean alterCubeTable(String table, Table hiveTable, AbstractCubeTable } return columnsChanged; } -// -// /** -// * -// * @param virtualTable table name -// * @param hiveTable hive table -// * @param cubeTable lens cube table -// * @throws LensException -// */ -// private void alterCubeVirtualFactTable(String virtualTable, Table hiveTable, AbstractCubeTable cubeTable) throws -// LensException { -// -// hiveTable.getParameters().putAll(cubeTable.getProperties()); -// hiveTable.getTTable().getParameters().putAll(cubeTable.getProperties()); -// try { -// getClient().alterTable(virtualTable, hiveTable, null); -// } catch (Exception e) { -// throw new LensException(e); -// } -// } public void pushHiveTable(Table hiveTable) throws HiveException, LensException { alterHiveTable(hiveTable.getTableName(), hiveTable); @@ -2776,9 +2652,7 @@ public void alterCubeFactTable(String factTableName, FactTable cubeFactTable, } updateFactCache(factTableName); - if(cubeFactTable.getTableType().equals(CubeTableType.FACT)) { - updateAllVirtualFacts(getCubeFact(factTableName)); - } + updateAllVirtualFacts(getCubeFact(factTableName)); } public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException { @@ -2813,7 +2687,16 @@ private void updateSegmentationCache(String segmentName) throws HiveException, L private void updateFactCache(String factTableName) throws LensException { if (enableCaching) { - allFactTables.put(factTableName.trim().toLowerCase(), getCubeFact(factTableName)); + Table factTbl = getTableWithTypeFailFast(factTableName, CubeTableType.FACT); + FactTable refreshedTable; + if (factTbl.getParameters().get(getSourceFactNameKey(factTableName)) != null){ + String sourceFactName = factTbl.getParameters().get(getSourceFactNameKey(factTableName)); + refreshedTable = new CubeVirtualFactTable(refreshTable(factTableName), + (CubeFactTable) getCubeFact(sourceFactName)); + }else { + refreshedTable = new CubeFactTable(refreshTable(factTableName)); + } + allFactTables.put(factTableName.trim().toLowerCase(), refreshedTable); } } @@ -2832,13 +2715,6 @@ private void updateAllVirtualFacts(FactTable cubeFactTable) throws LensException } } } -// -// private void updateVirtualFactCache(String factTableName, CubeFactTable cubeFactTable) throws LensException { -// if (enableCaching) { -// allVirtualFactTables.put(factTableName.trim().toLowerCase(), -// new CubeVirtualFactTable(refreshTable(factTableName), cubeFactTable )); -// } -// } private void updateDimCache(String dimTblName) throws LensException { if (enableCaching) { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index 0fc7c36be..adfe8aabb 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -78,6 +78,7 @@ public void alterWeight(double weight) { @Override protected void addProperties() { getProperties().put(MetastoreConstants.TABLE_TYPE_KEY, getTableType().name()); + getProperties().put(MetastoreUtil.getSourceFactNameKey(this.getName()), this.sourceCubeFactTable.getName()); if (virtualFactWeight.isPresent()) { getProperties().put(MetastoreUtil.getCubeTableWeightKey(this.getName()), String.valueOf(virtualFactWeight.get())); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java index 422d90b72..1d088ecb7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java @@ -18,12 +18,13 @@ */ package org.apache.lens.cube.metadata; -import java.util.*; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import com.google.common.collect.Lists; - public interface FactTable extends Named { public Map> getUpdatePeriods(); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java index 258c3c0f4..dd3fae955 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java @@ -18,8 +18,6 @@ */ package org.apache.lens.cube.metadata; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 221e24950..4436c9655 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -1267,11 +1267,10 @@ public void testVirtualCubeFact() throws Exception { Table virtualTbl = client.getHiveTable(virtualFactName); assertTrue(client.isVirtualFactTable(virtualTbl)); assertTrue(client.isVirtualFactTableForCube(virtualTbl, CUBE_NAME)); - - //get virtual fact - assertEquals(client.getAllFacts(client.getCube(CUBE_NAME)).get(0).getName(), virtualFactName.toLowerCase()); - assertEquals(client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(), - virtualFactName.toLowerCase()); +// +// //get virtual fact +// assertTrue(client.getAllFacts(client.getCube(CUBE_NAME)).contains(virtualFactName.toLowerCase())); +// assertTrue(client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)).contains(virtualFactName.toLowerCase(); CubeVirtualFactTable actualcubeVirtualFact = (CubeVirtualFactTable) (client.getCubeFact(virtualFactName)); assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt index caffda7d6..fce41fe28 100644 --- a/src/site/apt/user/cli.apt +++ b/src/site/apt/user/cli.apt @@ -244,7 +244,7 @@ User CLI Commands *--+--+ |fact get storage [--fact_name] \ [--storage_name] \|describe storage <<>> of fact <<>>\ | *--+--+ -|fact list partitions [--fact_name] \ [--storage_name] \ [[--filter] \]|get all partitions associated with fact <<>>, storage <<>> filtered by <<>>\ | +|fact list partitions [--fact_name] \ [--storage_name] \ [--filter] \|get all partitions associated with fact <<>>, storage <<>> filtered by <<>>\ | *--+--+ |fact list storage [--fact_name] \|display list of storages associated to fact <<>>\ | *--+--+ From 1740db362a4b8baa8d5f686fb42b359abf8a7641 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Fri, 12 May 2017 14:20:11 +0530 Subject: [PATCH 24/71] Fix in storage candidate --- .../parse/StorageCandidateHQLContext.java | 21 +++++++++++++++++-- .../apache/lens/cube/parse/StorageUtil.java | 12 ----------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java index 05d3eefec..a711d1d38 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java @@ -131,8 +131,25 @@ private void updateAnswerableSelectColumns() throws LensException { protected void setMissingExpressions() throws LensException { setFrom(getFromTable()); String whereString = genWhereClauseWithDimPartitions(getWhere()); - setWhere(StorageUtil.joinWithAnd(this.storageCandidate.getFact().getProperties().get(VIRTUAL_FACT_FILTER) != null - ? this.storageCandidate.getFact().getProperties().get(VIRTUAL_FACT_FILTER) :null, whereString)); + StringBuilder whereStringBuilder; + + if (whereString != null) { + whereStringBuilder = new StringBuilder(whereString); + } else { + whereStringBuilder = new StringBuilder(); + } + + if(this.storageCandidate.getFact().getProperties().get(VIRTUAL_FACT_FILTER) != null) { + appendWhereClause(whereStringBuilder, + this.storageCandidate.getFact().getProperties().get(VIRTUAL_FACT_FILTER), whereString != null); + } + + if (whereStringBuilder.length() == 0) { + setWhere(null); + }else { + setWhere(whereStringBuilder.toString()); + } + if (isRoot()) { if (Objects.equals(getStorageCandidate(), getCubeQueryContext().getPickedCandidate())) { updateAnswerableSelectColumns(); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java index fbf1740a0..871354741 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java @@ -83,18 +83,6 @@ public static String getNotLatestClauseForDimensions(String alias, Set t return sb.toString(); } - public static String joinWithAnd(String... clauses) { - StringBuilder sb = new StringBuilder(); - String sep = "(("; - for (String clause : clauses) { - if (clause != null && !clause.isEmpty()) { - sb.append(sep).append(clause); - sep = ") AND ("; - } - } - return sb.append(sep.equals("((") ? "" : "))").toString(); - } - /** * Get minimal set of storages which cover the queried partitions * From fd9b9b18306d18b6915f7bccb3d2b8f430f856a6 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Fri, 12 May 2017 14:44:12 +0530 Subject: [PATCH 25/71] Fix in storage candidate --- .../src/main/java/org/apache/lens/cube/parse/StorageUtil.java | 2 ++ .../test/java/org/apache/lens/cube/parse/TestCubeRewriter.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java index 871354741..506973ebb 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java @@ -19,6 +19,8 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.metadata.DateUtil.WSPACE; +import static org.apache.lens.cube.metadata.DateUtil.relativeToAbsolute; +import static org.apache.lens.cube.metadata.MetastoreConstants.VIRTUAL_FACT_FILTER; import java.util.*; import java.util.regex.Matcher; diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 083465dce..616e9fed4 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -153,7 +153,7 @@ public void testVirtualFactCubeQuery() throws Exception { CubeQueryContext rewrittenQuery = rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", - "( dim1 = 10 )", null, getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, + null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, "C1_summary1")); String hql = rewrittenQuery.toHQL(); compareQueries(hql, expected); From 76419139d4ed5035b478c766cf96501aa3c77040 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Fri, 12 May 2017 14:52:46 +0530 Subject: [PATCH 26/71] Changes for vf --- .../main/java/org/apache/lens/cube/parse/StorageCandidate.java | 1 - .../org/apache/lens/cube/parse/StorageCandidateHQLContext.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index e2f1bdf03..f8a65067c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -18,7 +18,6 @@ */ package org.apache.lens.cube.parse; -import static org.apache.lens.cube.metadata.MetastoreConstants.VIRTUAL_FACT_FILTER; import static java.util.Comparator.naturalOrder; import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java index a711d1d38..02bfd67e5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java @@ -139,7 +139,7 @@ protected void setMissingExpressions() throws LensException { whereStringBuilder = new StringBuilder(); } - if(this.storageCandidate.getFact().getProperties().get(VIRTUAL_FACT_FILTER) != null) { + if (this.storageCandidate.getFact().getProperties().get(VIRTUAL_FACT_FILTER) != null) { appendWhereClause(whereStringBuilder, this.storageCandidate.getFact().getProperties().get(VIRTUAL_FACT_FILTER), whereString != null); } From da45d176292136621684b12a492926676aaf997e Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Fri, 12 May 2017 15:11:44 +0530 Subject: [PATCH 27/71] Changes for vf --- .../src/main/java/org/apache/lens/cube/parse/StorageUtil.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java index 506973ebb..871354741 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java @@ -19,8 +19,6 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.metadata.DateUtil.WSPACE; -import static org.apache.lens.cube.metadata.DateUtil.relativeToAbsolute; -import static org.apache.lens.cube.metadata.MetastoreConstants.VIRTUAL_FACT_FILTER; import java.util.*; import java.util.regex.Matcher; From cd387d6b7cf4313408f95be52db096a24202839a Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Mon, 15 May 2017 15:11:31 +0530 Subject: [PATCH 28/71] Changes for vf --- .../lens/cube/metadata/AbstractCubeTable.java | 9 -- .../lens/cube/metadata/CubeFactTable.java | 6 +- .../cube/metadata/CubeMetastoreClient.java | 90 +++++++++++++---- .../cube/metadata/CubeVirtualFactTable.java | 4 +- .../apache/lens/cube/metadata/FactTable.java | 98 +++++++++++++++++++ .../apache/lens/cube/parse/CubeTestSetup.java | 2 +- 6 files changed, 175 insertions(+), 34 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java index 07ed31bdf..624b29463 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java @@ -205,13 +205,4 @@ public Date now() { return new Date(); } - - public static String getFactCubeName(String factName, Map props) { - return props.get(MetastoreUtil.getFactCubeNameKey(factName)); - } - - protected static void addCubeNames(String factName, Map props, String cubeName) { - props.put(MetastoreUtil.getFactCubeNameKey(factName), cubeName); - } - } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index 1407fd85e..eeee02bfc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -44,7 +44,7 @@ public class CubeFactTable extends AbstractCubeTable implements FactTable { public CubeFactTable(Table hiveTable) { super(hiveTable); this.storageUpdatePeriods = getUpdatePeriods(getName(), getProperties()); - this.cubeName = getFactCubeName(getName(), getProperties()); + this.cubeName = this.getProperties().get(MetastoreUtil.getFactCubeNameKey(getName())); this.storagePrefixUpdatePeriodMap = getUpdatePeriodMap(getName(), getProperties()); } @@ -88,7 +88,7 @@ public boolean hasColumn(String column) { @Override protected void addProperties() { super.addProperties(); - addCubeNames(getName(), getProperties(), cubeName); + this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName); addUpdatePeriodProperies(getName(), getProperties(), storageUpdatePeriods); addStorageTableProperties(getName(), getProperties(), storagePrefixUpdatePeriodMap); } @@ -366,7 +366,7 @@ public void addColumns(Collection columns) { */ public void alterCubeName(String cubeName) { this.cubeName = cubeName; - addCubeNames(getName(), getProperties(), cubeName); + this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName); } public String getDataCompletenessTag() { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 05cf10bc5..3bec7f319 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -90,11 +90,8 @@ private CubeMetastoreClient(HiveConf conf) { // map from fact name to fact table private final Map allFactTables = Maps.newConcurrentMap(); // map from fact name to all virtual fact tables - private final Map> factToVirtualFactTables = Maps.newConcurrentMap(); -// // map from virtual fact name to virtual fact table -// private final Map allVirtualFactTables = Maps.newConcurrentMap(); + private final Map> factToVirtualFactMapping = Maps.newConcurrentMap(); private volatile boolean allFactTablesPopulated = false; - private volatile boolean allVirtualFactTablesPopulated = false; //map from segmentation name to segmentation private final Map allSegmentations = Maps.newConcurrentMap(); private volatile boolean allSegmentationPopulated = false; @@ -1660,7 +1657,7 @@ boolean isFactTable(Table tbl) { boolean isFactTableForCube(Table tbl, String cube) { - return isFactTable(tbl) && CubeFactTable.getFactCubeName(tbl.getTableName(), tbl.getParameters()) + return isFactTable(tbl) && tbl.getParameters().get(MetastoreUtil.getFactCubeNameKey(tbl.getTableName())) .equalsIgnoreCase(cube.toLowerCase()); } @@ -1683,7 +1680,7 @@ boolean isVirtualFactTable(Table tbl) { boolean isVirtualFactTableForCube(Table tbl, String cube) { - return isVirtualFactTable(tbl) && CubeFactTable.getFactCubeName(tbl.getTableName(), tbl.getParameters()) + return isVirtualFactTable(tbl) && tbl.getParameters().get(MetastoreUtil.getFactCubeNameKey(tbl.getTableName())) .equalsIgnoreCase(cube.toLowerCase()); } @@ -1998,13 +1995,13 @@ private FactTable getCubeFact(String tableName, boolean throwException) throws L String sourceFactName = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); if (sourceFactName != null) { fact = new CubeVirtualFactTable(tbl, (CubeFactTable) getCubeFact(sourceFactName)); - if (factToVirtualFactTables.get(sourceFactName) != null) { - List prevList = factToVirtualFactTables.get(sourceFactName); + if (factToVirtualFactMapping.get(sourceFactName) != null) { + List prevList = factToVirtualFactMapping.get(sourceFactName); prevList.add(tableName); }else{ List newList = new ArrayList<>(); newList.add(tableName); - factToVirtualFactTables.put(sourceFactName, newList); + factToVirtualFactMapping.put(sourceFactName, newList); } } else { fact = new CubeFactTable(tbl); @@ -2196,6 +2193,38 @@ public Collection getAllFacts() throws LensException { } } + /** + * Get all facts in metastore (virtual facts optional) + * @param includeVirtualFacts set true for including virtual facts + * @return List of Cube Fact Table objects + * @throws LensException + */ + public Collection getAllFacts(boolean includeVirtualFacts) throws LensException { + if (!allFactTablesPopulated) { + List facts = new ArrayList<>(); + try { + for (String table : getAllHiveTableNames()) { + FactTable fact = getCubeFact(table, false); + if (fact != null) { + if(fact.getProperties().get(getSourceFactNameKey(fact.getName())) != null){ //is virtual fact + if(includeVirtualFacts) { + facts.add(fact); + } + }else { + facts.add(fact); + } + } + } + } catch (HiveException e) { + throw new LensException("Could not get all fact tables", e); + } + allFactTablesPopulated = enableCaching; + return facts; + } else { + return allFactTables.values(); + } + } + /** * Get all segmentations in metastore * @@ -2222,8 +2251,6 @@ public Collection getAllSegmentations() throws LensException { } } - - private Collection getAllHiveTableNames() throws HiveException, LensException { if (!allTablesPopulated) { List allTables = getClient().getAllTables(); @@ -2262,6 +2289,31 @@ public List getAllFacts(CubeInterface cube) throws LensException { return cubeFacts; } + /** + * Get all facts of cube (optional virtual facts) + * + * @param cube Cube object + * @param includeVirtualFacts set true for virtual facts + * @return List of fact tables with optional virtual facts + * @throws LensException + */ + public List getAllFacts(CubeInterface cube, boolean includeVirtualFacts) throws LensException { + String cubeName = null; + if (cube != null) { + if (cube instanceof DerivedCube) { + cube = ((DerivedCube) cube).getParent(); + } + cubeName = cube.getName(); + } + List cubeFacts = new ArrayList<>(); + for (FactTable fact : getAllFacts(includeVirtualFacts)) { + if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { + cubeFacts.add(fact); + } + } + return cubeFacts; + } + public List getAllSegmentations(CubeInterface cube) throws LensException { String cubeName = null; if (cube != null) { @@ -2499,9 +2551,9 @@ public void dropFact(String factName, boolean cascade) throws LensException { private void dropAllVirtualFactTablesFromCache(String cubeFactTableName) throws LensException { if (enableCaching) { cubeFactTableName = cubeFactTableName.trim().toLowerCase(); - if (factToVirtualFactTables.get(cubeFactTableName) != null) { - List virtualFactTableNames = factToVirtualFactTables.get(cubeFactTableName); - factToVirtualFactTables.remove(cubeFactTableName); + if (factToVirtualFactMapping.get(cubeFactTableName) != null) { + List virtualFactTableNames = factToVirtualFactMapping.get(cubeFactTableName); + factToVirtualFactMapping.remove(cubeFactTableName); for (String vf : virtualFactTableNames) { dropVirtualFact(vf.trim().toLowerCase()); } @@ -2519,9 +2571,9 @@ public void dropVirtualFact(String virtualFactName) throws LensException { virtualFactName = virtualFactName.trim().toLowerCase(); Table virtualTbl = getTableWithTypeFailFast(virtualFactName, CubeTableType.FACT); String sourceFactTable = virtualTbl.getParameters().get(getSourceFactNameKey(virtualTbl.getTableName())); - if (factToVirtualFactTables.get(sourceFactTable) != null - && factToVirtualFactTables.get(sourceFactTable).contains(virtualFactName)) { - factToVirtualFactTables.get(sourceFactTable).remove(virtualFactName); + if (factToVirtualFactMapping.get(sourceFactTable) != null + && factToVirtualFactMapping.get(sourceFactTable).contains(virtualFactName)) { + factToVirtualFactMapping.get(sourceFactTable).remove(virtualFactName); } dropHiveTable(virtualFactName); allFactTables.remove(virtualFactName.trim().toLowerCase()); @@ -2703,9 +2755,9 @@ private void updateFactCache(String factTableName) throws LensException { private void updateAllVirtualFacts(FactTable cubeFactTable) throws LensException { if (enableCaching) { String cubeFactTableName = cubeFactTable.getName().trim().toLowerCase(); - if (factToVirtualFactTables.get(cubeFactTableName) != null) { + if (factToVirtualFactMapping.get(cubeFactTableName) != null) { synchronized (allFactTables) { - List virtualFactTableNames = factToVirtualFactTables.get(cubeFactTableName); + List virtualFactTableNames = factToVirtualFactMapping.get(cubeFactTableName); for (String vf : virtualFactTableNames) { CubeVirtualFactTable cvf = (CubeVirtualFactTable) getCubeFact(vf); cvf.setSourceCubeFactTable(cubeFactTable); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index ce6a1c956..4e9311692 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -45,7 +45,7 @@ public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable public CubeVirtualFactTable(Table hiveTable, CubeFactTable sourceCubeFactTable) { super(hiveTable); - this.cubeName = getFactCubeName(getName(), getProperties()); + this.cubeName = this.getProperties().get(MetastoreUtil.getFactCubeNameKey(getName())); this.sourceCubeFactTable = sourceCubeFactTable; this.virtualFactWeight = Optional.absent(); @@ -82,7 +82,7 @@ protected void addProperties() { if (virtualFactWeight.isPresent()) { getProperties().put(MetastoreUtil.getCubeTableWeightKey(this.getName()), String.valueOf(virtualFactWeight.get())); } - addCubeNames(getName(), getProperties(), cubeName); + this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName); } @Override diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java index 1d088ecb7..1789061bd 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java @@ -25,44 +25,142 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; +/** + * The Fact table interface + */ public interface FactTable extends Named { + /** + * Get map of storage to update period mapping + * + * @return Map of storage to set of update periods + */ public Map> getUpdatePeriods(); + /** + * Cube to which this fact belongs to + * + * @return the cube string + */ public String getCubeName(); + /** + * The set of Storage names + * + * @return set of strings + */ public Set getStorages(); + /** + *The type of the fact + * + * @return table type {@link CubeTableType} + */ public CubeTableType getTableType(); + /** + * Config properties + * + * @return map of string, string + */ public Map getProperties(); + /** + * Valid columns of the fact + * + * @return list of column names + */ public List getValidColumns(); + /** + * Weight of the fact + * + * @return weight of the fact in double + */ public double weight(); + /** + * Set of all the columns names of the fact + * + * @return set of column names + */ public Set getAllFieldNames(); + /** + *tag for checking data completeness + * + * @return Tag String + */ public String getDataCompletenessTag(); + /** + * List of columns of the fact + * + * @return set of {@link FieldSchema} + */ public List getColumns(); + /** + * Is Aggregated Fact + * + * @return true if fact is Aggregated , false otherwise + */ public boolean isAggregated(); + /** + * Absolute start time of the fact + * + * @return Absolute Start time of the fact {@link Date} + */ public Date getAbsoluteStartTime(); + /** + * Relative start time of the fact + * + * @return Relative Start time of the fact {@link Date} + */ public Date getRelativeStartTime(); + /** + * Start time of the fact + * + * @return Start time of the fact {@link Date} + */ public Date getStartTime(); + /** + * Absolute end time of the fact + * + * @return Absolute End time of the fact {@link Date} + */ public Date getAbsoluteEndTime(); + /** + * Relative End time of the Fact + * + * @return Relative end time of the fact {@link Date} + */ public Date getRelativeEndTime(); + /** + * End time of the fact + * + * @return End time of the fact {@link Date} + */ public Date getEndTime(); + /** + * Is Virtual Fact + * + * @return true if fact is a virtual fact, false otherwise + */ public boolean isVirtualFact(); + /** + * Storage name of the fact + * + * @return Storage name of the fact + */ public String getStorageFactName(); } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index f2e6c4290..6a6810f50 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -835,7 +835,7 @@ private void dump(CubeMetastoreClient client) throws LensException, IOException // bw.write(ToXMLString.toString(JAXBUtils.xCubeFromHiveCube(cubeInterface))); // } // } - for (FactTable factTable : client.getAllFacts()) { + for (FactTable factTable : client.getAllFacts(false)) { CubeFactTable cubeFactTable = (CubeFactTable) factTable; try(BufferedWriter bw = new BufferedWriter(new FileWriter(getClass() .getResource("/schema/facts").getPath()+"/"+cubeFactTable.getName()+".xml"))) { From b7d71cef2dcfca18c31e4163a928b63ea0b572bb Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Mon, 15 May 2017 15:37:31 +0530 Subject: [PATCH 29/71] Changes for vf --- .../cube/metadata/CubeMetastoreClient.java | 10 ++-- .../lens/cube/parse/QueriedPhraseContext.java | 54 ------------------- .../apache/lens/cube/parse/CubeTestSetup.java | 2 +- 3 files changed, 7 insertions(+), 59 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 3bec7f319..36b2d6a1d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -53,6 +53,8 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.thrift.TException; +import com.google.common.base.Optional; + import org.jvnet.jaxb2_commons.lang.Equals; import org.jvnet.jaxb2_commons.lang.HashCode; import org.jvnet.jaxb2_commons.lang.ToString; @@ -379,7 +381,7 @@ public void createVirtualFactTable(String cubeName, String virtualFactName, Stri Map properties) throws LensException { FactTable sourceFact = getCubeFact(sourceFactName); - com.google.common.base.Optional optionalWeight = com.google.common.base.Optional.fromNullable(weight); + Optional optionalWeight = Optional.fromNullable(weight); CubeVirtualFactTable factTable = new CubeVirtualFactTable(cubeName, virtualFactName, optionalWeight, properties, sourceFact); @@ -2206,11 +2208,11 @@ public Collection getAllFacts(boolean includeVirtualFacts) throws Len for (String table : getAllHiveTableNames()) { FactTable fact = getCubeFact(table, false); if (fact != null) { - if(fact.getProperties().get(getSourceFactNameKey(fact.getName())) != null){ //is virtual fact - if(includeVirtualFacts) { + if (fact.getProperties().get(getSourceFactNameKey(fact.getName())) != null) { //is virtual fact + if (includeVirtualFacts) { facts.add(fact); } - }else { + } else { facts.add(fact); } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java index 72a64f74d..578939742 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -136,58 +136,4 @@ public boolean isEvaluable(StorageCandidate candidate) throws LensException { } return true; } - - private static boolean isColumnAvailableInRange(final TimeRange range, Date startTime, Date endTime) { - return (isColumnAvailableFrom(range.getFromDate(), startTime) - && isColumnAvailableTill(range.getToDate(), endTime)); - } - - private static boolean isColumnAvailableFrom(@NonNull final Date date, Date startTime) { - return (startTime == null) || date.equals(startTime) || date.after(startTime); - } - - private static boolean isColumnAvailableTill(@NonNull final Date date, Date endTime) { - return (endTime == null) || date.equals(endTime) || date.before(endTime); - } - - public static boolean isFactColumnValidForRange(CubeQueryContext cubeql, StorageCandidate sc, String col) { - for (TimeRange range : cubeql.getTimeRanges()) { - if (!isColumnAvailableInRange(range, getFactColumnStartTime(sc.getFact(), col), - getFactColumnEndTime(sc.getFact(), col))) { - return false; - } - } - return true; - } - - public static Date getFactColumnStartTime(FactTable ft, String factCol) { - Date startTime = null; - for (String key : ft.getProperties().keySet()) { - if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) { - String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_START_TIME_PFX); - if (factCol.equals(propCol)) { - startTime = MetastoreUtil.getDateFromProperty(ft.getProperties().get(key), false, true); - } - } - } - return startTime; - } - - public static Date getFactColumnEndTime(FactTable ft, String factCol) { - Date endTime = null; - for (String key : ft.getProperties().keySet()) { - if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) { - String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_END_TIME_PFX); - if (factCol.equals(propCol)) { - endTime = MetastoreUtil.getDateFromProperty(ft.getProperties().get(key), false, true); - } - } - } - return endTime; - } - - static boolean checkForColumnExistsAndValidForRange(StorageCandidate sc, String column, CubeQueryContext cubeql) { - return (sc.getColumns().contains(column) && isFactColumnValidForRange(cubeql, sc, column)); - } - } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 6a6810f50..f2e6c4290 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -835,7 +835,7 @@ private void dump(CubeMetastoreClient client) throws LensException, IOException // bw.write(ToXMLString.toString(JAXBUtils.xCubeFromHiveCube(cubeInterface))); // } // } - for (FactTable factTable : client.getAllFacts(false)) { + for (FactTable factTable : client.getAllFacts()) { CubeFactTable cubeFactTable = (CubeFactTable) factTable; try(BufferedWriter bw = new BufferedWriter(new FileWriter(getClass() .getResource("/schema/facts").getPath()+"/"+cubeFactTable.getName()+".xml"))) { From a4eef3263bf2e364dbdd136e55ac8529eb2726b6 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Mon, 15 May 2017 15:47:53 +0530 Subject: [PATCH 30/71] Changes for vf --- .../org/apache/lens/cube/metadata/CubeMetastoreClient.java | 4 ++-- .../org/apache/lens/cube/parse/QueriedPhraseContext.java | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 36b2d6a1d..284cc608c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -53,15 +53,15 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.thrift.TException; -import com.google.common.base.Optional; - import org.jvnet.jaxb2_commons.lang.Equals; import org.jvnet.jaxb2_commons.lang.HashCode; import org.jvnet.jaxb2_commons.lang.ToString; +import com.google.common.base.Optional; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; + import lombok.extern.slf4j.Slf4j; /** diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java index 578939742..be64d6a10 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -19,19 +19,12 @@ package org.apache.lens.cube.parse; import java.util.*; - -import org.apache.lens.cube.metadata.FactTable; -import org.apache.lens.cube.metadata.MetastoreConstants; -import org.apache.lens.cube.metadata.MetastoreUtil; -import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.server.api.error.LensException; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.parse.ASTNode; import lombok.Data; import lombok.EqualsAndHashCode; -import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @Data From 3fc606bc2914125f51f2f3834d3e023243cc4d2f Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Mon, 15 May 2017 16:02:49 +0530 Subject: [PATCH 31/71] Changes for vf --- .../java/org/apache/lens/cube/parse/QueriedPhraseContext.java | 1 + 1 file changed, 1 insertion(+) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java index be64d6a10..69e3e7308 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -19,6 +19,7 @@ package org.apache.lens.cube.parse; import java.util.*; + import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.hive.ql.parse.ASTNode; From 5b6936e38cae90104900175c4fa48ac517d9a155 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Tue, 16 May 2017 17:03:41 +0530 Subject: [PATCH 32/71] review changes --- .../lens/cube/metadata/CubeFactTable.java | 2 +- .../cube/metadata/CubeMetastoreClient.java | 6 +-- .../cube/metadata/CubeVirtualFactTable.java | 39 +++++++++++++------ .../apache/lens/cube/metadata/FactTable.java | 2 +- .../lens/cube/parse/StorageCandidate.java | 11 ++---- .../parse/StorageCandidateHQLContext.java | 15 +------ 6 files changed, 38 insertions(+), 37 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index eeee02bfc..ba18ddb69 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -388,7 +388,7 @@ public boolean isVirtualFact() { } @Override - public String getStorageFactName() { + public String getSourceFactName() { return this.getName(); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 284cc608c..5ba2b50f1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -128,7 +128,7 @@ public boolean isDataCompletenessCheckEnabled() { /** extract storage name from fact and storage table name. String operation */ private String extractStorageName(FactTable fact, String storageTableName) throws LensException { - int ind = storageTableName.lastIndexOf(fact.getStorageFactName()); + int ind = storageTableName.lastIndexOf(fact.getSourceFactName()); if (ind <= 0) { throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); } @@ -1427,7 +1427,7 @@ public boolean tableExists(String tblName) throws HiveException { public boolean factPartitionExists(FactTable fact, FactPartition part, String storageTableName) throws HiveException, LensException { String storage = extractStorageName(fact, storageTableName); - return partitionTimelineCache.partitionTimeExists(fact.getStorageFactName(), storage, + return partitionTimelineCache.partitionTimeExists(fact.getSourceFactName(), storage, part.getPeriod(), part.getPartCol(), part.getPartSpec()); } @@ -2394,7 +2394,7 @@ public Set getStorageTables(FactTable factTable, String storage) throws Set uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : factTable.getUpdatePeriods().get(storage)) { - String factName = factTable.getStorageFactName(); + String factName = factTable.getSourceFactName(); uniqueStorageTables.add(getStorageTableName(factName, storage, updatePeriod)); } return uniqueStorageTables; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index 4e9311692..30765c9ba 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -20,12 +20,14 @@ import java.util.*; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; import com.google.common.base.Optional; import com.google.common.collect.Lists; +import io.netty.util.internal.StringUtil; import lombok.Getter; import lombok.Setter; @@ -43,12 +45,11 @@ public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable COLUMNS.add(new FieldSchema("dummy", "string", "dummy column")); } - public CubeVirtualFactTable(Table hiveTable, CubeFactTable sourceCubeFactTable) { + public CubeVirtualFactTable(Table hiveTable, FactTable sourceCubeFactTable) { super(hiveTable); this.cubeName = this.getProperties().get(MetastoreUtil.getFactCubeNameKey(getName())); this.sourceCubeFactTable = sourceCubeFactTable; - this.virtualFactWeight = Optional.absent(); String wtStr = getProperties().get(MetastoreUtil.getCubeTableWeightKey(getName())); if (wtStr != null) { this.virtualFactWeight = Optional.of(Double.parseDouble(wtStr)); @@ -131,13 +132,21 @@ public double weight() { } public Date getAbsoluteStartTime() { - return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME), - false, true); + String absoluteStartTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME); + Date absoluteDate = null; + if(StringUtils.isNotBlank(absoluteStartTime)) { + absoluteDate = MetastoreUtil.getDateFromProperty(absoluteStartTime, false, true); + } + return absoluteDate == null ? this.sourceCubeFactTable.getAbsoluteStartTime() : absoluteDate; } public Date getRelativeStartTime() { - return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_RELATIVE_START_TIME), - true, true); + String relativeStartTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME); + Date relativeDate = null; + if(StringUtils.isNotBlank(relativeStartTime)) { + relativeDate = MetastoreUtil.getDateFromProperty(relativeStartTime, true, true); + } + return relativeDate == null ? this.sourceCubeFactTable.getRelativeStartTime() : relativeDate; } public Date getStartTime() { @@ -145,13 +154,21 @@ public Date getStartTime() { } public Date getAbsoluteEndTime() { - return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_END_TIME), - false, false); + String absoluteEndTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_END_TIME); + Date absoluteDate = null; + if(StringUtils.isNotBlank(absoluteEndTime)) { + absoluteDate = MetastoreUtil.getDateFromProperty(absoluteEndTime, false, false); + } + return absoluteDate == null ? this.sourceCubeFactTable.getAbsoluteEndTime() : absoluteDate; } public Date getRelativeEndTime() { - return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_RELATIVE_END_TIME), - true, false); + String relativeEndTime = this.getProperties().get(MetastoreConstants.FACT_RELATIVE_END_TIME); + Date relativeDate = null; + if(StringUtils.isNotBlank(relativeEndTime)) { + relativeDate = MetastoreUtil.getDateFromProperty(relativeEndTime, true, false); + } + return relativeDate == null ? this.sourceCubeFactTable.getRelativeEndTime() : relativeDate; } public Date getEndTime() { @@ -164,7 +181,7 @@ public boolean isVirtualFact() { } @Override - public String getStorageFactName() { + public String getSourceFactName() { return this.sourceCubeFactTable.getName(); } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java index 1789061bd..94c5f7909 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java @@ -161,6 +161,6 @@ public interface FactTable extends Named { * * @return Storage name of the fact */ - public String getStorageFactName(); + public String getSourceFactName(); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index f8a65067c..322120ab1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -180,7 +180,7 @@ public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, throw new IllegalArgumentException("Cube,fact and storageName should be non null"); } this.storageName = storageName; - this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(getStorageFactName(fact), storageName); + this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getSourceFactName(), storageName); this.name = fact.getName(); this.processTimePartCol = getConf().get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); String formatStr = getConf().get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); @@ -197,11 +197,6 @@ public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, setStorageStartAndEndDate(); } - public static String getStorageFactName(FactTable fact) { - return fact.getProperties().get(MetastoreUtil.getSourceFactNameKey(fact.getName()))!= null - ? ((CubeVirtualFactTable) fact).getSourceCubeFactTable().getName() : fact.getName(); - } - String getTimeRangeWhereClasue(TimeRangeWriter rangeWriter, TimeRange range) throws LensException { String rangeWhere = rangeWriter.getTimeRangeWhereClause( @@ -256,8 +251,8 @@ void setStorageStartAndEndDate() throws LensException { List startDates = new ArrayList<>(); List endDates = new ArrayList<>(); for (String storageTablePrefix : getValidStorageTableNames()) { - startDates.add(getCubeMetastoreClient().getStorageTableStartDate(storageTablePrefix, getStorageFactName(fact))); - endDates.add(getCubeMetastoreClient().getStorageTableEndDate(storageTablePrefix, getStorageFactName(fact))); + startDates.add(getCubeMetastoreClient().getStorageTableStartDate(storageTablePrefix, fact.getSourceFactName())); + endDates.add(getCubeMetastoreClient().getStorageTableEndDate(storageTablePrefix, fact.getSourceFactName())); } this.startTime = Collections.min(startDates); this.endTime = Collections.max(endDates); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java index 02bfd67e5..c5351965a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java @@ -131,24 +131,13 @@ private void updateAnswerableSelectColumns() throws LensException { protected void setMissingExpressions() throws LensException { setFrom(getFromTable()); String whereString = genWhereClauseWithDimPartitions(getWhere()); - StringBuilder whereStringBuilder; - - if (whereString != null) { - whereStringBuilder = new StringBuilder(whereString); - } else { - whereStringBuilder = new StringBuilder(); - } + StringBuilder whereStringBuilder = (whereString != null) ? new StringBuilder(whereString) : new StringBuilder(); if (this.storageCandidate.getFact().getProperties().get(VIRTUAL_FACT_FILTER) != null) { appendWhereClause(whereStringBuilder, this.storageCandidate.getFact().getProperties().get(VIRTUAL_FACT_FILTER), whereString != null); } - - if (whereStringBuilder.length() == 0) { - setWhere(null); - }else { - setWhere(whereStringBuilder.toString()); - } + setWhere(whereStringBuilder.length() == 0 ? null : whereStringBuilder.toString()); if (isRoot()) { if (Objects.equals(getStorageCandidate(), getCubeQueryContext().getPickedCandidate())) { From e0ff6fbb7854c7c81c79afc04994834e3431cf76 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Tue, 16 May 2017 17:46:59 +0530 Subject: [PATCH 33/71] review changes --- .../java/org/apache/lens/cube/metadata/CubeFactTable.java | 6 +++--- .../org/apache/lens/cube/metadata/CubeVirtualFactTable.java | 2 +- .../main/java/org/apache/lens/cube/metadata/FactTable.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index ba18ddb69..ed8ddcc97 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -77,7 +77,7 @@ public CubeFactTable(String cubeName, String factName, List columns } public boolean hasColumn(String column) { - List validColumns = getValidColumns(); + Set validColumns = getValidColumns(); if (validColumns != null) { return validColumns.contains(column); } else { @@ -268,10 +268,10 @@ public String getCubeName() { * * @return */ - public List getValidColumns() { + public Set getValidColumns() { String validColsStr = MetastoreUtil.getNamedStringValue(getProperties(), MetastoreUtil.getValidColumnsKey(getName())); - return validColsStr == null ? null : Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), ',')); + return validColsStr == null ? null : new HashSet<>(Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), ','))); } /** diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index 30765c9ba..33552f2bd 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -92,7 +92,7 @@ public CubeTableType getTableType() { } @Override - public List getValidColumns() { + public Set getValidColumns() { return this.sourceCubeFactTable.getValidColumns(); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java index 94c5f7909..f87cf4434 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java @@ -70,7 +70,7 @@ public interface FactTable extends Named { * * @return list of column names */ - public List getValidColumns(); + public Set getValidColumns(); /** * Weight of the fact From 069c40c90cccd42fb7596890970aab01c1bdc9af Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Tue, 16 May 2017 18:30:34 +0530 Subject: [PATCH 34/71] review changes --- .../java/org/apache/lens/cube/metadata/CubeFactTable.java | 3 ++- .../apache/lens/cube/metadata/CubeVirtualFactTable.java | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index ed8ddcc97..a3ef5747f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -271,7 +271,8 @@ public String getCubeName() { public Set getValidColumns() { String validColsStr = MetastoreUtil.getNamedStringValue(getProperties(), MetastoreUtil.getValidColumnsKey(getName())); - return validColsStr == null ? null : new HashSet<>(Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), ','))); + return validColsStr == null ? null : + new HashSet<>(Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), ','))); } /** diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index 33552f2bd..267a4fe6e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -134,7 +134,7 @@ public double weight() { public Date getAbsoluteStartTime() { String absoluteStartTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME); Date absoluteDate = null; - if(StringUtils.isNotBlank(absoluteStartTime)) { + if (StringUtils.isNotBlank(absoluteStartTime)) { absoluteDate = MetastoreUtil.getDateFromProperty(absoluteStartTime, false, true); } return absoluteDate == null ? this.sourceCubeFactTable.getAbsoluteStartTime() : absoluteDate; @@ -143,7 +143,7 @@ public Date getAbsoluteStartTime() { public Date getRelativeStartTime() { String relativeStartTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME); Date relativeDate = null; - if(StringUtils.isNotBlank(relativeStartTime)) { + if (StringUtils.isNotBlank(relativeStartTime)) { relativeDate = MetastoreUtil.getDateFromProperty(relativeStartTime, true, true); } return relativeDate == null ? this.sourceCubeFactTable.getRelativeStartTime() : relativeDate; @@ -156,7 +156,7 @@ public Date getStartTime() { public Date getAbsoluteEndTime() { String absoluteEndTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_END_TIME); Date absoluteDate = null; - if(StringUtils.isNotBlank(absoluteEndTime)) { + if (StringUtils.isNotBlank(absoluteEndTime)) { absoluteDate = MetastoreUtil.getDateFromProperty(absoluteEndTime, false, false); } return absoluteDate == null ? this.sourceCubeFactTable.getAbsoluteEndTime() : absoluteDate; @@ -165,7 +165,7 @@ public Date getAbsoluteEndTime() { public Date getRelativeEndTime() { String relativeEndTime = this.getProperties().get(MetastoreConstants.FACT_RELATIVE_END_TIME); Date relativeDate = null; - if(StringUtils.isNotBlank(relativeEndTime)) { + if (StringUtils.isNotBlank(relativeEndTime)) { relativeDate = MetastoreUtil.getDateFromProperty(relativeEndTime, true, false); } return relativeDate == null ? this.sourceCubeFactTable.getRelativeEndTime() : relativeDate; From 46dd86f4a645671e435bd646253814d9fae36d8b Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Tue, 16 May 2017 18:53:57 +0530 Subject: [PATCH 35/71] review changes --- .../java/org/apache/lens/cube/metadata/CubeFactTable.java | 4 ++-- .../org/apache/lens/cube/metadata/CubeVirtualFactTable.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index a3ef5747f..5df028893 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -271,8 +271,8 @@ public String getCubeName() { public Set getValidColumns() { String validColsStr = MetastoreUtil.getNamedStringValue(getProperties(), MetastoreUtil.getValidColumnsKey(getName())); - return validColsStr == null ? null : - new HashSet<>(Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), ','))); + return validColsStr == null ? + null : new HashSet<>(Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), ','))); } /** diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index 267a4fe6e..1fc74b04b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -27,7 +27,6 @@ import com.google.common.base.Optional; import com.google.common.collect.Lists; -import io.netty.util.internal.StringUtil; import lombok.Getter; import lombok.Setter; From 936671efd789624f0dc250c602f62180ac73150c Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Tue, 16 May 2017 19:04:25 +0530 Subject: [PATCH 36/71] review changes --- .../java/org/apache/lens/cube/metadata/CubeFactTable.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index 5df028893..c57a9c148 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -271,8 +271,8 @@ public String getCubeName() { public Set getValidColumns() { String validColsStr = MetastoreUtil.getNamedStringValue(getProperties(), MetastoreUtil.getValidColumnsKey(getName())); - return validColsStr == null ? - null : new HashSet<>(Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), ','))); + return validColsStr == null ? null : new HashSet<>(Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), + ','))); } /** From 1e5954e34ec49a735ca0c29a17b69793b6221a1c Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 17 May 2017 17:22:08 +0530 Subject: [PATCH 37/71] review changes --- lens-api/src/main/resources/lens-errors.conf | 6 +++ .../lens/cube/error/LensCubeErrorCode.java | 3 +- .../cube/metadata/CubeMetastoreClient.java | 43 ++++++++++++++----- .../lens/cube/metadata/MetastoreUtil.java | 5 ++- .../lens/cube/parse/StorageCandidate.java | 13 +++--- .../metadata/TestCubeMetastoreClient.java | 14 +++--- .../metastore/CubeMetastoreServiceImpl.java | 17 -------- .../server/metastore/MetastoreResource.java | 16 +------ 8 files changed, 59 insertions(+), 58 deletions(-) diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf index 43de1e991..fafd65501 100644 --- a/lens-api/src/main/resources/lens-errors.conf +++ b/lens-api/src/main/resources/lens-errors.conf @@ -384,6 +384,12 @@ lensCubeErrorsForMetastore = [ errorMsg = "Partition filter can not be null or empty" } + { + errorCode = 3106 + httpStatusCode = ${BAD_REQUEST} + errorMsg = "Fact %s not of type %s" + } + ] lensDriverErrors = [ diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java index 32b9db35e..ed076e250 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java @@ -65,7 +65,8 @@ public enum LensCubeErrorCode { TIMELINE_ABSENT(3102, 100), EXPRESSION_NOT_PARSABLE(3103, 1500), ENTITY_NOT_FOUND(3104, 1500), - NO_PARTITION_FILTER(3105, 1500); + NO_PARTITION_FILTER(3105, 1500), + ENTITY_TYPE_NOT_AS_EXPECTED(3106, 1500); public LensErrorInfo getLensErrorInfo() { return this.errorInfo; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 5ba2b50f1..2b85704c0 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -35,6 +35,7 @@ import org.apache.lens.cube.metadata.timeline.PartitionTimeline; import org.apache.lens.cube.metadata.timeline.PartitionTimelineFactory; import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.LensErrorInfo; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.DataCompletenessChecker; import org.apache.lens.server.api.util.LensUtil; @@ -91,7 +92,7 @@ private CubeMetastoreClient(HiveConf conf) { private volatile boolean allDimTablesPopulated = false; // map from fact name to fact table private final Map allFactTables = Maps.newConcurrentMap(); - // map from fact name to all virtual fact tables + // map from fact name to all virtual fact tables, any changes to facts must reflect in all of its virtual facts private final Map> factToVirtualFactMapping = Maps.newConcurrentMap(); private volatile boolean allFactTablesPopulated = false; //map from segmentation name to segmentation @@ -185,7 +186,7 @@ public List getTimelines(String factName, String storage, Str throws LensException, HiveException { UpdatePeriod updatePeriod = updatePeriodStr == null ? null : UpdatePeriod.valueOf(updatePeriodStr.toUpperCase()); List ret = Lists.newArrayList(); - CubeFactTable fact = (CubeFactTable) getCubeFact(factName); + CubeFactTable fact = getCubeFactTable(factName); List storageList = Lists.newArrayList(); if (storage != null) { storageList.add(storage); @@ -1764,7 +1765,7 @@ boolean isDimension(Table tbl) { } public XFactTable getXFactTable(String tableName) throws LensException { - return getXFactTable((CubeFactTable) getCubeFact(tableName)); + return getXFactTable(getCubeFactTable(tableName)); } public XFactTable getXFactTable(CubeFactTable cft) throws LensException { @@ -1996,7 +1997,7 @@ private FactTable getCubeFact(String tableName, boolean throwException) throws L if (tbl != null){ String sourceFactName = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); if (sourceFactName != null) { - fact = new CubeVirtualFactTable(tbl, (CubeFactTable) getCubeFact(sourceFactName)); + fact = new CubeVirtualFactTable(tbl, getCubeFactTable(sourceFactName)); if (factToVirtualFactMapping.get(sourceFactName) != null) { List prevList = factToVirtualFactMapping.get(sourceFactName); prevList.add(tableName); @@ -2023,7 +2024,7 @@ private FactTable getCubeFact(String tableName, boolean throwException) throws L private FactTable getFactTable(Table tbl) throws LensException { String sourceFact = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); if (sourceFact != null) { - return new CubeVirtualFactTable(tbl, (CubeFactTable) getCubeFact(sourceFact)); + return new CubeVirtualFactTable(tbl, getCubeFactTable(sourceFact)); } else { return new CubeFactTable(tbl); } @@ -2547,10 +2548,10 @@ public void dropFact(String factName, boolean cascade) throws LensException { } dropHiveTable(factName); allFactTables.remove(factName.trim().toLowerCase()); - dropAllVirtualFactTablesFromCache(factName); + dropAllVirtualFactTables(factName); } - private void dropAllVirtualFactTablesFromCache(String cubeFactTableName) throws LensException { + private void dropAllVirtualFactTables(String cubeFactTableName) throws LensException { if (enableCaching) { cubeFactTableName = cubeFactTableName.trim().toLowerCase(); if (factToVirtualFactMapping.get(cubeFactTableName) != null) { @@ -2604,7 +2605,7 @@ public void dropStorageFromFact(String factName, String storage) throws LensExce } private void dropHiveTablesForStorage(String factName, String storage) throws LensException{ - CubeFactTable cft = (CubeFactTable) getCubeFact(factName); + CubeFactTable cft = (CubeFactTable) getCubeFactTable(factName); Set droppedTables = new HashSet<>(); for (Map.Entry updatePeriodEntry : cft.getStoragePrefixUpdatePeriodMap().get(storage).entrySet()) { UpdatePeriod updatePeriod = (UpdatePeriod) updatePeriodEntry.getKey(); @@ -2620,7 +2621,7 @@ private void dropStorageFromFact(String factName, String storage, boolean update throws LensException { dropHiveTablesForStorage(factName, storage); if (updateFact) { - CubeFactTable cft = (CubeFactTable) getCubeFact(factName); + CubeFactTable cft = getCubeFactTable(factName); cft.dropStorage(storage); alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft); updateFactCache(factName); @@ -2746,7 +2747,7 @@ private void updateFactCache(String factTableName) throws LensException { if (factTbl.getParameters().get(getSourceFactNameKey(factTableName)) != null){ String sourceFactName = factTbl.getParameters().get(getSourceFactNameKey(factTableName)); refreshedTable = new CubeVirtualFactTable(refreshTable(factTableName), - (CubeFactTable) getCubeFact(sourceFactName)); + getCubeFactTable(sourceFactName)); }else { refreshedTable = new CubeFactTable(refreshTable(factTableName)); } @@ -2754,6 +2755,26 @@ private void updateFactCache(String factTableName) throws LensException { } } + private CubeFactTable getCubeFactTable(String factName) throws LensException { + FactTable factTable = getCubeFact(factName); + if(factTable instanceof CubeFactTable) { + return (CubeFactTable) factTable; + }else { + throw new LensException(new LensException(LensCubeErrorCode.ENTITY_TYPE_NOT_AS_EXPECTED.getLensErrorInfo(), + factName, "CubeFactTable")); + } + } + + private CubeVirtualFactTable getCubeVirtualFactTable(String factName) throws LensException { + FactTable factTable = getCubeFact(factName); + if(factTable instanceof CubeVirtualFactTable) { + return (CubeVirtualFactTable) factTable; + }else { + throw new LensException(new LensException(LensCubeErrorCode.ENTITY_TYPE_NOT_AS_EXPECTED.getLensErrorInfo(), + factName, "CubeVirtualFactTable")); + } + } + private void updateAllVirtualFacts(FactTable cubeFactTable) throws LensException { if (enableCaching) { String cubeFactTableName = cubeFactTable.getName().trim().toLowerCase(); @@ -2761,7 +2782,7 @@ private void updateAllVirtualFacts(FactTable cubeFactTable) throws LensException synchronized (allFactTables) { List virtualFactTableNames = factToVirtualFactMapping.get(cubeFactTableName); for (String vf : virtualFactTableNames) { - CubeVirtualFactTable cvf = (CubeVirtualFactTable) getCubeFact(vf); + CubeVirtualFactTable cvf = getCubeVirtualFactTable(vf); cvf.setSourceCubeFactTable(cubeFactTable); allFactTables.put(vf.trim().toLowerCase(), cvf); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index 1f0d4cb29..fbc37ac0a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -49,6 +49,9 @@ import com.google.common.collect.Sets; +import lombok.extern.slf4j.Slf4j; + +@Slf4j public class MetastoreUtil { private MetastoreUtil() { @@ -413,7 +416,7 @@ public static Date getDateFromProperty(String prop, boolean relative, boolean st } } } catch (LensException e) { - // log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop); + log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop); } return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE; } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 322120ab1..8734884d4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -577,7 +577,6 @@ public boolean evaluateCompleteness(TimeRange timeRange, TimeRange queriedTimeRa Set rangeParts = getPartitions(timeRange, validUpdatePeriods, true, failOnPartialData, missingParts); String partCol = timeRange.getPartitionColumn(); boolean partColNotSupported = rangeParts.isEmpty(); - String storageTableName = getStorageTable(); if (storagePruningMsgs.containsKey(this)) { List causes = storagePruningMsgs.get(this); @@ -594,11 +593,13 @@ public boolean evaluateCompleteness(TimeRange timeRange, TimeRange queriedTimeRa String sep = ""; while (rangeParts.isEmpty()) { String timeDim = cubeQueryContext.getBaseCube().getTimeDimOfPartitionColumn(partCol); - if (partColNotSupported && !((CubeFactTable)getFact()).hasColumn(timeDim)) { - unsupportedTimeDims.add( - cubeQueryContext.getBaseCube().getTimeDimOfPartitionColumn(timeRange.getPartitionColumn()) - ); - break; + if(getFact() instanceof CubeFactTable) { + if (partColNotSupported && !((CubeFactTable) getFact()).hasColumn(timeDim)) { + unsupportedTimeDims.add( + cubeQueryContext.getBaseCube().getTimeDimOfPartitionColumn(timeRange.getPartitionColumn()) + ); + break; + } } TimeRange fallBackRange = getFallbackRange(prevRange, this.getFact().getName(), cubeQueryContext); log.info("No partitions for range:{}. fallback range: {}", timeRange, fallBackRange); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 49cf18424..9c4893be5 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -82,6 +82,7 @@ public class TestCubeMetastoreClient { private static Set moreCubeDimensions = Sets.newHashSet(); private static Set hourlyAndDaily = Sets.newHashSet(HOURLY, DAILY); private static final String CUBE_NAME = "testMetastoreCube"; + private static final String VIRTUAL_CUBE_NAME = "testMetastoreVirtualCube"; private static final String CUBE_NAME_WITH_PROPS = "testMetastoreCubeWithProps"; private static final String DERIVED_CUBE_NAME = "derivedTestMetastoreCube"; private static final String DERIVED_CUBE_NAME_WITH_PROPS = "derivedTestMetastoreCubeWithProps"; @@ -1257,26 +1258,25 @@ public void testVirtualCubeFact() throws Exception { Map virtualFactPropertiesMap = getHashMap("name1", "newvalue1"); - CubeVirtualFactTable cubeVirtualFact = new CubeVirtualFactTable(CUBE_NAME, virtualFactName, + CubeVirtualFactTable cubeVirtualFact = new CubeVirtualFactTable(VIRTUAL_CUBE_NAME, virtualFactName, com.google.common.base.Optional.fromNullable(null), virtualFactPropertiesMap, sourceFact); // create virtual cube fact - client.createVirtualFactTable(CUBE_NAME, virtualFactName, sourceFactName, null, virtualFactPropertiesMap); + client.createVirtualFactTable(VIRTUAL_CUBE_NAME, virtualFactName, sourceFactName, null, virtualFactPropertiesMap); assertTrue(client.tableExists(virtualFactName)); Table virtualTbl = client.getHiveTable(virtualFactName); assertTrue(client.isVirtualFactTable(virtualTbl)); assertTrue(client.isVirtualFactTableForCube(virtualTbl, CUBE_NAME)); -// -// //get virtual fact -// assertTrue(client.getAllFacts(client.getCube(CUBE_NAME)).contains(virtualFactName.toLowerCase())); -// assertTrue(client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)).contains(virtualFactName.toLowerCase(); + + //get virtual fact + assertTrue(client.getAllFacts(client.getCube(VIRTUAL_CUBE_NAME)).get(0).getName().equals(virtualFactName.trim().toLowerCase())); CubeVirtualFactTable actualcubeVirtualFact = (CubeVirtualFactTable) (client.getCubeFact(virtualFactName)); assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); //alter virtual fact Map alterVirtualFactPropertiesMap = getHashMap("name1", "newvalue2", "name3", "value3"); - cubeVirtualFact = new CubeVirtualFactTable(CUBE_NAME, virtualFactName, + cubeVirtualFact = new CubeVirtualFactTable(VIRTUAL_CUBE_NAME, virtualFactName, com.google.common.base.Optional.fromNullable(null), alterVirtualFactPropertiesMap, sourceFact); client.alterVirtualCubeFactTable(cubeVirtualFact); diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index 79b964906..3f91c7093 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -485,23 +485,6 @@ public void dropVirtualFactTable(LensSessionHandle sessionid, String virtualFact log.info("Dropped virtual fact table " + virtualFact); } } -// -// @Override -// public List getAllVirtualFactNames(LensSessionHandle sessionid, String cubeName) throws LensException { -// try (SessionContext ignored = new SessionContext(sessionid)){ -// CubeMetastoreClient client = getClient(sessionid); -// CubeInterface fact = client.getCube(cubeName); -// if (cubeName != null && fact == null) { -// throw new LensException("Could not get table: " + cubeName + " as a cube"); -// } -// Collection virtualFacts = client.getAllFacts(fact); -// List factNames = new ArrayList<>(virtualFacts.size()); -// for (FactTable cft : virtualFacts) { -// factNames.add(cft.getName()); -// } -// return factNames; -// } -// } @Override public List getAllSegmentations(LensSessionHandle sessionid, String cubeName) throws LensException { diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index 07c5c2237..da049ca05 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -183,7 +183,7 @@ public void doDelete(LensSessionHandle sessionid, String entityName, Boolean cas }, VIRTUALFACT { @Override public List doGetAll(LensSessionHandle sessionid) throws LensException { - return getSvc().getAllFactNames(sessionid, null); + throw new NotImplementedException(); } @Override @@ -724,20 +724,6 @@ public StringList getAllFacts(@QueryParam("sessionid") LensSessionHandle session return Entity.FACT.getAll(sessionid); } - /** - * Get all virtual fact tables in the metastore in the current database - * - * @param sessionid The sessionid in which user is working - * @return StringList consisting of all virtual fact table names - */ - @GET - @Path("/virtualfacts") - public StringList getAllVirtualFacts(@QueryParam("sessionid") LensSessionHandle sessionid) throws LensException { - checkSessionId(sessionid); - return Entity.VIRTUALFACT.getAll(sessionid); - } - - /** * Get all segmentations in the current database * From 6e0fcb6512382f66717cef388ff780dc29173dfb Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 17 May 2017 19:22:17 +0530 Subject: [PATCH 38/71] review changes --- .../cube/metadata/CubeMetastoreClient.java | 10 +++---- .../metadata/TestCubeMetastoreClient.java | 28 ++++++++++--------- .../apache/lens/cube/parse/CubeTestSetup.java | 20 ++++++------- .../metastore/CubeMetastoreServiceImpl.java | 10 +++---- 4 files changed, 35 insertions(+), 33 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 2b85704c0..117d390a0 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -2597,7 +2597,7 @@ public void dropSegmentation(String segName) throws LensException { * @throws LensException */ public void dropStorageFromFact(String factName, String storage) throws LensException { - CubeFactTable cft = (CubeFactTable) getCubeFact(factName); + CubeFactTable cft = getCubeFactTable(factName); dropHiveTablesForStorage(factName, storage); cft.dropStorage(storage); alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft); @@ -2605,7 +2605,7 @@ public void dropStorageFromFact(String factName, String storage) throws LensExce } private void dropHiveTablesForStorage(String factName, String storage) throws LensException{ - CubeFactTable cft = (CubeFactTable) getCubeFactTable(factName); + CubeFactTable cft = getCubeFactTable(factName); Set droppedTables = new HashSet<>(); for (Map.Entry updatePeriodEntry : cft.getStoragePrefixUpdatePeriodMap().get(storage).entrySet()) { UpdatePeriod updatePeriod = (UpdatePeriod) updatePeriodEntry.getKey(); @@ -2755,7 +2755,7 @@ private void updateFactCache(String factTableName) throws LensException { } } - private CubeFactTable getCubeFactTable(String factName) throws LensException { + public CubeFactTable getCubeFactTable(String factName) throws LensException { FactTable factTable = getCubeFact(factName); if(factTable instanceof CubeFactTable) { return (CubeFactTable) factTable; @@ -2765,7 +2765,7 @@ private CubeFactTable getCubeFactTable(String factName) throws LensException { } } - private CubeVirtualFactTable getCubeVirtualFactTable(String factName) throws LensException { + public CubeVirtualFactTable getCubeVirtualFactTable(String factName) throws LensException { FactTable factTable = getCubeFact(factName); if(factTable instanceof CubeVirtualFactTable) { return (CubeVirtualFactTable) factTable; @@ -2898,7 +2898,7 @@ private String getStorageTablePrefixFromStorage(String factOrDimTableName, Strin return storage; } if (isFactTable(factOrDimTableName)) { - return ((CubeFactTable)getCubeFact(factOrDimTableName)).getTablePrefix(storage, updatePeriod); + return (getCubeFactTable(factOrDimTableName)).getTablePrefix(storage, updatePeriod); } else { return storage; } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 9c4893be5..3dc2168d8 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -151,6 +151,7 @@ public static void setup() throws HiveException, AlreadyExistsException, LensExc public static void teardown() throws Exception { // Drop the cube client.dropCube(CUBE_NAME); + client.dropCube(VIRTUAL_CUBE_NAME); client = CubeMetastoreClient.getInstance(conf); assertFalse(client.tableExists(CUBE_NAME)); @@ -1228,6 +1229,7 @@ public void testCubeFact() throws Exception { @Test(priority = 2) public void testVirtualCubeFact() throws Exception { + client.createCube(VIRTUAL_CUBE_NAME, cubeMeasures, cubeDimensions); String sourceFactName = "testMetastoreFact1"; List factColumns = new ArrayList<>(cubeMeasures.size()); for (CubeMeasure measure : cubeMeasures) { @@ -1266,7 +1268,7 @@ public void testVirtualCubeFact() throws Exception { assertTrue(client.tableExists(virtualFactName)); Table virtualTbl = client.getHiveTable(virtualFactName); assertTrue(client.isVirtualFactTable(virtualTbl)); - assertTrue(client.isVirtualFactTableForCube(virtualTbl, CUBE_NAME)); + assertTrue(client.isVirtualFactTableForCube(virtualTbl, VIRTUAL_CUBE_NAME)); //get virtual fact assertTrue(client.getAllFacts(client.getCube(VIRTUAL_CUBE_NAME)).get(0).getName().equals(virtualFactName.trim().toLowerCase())); @@ -1458,7 +1460,7 @@ public void testAlterCubeFact() throws Exception { updatePeriodStoragePrefix.put(DAILY, c4); factTable.addStorage(c4, hourlyAndDaily, updatePeriodStoragePrefix); client.alterCubeFactTable(factName, factTable, storageTables, new HashMap()); - CubeFactTable altered2 = (CubeFactTable) client.getCubeFact(factName); + CubeFactTable altered2 = client.getCubeFactTable(factName); assertTrue(client.tableExists(c1TableName)); Table alteredC1Table = client.getTable(c1TableName); assertEquals(alteredC1Table.getInputFormatClass(), SequenceFileInputFormat.class); @@ -1485,7 +1487,7 @@ public void testAlterCubeFact() throws Exception { Map storageTableDescMap = new HashMap<>(); storageTableDescMap.put(c3, s1); client.addStorage(altered2, c3, hourlyAndDaily, storageTableDescMap, updatePeriodStoragePrefix); - CubeFactTable altered3 = (CubeFactTable) client.getCubeFact(factName); + CubeFactTable altered3 = client.getCubeFactTable(factName); assertTrue(altered3.getStorages().contains("C3")); assertTrue(altered3.getUpdatePeriods().get("C3").equals(hourlyAndDaily)); String storageTableName = getFactOrDimtableStorageTableName(factName, c3); @@ -2942,8 +2944,8 @@ public void testCubeDimWithTwoStorages() throws Exception { public void testCaching() throws HiveException, LensException { client = CubeMetastoreClient.getInstance(conf); CubeMetastoreClient client2 = CubeMetastoreClient.getInstance(new HiveConf(TestCubeMetastoreClient.class)); - assertEquals(5, client.getAllCubes().size()); - assertEquals(5, client2.getAllCubes().size()); + assertEquals(6, client.getAllCubes().size()); + assertEquals(6, client2.getAllCubes().size()); defineCube("testcache1", "testcache2", "derived1", "derived2"); client.createCube("testcache1", cubeMeasures, cubeDimensions); @@ -2952,18 +2954,18 @@ public void testCaching() throws HiveException, LensException { client.createDerivedCube("testcache2", "derived2", measures, dimensions, CUBE_PROPERTIES, 0L); assertNotNull(client.getCube("testcache1")); assertNotNull(client2.getCube("testcache1")); - assertEquals(9, client.getAllCubes().size()); - assertEquals(9, client2.getAllCubes().size()); + assertEquals(10, client.getAllCubes().size()); + assertEquals(10, client2.getAllCubes().size()); client2 = CubeMetastoreClient.getInstance(conf); - assertEquals(9, client.getAllCubes().size()); - assertEquals(9, client2.getAllCubes().size()); + assertEquals(10, client.getAllCubes().size()); + assertEquals(10, client2.getAllCubes().size()); conf.setBoolean(MetastoreConstants.METASTORE_ENABLE_CACHING, false); client = CubeMetastoreClient.getInstance(conf); client2 = CubeMetastoreClient.getInstance(conf); - assertEquals(9, client.getAllCubes().size()); - assertEquals(9, client2.getAllCubes().size()); + assertEquals(10, client.getAllCubes().size()); + assertEquals(10, client2.getAllCubes().size()); defineCube("testcache3", "testcache4", "dervied3", "derived4"); client.createCube("testcache3", cubeMeasures, cubeDimensions); client.createCube("testcache4", cubeMeasures, cubeDimensions, CUBE_PROPERTIES); @@ -2971,8 +2973,8 @@ public void testCaching() throws HiveException, LensException { client.createDerivedCube("testcache4", "derived4", measures, dimensions, CUBE_PROPERTIES, 0L); assertNotNull(client.getCube("testcache3")); assertNotNull(client2.getCube("testcache3")); - assertEquals(13, client.getAllCubes().size()); - assertEquals(13, client2.getAllCubes().size()); + assertEquals(14, client.getAllCubes().size()); + assertEquals(14, client2.getAllCubes().size()); conf.setBoolean(MetastoreConstants.METASTORE_ENABLE_CACHING, true); client = CubeMetastoreClient.getInstance(conf); } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index f2e6c4290..48869c2bd 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -583,7 +583,7 @@ private void assertTestFactTimelineClass(CubeMetastoreClient client) throws Exce client.clearHiveTableCache(); - CubeFactTable fact = (CubeFactTable) client.getCubeFact(factName); + CubeFactTable fact = client.getCubeFactTable(factName); Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c1))); assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); for (UpdatePeriod period : Lists.newArrayList(MINUTELY, HOURLY, DAILY, MONTHLY, YEARLY, QUARTERLY)) { @@ -632,7 +632,7 @@ private void assertTimeline(CubeMetastoreClient client, String factName, String private void createCubeCheapFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "cheapFact"; - CubeFactTable fact = (CubeFactTable) client.getCubeFact(factName); + CubeFactTable fact = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); @@ -664,7 +664,7 @@ private void createCubeCheapFactPartitions(CubeMetastoreClient client) throws Hi private void createTestFact2Partitions(CubeMetastoreClient client) throws Exception { String factName = "testFact2"; - CubeFactTable fact = (CubeFactTable) client.getCubeFact(factName); + CubeFactTable fact = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); @@ -752,7 +752,7 @@ private void createTestFact2Partitions(CubeMetastoreClient client) throws Except private void createTestFact2RawPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "testFact2_raw"; - CubeFactTable fact2 = (CubeFactTable) client.getCubeFact(factName); + CubeFactTable fact2 = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); @@ -835,7 +835,7 @@ private void dump(CubeMetastoreClient client) throws LensException, IOException // bw.write(ToXMLString.toString(JAXBUtils.xCubeFromHiveCube(cubeInterface))); // } // } - for (FactTable factTable : client.getAllFacts()) { + for (FactTable factTable : client.getAllFacts(false)) { CubeFactTable cubeFactTable = (CubeFactTable) factTable; try(BufferedWriter bw = new BufferedWriter(new FileWriter(getClass() .getResource("/schema/facts").getPath()+"/"+cubeFactTable.getName()+".xml"))) { @@ -870,26 +870,26 @@ public void dropSources(HiveConf conf, String dbName) throws Exception { private void createSummaryPartitions(CubeMetastoreClient client) throws Exception { String factName = "summary1"; - CubeFactTable fact1 = (CubeFactTable) client.getCubeFact(factName); + CubeFactTable fact1 = client.getCubeFactTable(factName); createPIEParts(client, fact1, c2); factName = "summary2"; - CubeFactTable fact2 = (CubeFactTable) client.getCubeFact(factName); + CubeFactTable fact2 = client.getCubeFactTable(factName); createPIEParts(client, fact2, c2); factName = "summary3"; - CubeFactTable fact3 = (CubeFactTable) client.getCubeFact(factName); + CubeFactTable fact3 = client.getCubeFactTable(factName); createPIEParts(client, fact3, c2); factName = "summary4"; - CubeFactTable fact4 = (CubeFactTable) client.getCubeFact(factName); + CubeFactTable fact4 = client.getCubeFactTable(factName); createPIEParts(client, fact4, c2); } private void createBaseCubeFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "testFact5_RAW_BASE"; - CubeFactTable fact = (CubeFactTable) client.getCubeFact(factName); + CubeFactTable fact = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index 3f91c7093..9b883d4ac 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -323,7 +323,7 @@ public List getAllDimTableNames(LensSessionHandle sessionid, String dime public void dropAllStoragesOfFact(LensSessionHandle sessionid, String factName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); - CubeFactTable tab = (CubeFactTable) msClient.getCubeFact(factName); + CubeFactTable tab = msClient.getCubeFactTable(factName); int total = tab.getStorages().size(); int i = 0; List storageNames = new ArrayList<>(tab.getStorages()); @@ -513,7 +513,7 @@ public List getStoragesOfFact(LensSessionHandle sessionid, String fact) throw new NotFoundException("Not a fact table " + fact); } - CubeFactTable cft = (CubeFactTable) msClient.getCubeFact(fact); + CubeFactTable cft = msClient.getCubeFactTable(fact); if (cft != null) { return new ArrayList<>(cft.getStorages()); } else { @@ -565,7 +565,7 @@ public void addStorageToFact(LensSessionHandle sessionid, String fact, XStorageT CubeMetastoreClient msClient = getClient(sessionid); XStorageTables tables = new XStorageTables(); tables.getStorageTable().add(storageTable); - msClient.addStorage((CubeFactTable) msClient.getCubeFact(fact), storageTable.getStorageName(), updatePeriods, + msClient.addStorage(msClient.getCubeFactTable(fact), storageTable.getStorageName(), updatePeriods, JAXBUtils.tableDescPrefixMapFromXStorageTables(tables), JAXBUtils.storageTablePrefixMapOfStorage(tables).get(storageTable.getStorageName())); log.info("Added storage " + storageTable.getStorageName() + ":" + updatePeriods + " for fact " + fact); @@ -586,7 +586,7 @@ public void dropStorageOfFact(LensSessionHandle sessionid, String fact, String s private CubeFactTable checkFactStorage(LensSessionHandle sessionid, String fact, String storage) throws HiveException, LensException { CubeMetastoreClient client = getClient(sessionid); - CubeFactTable factTable = (CubeFactTable) client.getCubeFact(fact); + CubeFactTable factTable = client.getCubeFactTable(fact); client.verifyStorageExists(factTable, storage); return factTable; } @@ -595,7 +595,7 @@ private Set getAllTablesForStorage(LensSessionHandle sessionHandle, Stri throws LensException { Set storageTableNames = new HashSet<>(); if (getClient(sessionHandle).isFactTable(fact)) { - CubeFactTable cft = (CubeFactTable) getClient(sessionHandle).getCubeFact(fact); + CubeFactTable cft = getClient(sessionHandle).getCubeFactTable(fact); Map storageMap = cft.getStoragePrefixUpdatePeriodMap().get(storageName); for (Map.Entry entry : storageMap.entrySet()) { storageTableNames.add(MetastoreUtil.getStorageTableName(fact, Storage.getPrefix((String) entry.getValue()))); From f01d7a1db53b5b01f5e4b4106e0b005b302862b6 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 17 May 2017 19:35:12 +0530 Subject: [PATCH 39/71] review changes --- .../apache/lens/cube/metadata/CubeMetastoreClient.java | 9 ++++----- .../org/apache/lens/cube/parse/StorageCandidate.java | 2 +- .../lens/cube/metadata/TestCubeMetastoreClient.java | 6 ++++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 117d390a0..1db8ba874 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -35,7 +35,6 @@ import org.apache.lens.cube.metadata.timeline.PartitionTimeline; import org.apache.lens.cube.metadata.timeline.PartitionTimelineFactory; import org.apache.lens.server.api.LensConfConstants; -import org.apache.lens.server.api.LensErrorInfo; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.DataCompletenessChecker; import org.apache.lens.server.api.util.LensUtil; @@ -2757,9 +2756,9 @@ private void updateFactCache(String factTableName) throws LensException { public CubeFactTable getCubeFactTable(String factName) throws LensException { FactTable factTable = getCubeFact(factName); - if(factTable instanceof CubeFactTable) { + if (factTable instanceof CubeFactTable) { return (CubeFactTable) factTable; - }else { + } else { throw new LensException(new LensException(LensCubeErrorCode.ENTITY_TYPE_NOT_AS_EXPECTED.getLensErrorInfo(), factName, "CubeFactTable")); } @@ -2767,9 +2766,9 @@ public CubeFactTable getCubeFactTable(String factName) throws LensException { public CubeVirtualFactTable getCubeVirtualFactTable(String factName) throws LensException { FactTable factTable = getCubeFact(factName); - if(factTable instanceof CubeVirtualFactTable) { + if (factTable instanceof CubeVirtualFactTable) { return (CubeVirtualFactTable) factTable; - }else { + } else { throw new LensException(new LensException(LensCubeErrorCode.ENTITY_TYPE_NOT_AS_EXPECTED.getLensErrorInfo(), factName, "CubeVirtualFactTable")); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 8734884d4..e27e77bec 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -593,7 +593,7 @@ public boolean evaluateCompleteness(TimeRange timeRange, TimeRange queriedTimeRa String sep = ""; while (rangeParts.isEmpty()) { String timeDim = cubeQueryContext.getBaseCube().getTimeDimOfPartitionColumn(partCol); - if(getFact() instanceof CubeFactTable) { + if (getFact() instanceof CubeFactTable) { if (partColNotSupported && !((CubeFactTable) getFact()).hasColumn(timeDim)) { unsupportedTimeDims.add( cubeQueryContext.getBaseCube().getTimeDimOfPartitionColumn(timeRange.getPartitionColumn()) diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 3dc2168d8..f33c719d4 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -1264,14 +1264,16 @@ public void testVirtualCubeFact() throws Exception { com.google.common.base.Optional.fromNullable(null), virtualFactPropertiesMap, sourceFact); // create virtual cube fact - client.createVirtualFactTable(VIRTUAL_CUBE_NAME, virtualFactName, sourceFactName, null, virtualFactPropertiesMap); + client.createVirtualFactTable(VIRTUAL_CUBE_NAME, virtualFactName, sourceFactName, null, + virtualFactPropertiesMap); assertTrue(client.tableExists(virtualFactName)); Table virtualTbl = client.getHiveTable(virtualFactName); assertTrue(client.isVirtualFactTable(virtualTbl)); assertTrue(client.isVirtualFactTableForCube(virtualTbl, VIRTUAL_CUBE_NAME)); //get virtual fact - assertTrue(client.getAllFacts(client.getCube(VIRTUAL_CUBE_NAME)).get(0).getName().equals(virtualFactName.trim().toLowerCase())); + assertTrue(client.getAllFacts(client.getCube(VIRTUAL_CUBE_NAME)).get(0).getName().equals(virtualFactName.trim() + .toLowerCase())); CubeVirtualFactTable actualcubeVirtualFact = (CubeVirtualFactTable) (client.getCubeFact(virtualFactName)); assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); From 239d0ff9770e807876aacb3314bb22757b828379 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 17 May 2017 20:25:16 +0530 Subject: [PATCH 40/71] review changes --- .../org/apache/lens/client/LensClient.java | 12 ++++++ .../lens/client/LensMetadataClient.java | 40 +++++++++++++++++++ .../metastore/TestMetastoreService.java | 6 +-- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java index 4f8da06d1..93f413a96 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java @@ -454,6 +454,10 @@ public APIResult createFactTable(String factSpec) { return mc.createFactTable(factSpec); } + public APIResult createVirtualFactTable(String vfactSpec) { + return mc.createVirtualFactTable(vfactSpec); + } + public APIResult createCube(String cubeSpec) { return mc.createCube(cubeSpec); } @@ -482,6 +486,10 @@ public APIResult dropFactTable(String fact, boolean cascade) { return mc.dropFactTable(fact, cascade); } + public APIResult dropVirtualFactTable(String fact, boolean cascade) { + return mc.dropVirtualFactTable(fact); + } + public APIResult dropCube(String cube) { return mc.dropCube(cube); } @@ -498,6 +506,10 @@ public APIResult updateFactTable(String factName, String factSpec) { return mc.updateFactTable(factName, factSpec); } + public APIResult updateVirtualFactTable(String factName, String factSpec) { + return mc.updateFactTable(factName, factSpec); + } + public APIResult updateDimensionTable(String dimName, String dimSpec) { return mc.updateDimensionTable(dimName, dimSpec); } diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java index f077c9cf2..037f0af79 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java @@ -461,6 +461,22 @@ public APIResult createFactTable(String factSpec) { } } + public APIResult createVirtualFactTable(XVirtualFactTable f) { + WebTarget target = getMetastoreWebTarget(); + return translate(target.path("virtualfacts") + .queryParam("sessionid", this.connection.getSessionHandle()) + .request(MediaType.APPLICATION_XML) + .post(Entity.xml(new GenericEntity>(objFact.createXVirtualFactTable(f)){}))); + } + + public APIResult createVirtualFactTable(String vfactSpec) { + try { + return createVirtualFactTable(this.readFromXML(vfactSpec)); + } catch (JAXBException | IOException e) { + return failureAPIResult(e); + } + } + public APIResult createSegmentation(XSegmentation seg) { WebTarget target = getMetastoreWebTarget(); return translate(target.path("segmentations") @@ -494,6 +510,22 @@ public APIResult updateFactTable(String factName, String table) { } } + public APIResult updateVirtualFactTable(String factName, XVirtualFactTable table) { + WebTarget target = getMetastoreWebTarget(); + return translate(target.path("virtualfacts").path(factName) + .queryParam("sessionid", this.connection.getSessionHandle()) + .request(MediaType.APPLICATION_XML_TYPE) + .put(Entity.xml(new GenericEntity>(objFact.createXVirtualFactTable(table)){}))); + } + + public APIResult updateVirtualFactTable(String factName, String table) { + try { + return updateVirtualFactTable(factName, this.readFromXML(table)); + } catch (JAXBException | IOException e) { + return failureAPIResult(e); + } + } + public APIResult updateSegmentation(String segName, XSegmentation seg) { WebTarget target = getMetastoreWebTarget(); return translate(target.path("segmentations").path(segName) @@ -525,6 +557,14 @@ public APIResult dropFactTable(String factName) { return dropFactTable(factName, false); } + public APIResult dropVirtualFactTable(String factName) { + WebTarget target = getMetastoreWebTarget(); + return translate(target.path("virtualfacts").path(factName) + .queryParam("sessionid", this.connection.getSessionHandle()) + .request(MediaType.APPLICATION_XML) + .delete()); + } + public APIResult dropSegmentation(String segName) { WebTarget target = getMetastoreWebTarget(); return translate(target.path("segmentations").path(segName) diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index b0f1824a4..cab0a7b87 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -2280,10 +2280,10 @@ public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { APIResult.class); assertSuccess(result); - // Get all virtual fact names, this should contain the virtual fact table - StringList virtualFactNames = target().path("metastore/virtualfacts") + // Get all fact names, this should contain the virtual fact table + StringList factNames = target().path("metastore/facts") .queryParam("sessionid", lensSessionId).request(mediaType).get(StringList.class); - assertTrue(virtualFactNames.getElements().contains(virtualFactTable.toLowerCase())); + assertTrue(factNames.getElements().contains(virtualFactTable.toLowerCase())); // Get the created table JAXBElement gotFactElement = target().path("metastore/virtualfacts").path(virtualFactTable) From eb4c034b32877fc92567d394c4fd21ffe8c44027 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 17 May 2017 21:56:36 +0530 Subject: [PATCH 41/71] review changes --- .../src/main/java/org/apache/lens/client/LensClient.java | 6 +++++- .../java/org/apache/lens/client/LensMetadataClient.java | 8 ++++++++ .../apache/lens/server/metastore/MetastoreResource.java | 7 ++----- .../lens/server/metastore/TestMetastoreService.java | 2 +- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java index 93f413a96..520dde587 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java @@ -486,10 +486,14 @@ public APIResult dropFactTable(String fact, boolean cascade) { return mc.dropFactTable(fact, cascade); } - public APIResult dropVirtualFactTable(String fact, boolean cascade) { + public APIResult dropVirtualFactTable(String fact) { return mc.dropVirtualFactTable(fact); } + public APIResult dropAllVirtualFacts() { + return mc.dropAllVirtualFactTables(); + } + public APIResult dropCube(String cube) { return mc.dropCube(cube); } diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java index 037f0af79..436417aa6 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java @@ -565,6 +565,14 @@ public APIResult dropVirtualFactTable(String factName) { .delete()); } + public APIResult dropAllVirtualFactTables() { + WebTarget target = getMetastoreWebTarget(); + return translate(target.path("virtualfacts") + .queryParam("sessionid", this.connection.getSessionHandle()) + .request(MediaType.APPLICATION_XML) + .delete()); + } + public APIResult dropSegmentation(String segName) { WebTarget target = getMetastoreWebTarget(); return translate(target.path("segmentations").path(segName) diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index da049ca05..27c39225a 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -759,16 +759,13 @@ public APIResult deleteAllFacts(@QueryParam("sessionid") LensSessionHandle sessi * Delete all virtual fact tables * * @param sessionid The sessionid in which user is working - * @param cascade if set to true, all the underlying tables will be dropped, if set to false, only the virtual fact - * table will be dropped * @return APIResult with state {@link Status#SUCCEEDED} in case of successful delete. APIResult with state {@link * Status#FAILED} in case of delete failure. APIResult with state {@link Status#PARTIAL} in case of partial delete. */ @DELETE @Path("virtualfacts") - public APIResult deleteAllVirtualFacts(@QueryParam("sessionid") LensSessionHandle sessionid, - @DefaultValue("false") @QueryParam("cascade") boolean cascade) throws LensException { - return Entity.VIRTUALFACT.deleteAll(sessionid, cascade); + public APIResult deleteAllVirtualFacts(@QueryParam("sessionid") LensSessionHandle sessionid) throws LensException { + return Entity.VIRTUALFACT.deleteAll(sessionid, false); } /** diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index cab0a7b87..1439d5087 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -2384,7 +2384,7 @@ public void testUpdateVirtualFactTable(MediaType mediaType) throws Exception { assertSuccess(result); // Get all virtual fact names, this should contain the virtual fact table - StringList virtualFactNames = target().path("metastore/virtualfacts") + StringList virtualFactNames = target().path("metastore/facts") .queryParam("sessionid", lensSessionId).request(mediaType).get(StringList.class); assertTrue(virtualFactNames.getElements().contains(table.toLowerCase())); From 1126f8357c99b712687979fbabc1558d6563a590 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 18 May 2017 12:24:31 +0530 Subject: [PATCH 42/71] review changes --- .../apache/lens/server/metastore/TestMetastoreService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index 1439d5087..a35381665 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -2383,10 +2383,10 @@ public void testUpdateVirtualFactTable(MediaType mediaType) throws Exception { APIResult.class); assertSuccess(result); - // Get all virtual fact names, this should contain the virtual fact table - StringList virtualFactNames = target().path("metastore/facts") + // Get all fact names, this should contain the virtual fact table + StringList factNames = target().path("metastore/facts") .queryParam("sessionid", lensSessionId).request(mediaType).get(StringList.class); - assertTrue(virtualFactNames.getElements().contains(table.toLowerCase())); + assertTrue(factNames.getElements().contains(table.toLowerCase())); // Get the created table JAXBElement gotFactElement = target().path("metastore/virtualfacts").path(table) From 71000efce1d3cb309ec78b188e5d8bdc69ffe3c7 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 18 May 2017 12:27:15 +0530 Subject: [PATCH 43/71] review changes --- .../org/apache/lens/cube/metadata/CubeMetastoreClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 1db8ba874..1270de7d9 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -2760,7 +2760,7 @@ public CubeFactTable getCubeFactTable(String factName) throws LensException { return (CubeFactTable) factTable; } else { throw new LensException(new LensException(LensCubeErrorCode.ENTITY_TYPE_NOT_AS_EXPECTED.getLensErrorInfo(), - factName, "CubeFactTable")); + factName, "Fact")); } } @@ -2770,7 +2770,7 @@ public CubeVirtualFactTable getCubeVirtualFactTable(String factName) throws Lens return (CubeVirtualFactTable) factTable; } else { throw new LensException(new LensException(LensCubeErrorCode.ENTITY_TYPE_NOT_AS_EXPECTED.getLensErrorInfo(), - factName, "CubeVirtualFactTable")); + factName, "VirtualFact")); } } From f6595ed58dc05ca8be07d4c7e8e2eb4569f5a2e1 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Fri, 19 May 2017 12:25:48 +0530 Subject: [PATCH 44/71] More changes for vf --- lens-api/src/main/resources/cube-0.1.xsd | 188 +++++++++-------- .../cube/metadata/CubeMetastoreClient.java | 189 ++++++++++-------- .../metadata/TestCubeMetastoreClient.java | 6 +- .../api/metastore/CubeMetastoreService.java | 55 +---- .../metastore/CubeMetastoreServiceImpl.java | 55 +---- .../server/metastore/MetastoreResource.java | 112 +---------- .../metastore/TestMetastoreService.java | 88 ++++---- 7 files changed, 272 insertions(+), 421 deletions(-) diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index b50ef1da5..1f54c752a 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -168,76 +168,6 @@ - - - - - Virtual fact extends fact. It can override the cube of its source fact. It can have different - properties associated with it than its source. - Properties that can be set for a virtual fact are : - - Filters which would be added in query rewriting - - - - - - - Properties that can be set for a virtual fact are : - 1. cube.fact.query.where.filter : filter string that needs to be added in WHERE clause. This string would be added as an additional - filter when the query is being constructed in the cube query writing phase. - 2. cube.fact.absolute.start.time: start time of the fact. For queries that ask for time before this, - this fact is not a candidate. Time format can be as you would specify in the time_range_in clause. - i.e. yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]] - 3. cube.fact.relative.start.time: Here you can specify fact's relative validity relative to current time. - Useful if you want to specify e.g. this fact is valid for today - 90 days. Can be specified as just - a time difference e.g. "-90 days". Or can be specified in relative syntax. - e.g. now.year or now.day - 6 hour etc. - 4. cube.fact.absolute.end.time: If you're deprecating this fact, put the final date till which the data of - the fact will be valid here. Format same as absolute start time. - 5. cube.fact.relative.end.time: You can specify the end date for fact table - relative to current date e.g. fact table is valid for next 90days starting from today. - This can be specified as just a time difference e.g. "+90 days" - - - - - - - - - The Source fact name over which the Virtual fact is defined. - - - - - - - The base cube's name to which the Virtual fact is associated. - - - - - - - The virtual fact table name. - - - - - - - The weight of the fact table. LENS will use this attribute to decide the lightest table to query when there - are more than one eligible tables. If not defined, the source fact weight would be picked up. - - - - - - - - - - @@ -1287,9 +1217,111 @@ + + + + + + XCube can either be a Base cube for which the user would give the full specification of the + measures and dimensions + or can be a Derived cube, for which the user would specify only the measure names, + dimension names and parent. + Derived cube can have subset of measure names and dimension names associated with its + parent. It can have + different properties associated with it than the parent. + + + + + + + + + + + + + + + + + + The cube's name to which the fact is associated. + + + + + + + + + + + + Virtual fact extends fact. It can override the cube of its source fact. It can have different + properties associated with it than its source. + Properties that can be set for a virtual fact are : + - Filters which would be added in query rewriting + + + + + + + Properties that can be set for a virtual fact are : + 1. cube.fact.query.where.filter : filter string that needs to be added in WHERE clause. This string + would be added as an additional + filter when the query is being constructed in the cube query writing phase. + 2. cube.fact.absolute.start.time: start time of the fact. For queries that ask for time before this, + this fact is not a candidate. Time format can be as you would specify in the time_range_in clause. + i.e. yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]] + 3. cube.fact.relative.start.time: Here you can specify fact's relative validity relative to current + time. + Useful if you want to specify e.g. this fact is valid for today - 90 days. Can be specified as just + a time difference e.g. "-90 days". Or can be specified in relative syntax. + e.g. now.year or now.day - 6 hour etc. + 4. cube.fact.absolute.end.time: If you're deprecating this fact, put the final date till which the data + of + the fact will be valid here. Format same as absolute start time. + 5. cube.fact.relative.end.time: You can specify the end date for fact table + relative to current date e.g. fact table is valid for next 90days starting from today. + This can be specified as just a time difference e.g. "+90 days" + + + + + + + + + + The Source fact name over which the Virtual fact is defined. + + + + + + + The weight of the fact table. LENS will use this attribute to decide the lightest table to query when + there + are more than one eligible tables. If not defined, the source fact weight would be picked up. + + + + + + + + + + + + + + Fact table that is associated to a base cube. The columns in the fact table will be a subset of @@ -1334,20 +1366,6 @@ - - - - The fact table name. - - - - - - - The base cube's name to which the fact_table is associated. - - - @@ -1361,6 +1379,8 @@ + + diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 1270de7d9..f24706d10 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -309,7 +309,7 @@ public void createCubeFactTable(String cubeName, String factName, List void createEntity(T entity) thro createCube((XCube)entity); } else if (entity instanceof XDimension) { createDimension((XDimension) entity); - } else if (entity instanceof XFactTable) { - createCubeFactTable((XFactTable) entity); + } else if (entity instanceof XFact) { + createFactTable((XFact) entity); } else if (entity instanceof XDimensionTable) { createCubeDimensionTable((XDimensionTable) entity); } else if (entity instanceof XSegmentation) { createSegmentation((XSegmentation) entity); - }else if (entity instanceof XVirtualFactTable) { - createVirtualFactTable((XVirtualFactTable) entity); } else { throw new LensException("Unable to create entity " + entity + " as it's unrecognizable: "+ entity.getClass()); } @@ -341,45 +339,46 @@ public void updateEntity(String name, T alterCube((XCube)entity); } else if (entity instanceof XDimension) { alterDimension((XDimension) entity); - } else if (entity instanceof XFactTable) { - alterCubeFactTable((XFactTable) entity); + } else if (entity instanceof XFact) { + alterCubeFactTable((XFact) entity); } else if (entity instanceof XDimensionTable) { alterCubeDimensionTable((XDimensionTable) entity); } else if (entity instanceof XSegmentation) { alterSegmentation((XSegmentation) entity); - } else if (entity instanceof XVirtualFactTable) { - alterVirtualCubeFactTable((XVirtualFactTable) entity); } else { throw new LensException("Unable to alter entity " + entity + " as it's unrecognizable: " + entity.getClass()); } } - public static Map addFactColStartTimePropertyToFactProperties(XFactTable fact) { Map props = new HashMap(); props.putAll(JAXBUtils.mapFromXProperties(fact.getProperties())); props.putAll(JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); return props; } - public void createCubeFactTable(XFactTable fact) throws LensException { - createCubeFactTable(fact.getCubeName(), - fact.getName(), - JAXBUtils.fieldSchemaListFromColumns(fact.getColumns()), - JAXBUtils.getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()), - fact.getWeight(), - addFactColStartTimePropertyToFactProperties(fact), - JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), - JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables())); - } - public void createVirtualFactTable(XVirtualFactTable virtualFact) throws LensException { - createVirtualFactTable(virtualFact.getCubeName(), virtualFact.getName(), virtualFact.getSourceFactName(), - virtualFact.getWeight(), JAXBUtils.mapFromXProperties(virtualFact.getProperties())); + public void createFactTable(XFact fact) throws LensException { + + if (fact instanceof XVirtualFactTable) { + XVirtualFactTable xvf = (XVirtualFactTable) fact; + createVirtualFactTable(xvf.getCubeName(), xvf.getName(), xvf.getSourceFactName(), + xvf.getWeight(), JAXBUtils.mapFromXProperties(xvf.getProperties())); + } else { + XFactTable xf = (XFactTable) fact; + createCubeFactTable(fact.getCubeName(), + fact.getName(), + JAXBUtils.fieldSchemaListFromColumns(xf.getColumns()), + JAXBUtils.getFactUpdatePeriodsFromStorageTables(xf.getStorageTables()), + xf.getWeight(), + addFactColStartTimePropertyToFactProperties(xf), + JAXBUtils.tableDescPrefixMapFromXStorageTables(xf.getStorageTables()), + JAXBUtils.storageTablePrefixMapOfStorage(xf.getStorageTables())); + } } public void createVirtualFactTable(String cubeName, String virtualFactName, String sourceFactName, Double weight, Map properties) throws LensException { - FactTable sourceFact = getCubeFact(sourceFactName); + FactTable sourceFact = getFactTable(sourceFactName); Optional optionalWeight = Optional.fromNullable(weight); @@ -387,7 +386,7 @@ public void createVirtualFactTable(String cubeName, String virtualFactName, Stri optionalWeight, properties, sourceFact); createCubeTable(factTable, null); // do a get to update cache - getCubeFact(virtualFactName); + getFactTable(virtualFactName); } @@ -458,7 +457,7 @@ public TreeMap> ge private void loadTimeLines(String fact, String storage, String timeLineKey) throws LensException, HiveException { Set uniqueStorageTables = new HashSet<>(); Map updatePeriodTableName = new HashMap<>(); - for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) { + for (UpdatePeriod updatePeriod : getFactTable(fact).getUpdatePeriods().get(storage)) { String storageTableName = getStorageTableName(fact, storage, updatePeriod); updatePeriodTableName.put(updatePeriod, storageTableName); Table storageTable = getTable(storageTableName); @@ -493,7 +492,7 @@ private void ensureEntryForTimeLineKey(String fact, String storage, UpdatePeriod // Not found in table properties either, compute from all partitions of the fact-storage table. // First make sure all combinations of update period and partition column have an entry even // if no partitions exist - if (getCubeFact(fact).getUpdatePeriods() != null && getCubeFact(fact).getUpdatePeriods().get(storage) != null) { + if (getFactTable(fact).getUpdatePeriods() != null && getFactTable(fact).getUpdatePeriods().get(storage) != null) { log.info("loading from all partitions: {}", storageTableName); Table storageTable = getTable(storageTableName); for (String partCol : getTimePartColNamesOfTable(storageTable)) { @@ -864,7 +863,7 @@ public void createCubeFactTable(String cubeName, String factName, List addPartitions(String factOrDimTable, String storageName, public Date getStorageTableStartDate(String storageTable, String factTableName) throws LensException { List startDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableStartTimesKey()); - startDates.add(getCubeFact(factTableName).getStartTime()); + startDates.add(getFactTable(factTableName).getStartTime()); return Collections.max(startDates); } public Date getStorageTableEndDate(String storageTable, String factTableName) throws LensException { List endDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableEndTimesKey()); - endDates.add((getCubeFact(factTableName)).getEndTime()); + endDates.add((getFactTable(factTableName)).getEndTime()); return Collections.min(endDates); } @@ -1763,47 +1762,57 @@ boolean isDimension(Table tbl) { return CubeTableType.DIMENSION.name().equals(tableType); } - public XFactTable getXFactTable(String tableName) throws LensException { - return getXFactTable(getCubeFactTable(tableName)); - } - public XFactTable getXFactTable(CubeFactTable cft) throws LensException { - - XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft); - Map> storageMap = cft.getStoragePrefixUpdatePeriodMap(); - for (String storageName : cft.getStorages()) { - Set updatePeriods = cft.getUpdatePeriods().get(storageName); - // This map tells if there are different tables for different update period. - Map updatePeriodToTableMap = storageMap.get(storageName); - Set tableNames = new HashSet<>(); - for (UpdatePeriod updatePeriod : updatePeriods) { - tableNames.add(updatePeriodToTableMap.get(updatePeriod)); - } - if (tableNames.size() <= 1) { - XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( - getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName))); - tblElement.setStorageName(storageName); - for (UpdatePeriod p : updatePeriods) { - tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); + public XFact getXFactTable(String tableName) throws LensException { + return getXFactTable(getFactTable(tableName)); + } + public XFact getXFactTable(FactTable ft) throws LensException { + + XFact fact; + if(ft.isVirtualFact()){ + CubeVirtualFactTable cvft = (CubeVirtualFactTable) ft; + XVirtualFactTable factTable = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cvft); + factTable.setSourceFactName(cvft.getSourceCubeFactTable().getName()); + fact = factTable; + }else { + CubeFactTable cft = (CubeFactTable) ft; + XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft); + Map> storageMap = cft.getStoragePrefixUpdatePeriodMap(); + for (String storageName : cft.getStorages()) { + Set updatePeriods = cft.getUpdatePeriods().get(storageName); + // This map tells if there are different tables for different update period. + Map updatePeriodToTableMap = storageMap.get(storageName); + Set tableNames = new HashSet<>(); + for (UpdatePeriod updatePeriod : updatePeriods) { + tableNames.add(updatePeriodToTableMap.get(updatePeriod)); } - factTable.getStorageTables().getStorageTable().add(tblElement); - } else { - // Multiple storage tables. - XStorageTableElement tblElement = new XStorageTableElement(); - tblElement.setStorageName(storageName); - XUpdatePeriods xUpdatePeriods = new XUpdatePeriods(); - tblElement.setUpdatePeriods(xUpdatePeriods); - for (Map.Entry entry : updatePeriodToTableMap.entrySet()) { - XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); - updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable( - this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), + if (tableNames.size() <= 1) { + XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( + getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName))); + tblElement.setStorageName(storageName); + for (UpdatePeriod p : updatePeriods) { + tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); + } + factTable.getStorageTables().getStorageTable().add(tblElement); + } else { + // Multiple storage tables. + XStorageTableElement tblElement = new XStorageTableElement(); + tblElement.setStorageName(storageName); + XUpdatePeriods xUpdatePeriods = new XUpdatePeriods(); + tblElement.setUpdatePeriods(xUpdatePeriods); + for (Map.Entry entry : updatePeriodToTableMap.entrySet()) { + XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); + updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable( + this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), (String) entry.getValue())))); - updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod)entry.getKey()).name())); - xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor); + updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod) entry.getKey()).name())); + xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor); + } + factTable.getStorageTables().getStorageTable().add(tblElement); } - factTable.getStorageTables().getStorageTable().add(tblElement); } + fact = factTable; } - return factTable; + return fact; } public Segmentation getSegmentationTable(String tableName) throws HiveException, LensException { @@ -1983,10 +1992,10 @@ private Dimension getDimension(String tableName, boolean throwException) throws * @return Returns cube is table name passed is a cube * @throws LensException if there is no cube by the name */ - public FactTable getCubeFact(String tableName) throws LensException { - return getCubeFact(tableName, true); + public FactTable getFactTable(String tableName) throws LensException { + return getFactTable(tableName, true); } - private FactTable getCubeFact(String tableName, boolean throwException) throws LensException { + private FactTable getFactTable(String tableName, boolean throwException) throws LensException { tableName = tableName.trim().toLowerCase(); FactTable fact = allFactTables.get(tableName); if (fact == null) { @@ -2180,7 +2189,7 @@ public Collection getAllFacts() throws LensException { List facts = new ArrayList<>(); try { for (String table : getAllHiveTableNames()) { - FactTable fact = getCubeFact(table, false); + FactTable fact = getFactTable(table, false); if (fact != null) { facts.add(fact); } @@ -2206,7 +2215,7 @@ public Collection getAllFacts(boolean includeVirtualFacts) throws Len List facts = new ArrayList<>(); try { for (String table : getAllHiveTableNames()) { - FactTable fact = getCubeFact(table, false); + FactTable fact = getFactTable(table, false); if (fact != null) { if (fact.getProperties().get(getSourceFactNameKey(fact.getName())) != null) { //is virtual fact if (includeVirtualFacts) { @@ -2539,7 +2548,7 @@ public void dropDimension(String dimName) throws LensException { */ public void dropFact(String factName, boolean cascade) throws LensException { getTableWithTypeFailFast(factName, CubeTableType.FACT); - FactTable fact = getCubeFact(factName); + FactTable fact = getFactTable(factName); if (cascade) { for (String storage : fact.getStorages()) { dropStorageFromFact(factName, storage, false); @@ -2547,7 +2556,15 @@ public void dropFact(String factName, boolean cascade) throws LensException { } dropHiveTable(factName); allFactTables.remove(factName.trim().toLowerCase()); - dropAllVirtualFactTables(factName); + if(fact.isVirtualFact()) { + String sourceFactTable = fact.getProperties().get(getSourceFactNameKey(fact.getName())); + if (factToVirtualFactMapping.get(sourceFactTable) != null + && factToVirtualFactMapping.get(sourceFactTable).contains(fact.getName())) { + factToVirtualFactMapping.get(sourceFactTable).remove(fact.getName()); + } + }else { + dropAllVirtualFactTables(factName); + } } private void dropAllVirtualFactTables(String cubeFactTableName) throws LensException { @@ -2670,16 +2687,20 @@ public void dropDimensionTable(String dimTblName, boolean cascade) throws LensEx dropHiveTable(dimTblName); allDimTables.remove(dimTblName.trim().toLowerCase()); } - public void alterCubeFactTable(XFactTable fact) throws LensException, HiveException { - alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact), - JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()), - JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns())); - } - public void alterVirtualCubeFactTable(XVirtualFactTable virtualFact) throws LensException, HiveException { - alterCubeFactTable(virtualFact.getName(), JAXBUtils.cubeVirtualFactFromFactTable(virtualFact, - getCubeFact(virtualFact.getSourceFactName())), null, new HashMap<>()); + public void alterCubeFactTable(XFact fact) throws LensException, HiveException { + if (fact instanceof XVirtualFactTable) { + XVirtualFactTable xvf = (XVirtualFactTable) fact; + alterCubeFactTable(xvf.getName(), JAXBUtils.cubeVirtualFactFromFactTable(xvf, + getFactTable(xvf.getSourceFactName())), null, new HashMap<>()); + } else { + XFactTable xf = (XFactTable) fact; + alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(xf), + JAXBUtils.tableDescPrefixMapFromXStorageTables(xf.getStorageTables()), + JAXBUtils.columnStartAndEndTimeFromXColumns(xf.getColumns())); + } } + /** * Alter a cubefact with new definition and alter underlying storage tables as well. * @@ -2706,7 +2727,7 @@ public void alterCubeFactTable(String factTableName, FactTable cubeFactTable, } updateFactCache(factTableName); - updateAllVirtualFacts(getCubeFact(factTableName)); + updateAllVirtualFacts(getFactTable(factTableName)); } public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException { @@ -2755,7 +2776,7 @@ private void updateFactCache(String factTableName) throws LensException { } public CubeFactTable getCubeFactTable(String factName) throws LensException { - FactTable factTable = getCubeFact(factName); + FactTable factTable = getFactTable(factName); if (factTable instanceof CubeFactTable) { return (CubeFactTable) factTable; } else { @@ -2765,7 +2786,7 @@ public CubeFactTable getCubeFactTable(String factName) throws LensException { } public CubeVirtualFactTable getCubeVirtualFactTable(String factName) throws LensException { - FactTable factTable = getCubeFact(factName); + FactTable factTable = getFactTable(factName); if (factTable instanceof CubeVirtualFactTable) { return (CubeVirtualFactTable) factTable; } else { diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index f33c719d4..ddd2c9a89 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -1275,7 +1275,7 @@ public void testVirtualCubeFact() throws Exception { assertTrue(client.getAllFacts(client.getCube(VIRTUAL_CUBE_NAME)).get(0).getName().equals(virtualFactName.trim() .toLowerCase())); - CubeVirtualFactTable actualcubeVirtualFact = (CubeVirtualFactTable) (client.getCubeFact(virtualFactName)); + CubeVirtualFactTable actualcubeVirtualFact = (CubeVirtualFactTable) (client.getFactTable(virtualFactName)); assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); //alter virtual fact @@ -1284,7 +1284,7 @@ public void testVirtualCubeFact() throws Exception { com.google.common.base.Optional.fromNullable(null), alterVirtualFactPropertiesMap, sourceFact); client.alterVirtualCubeFactTable(cubeVirtualFact); - actualcubeVirtualFact = (CubeVirtualFactTable) client.getCubeFact(virtualFactName); + actualcubeVirtualFact = (CubeVirtualFactTable) client.getFactTable(virtualFactName); assertEquals(actualcubeVirtualFact.getProperties().get("name1"), "newvalue2"); assertEquals(actualcubeVirtualFact.getProperties().get("name3"), "value3"); assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); @@ -1294,7 +1294,7 @@ public void testVirtualCubeFact() throws Exception { sourceFact.alterColumn(newcol); sourceFact.alterWeight(100); client.alterCubeFactTable(sourceFact.getName(), sourceFact, storageTables, new HashMap()); - actualcubeVirtualFact = (CubeVirtualFactTable) client.getCubeFact(virtualFactName); + actualcubeVirtualFact = (CubeVirtualFactTable) client.getFactTable(virtualFactName); assertTrue(actualcubeVirtualFact.getColumns().contains(newcol)); assertEquals(actualcubeVirtualFact.weight(), 100.0); assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java index 729041321..9d1653320 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java @@ -378,10 +378,10 @@ int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, St * * @param sessionid The sessionid * @param fact The fact table name - * @return {@link XFactTable} + * @return {@link XFact} * @throws LensException */ - XFactTable getFactTable(LensSessionHandle sessionid, String fact) throws LensException; + XFact getFactTable(LensSessionHandle sessionid, String fact) throws LensException; /** * Create fact table @@ -390,7 +390,7 @@ int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, St * @param fact The fact table definition * @throws LensException */ - void createFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException; + void createFactTable(LensSessionHandle sessionid, XFact fact) throws LensException; /** * Update/Alter fact table @@ -399,7 +399,7 @@ int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, St * @param fact The fact table's new definition * @throws LensException */ - void updateFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException; + void updateFactTable(LensSessionHandle sessionid, XFact fact) throws LensException; /** * Drop fact table. @@ -421,53 +421,6 @@ int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, St */ List getAllFactNames(LensSessionHandle sessionid, String cubeName) throws LensException; - /** - * Get virtual fact table given by name - * - * @param sessionid The sessionid - * @param virtualFact The virtual fact table name - * @return {@link XFactTable} - * @throws LensException - */ - XVirtualFactTable getVirtualFactTable(LensSessionHandle sessionid, String virtualFact) throws LensException; - - /** - * Create virtual fact table - * - * @param sessionid The sessionid - * @param fact The virtual fact table definition - * @throws LensException - */ - void createVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable fact) throws LensException; - - /** - * Update/Alter virtual fact table - * - * @param sessionid The sessionid - * @param virtualFact The virtual fact table's new definition - * @throws LensException - */ - void updateVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable virtualFact) throws LensException; - - /** - * Drop virtual fact table. - * - * @param sessionid The sessionid - * @param virtualFact The virtual fact table name - * @throws LensException - */ - void dropVirtualFactTable(LensSessionHandle sessionid, String virtualFact) throws LensException; - -// /** -// * Get all virtual fact names -// * -// * @param sessionid The sessionid -// * @param cubeName optional filter filter facts by cube name. -// * @return List of virtual fact table names -// * @throws LensException -// */ -// List getAllVirtualFactNames(LensSessionHandle sessionid, String cubeName) throws LensException; - /** * Get all storages of fact * diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index 9b883d4ac..0a8635c3e 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -354,7 +354,7 @@ public void dropStorageOfDimTable(LensSessionHandle sessionid, String dimTblName } @Override - public XFactTable getFactTable(LensSessionHandle sessionid, String fact) throws LensException { + public XFact getFactTable(LensSessionHandle sessionid, String fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ return getClient(sessionid).getXFactTable(fact); } @@ -371,15 +371,15 @@ public XSegmentation getSegmentation(LensSessionHandle sessionid, String cubeSeg @Override - public void createFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { + public void createFactTable(LensSessionHandle sessionid, XFact fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).createCubeFactTable(fact); + getClient(sessionid).createFactTable(fact); log.info("Created fact table " + fact.getName()); } } @Override - public void updateFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { + public void updateFactTable(LensSessionHandle sessionid, XFact fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ getClient(sessionid).alterCubeFactTable(fact); log.info("Updated fact table " + fact.getName()); @@ -441,51 +441,6 @@ public List getAllFactNames(LensSessionHandle sessionid, String cubeName } } - @Override - public XVirtualFactTable getVirtualFactTable(LensSessionHandle sessionid, String virtualFact) throws LensException { - try (SessionContext ignored = new SessionContext(sessionid)) { - CubeMetastoreClient msClient = getClient(sessionid); - CubeVirtualFactTable cft = (CubeVirtualFactTable) msClient.getCubeFact(virtualFact); - XVirtualFactTable factTable = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cft); - factTable.setSourceFactName(cft.getSourceCubeFactTable().getName()); - return factTable; - } - } - - @Override - public void createVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable fact) throws LensException { - try (SessionContext ignored = new SessionContext(sessionid)) { - getClient(sessionid).createVirtualFactTable(fact.getCubeName(), - fact.getName(), - fact.getSourceFactName(), - fact.getWeight(), - JAXBUtils.mapFromXProperties(fact.getProperties())); - log.info("Created virtual fact table " + fact.getName()); - } - } - - @Override - public void updateVirtualFactTable(LensSessionHandle sessionid, XVirtualFactTable virtualFact) throws LensException { - try (SessionContext ignored = new SessionContext(sessionid)) { - CubeVirtualFactTable cubeVirtualFactTable = cubeVirtualFactFromFactTable(virtualFact, - getClient(sessionid).getCubeFact(virtualFact.getSourceFactName())); - - getClient(sessionid).alterVirtualCubeFactTable(cubeVirtualFactTable); - log.info("Updated virtual fact table " + virtualFact.getName()); - } catch (HiveException e) { - throw new LensException(e); - } - } - - - @Override - public void dropVirtualFactTable(LensSessionHandle sessionid, String virtualFact) throws LensException { - try (SessionContext ignored = new SessionContext(sessionid)) { - getClient(sessionid).dropVirtualFact(virtualFact); - log.info("Dropped virtual fact table " + virtualFact); - } - } - @Override public List getAllSegmentations(LensSessionHandle sessionid, String cubeName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ @@ -526,7 +481,7 @@ public XStorageTableElement getStorageOfFact(LensSessionHandle sessionid, String throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); - FactTable factTable = msClient.getCubeFact(fact); + FactTable factTable = msClient.getFactTable(fact); Set updatePeriods = factTable.getUpdatePeriods().get(storageName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName))); diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index 27c39225a..0af6a34d2 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -180,20 +180,6 @@ public void doDelete(LensSessionHandle sessionid, String entityName, Boolean cas getSvc().dropDimensionTable(sessionid, entityName, cascade); } } - }, VIRTUALFACT { - @Override - public List doGetAll(LensSessionHandle sessionid) throws LensException { - throw new NotImplementedException(); - } - - @Override - public void doDelete(LensSessionHandle sessionid, String entityName, Boolean cascade) throws LensException { - if (cascade == null) { - getSvc().dropVirtualFactTable(sessionid, entityName); - } else { - throw new NotImplementedException(); - } - } }; public abstract List doGetAll(LensSessionHandle sessionid) throws LensException; @@ -755,19 +741,6 @@ public APIResult deleteAllFacts(@QueryParam("sessionid") LensSessionHandle sessi return Entity.FACT.deleteAll(sessionid, cascade); } - /** - * Delete all virtual fact tables - * - * @param sessionid The sessionid in which user is working - * @return APIResult with state {@link Status#SUCCEEDED} in case of successful delete. APIResult with state {@link - * Status#FAILED} in case of delete failure. APIResult with state {@link Status#PARTIAL} in case of partial delete. - */ - @DELETE - @Path("virtualfacts") - public APIResult deleteAllVirtualFacts(@QueryParam("sessionid") LensSessionHandle sessionid) throws LensException { - return Entity.VIRTUALFACT.deleteAll(sessionid, false); - } - /** * Delete all segmentations * @@ -786,31 +759,15 @@ public APIResult deleteAllSegmentations(@QueryParam("sessionid") LensSessionHand * * @param sessionid The sessionid in which user is working * @param factName The fact table name - * @return JAXB representation of {@link XFactTable} + * @return JAXB representation of {@link XFact} */ @GET @Path("/facts/{factName}") - public JAXBElement getFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, + public JAXBElement getFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, @PathParam("factName") String factName) throws LensException { checkSessionId(sessionid); - return X_CUBE_OBJECT_FACTORY.createXFactTable(getSvc().getFactTable(sessionid, factName)); - } - - /** - * Get the virtual fact table specified by name - * - * @param sessionid The sessionid in which user is working - * @param factName The virtual fact table name - * @return JAXB representation of {@link XFactTable} - */ - @GET - @Path("/virtualfacts/{virtualFactName}") - public JAXBElement getVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, - @PathParam("virtualFactName") String factName) - throws LensException { - checkSessionId(sessionid); - return X_CUBE_OBJECT_FACTORY.createXVirtualFactTable(getSvc().getVirtualFactTable(sessionid, factName)); + return X_CUBE_OBJECT_FACTORY.createXFact(getSvc().getFactTable(sessionid, factName)); } /** @@ -833,13 +790,13 @@ public JAXBElement getSegmentation(@QueryParam("sessionid") LensS * Create a new fact tabble * * @param sessionid The sessionid in which user is working - * @param fact The {@link XFactTable} representation of the fact table definition + * @param fact The {@link XFact} representation of the fact table definition * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if create was successful. {@link APIResult} with * state {@link Status#FAILED}, if create has failed */ @POST @Path("/facts") - public APIResult createFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, XFactTable fact) + public APIResult createFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, XFact fact) throws LensException { checkSessionId(sessionid); log.info("Create fact table"); @@ -847,25 +804,6 @@ public APIResult createFactTable(@QueryParam("sessionid") LensSessionHandle sess return success(); } - - /** - * Create a new virtual fact tabble - * - * @param sessionid The sessionid in which user is working - * @param fact The {@link XFactTable} representation of the fact table definition - * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if create was successful. {@link APIResult} with - * state {@link Status#FAILED}, if create has failed - */ - @POST - @Path("/virtualfacts") - public APIResult createVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, XVirtualFactTable fact) - throws LensException { - checkSessionId(sessionid); - log.info("Create virtual fact table"); - getSvc().createVirtualFactTable(sessionid, fact); - return success(); - } - /** * Create a new segmentation * @@ -884,39 +822,19 @@ public APIResult createSegmentation(@QueryParam("sessionid") LensSessionHandle s return success(); } - - /** - * Update virtualFact table definition - * - * @param sessionid The sessionid in which user is working - * @param factName name of the virtual virtualFact table - * @param virtualFact The {@link XFactTable} representation of the updated virtual virtualFact table definition - * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful. {@link APIResult} with - * state {@link Status#FAILED}, if update has failed - */ - @PUT - @Path("/virtualfacts/{virtualFactName}") - public APIResult updateVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, - @PathParam("factName") String factName, XVirtualFactTable virtualFact) - throws LensException { - checkSessionId(sessionid); - getSvc().updateVirtualFactTable(sessionid, virtualFact); - return success(); - } - /** * Update fact table definition * * @param sessionid The sessionid in which user is working * @param factName name of the fact table - * @param fact The {@link XFactTable} representation of the updated fact table definition + * @param fact The {@link XFact} representation of the updated fact table definition * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful. {@link APIResult} with * state {@link Status#FAILED}, if update has failed */ @PUT @Path("/facts/{factName}") public APIResult updateFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, - @PathParam("factName") String factName, XFactTable fact) + @PathParam("factName") String factName, XFact fact) throws LensException { checkSessionId(sessionid); getSvc().updateFactTable(sessionid, fact); @@ -960,22 +878,6 @@ public APIResult dropFactTable(@QueryParam("sessionid") LensSessionHandle sessio return Entity.FACT.delete(sessionid, factName, cascade); } - /** - * Drop the virtual fact table, specified by name - * - * @param sessionid The sessionid in which user is working - * @param virtualFactName The virtual fact table name - * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if drop was successful. {@link APIResult} with state - * {@link Status#FAILED}, if drop has failed - */ - @DELETE - @Path("/virtualfacts/{virtualFactName}") - public APIResult dropVirtualFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, - @PathParam("virtualFactName") String virtualFactName) - throws LensException { - return Entity.VIRTUALFACT.delete(sessionid, virtualFactName); - } - /** * Drop the segmentation, specified by name * diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index a35381665..ab2403346 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -1895,7 +1895,7 @@ public void testCreateFactTableWithMultipleUpdatePeriods(MediaType mediaType) th .add(createStorageTblWithMultipleTableDescriptors("S1", tables, updatePeriods)); APIResult result = target().path("metastore").path("facts").queryParam("sessionid", lensSessionId) .request(mediaType) - .post(Entity.entity(new GenericEntity>(cubeObjectFactory.createXFactTable(f)) { + .post(Entity.entity(new GenericEntity>(cubeObjectFactory.createXFact(f)) { }, mediaType), APIResult.class); assertSuccess(result); @@ -1904,10 +1904,10 @@ public void testCreateFactTableWithMultipleUpdatePeriods(MediaType mediaType) th assertTrue(factNames.getElements().contains(table.toLowerCase())); // Get the created tables - JAXBElement gotFactElement = target().path("metastore/facts").path(table) - .queryParam("sessionid", lensSessionId).request(mediaType).get(new GenericType>() { + JAXBElement gotFactElement = target().path("metastore/facts").path(table) + .queryParam("sessionid", lensSessionId).request(mediaType).get(new GenericType>() { }); - XFactTable gotFact = gotFactElement.getValue(); + XFactTable gotFact = (XFactTable) gotFactElement.getValue(); assertTrue(gotFact.getName().equalsIgnoreCase(table)); assertEquals(gotFact.getWeight(), 10.0); @@ -1933,15 +1933,15 @@ public void testCreateFactTableWithMultipleUpdatePeriods(MediaType mediaType) th // Update result = target().path("metastore").path("facts").path(table).queryParam("sessionid", lensSessionId) .request(mediaType) - .put(Entity.entity(new GenericEntity>(cubeObjectFactory.createXFactTable(update)) { + .put(Entity.entity(new GenericEntity>(cubeObjectFactory.createXFact(update)) { }, mediaType), APIResult.class); assertSuccess(result); // Get the updated table gotFactElement = target().path("metastore/facts").path(table).queryParam("sessionid", lensSessionId) - .request(mediaType).get(new GenericType>() { + .request(mediaType).get(new GenericType>() { }); - gotFact = gotFactElement.getValue(); + gotFact = (XFactTable) gotFactElement.getValue(); CubeFactTable ucf = JAXBUtils.cubeFactFromFactTable(gotFact); assertEquals(ucf.weight(), 20.0); assertTrue(ucf.getUpdatePeriods().get("S1").contains(HOURLY)); @@ -2068,7 +2068,7 @@ public void testCreateFactTable(MediaType mediaType) throws Exception { .path("facts").queryParam("sessionid", lensSessionId) .request(mediaType) .post(Entity.entity( - new GenericEntity>(cubeObjectFactory.createXFactTable(f)){}, mediaType), + new GenericEntity>(cubeObjectFactory.createXFact(f)){}, mediaType), APIResult.class); assertSuccess(result); @@ -2078,10 +2078,10 @@ public void testCreateFactTable(MediaType mediaType) throws Exception { assertTrue(factNames.getElements().contains(table.toLowerCase())); // Get the created table - JAXBElement gotFactElement = target().path("metastore/facts").path(table) + JAXBElement gotFactElement = target().path("metastore/facts").path(table) .queryParam("sessionid", lensSessionId).request(mediaType) - .get(new GenericType>() {}); - XFactTable gotFact = gotFactElement.getValue(); + .get(new GenericType>() {}); + XFactTable gotFact = (XFactTable) gotFactElement.getValue(); assertTrue(gotFact.getName().equalsIgnoreCase(table)); assertEquals(gotFact.getWeight(), 10.0); CubeFactTable cf = JAXBUtils.cubeFactFromFactTable(gotFact); @@ -2151,15 +2151,15 @@ public void testUpdateFactTable(MediaType mediaType) throws Exception { .path("facts").queryParam("sessionid", lensSessionId) .request(mediaType) .post(Entity.entity( - new GenericEntity>(cubeObjectFactory.createXFactTable(f)){}, mediaType), + new GenericEntity>(cubeObjectFactory.createXFact(f)){}, mediaType), APIResult.class); assertSuccess(result); // Get the created table - JAXBElement gotFactElement = target().path("metastore/facts").path(table) + JAXBElement gotFactElement = target().path("metastore/facts").path(table) .queryParam("sessionid", lensSessionId).request(mediaType) - .get(new GenericType>() {}); - XFactTable gotFact = gotFactElement.getValue(); + .get(new GenericType>() {}); + XFactTable gotFact = (XFactTable) gotFactElement.getValue(); assertTrue(gotFact.getName().equalsIgnoreCase(table)); assertEquals(gotFact.getWeight(), 10.0); CubeFactTable cf = JAXBUtils.cubeFactFromFactTable(gotFact); @@ -2178,7 +2178,7 @@ public void testUpdateFactTable(MediaType mediaType) throws Exception { // Update result = target().path("metastore").path("facts").path(table) .queryParam("sessionid", lensSessionId).request(mediaType) - .put(Entity.entity(new GenericEntity>(cubeObjectFactory.createXFactTable(update)){}, + .put(Entity.entity(new GenericEntity>(cubeObjectFactory.createXFact(update)){}, mediaType), APIResult.class); assertSuccess(result); @@ -2186,8 +2186,8 @@ public void testUpdateFactTable(MediaType mediaType) throws Exception { // Get the updated table gotFactElement = target().path("metastore/facts").path(table) .queryParam("sessionid", lensSessionId).request(mediaType) - .get(new GenericType>() {}); - gotFact = gotFactElement.getValue(); + .get(new GenericType>() {}); + gotFact = (XFactTable) gotFactElement.getValue(); CubeFactTable ucf = JAXBUtils.cubeFactFromFactTable(gotFact); assertEquals(ucf.weight(), 20.0); @@ -2262,7 +2262,7 @@ public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { .path("facts").queryParam("sessionid", lensSessionId) .request(mediaType) .post(Entity.entity( - new GenericEntity>(cubeObjectFactory.createXFactTable(source)) { + new GenericEntity>(cubeObjectFactory.createXFact(source)) { }, mediaType), APIResult.class); assertSuccess(result); @@ -2272,10 +2272,10 @@ public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { result = target() .path("metastore") - .path("virtualfacts").queryParam("sessionid", lensSessionId) + .path("facts").queryParam("sessionid", lensSessionId) .request(mediaType) .post(Entity.entity( - new GenericEntity>(cubeObjectFactory.createXVirtualFactTable(f)) { + new GenericEntity>(cubeObjectFactory.createXFact(f)) { }, mediaType), APIResult.class); assertSuccess(result); @@ -2286,11 +2286,11 @@ public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { assertTrue(factNames.getElements().contains(virtualFactTable.toLowerCase())); // Get the created table - JAXBElement gotFactElement = target().path("metastore/virtualfacts").path(virtualFactTable) + JAXBElement gotFactElement = target().path("metastore/facts").path(virtualFactTable) .queryParam("sessionid", lensSessionId).request(mediaType) - .get(new GenericType>() { + .get(new GenericType>() { }); - XVirtualFactTable gotFact = gotFactElement.getValue(); + XVirtualFactTable gotFact = (XVirtualFactTable) gotFactElement.getValue(); assertTrue(gotFact.getName().equalsIgnoreCase(virtualFactTable)); assertEquals(gotFact.getWeight(), 10.0); CubeVirtualFactTable cvf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, @@ -2313,7 +2313,7 @@ public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { assertTrue(cvf.getUpdatePeriods().get("VS2").contains(DAILY)); // drop the virtual fact table - result = target().path("metastore").path("virtualfacts").path(virtualFactTable) + result = target().path("metastore").path("facts").path(virtualFactTable) .queryParam("sessionid", lensSessionId).request(mediaType) .delete(APIResult.class); @@ -2329,7 +2329,7 @@ public void testCreateVirtualFactTable(MediaType mediaType) throws Exception { // Drop again, this time it should give a 404 try { - target().path("metastore").path("virtualfacts").path(virtualFactTable) + target().path("metastore").path("facts").path(virtualFactTable) .queryParam("sessionid", lensSessionId).request(mediaType) .delete(APIResult.class); fail("Expected 404"); @@ -2365,7 +2365,7 @@ public void testUpdateVirtualFactTable(MediaType mediaType) throws Exception { .path("facts").queryParam("sessionid", lensSessionId) .request(mediaType) .post(Entity.entity( - new GenericEntity>(cubeObjectFactory.createXFactTable(source)) { + new GenericEntity>(cubeObjectFactory.createXFact(source)) { }, mediaType), APIResult.class); assertSuccess(result); @@ -2375,10 +2375,10 @@ public void testUpdateVirtualFactTable(MediaType mediaType) throws Exception { result = target() .path("metastore") - .path("virtualfacts").queryParam("sessionid", lensSessionId) + .path("facts").queryParam("sessionid", lensSessionId) .request(mediaType) .post(Entity.entity( - new GenericEntity>(cubeObjectFactory.createXVirtualFactTable(f)) { + new GenericEntity>(cubeObjectFactory.createXFact(f)) { }, mediaType), APIResult.class); assertSuccess(result); @@ -2389,11 +2389,11 @@ public void testUpdateVirtualFactTable(MediaType mediaType) throws Exception { assertTrue(factNames.getElements().contains(table.toLowerCase())); // Get the created table - JAXBElement gotFactElement = target().path("metastore/virtualfacts").path(table) + JAXBElement gotFactElement = target().path("metastore/facts").path(table) .queryParam("sessionid", lensSessionId).request(mediaType) - .get(new GenericType>() { + .get(new GenericType>() { }); - XVirtualFactTable gotFact = gotFactElement.getValue(); + XVirtualFactTable gotFact = (XVirtualFactTable) gotFactElement.getValue(); assertTrue(gotFact.getName().equalsIgnoreCase(table)); assertEquals(gotFact.getWeight(), 10.0); CubeVirtualFactTable cvf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, @@ -2404,24 +2404,24 @@ public void testUpdateVirtualFactTable(MediaType mediaType) throws Exception { XVirtualFactTable update = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cvf); // Update - result = target().path("metastore").path("virtualfacts").path(table) + result = target().path("metastore").path("facts").path(table) .queryParam("sessionid", lensSessionId).request(mediaType) - .put(Entity.entity(new GenericEntity>(cubeObjectFactory - .createXVirtualFactTable(update)) {}, mediaType), APIResult.class); + .put(Entity.entity(new GenericEntity>(cubeObjectFactory + .createXFact(update)) {}, mediaType), APIResult.class); assertSuccess(result); // Get the updated table - gotFactElement = target().path("metastore/virtualfacts").path(table) + gotFactElement = target().path("metastore/facts").path(table) .queryParam("sessionid", lensSessionId).request(mediaType) - .get(new GenericType>() {}); - gotFact = gotFactElement.getValue(); + .get(new GenericType>() {}); + gotFact = (XVirtualFactTable) gotFactElement.getValue(); CubeVirtualFactTable ucf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, JAXBUtils.cubeFactFromFactTable(source)); assertEquals(ucf.weight(), 20.0); // drop the virtual fact table - result = target().path("metastore").path("virtualfacts").path(table) + result = target().path("metastore").path("facts").path(table) .queryParam("sessionid", lensSessionId).request(mediaType) .delete(APIResult.class); @@ -2437,7 +2437,7 @@ public void testUpdateVirtualFactTable(MediaType mediaType) throws Exception { // Drop again, this time it should give a 404 try { - target().path("metastore").path("virtualfacts").path(table) + target().path("metastore").path("facts").path(table) .queryParam("sessionid", lensSessionId).request(mediaType) .delete(APIResult.class); fail("Expected 404"); @@ -2471,7 +2471,7 @@ public void testFactStorages(MediaType mediaType) throws Exception { .path("facts").queryParam("sessionid", lensSessionId) .request(mediaType) .post(Entity.entity( - new GenericEntity>(cubeObjectFactory.createXFactTable(f)){}, mediaType), + new GenericEntity>(cubeObjectFactory.createXFact(f)){}, mediaType), APIResult.class); assertSuccess(result); @@ -2669,7 +2669,7 @@ public void testSkipFactStoragePartitions(MediaType mediaType) throws Exception .path("facts").queryParam("sessionid", lensSessionId) .request(mediaType) .post(Entity.entity( - new GenericEntity>(cubeObjectFactory.createXFactTable(f)){}, mediaType), + new GenericEntity>(cubeObjectFactory.createXFact(f)){}, mediaType), APIResult.class); assertSuccess(result); @@ -2722,7 +2722,7 @@ public void testFactStoragePartitions(MediaType mediaType) throws Exception { .path("facts").queryParam("sessionid", lensSessionId) .request(mediaType) .post(Entity.entity( - new GenericEntity>(cubeObjectFactory.createXFactTable(f)){}, mediaType), + new GenericEntity>(cubeObjectFactory.createXFact(f)){}, mediaType), APIResult.class); assertSuccess(result); @@ -3374,7 +3374,7 @@ private void createTestFactAndStorageTable(final String cubeName, final String[] .path("facts").queryParam("sessionid", lensSessionId) .request(mediaType) .post(Entity.entity( - new GenericEntity>(cubeObjectFactory.createXFactTable(f)) { + new GenericEntity>(cubeObjectFactory.createXFact(f)) { }, mediaType), APIResult.class); assertSuccess(result); From 903adf98726c97dc5ba1d6421aa594a784eda0d9 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Fri, 19 May 2017 12:35:35 +0530 Subject: [PATCH 45/71] More changes for vf --- .../apache/lens/cube/metadata/CubeMetastoreClient.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index f24706d10..9cfbb616d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -1768,12 +1768,12 @@ public XFact getXFactTable(String tableName) throws LensException { public XFact getXFactTable(FactTable ft) throws LensException { XFact fact; - if(ft.isVirtualFact()){ + if (ft.isVirtualFact()) { CubeVirtualFactTable cvft = (CubeVirtualFactTable) ft; XVirtualFactTable factTable = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cvft); factTable.setSourceFactName(cvft.getSourceCubeFactTable().getName()); fact = factTable; - }else { + } else { CubeFactTable cft = (CubeFactTable) ft; XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft); Map> storageMap = cft.getStoragePrefixUpdatePeriodMap(); @@ -2556,13 +2556,13 @@ public void dropFact(String factName, boolean cascade) throws LensException { } dropHiveTable(factName); allFactTables.remove(factName.trim().toLowerCase()); - if(fact.isVirtualFact()) { + if (fact.isVirtualFact()) { String sourceFactTable = fact.getProperties().get(getSourceFactNameKey(fact.getName())); if (factToVirtualFactMapping.get(sourceFactTable) != null && factToVirtualFactMapping.get(sourceFactTable).contains(fact.getName())) { factToVirtualFactMapping.get(sourceFactTable).remove(fact.getName()); } - }else { + } else { dropAllVirtualFactTables(factName); } } From d99e9edf53f583612abb7c022d14f4daf370e5bb Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Fri, 19 May 2017 14:46:30 +0530 Subject: [PATCH 46/71] More changes for vf --- .../lens/server/common/FormDataMultiPartFactory.java | 6 +++--- .../org/apache/lens/server/common/RestAPITestUtil.java | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/lens-server/src/test/java/org/apache/lens/server/common/FormDataMultiPartFactory.java b/lens-server/src/test/java/org/apache/lens/server/common/FormDataMultiPartFactory.java index 9a1d79d56..13b17ceb2 100644 --- a/lens-server/src/test/java/org/apache/lens/server/common/FormDataMultiPartFactory.java +++ b/lens-server/src/test/java/org/apache/lens/server/common/FormDataMultiPartFactory.java @@ -24,7 +24,7 @@ import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.metastore.ObjectFactory; -import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XFact; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -83,11 +83,11 @@ public static FormDataMultiPart createFormDataMultiPartForSession( } public static FormDataMultiPart createFormDataMultiPartForFact(final LensSessionHandle sessionId, - final XFactTable xFactTable, MediaType mt) { + final XFact xFact, MediaType mt) { final FormDataMultiPart mp = new FormDataMultiPart(); mp.bodyPart(getSessionIdFormDataBodyPart(sessionId, mt)); - mp.bodyPart(getFormDataBodyPart("fact", "fact", cubeObjectFactory.createXFactTable(xFactTable), mt)); + mp.bodyPart(getFormDataBodyPart("fact", "fact", cubeObjectFactory.createXFact(xFact), mt)); return mp; } diff --git a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java index df4e07a00..13bde25e4 100644 --- a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java +++ b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java @@ -36,10 +36,7 @@ import org.apache.lens.api.APIResult; import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; -import org.apache.lens.api.metastore.ObjectFactory; -import org.apache.lens.api.metastore.XCube; -import org.apache.lens.api.metastore.XFactTable; -import org.apache.lens.api.metastore.XStorage; +import org.apache.lens.api.metastore.*; import org.apache.lens.api.query.*; import org.apache.lens.api.result.LensAPIResult; @@ -157,11 +154,11 @@ public static void createCubeFailFast(final WebTarget target, final LensSessionH } public static void createFactFailFast(final WebTarget target, final LensSessionHandle sessionId, - final XFactTable factTable, MediaType mt) { + final XFactTable fact, MediaType mt) { APIResult result = target.path("metastore").path("facts").queryParam("sessionid", sessionId) .request(mt).post(Entity.entity( - new GenericEntity>(cubeObjectFactory.createXFactTable(factTable)) { + new GenericEntity>(cubeObjectFactory.createXFact(fact)) { }, mt), APIResult.class); checkResponse(result); From 8503d0c6ac62de96ee382e5e2f0a5cae9840afd3 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Mon, 22 May 2017 12:12:54 +0530 Subject: [PATCH 47/71] vf changes --- .../lens/cli/commands/LensFactCommands.java | 6 +- .../org/apache/lens/client/LensClient.java | 18 +--- .../lens/client/LensMetadataClient.java | 66 ++------------- src/site/apt/user/cli.apt | 84 +++++++++---------- 4 files changed, 55 insertions(+), 119 deletions(-) diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java index a01d6c022..a8729980c 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.lens.api.APIResult; -import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XFact; import org.apache.lens.api.metastore.XPartition; import org.apache.lens.api.metastore.XStorageTableElement; import org.apache.lens.cli.commands.annotations.UserDocumentation; @@ -39,7 +39,7 @@ @Component @UserDocumentation(title = "Commands for Facts Management", description = "These command provide CRUD for facts, associated storages, and fact partitions") -public class LensFactCommands extends LogicalTableCrudCommand { +public class LensFactCommands extends LogicalTableCrudCommand { /** * Show facts. @@ -352,7 +352,7 @@ protected APIResult doCreate(String path, boolean ignoreIfExists) { } @Override - protected XFactTable doRead(String name) { + protected XFact doRead(String name) { return getClient().getFactTable(name); } diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java index 520dde587..1de99fe04 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java @@ -454,10 +454,6 @@ public APIResult createFactTable(String factSpec) { return mc.createFactTable(factSpec); } - public APIResult createVirtualFactTable(String vfactSpec) { - return mc.createVirtualFactTable(vfactSpec); - } - public APIResult createCube(String cubeSpec) { return mc.createCube(cubeSpec); } @@ -486,14 +482,6 @@ public APIResult dropFactTable(String fact, boolean cascade) { return mc.dropFactTable(fact, cascade); } - public APIResult dropVirtualFactTable(String fact) { - return mc.dropVirtualFactTable(fact); - } - - public APIResult dropAllVirtualFacts() { - return mc.dropAllVirtualFactTables(); - } - public APIResult dropCube(String cube) { return mc.dropCube(cube); } @@ -510,10 +498,6 @@ public APIResult updateFactTable(String factName, String factSpec) { return mc.updateFactTable(factName, factSpec); } - public APIResult updateVirtualFactTable(String factName, String factSpec) { - return mc.updateFactTable(factName, factSpec); - } - public APIResult updateDimensionTable(String dimName, String dimSpec) { return mc.updateDimensionTable(dimName, dimSpec); } @@ -530,7 +514,7 @@ public APIResult updateDimension(String dimName, String dimSpec) { return mc.updateDimension(dimName, dimSpec); } - public XFactTable getFactTable(String factName) { + public XFact getFactTable(String factName) { return mc.getFactTable(factName); } diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java index 436417aa6..8a05952e8 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java @@ -425,12 +425,12 @@ public APIResult deleteAllSegmentations() { .delete()); } - public XFactTable getFactTable(String factTableName) { + public XFact getFactTable(String factTableName) { WebTarget target = getMetastoreWebTarget(); - JAXBElement table = target.path("facts").path(factTableName) + JAXBElement table = target.path("facts").path(factTableName) .queryParam("sessionid", this.connection.getSessionHandle()) .request(MediaType.APPLICATION_XML) - .get(new GenericType>() { + .get(new GenericType>() { }); return table.getValue(); } @@ -445,33 +445,17 @@ public XSegmentation getSegmentation(String segName) { return seg.getValue(); } - public APIResult createFactTable(XFactTable f) { + public APIResult createFactTable(XFact f) { WebTarget target = getMetastoreWebTarget(); return translate(target.path("facts") .queryParam("sessionid", this.connection.getSessionHandle()) .request(MediaType.APPLICATION_XML) - .post(Entity.xml(new GenericEntity>(objFact.createXFactTable(f)){}))); + .post(Entity.xml(new GenericEntity>(objFact.createXFact(f)){}))); } public APIResult createFactTable(String factSpec) { try { - return createFactTable(this.readFromXML(factSpec)); - } catch (JAXBException | IOException e) { - return failureAPIResult(e); - } - } - - public APIResult createVirtualFactTable(XVirtualFactTable f) { - WebTarget target = getMetastoreWebTarget(); - return translate(target.path("virtualfacts") - .queryParam("sessionid", this.connection.getSessionHandle()) - .request(MediaType.APPLICATION_XML) - .post(Entity.xml(new GenericEntity>(objFact.createXVirtualFactTable(f)){}))); - } - - public APIResult createVirtualFactTable(String vfactSpec) { - try { - return createVirtualFactTable(this.readFromXML(vfactSpec)); + return createFactTable(this.readFromXML(factSpec)); } catch (JAXBException | IOException e) { return failureAPIResult(e); } @@ -494,33 +478,17 @@ public APIResult createSegmentation(String segSpec) { } } - public APIResult updateFactTable(String factName, XFactTable table) { + public APIResult updateFactTable(String factName, XFact table) { WebTarget target = getMetastoreWebTarget(); return translate(target.path("facts").path(factName) .queryParam("sessionid", this.connection.getSessionHandle()) .request(MediaType.APPLICATION_XML_TYPE) - .put(Entity.xml(new GenericEntity>(objFact.createXFactTable(table)){}))); + .put(Entity.xml(new GenericEntity>(objFact.createXFact(table)){}))); } public APIResult updateFactTable(String factName, String table) { try { - return updateFactTable(factName, this.readFromXML(table)); - } catch (JAXBException | IOException e) { - return failureAPIResult(e); - } - } - - public APIResult updateVirtualFactTable(String factName, XVirtualFactTable table) { - WebTarget target = getMetastoreWebTarget(); - return translate(target.path("virtualfacts").path(factName) - .queryParam("sessionid", this.connection.getSessionHandle()) - .request(MediaType.APPLICATION_XML_TYPE) - .put(Entity.xml(new GenericEntity>(objFact.createXVirtualFactTable(table)){}))); - } - - public APIResult updateVirtualFactTable(String factName, String table) { - try { - return updateVirtualFactTable(factName, this.readFromXML(table)); + return updateFactTable(factName, this.readFromXML(table)); } catch (JAXBException | IOException e) { return failureAPIResult(e); } @@ -557,22 +525,6 @@ public APIResult dropFactTable(String factName) { return dropFactTable(factName, false); } - public APIResult dropVirtualFactTable(String factName) { - WebTarget target = getMetastoreWebTarget(); - return translate(target.path("virtualfacts").path(factName) - .queryParam("sessionid", this.connection.getSessionHandle()) - .request(MediaType.APPLICATION_XML) - .delete()); - } - - public APIResult dropAllVirtualFactTables() { - WebTarget target = getMetastoreWebTarget(); - return translate(target.path("virtualfacts") - .queryParam("sessionid", this.connection.getSessionHandle()) - .request(MediaType.APPLICATION_XML) - .delete()); - } - public APIResult dropSegmentation(String segName) { WebTarget target = getMetastoreWebTarget(); return translate(target.path("segmentations").path(segName) diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt index fce41fe28..d246aba62 100644 --- a/src/site/apt/user/cli.apt +++ b/src/site/apt/user/cli.apt @@ -389,48 +389,48 @@ User CLI Commands *--+--+ |<>|<>| *--+--+ -|schema/create schema [--db] \ [--path/--file] \|Parses the specified resource file and executes commands for creation/updation of schema \ | -| |Expected\ structure\ is\ \ | -| |. \ | -| |\|--\ storages \ | -| |\|\ \ \|--\ storage1.xml \ | -| |\|\ \ \|--\ storage2.xml \ | -| |\| \ | -| |\|--\ dimensions \ | -| |\|\ \ \|--\ dim1.xml \ | -| |\|\ \ \|--\ dim2.xml \ | -| |\| \ | -| |\|--\ cubes \ | -| |\|\ \ \|--\ base \ | -| |\|\ \ \|\ \ \|--\ base_cube1.xml \ | -| |\|\ \ \|\ \ \|--\ base_cube2.xml \ | -| |\|\ \ \| \ | -| |\|\ \ \|--\ derived \ | -| |\|\ \ \|\ \ \|--\ derived_cube1.xml \ | -| |\|\ \ \|\ \ \|--\ derived_cube2.xml \ | -| |\|\ \ \| \ | -| |\|\ \ \|--\ independent_cube1.xml \ | -| |\|\ \ \|--\ independent_cube2.xml \ | -| |\| \ | -| |\|--\ dimensiontables \ | -| |\|\ \ \|--\ dimtable1.xml \ | -| |\|\ \ \|--\ dimtable2.xml \ | -| |\| \ | -| |\|--\ dimtables \ | -| |\|\ \ \|--\ dimtable3.xml \ | -| |\|\ \ \|--\ dimtable4.xml \ | -| |\| \ | -| |\|--\ facts \ | -| |\ \ \ \|--\ fact1.xml \ | -| |\ \ \ \|--\ fact2.xml \ | -| | \ | -| | \ | -| |If\ your\ cubes\ are\ divided\ between\ base\ and\ derived\ cubes, \ | -| |it\ makes\ sense\ to\ seperate\ into\ two\ directories,\ since\ derived\ cubes\ can't\ be\ created\ unless\ base\ cube\ exists.\ | -| |In\ the\ other\ case\ you\ can\ keep\ them\ in\ the\ cubes\ directory\ itself. \ | -| |For\ dimtables,\ you\ can\ keep\ your\ schema\ files\ in\ a\ directory\ named\ either\ dimtables\ or\ dimensiontables. \ | -| |Each\ of\ these\ directories\ is\ optional\ and\ the\ order\ of\ processing\ is\ top\ to\ bottom. \ | -| |CLI\ will\ let\ you\ know\ in\ case\ of\ any\ errors\ and\ proceed\ further\ without\ failing\ in\ between. \ | +|schema/create schema [--db] \ [--path/--file] \ [[--type] \] [[--name] \]|Parses the specified resource file and executes commands for creation/updation of schema. If <<>> is provided, only schema types matching that will be worked upon. If <<>> is provided, then only those files that contain the filter value will be worked upon. \ | +| |Expected\ directory\ structure\ is\ \ | +| |. \ | +| |\|--\ storages \ | +| |\|\ \ \|--\ storage1.xml \ | +| |\|\ \ \|--\ storage2.xml \ | +| |\| \ | +| |\|--\ dimensions \ | +| |\|\ \ \|--\ dim1.xml \ | +| |\|\ \ \|--\ dim2.xml \ | +| |\| \ | +| |\|--\ cubes \ | +| |\|\ \ \|--\ base \ | +| |\|\ \ \|\ \ \|--\ base_cube1.xml \ | +| |\|\ \ \|\ \ \|--\ base_cube2.xml \ | +| |\|\ \ \| \ | +| |\|\ \ \|--\ derived \ | +| |\|\ \ \|\ \ \|--\ derived_cube1.xml \ | +| |\|\ \ \|\ \ \|--\ derived_cube2.xml \ | +| |\|\ \ \| \ | +| |\|\ \ \|--\ independent_cube1.xml \ | +| |\|\ \ \|--\ independent_cube2.xml \ | +| |\| \ | +| |\|--\ dimensiontables \ | +| |\|\ \ \|--\ dimtable1.xml \ | +| |\|\ \ \|--\ dimtable2.xml \ | +| |\| \ | +| |\|--\ dimtables \ | +| |\|\ \ \|--\ dimtable3.xml \ | +| |\|\ \ \|--\ dimtable4.xml \ | +| |\| \ | +| |\|--\ facts \ | +| |\ \ \ \|--\ fact1.xml \ | +| |\ \ \ \|--\ fact2.xml \ | +| | \ | +| | \ | +| |If\ your\ cubes\ are\ divided\ between\ base\ and\ derived\ cubes, \ | +| |it\ makes\ sense\ to\ seperate\ into\ two\ directories,\ since\ derived\ cubes\ can't\ be\ created\ unless\ base\ cube\ exists. \ | +| |In\ the\ other\ case\ you\ can\ keep\ them\ in\ the\ cubes\ directory\ itself. \ | +| |For\ dimtables,\ you\ can\ keep\ your\ schema\ files\ in\ a\ directory\ named\ either\ dimtables\ or\ dimensiontables. \ | +| |Each\ of\ these\ directories\ is\ optional\ and\ the\ order\ of\ processing\ is\ top\ to\ bottom. \ | +| |CLI\ will\ let\ you\ know\ in\ case\ of\ any\ errors\ and\ proceed\ further\ without\ failing\ in\ between. \ | *--+--+ <> From 9bd44422b1e5ea99a33081458dc87990b1f893b7 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Mon, 22 May 2017 13:40:20 +0530 Subject: [PATCH 48/71] vf changes --- lens-examples/src/test/resources/yaml/fact1.yaml | 4 ++-- lens-examples/src/test/resources/yaml/fact2.yaml | 4 ++-- lens-examples/src/test/resources/yaml/rawfact.yaml | 4 ++-- .../src/test/resources/yaml/sales-aggr-continuous-fact.yaml | 4 ++-- lens-examples/src/test/resources/yaml/sales-aggr-fact1.yaml | 4 ++-- lens-examples/src/test/resources/yaml/sales-aggr-fact2.yaml | 4 ++-- lens-examples/src/test/resources/yaml/sales-raw-fact.yaml | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lens-examples/src/test/resources/yaml/fact1.yaml b/lens-examples/src/test/resources/yaml/fact1.yaml index 9c1c527a1..1ad539e02 100644 --- a/lens-examples/src/test/resources/yaml/fact1.yaml +++ b/lens-examples/src/test/resources/yaml/fact1.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: fact1 +cubeName: sample_cube columns: dim1: type: INT measure2: type: BIGINT @@ -37,6 +39,4 @@ storageTables: external: true tableLocation: /tmp/examples/fact1_local fieldDelimiter: , -name: fact1 -cubeName: sample_cube weight: 100.0 diff --git a/lens-examples/src/test/resources/yaml/fact2.yaml b/lens-examples/src/test/resources/yaml/fact2.yaml index 6fadd11cc..4eed1ae10 100644 --- a/lens-examples/src/test/resources/yaml/fact2.yaml +++ b/lens-examples/src/test/resources/yaml/fact2.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: fact2 +cubeName: sample_cube columns: dim1: type: INT dim3: type: INT @@ -32,6 +34,4 @@ storageTables: external: true tableLocation: /tmp/examples/fact2_local fieldDelimiter: , -name: fact2 -cubeName: sample_cube weight: 200.0 diff --git a/lens-examples/src/test/resources/yaml/rawfact.yaml b/lens-examples/src/test/resources/yaml/rawfact.yaml index f7b2d55e2..47645481c 100644 --- a/lens-examples/src/test/resources/yaml/rawfact.yaml +++ b/lens-examples/src/test/resources/yaml/rawfact.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: rawfact +cubeName: sample_cube columns: dim1: type: INT dim2: type: INT @@ -34,6 +36,4 @@ storageTables: external: true tableLocation: /tmp/examples/rawfact fieldDelimiter: , -name: rawfact -cubeName: sample_cube weight: 500.0 diff --git a/lens-examples/src/test/resources/yaml/sales-aggr-continuous-fact.yaml b/lens-examples/src/test/resources/yaml/sales-aggr-continuous-fact.yaml index f95f91bac..4fb716585 100644 --- a/lens-examples/src/test/resources/yaml/sales-aggr-continuous-fact.yaml +++ b/lens-examples/src/test/resources/yaml/sales-aggr-continuous-fact.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: sales_aggr_continuous_fact +cubeName: sales columns: order_time: type: TIMESTAMP delivery_time: type: TIMESTAMP @@ -41,6 +43,4 @@ storageTables: tableLocation: /tmp/db-storage.db fieldDelimiter: , storageHandlerName: org.apache.lens.storage.db.DBStorageHandler -name: sales_aggr_continuous_fact -cubeName: sales weight: 50.0 diff --git a/lens-examples/src/test/resources/yaml/sales-aggr-fact1.yaml b/lens-examples/src/test/resources/yaml/sales-aggr-fact1.yaml index 02f65ff49..80178f8c2 100644 --- a/lens-examples/src/test/resources/yaml/sales-aggr-fact1.yaml +++ b/lens-examples/src/test/resources/yaml/sales-aggr-fact1.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: sales_aggr_fact1 +cubeName: sales columns: order_time: type: TIMESTAMP delivery_time: type: TIMESTAMP @@ -57,6 +59,4 @@ storageTables: tableLocation: /tmp/db-storage.db fieldDelimiter: , storageHandlerName: org.apache.lens.storage.db.DBStorageHandler -name: sales_aggr_fact1 -cubeName: sales weight: 300.0 diff --git a/lens-examples/src/test/resources/yaml/sales-aggr-fact2.yaml b/lens-examples/src/test/resources/yaml/sales-aggr-fact2.yaml index ff47d36ed..ff44dd0b2 100644 --- a/lens-examples/src/test/resources/yaml/sales-aggr-fact2.yaml +++ b/lens-examples/src/test/resources/yaml/sales-aggr-fact2.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: sales_aggr_fact2 +cubeName: sales columns: order_time: type: TIMESTAMP delivery_time: type: TIMESTAMP @@ -53,6 +55,4 @@ storageTables: tableLocation: /tmp/db-storage.db fieldDelimiter: , storageHandlerName: org.apache.lens.storage.db.DBStorageHandler -name: sales_aggr_fact2 -cubeName: sales weight: 100.0 diff --git a/lens-examples/src/test/resources/yaml/sales-raw-fact.yaml b/lens-examples/src/test/resources/yaml/sales-raw-fact.yaml index 15becfc33..20106b66b 100644 --- a/lens-examples/src/test/resources/yaml/sales-raw-fact.yaml +++ b/lens-examples/src/test/resources/yaml/sales-raw-fact.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: sales_raw_fact +cubeName: sales columns: order_time: type: TIMESTAMP delivery_time: type: TIMESTAMP @@ -46,6 +48,4 @@ storageTables: external: true tableLocation: /tmp/examples/rawfact fieldDelimiter: , -name: sales_raw_fact -cubeName: sales weight: 500.0 From 5016add8259f2df8b052693d60c6984c9c0ff6e7 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Tue, 30 May 2017 17:25:01 +0530 Subject: [PATCH 49/71] test resource --- lens-api/src/main/resources/cube-0.1.xsd | 20 +-- .../apache/lens/cli/TestLensFactCommands.java | 129 ++++++++++++++++++ .../schema/cubes/base/virtual-cube.xml | 23 ++++ .../resources/schema/facts/virtual_fact.xml | 28 ++++ .../lens/cube/parse/TestCubeRewriter.java | 32 ++++- .../schema/facts/testfact_virtual.xml | 101 ++++++++++++++ 6 files changed, 316 insertions(+), 17 deletions(-) create mode 100644 lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml create mode 100644 lens-cli/src/test/resources/schema/facts/virtual_fact.xml create mode 100644 lens-cube/src/test/resources/schema/facts/testfact_virtual.xml diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 1f54c752a..a10d3f1ba 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -1222,25 +1222,13 @@ - XCube can either be a Base cube for which the user would give the full specification of the + XFact can either be a Cube Fact for which the user would give the full specification of the measures and dimensions - or can be a Derived cube, for which the user would specify only the measure names, - dimension names and parent. - Derived cube can have subset of measure names and dimension names associated with its - parent. It can have - different properties associated with it than the parent. + or can be a Virtual fact, for which the user would specify only the Source fact and filters if any. + Virtual fact derives its columns and storage specific details all from the source Fact. It can however have + different properties associated with it than the source. - - - - - - - - - - diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java index d31e25c26..6a8394658 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java @@ -59,11 +59,22 @@ public class TestLensFactCommands extends LensCliApplicationTest { @Test public void testFactCommands() throws IOException, URISyntaxException { createSampleCube(); + // createVirtualCube(); + addFact1Table(); updateFact1Table(); testFactStorageActions(); testFactPartitionActions(); + +// addVirtualFactTable(); +// updateVirtualFactTable(); +// dropVirtualFactTable(); + dropFact1Table(); + + //test virtual fact + + // dropVirtualCube(); dropSampleCube(); } @@ -76,10 +87,25 @@ private void createSampleCube() throws URISyntaxException { assertTrue(cubeList.contains("sample_cube"), cubeList); } + private void createVirtualCube() throws URISyntaxException { + URL cubeSpec = TestLensCubeCommands.class.getClassLoader().getResource("schema/cubes/base/virtual-cube.xml"); + String cubeList = getCubeCommand().showCubes(); + assertFalse(cubeList.contains("virtualcube"), cubeList); + getCubeCommand().createCube(new File(cubeSpec.toURI())); + cubeList = getCubeCommand().showCubes(); + assertTrue(cubeList.contains("virtualcube"), cubeList); + } + + private void dropSampleCube() { getCubeCommand().dropCube("sample_cube"); } + + private void dropVirtualCube() { + getCubeCommand().dropCube("virtualcube"); + } + private static LensFactCommands getCommand() { if (command == null) { LensClient client = new LensClient(); @@ -143,6 +169,41 @@ public static void addFact1Table() throws IOException { assertEquals("fact1", factList, "Fact1 table should be found"); } + /** + * Adds the virtual fact table. + * + * @throws IOException + */ + public static void addVirtualFactTable() throws IOException { + LensFactCommands command = getCommand(); + String factList = command.showFacts(null); + assertEquals(command.showFacts("sample_cube"), "No fact found for sample_cube"); + assertEquals(factList, "No fact found", "Fact tables should not be found"); + // add local storage before adding fact table + TestLensStorageCommands.addLocalStorage(FACT_LOCAL); + URL factSpec = TestLensFactCommands.class.getClassLoader().getResource("schema/facts/virtual_fact.xml"); + try { + command.createFact(new File(factSpec.toURI())); + } catch (Exception e) { + fail("Unable to create virtual fact table" + e.getMessage()); + } + factList = command.showFacts(null); + assertEquals(command.showFacts("sample_cube"), factList); + try { + assertEquals(command.showFacts("blah"), factList); + fail(); + } catch (NotFoundException e) { + log.info("blah is not a table", e); + } + try { + assertEquals(command.showFacts("virtualfact"), factList); + fail(); + } catch (NotFoundException e) { + log.info("virtualfact is a table, but not a cube table", e); + } + assertEquals("virtualfact", factList, "Virtualfact table should be found"); + } + /** * Update fact1 table. */ @@ -196,6 +257,60 @@ public static void updateFact1Table() { } + + /** + * Update virtual fact table. + */ + public static void updateVirtualFactTable() { + try { + LensFactCommands command = getCommand(); + URL factSpec = TestLensFactCommands.class.getClassLoader().getResource("schema/facts/virtual_fact.xml"); + StringBuilder sb = new StringBuilder(); + BufferedReader bufferedReader = new BufferedReader(new FileReader(factSpec.getFile())); + String s; + while ((s = bufferedReader.readLine()) != null) { + sb.append(s).append("\n"); + } + + bufferedReader.close(); + + String xmlContent = sb.toString(); + + xmlContent = xmlContent.replace("\n", + "" + "\n\n"); + + xmlContent = xmlContent.replace("", + "" + + "\n\n"); + + File newFile = new File("target/local-virtualfact.xml"); + Writer writer = new OutputStreamWriter(new FileOutputStream(newFile)); + writer.write(xmlContent); + writer.close(); + + String desc = command.describeFactTable("fact1"); + log.debug(desc); + String propString = "fact1.prop: f1"; + String propString1 = "fact1.prop1: f2"; + String propStringColStartTime = "cube.fact.col.start.time.measure4: 2015-01-01"; + assertTrue(desc.contains(propString)); + + command.updateFactTable("fact1", new File("target/local-fact1.xml")); + desc = command.describeFactTable("fact1"); + log.debug(desc); + assertTrue(desc.contains(propString), "The sample property value is not set"); + assertTrue(desc.contains(propString1), "The sample property value is not set"); + assertTrue(desc.contains(propStringColStartTime), "The sample property value is not set"); + + newFile.delete(); + + } catch (Throwable t) { + log.error("Updating of the fact1 table failed with ", t); + fail("Updating of the fact1 table failed with " + t.getMessage()); + } + + } + /** * Test fact storage actions. */ @@ -343,4 +458,18 @@ public static void dropFact1Table() { assertEquals(factList, "No fact found", "Fact tables should not be found"); TestLensStorageCommands.dropStorage(FACT_LOCAL); } + + + /** + * Drop virtualfact table. + */ + public static void dropVirtualFactTable() { + LensFactCommands command = getCommand(); + String factList = command.showFacts(null); + assertEquals("virtualfact", factList, "Virtualfact table should be found"); + command.dropFact("virtualfact", false); + factList = command.showFacts(null); + assertEquals(factList, "No fact found", "Fact tables should not be found"); + TestLensStorageCommands.dropStorage(FACT_LOCAL); + } } diff --git a/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml b/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml new file mode 100644 index 000000000..9002129ce --- /dev/null +++ b/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml @@ -0,0 +1,23 @@ + + + + diff --git a/lens-cli/src/test/resources/schema/facts/virtual_fact.xml b/lens-cli/src/test/resources/schema/facts/virtual_fact.xml new file mode 100644 index 000000000..93e4f0271 --- /dev/null +++ b/lens-cli/src/test/resources/schema/facts/virtual_fact.xml @@ -0,0 +1,28 @@ + + + + + + + + \ No newline at end of file diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 616e9fed4..7929f1109 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -147,7 +147,7 @@ public void testCubeQuery() throws Exception { @Test - public void testVirtualFactCubeQuery() throws Exception { + public void testVirtualFactCubeSimpleQuery() throws Exception { Configuration conf = getConf(); conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); CubeQueryContext rewrittenQuery = @@ -160,6 +160,36 @@ public void testVirtualFactCubeQuery() throws Exception { System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); } + + @Test + public void testVirtualFactCubeUnionQuery() throws Exception { + Configuration conf = getConf(); + conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); + String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", + null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, + "C1_summary1")); + String hql = rewrittenQuery.toHQL(); + compareQueries(hql, expected); + System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); + } + + + @Test + public void testVirtualFactCubeFactQuery() throws Exception { + Configuration conf = getConf(); + conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); + String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", + null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, + "C1_summary1")); + String hql = rewrittenQuery.toHQL(); + compareQueries(hql, expected); + System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); + } + @Test public void testMaxCoveringFact() throws Exception { Configuration conf = getConf(); diff --git a/lens-cube/src/test/resources/schema/facts/testfact_virtual.xml b/lens-cube/src/test/resources/schema/facts/testfact_virtual.xml new file mode 100644 index 000000000..7fb5ba534 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/testfact_virtual.xml @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MINUTELY + HOURLY + DAILY + + C1 + + + + + + + + + + + + dt + + + + + MINUTELY + HOURLY + DAILY + + C2 + + + + + + + + + + + + + pt + it + et + + + + \ No newline at end of file From 823cb52ee7c12bfe53d8916e5e77cf26546b5e7f Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Tue, 30 May 2017 17:40:02 +0530 Subject: [PATCH 50/71] sync with master and inmobi develop --- .../lens/cube/parse/TestCubeRewriter.java | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 7929f1109..13d4cb700 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -148,36 +148,6 @@ public void testCubeQuery() throws Exception { @Test public void testVirtualFactCubeSimpleQuery() throws Exception { - Configuration conf = getConf(); - conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); - CubeQueryContext rewrittenQuery = - rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); - String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", - null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, - "C1_summary1")); - String hql = rewrittenQuery.toHQL(); - compareQueries(hql, expected); - System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); - } - - - @Test - public void testVirtualFactCubeUnionQuery() throws Exception { - Configuration conf = getConf(); - conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); - CubeQueryContext rewrittenQuery = - rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); - String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", - null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, - "C1_summary1")); - String hql = rewrittenQuery.toHQL(); - compareQueries(hql, expected); - System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); - } - - - @Test - public void testVirtualFactCubeFactQuery() throws Exception { Configuration conf = getConf(); conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); CubeQueryContext rewrittenQuery = @@ -189,6 +159,36 @@ public void testVirtualFactCubeFactQuery() throws Exception { compareQueries(hql, expected); System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); } +// +// +// @Test +// public void testVirtualFactCubeUnionQuery() throws Exception { +// Configuration conf = getConf(); +// conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); +// CubeQueryContext rewrittenQuery = +// rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); +// String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", +// null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, +// "C1_summary1")); +// String hql = rewrittenQuery.toHQL(); +// compareQueries(hql, expected); +// System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); +// } +// +// +// @Test +// public void testVirtualFactCubeFactQuery() throws Exception { +// Configuration conf = getConf(); +// conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); +// CubeQueryContext rewrittenQuery = +// rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); +// String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", +// null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, +// "C1_summary1")); +// String hql = rewrittenQuery.toHQL(); +// compareQueries(hql, expected); +// System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); +// } @Test public void testMaxCoveringFact() throws Exception { From ab603a67565b1acd23738f026310f46ef942bff6 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Tue, 30 May 2017 18:20:06 +0530 Subject: [PATCH 51/71] sync with master and inmobi develop --- .../schema/facts/testfact_virtual.xml | 101 ------------------ 1 file changed, 101 deletions(-) delete mode 100644 lens-cube/src/test/resources/schema/facts/testfact_virtual.xml diff --git a/lens-cube/src/test/resources/schema/facts/testfact_virtual.xml b/lens-cube/src/test/resources/schema/facts/testfact_virtual.xml deleted file mode 100644 index 7fb5ba534..000000000 --- a/lens-cube/src/test/resources/schema/facts/testfact_virtual.xml +++ /dev/null @@ -1,101 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - MINUTELY - HOURLY - DAILY - - C1 - - - - - - - - - - - - dt - - - - - MINUTELY - HOURLY - DAILY - - C2 - - - - - - - - - - - - - pt - it - et - - - - \ No newline at end of file From 88517e08d513de01720279ab5d6558ef903c69ab Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 31 May 2017 11:07:14 +0530 Subject: [PATCH 52/71] Adding virtual fact cli test case --- .../apache/lens/cli/TestLensFactCommands.java | 70 ++++++++----------- .../schema/cubes/base/virtual-cube.xml | 7 ++ .../resources/schema/facts/virtual_fact.xml | 4 +- .../lens/cube/parse/TestCubeRewriter.java | 31 +------- .../resources/schema/facts/virtualfact2.xml | 25 +++++++ 5 files changed, 65 insertions(+), 72 deletions(-) create mode 100644 lens-cube/src/test/resources/schema/facts/virtualfact2.xml diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java index 6a8394658..265cd135c 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java @@ -59,22 +59,21 @@ public class TestLensFactCommands extends LensCliApplicationTest { @Test public void testFactCommands() throws IOException, URISyntaxException { createSampleCube(); - // createVirtualCube(); + createVirtualCube(); + //test base fact addFact1Table(); updateFact1Table(); testFactStorageActions(); testFactPartitionActions(); -// addVirtualFactTable(); -// updateVirtualFactTable(); -// dropVirtualFactTable(); - - dropFact1Table(); - //test virtual fact + addVirtualFactTable(); + updateVirtualFactTable(); + dropVirtualFactTable(); - // dropVirtualCube(); + dropFact1Table(); + dropVirtualCube(); dropSampleCube(); } @@ -141,9 +140,9 @@ public void cleanUp() { */ public static void addFact1Table() throws IOException { LensFactCommands command = getCommand(); - String factList = command.showFacts(null); + String factList = command.showFacts("sample_cube"); assertEquals(command.showFacts("sample_cube"), "No fact found for sample_cube"); - assertEquals(factList, "No fact found", "Fact tables should not be found"); + assertEquals(factList, "No fact found for sample_cube", "Fact tables should not be found"); // add local storage before adding fact table TestLensStorageCommands.addLocalStorage(FACT_LOCAL); URL factSpec = TestLensFactCommands.class.getClassLoader().getResource("schema/facts/fact1.xml"); @@ -152,7 +151,7 @@ public static void addFact1Table() throws IOException { } catch (Exception e) { fail("Unable to create fact table" + e.getMessage()); } - factList = command.showFacts(null); + factList = command.showFacts("sample_cube"); assertEquals(command.showFacts("sample_cube"), factList); try { assertEquals(command.showFacts("blah"), factList); @@ -176,19 +175,18 @@ public static void addFact1Table() throws IOException { */ public static void addVirtualFactTable() throws IOException { LensFactCommands command = getCommand(); - String factList = command.showFacts(null); - assertEquals(command.showFacts("sample_cube"), "No fact found for sample_cube"); - assertEquals(factList, "No fact found", "Fact tables should not be found"); + String factList = command.showFacts("virtualcube"); + assertEquals(command.showFacts("virtualcube"), "No fact found for virtualcube"); + assertEquals(factList, "No fact found for virtualcube", "Fact tables should not be found"); // add local storage before adding fact table - TestLensStorageCommands.addLocalStorage(FACT_LOCAL); URL factSpec = TestLensFactCommands.class.getClassLoader().getResource("schema/facts/virtual_fact.xml"); try { command.createFact(new File(factSpec.toURI())); } catch (Exception e) { fail("Unable to create virtual fact table" + e.getMessage()); } - factList = command.showFacts(null); - assertEquals(command.showFacts("sample_cube"), factList); + factList = command.showFacts("virtualcube"); + assertEquals(command.showFacts("virtualcube"), factList); try { assertEquals(command.showFacts("blah"), factList); fail(); @@ -276,37 +274,30 @@ public static void updateVirtualFactTable() { String xmlContent = sb.toString(); - xmlContent = xmlContent.replace("\n", - "" + "\n\n"); - - xmlContent = xmlContent.replace("", - "" - + "\n\n"); + xmlContent = xmlContent.replace("\n", + "" + "\n\n"); File newFile = new File("target/local-virtualfact.xml"); Writer writer = new OutputStreamWriter(new FileOutputStream(newFile)); writer.write(xmlContent); writer.close(); - String desc = command.describeFactTable("fact1"); + String desc = command.describeFactTable("virtualfact"); log.debug(desc); - String propString = "fact1.prop: f1"; - String propString1 = "fact1.prop1: f2"; - String propStringColStartTime = "cube.fact.col.start.time.measure4: 2015-01-01"; - assertTrue(desc.contains(propString)); + String propString = "virtualfact.prop: f1"; + String propString1 = "virtualfact.prop1: f2"; - command.updateFactTable("fact1", new File("target/local-fact1.xml")); - desc = command.describeFactTable("fact1"); + command.updateFactTable("virtualfact", new File("target/local-virtualfact.xml")); + desc = command.describeFactTable("virtualfact"); log.debug(desc); assertTrue(desc.contains(propString), "The sample property value is not set"); assertTrue(desc.contains(propString1), "The sample property value is not set"); - assertTrue(desc.contains(propStringColStartTime), "The sample property value is not set"); newFile.delete(); } catch (Throwable t) { - log.error("Updating of the fact1 table failed with ", t); - fail("Updating of the fact1 table failed with " + t.getMessage()); + log.error("Updating of the virtualfact table failed with ", t); + fail("Updating of the virtualfact table failed with " + t.getMessage()); } } @@ -451,11 +442,11 @@ private void verifyAndDeletePartitions() { */ public static void dropFact1Table() { LensFactCommands command = getCommand(); - String factList = command.showFacts(null); + String factList = command.showFacts("sample_cube"); assertEquals("fact1", factList, "Fact1 table should be found"); command.dropFact("fact1", false); - factList = command.showFacts(null); - assertEquals(factList, "No fact found", "Fact tables should not be found"); + factList = command.showFacts("sample_cube"); + assertEquals(factList, "No fact found for sample_cube", "Fact tables should not be found"); TestLensStorageCommands.dropStorage(FACT_LOCAL); } @@ -465,11 +456,10 @@ public static void dropFact1Table() { */ public static void dropVirtualFactTable() { LensFactCommands command = getCommand(); - String factList = command.showFacts(null); + String factList = command.showFacts("virtualcube"); assertEquals("virtualfact", factList, "Virtualfact table should be found"); command.dropFact("virtualfact", false); - factList = command.showFacts(null); - assertEquals(factList, "No fact found", "Fact tables should not be found"); - TestLensStorageCommands.dropStorage(FACT_LOCAL); + factList = command.showFacts("virtualcube"); + assertEquals(factList, "No fact found for virtualcube", "Virtual Fact tables should not be found"); } } diff --git a/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml b/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml index 9002129ce..8c57cee76 100644 --- a/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml +++ b/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml @@ -20,4 +20,11 @@ --> + + + + + + + diff --git a/lens-cli/src/test/resources/schema/facts/virtual_fact.xml b/lens-cli/src/test/resources/schema/facts/virtual_fact.xml index 93e4f0271..81e2676b4 100644 --- a/lens-cli/src/test/resources/schema/facts/virtual_fact.xml +++ b/lens-cli/src/test/resources/schema/facts/virtual_fact.xml @@ -19,10 +19,10 @@ under the License. --> - - + \ No newline at end of file diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 13d4cb700..65ca86134 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -159,36 +159,7 @@ public void testVirtualFactCubeSimpleQuery() throws Exception { compareQueries(hql, expected); System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); } -// -// -// @Test -// public void testVirtualFactCubeUnionQuery() throws Exception { -// Configuration conf = getConf(); -// conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); -// CubeQueryContext rewrittenQuery = -// rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); -// String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", -// null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, -// "C1_summary1")); -// String hql = rewrittenQuery.toHQL(); -// compareQueries(hql, expected); -// System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); -// } -// -// -// @Test -// public void testVirtualFactCubeFactQuery() throws Exception { -// Configuration conf = getConf(); -// conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); -// CubeQueryContext rewrittenQuery = -// rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); -// String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", -// null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, -// "C1_summary1")); -// String hql = rewrittenQuery.toHQL(); -// compareQueries(hql, expected); -// System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); -// } + @Test public void testMaxCoveringFact() throws Exception { diff --git a/lens-cube/src/test/resources/schema/facts/virtualfact2.xml b/lens-cube/src/test/resources/schema/facts/virtualfact2.xml new file mode 100644 index 000000000..82481873f --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/virtualfact2.xml @@ -0,0 +1,25 @@ + + + + + From 797f1c41c5f84b9abeda6b2f3fea7c1bae8d62cc Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 31 May 2017 17:42:27 +0530 Subject: [PATCH 53/71] Bug fixes --- .../lens/api/metastore/SchemaTraverser.java | 1 + .../lens/cli/commands/LensSchemaCommands.java | 18 ++++++------- .../cube/metadata/CubeMetastoreClient.java | 14 +++++------ .../schema/cubes/base/virtualcube.xml | 10 ++++---- .../resources/schema/facts/virtualfact2.xml | 25 ------------------- 5 files changed, 22 insertions(+), 46 deletions(-) delete mode 100644 lens-cube/src/test/resources/schema/facts/virtualfact2.xml diff --git a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java index 9564443bd..09c848a05 100644 --- a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java +++ b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java @@ -42,6 +42,7 @@ public class SchemaTraverser implements Runnable { types.put("cubes/derived", XDerivedCube.class); types.put("dimensions", XDimension.class); types.put("facts", XFactTable.class); + types.put("facts/virtual", XVirtualFactTable.class); types.put("dimtables", XDimensionTable.class); types.put("dimensiontables", XDimensionTable.class); types.put("dimensiontables", XDimensionTable.class); diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java index 20f313a48..aca1cf9dd 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java @@ -24,14 +24,7 @@ import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.lens.api.metastore.SchemaTraverser; -import org.apache.lens.api.metastore.XBaseCube; -import org.apache.lens.api.metastore.XDerivedCube; -import org.apache.lens.api.metastore.XDimension; -import org.apache.lens.api.metastore.XDimensionTable; -import org.apache.lens.api.metastore.XFactTable; -import org.apache.lens.api.metastore.XSegmentation; -import org.apache.lens.api.metastore.XStorage; +import org.apache.lens.api.metastore.*; import org.apache.lens.cli.commands.annotations.UserDocumentation; import org.springframework.beans.factory.annotation.Autowired; @@ -81,7 +74,12 @@ public class LensSchemaCommands implements CommandMarker { + "|\n" + "|-- facts\n" + " |-- fact1.xml\n" - + " |-- fact2.xml\n\n\n" + + " |-- fact2.xml\n" + + "| |\n" + + "| |-- virtual\n" + + "| | |-- virtual_fact1.xml\n" + + "| | |-- virtual_fact2.xml\n" + + "| |\n\n\n" + "If your cubes are divided between base and derived cubes,\nit makes sense to seperate into two directories, " + "since derived cubes can't be created unless base cube exists.\nIn the other case you can keep them in the cubes " + "directory itself.\nFor dimtables, you can keep your schema files in a directory named either dimtables or " @@ -114,6 +112,8 @@ public class LensSchemaCommands implements CommandMarker { UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s"); CREATE_COMMAND_MAP.put(XFactTable.class, "create fact --path %s"); UPDATE_COMMAND_MAP.put(XFactTable.class, "update fact --fact_name %s --path %s"); + CREATE_COMMAND_MAP.put(XVirtualFactTable.class, "create fact --path %s"); + UPDATE_COMMAND_MAP.put(XVirtualFactTable.class, "update fact --name %s --path %s"); CREATE_COMMAND_MAP.put(XSegmentation.class, "create segmentation --path %s"); UPDATE_COMMAND_MAP.put(XSegmentation.class, "update segmentation --name %s --path %s"); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index cb807610c..9eb69fd77 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -1652,8 +1652,7 @@ public boolean isFactTable(String tableName) throws LensException { } boolean isFactTable(Table tbl) { - String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY); - return CubeTableType.FACT.name().equals(tableType); + return tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) == null; } @@ -1675,11 +1674,9 @@ public boolean isVirtualFactTable(String virtualTableName) throws LensException } boolean isVirtualFactTable(Table tbl) { - String sourceFactName = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); - return sourceFactName != null; + return tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) != null; } - boolean isVirtualFactTableForCube(Table tbl, String cube) { return isVirtualFactTable(tbl) && tbl.getParameters().get(MetastoreUtil.getFactCubeNameKey(tbl.getTableName())) .equalsIgnoreCase(cube.toLowerCase()); @@ -2917,8 +2914,11 @@ private String getStorageTablePrefixFromStorage(String factOrDimTableName, Strin if (updatePeriod == null) { return storage; } - if (isFactTable(factOrDimTableName)) { - return (getCubeFactTable(factOrDimTableName)).getTablePrefix(storage, updatePeriod); + if (isVirtualFactTable(factOrDimTableName)) { + CubeFactTable sourceFact = (CubeFactTable)getCubeVirtualFactTable(factOrDimTableName).getSourceCubeFactTable(); + return sourceFact.getTablePrefix(storage, updatePeriod); + } else if(isFactTable(factOrDimTableName)) { + return getCubeFactTable(factOrDimTableName).getTablePrefix(storage,updatePeriod); } else { return storage; } diff --git a/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml index 2a5e0b0c5..fa35adc6d 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml @@ -20,12 +20,12 @@ --> - - - - + + + + - diff --git a/lens-cube/src/test/resources/schema/facts/virtualfact2.xml b/lens-cube/src/test/resources/schema/facts/virtualfact2.xml deleted file mode 100644 index 82481873f..000000000 --- a/lens-cube/src/test/resources/schema/facts/virtualfact2.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - From 3daf29601641c4afeedcecaa2164cea09f262a65 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 31 May 2017 17:59:39 +0530 Subject: [PATCH 54/71] Bug fixes --- .../apache/lens/cube/metadata/CubeMetastoreClient.java | 1 - .../test/resources/schema/cubes/base/virtualcube.xml | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 9eb69fd77..a20855306 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -1655,7 +1655,6 @@ boolean isFactTable(Table tbl) { return tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) == null; } - boolean isFactTableForCube(Table tbl, String cube) { return isFactTable(tbl) && tbl.getParameters().get(MetastoreUtil.getFactCubeNameKey(tbl.getTableName())) .equalsIgnoreCase(cube.toLowerCase()); diff --git a/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml index fa35adc6d..2a5e0b0c5 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml @@ -20,12 +20,12 @@ --> - - - - + + + + - From a07b354c00d926eed50f276f507219a13d6233ca Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 31 May 2017 18:15:11 +0530 Subject: [PATCH 55/71] Bug fixes --- .../org/apache/lens/cube/metadata/CubeMetastoreClient.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index a20855306..2588c263c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -1652,9 +1652,12 @@ public boolean isFactTable(String tableName) throws LensException { } boolean isFactTable(Table tbl) { - return tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) == null; + String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY); + return CubeTableType.FACT.name().equals(tableType) && + tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) == null; } + boolean isFactTableForCube(Table tbl, String cube) { return isFactTable(tbl) && tbl.getParameters().get(MetastoreUtil.getFactCubeNameKey(tbl.getTableName())) .equalsIgnoreCase(cube.toLowerCase()); From 5e9777119234d96d85167d0f76f16df7cb7b828b Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 31 May 2017 20:17:23 +0530 Subject: [PATCH 56/71] Adding virtual fact cli test case --- .../lens/cube/metadata/CubeMetastoreClient.java | 1 - .../cube/parse/TestCubeSegmentationRewriter.java | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 2588c263c..4a64cfb85 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -1657,7 +1657,6 @@ boolean isFactTable(Table tbl) { tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) == null; } - boolean isFactTableForCube(Table tbl, String cube) { return isFactTable(tbl) && tbl.getParameters().get(MetastoreUtil.getFactCubeNameKey(tbl.getTableName())) .equalsIgnoreCase(cube.toLowerCase()); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java index fe5260099..822caba28 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java @@ -126,13 +126,13 @@ public void testSegmentRewrite() throws Exception { */ @Test public void testFactUnionSegmentWithInnerUnion() throws Exception { - CubeQueryContext ctx = rewriteCtx("select cityid, segmsr1 from testcube where " + TWO_MONTHS_RANGE_UPTO_DAYS, - getConf()); + CubeQueryContext ctx = rewriteCtx("select cityid, segmsr1 from testcube where " + + "time_range_in(d_time, 'now.day-40day', 'now.day')", getConf()); String query1, query2, query3, query4; query1 = getExpectedQuery("testcube", "select testcube.cityid as alias0, sum(testcube.segmsr1) as alias1 from ", null, "group by testcube.cityid", getWhereForUpdatePeriods("testcube", "c0_b1fact1", - addDays(getDateWithOffset(UpdatePeriod.MONTHLY, -1), -1), getDateWithOffset(UpdatePeriod.DAILY, -10), + getDateWithOffset(UpdatePeriod.DAILY, -31), getDateWithOffset(UpdatePeriod.DAILY, -10), Sets.newHashSet(UpdatePeriod.MONTHLY, UpdatePeriod.DAILY))); query2 = getExpectedQuery("testcube", "select testcube.cityid as alias0, sum(testcube.segmsr1) as alias1 from ", null, "group by testcube.cityid", @@ -142,13 +142,13 @@ public void testFactUnionSegmentWithInnerUnion() throws Exception { query3 = getExpectedQuery("testcube", "select testcube.cityid as alias0, sum(testcube.segmsr1) as alias1 from ", null, "group by testcube.cityid", getWhereForUpdatePeriods("testcube", "c0_b2fact1", - addDays(getDateWithOffset(UpdatePeriod.MONTHLY, -1), -1), NOW, + getDateWithOffset(UpdatePeriod.DAILY, -31), NOW, Sets.newHashSet(UpdatePeriod.MONTHLY, UpdatePeriod.DAILY))); query4 = getExpectedQuery("testcube", "select testcube.cityid as alias0, sum(testcube.segmsr1) as alias1 from ", null, "group by testcube.cityid", getWhereForUpdatePeriods("testcube", "c0_b1b2fact1", - addDays(getDateWithOffset(UpdatePeriod.MONTHLY, -2), -1), - addDays(getDateWithOffset(UpdatePeriod.MONTHLY, -1), 0), + getDateWithOffset(UpdatePeriod.DAILY, -41), + getDateWithOffset(UpdatePeriod.DAILY, -30), Sets.newHashSet(UpdatePeriod.MONTHLY, UpdatePeriod.DAILY))); compareUnionQuery(ctx, "select testcube.alias0 as cityid, sum(testcube.alias1) as segmsr1 from (", ") AS testcube GROUP BY (testcube.alias0)", newArrayList(query1, query2, query3, query4)); From 172bdd3bfbb40d7dffbd60c16b87a538ea893309 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Wed, 31 May 2017 20:28:10 +0530 Subject: [PATCH 57/71] checkstyle fix --- .../apache/lens/cube/metadata/CubeMetastoreClient.java | 10 +++++----- .../lens/cube/parse/TestCubeSegmentationRewriter.java | 3 --- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 4a64cfb85..8c26df907 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -1653,8 +1653,8 @@ public boolean isFactTable(String tableName) throws LensException { boolean isFactTable(Table tbl) { String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY); - return CubeTableType.FACT.name().equals(tableType) && - tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) == null; + return CubeTableType.FACT.name().equals(tableType) + && tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) == null; } boolean isFactTableForCube(Table tbl, String cube) { @@ -2916,10 +2916,10 @@ private String getStorageTablePrefixFromStorage(String factOrDimTableName, Strin return storage; } if (isVirtualFactTable(factOrDimTableName)) { - CubeFactTable sourceFact = (CubeFactTable)getCubeVirtualFactTable(factOrDimTableName).getSourceCubeFactTable(); + CubeFactTable sourceFact = (CubeFactTable) getCubeVirtualFactTable(factOrDimTableName).getSourceCubeFactTable(); return sourceFact.getTablePrefix(storage, updatePeriod); - } else if(isFactTable(factOrDimTableName)) { - return getCubeFactTable(factOrDimTableName).getTablePrefix(storage,updatePeriod); + } else if (isFactTable(factOrDimTableName)) { + return getCubeFactTable(factOrDimTableName).getTablePrefix(storage, updatePeriod); } else { return storage; } diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java index 822caba28..9fa31dc1b 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java @@ -21,7 +21,6 @@ import static org.apache.lens.cube.metadata.DateFactory.NOW; import static org.apache.lens.cube.metadata.DateFactory.TWO_DAYS_RANGE; -import static org.apache.lens.cube.metadata.DateFactory.TWO_MONTHS_RANGE_UPTO_DAYS; import static org.apache.lens.cube.metadata.DateFactory.getDateWithOffset; import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.SEGMENTATION_PRUNED; import static org.apache.lens.cube.parse.CubeQueryConfUtil.DISABLE_AGGREGATE_RESOLVER; @@ -38,8 +37,6 @@ import static org.apache.lens.cube.parse.CubeTestSetup.getWhereForUpdatePeriods; import static org.apache.lens.cube.parse.TestCubeRewriter.compareQueries; -import static org.apache.commons.lang3.time.DateUtils.addDays; - import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; From 00577672b86245146e2106fa7a3cac31338467c6 Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Wed, 31 May 2017 19:50:52 +0530 Subject: [PATCH 58/71] Fixed expression resolver bug --- .../lens/cube/parse/ExpressionResolver.java | 16 +++++++--------- .../test/resources/schema/cubes/base/b1cube.xml | 2 +- .../test/resources/schema/cubes/base/b2cube.xml | 2 +- .../resources/schema/cubes/base/testcube.xml | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 7cad400bb..45523b999 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -518,15 +518,13 @@ void pruneExpressions() { } // Remove expressions for which denormalized columns are no more reachable esc.getDeNormCtx().pruneReferences(cubeql); - for (String table : esc.getDeNormCtx().getTableToRefCols().keySet()) { - Set nonReachableFields = esc.getDeNormCtx().getNonReachableReferenceFields(table); - if (!nonReachableFields.isEmpty()) { - log.info("Removing expression {} as columns {} are not available", esc, nonReachableFields); - iterator.remove(); - removedEsc.add(esc); - removed = true; - break; - } + if (esc.getDeNormCtx().getTableToRefCols().keySet().stream() + .map(esc.getDeNormCtx()::getNonReachableReferenceFields).noneMatch(Set::isEmpty)) { + log.info("Removing expression {} as all tables have non reachable fields", esc); + iterator.remove(); + removedEsc.add(esc); + removed = true; + break; } if (removed) { continue; diff --git a/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml index e3e99098f..fd25982c5 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml @@ -178,7 +178,7 @@ - + diff --git a/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml b/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml index e5cf2c089..ca6bcd70f 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml @@ -178,7 +178,7 @@ - + diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml index f123341f5..ef1370097 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml @@ -165,7 +165,7 @@ - + From 6918421d3a6ca1e9c0e1cd4d64f991a576907a22 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 1 Jun 2017 15:22:06 +0530 Subject: [PATCH 59/71] checkstyle fix --- .../lens/cube/metadata/DateFactory.java | 4 +-- .../lens/cube/parse/TestUnionQueries.java | 30 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java index 855f54a3a..984d405d8 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java @@ -219,8 +219,8 @@ public static boolean isZerothHour() { TWO_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -2, 0); TWO_MONTHS_RANGE_UPTO_DAYS = getTimeRangeString(MONTHLY, -2, 0, DAILY); TWO_MONTHS_RANGE_UPTO_HOURS = getTimeRangeString(MONTHLY, -2, 0, HOURLY); - THREE_MONTHS_RANGE_UPTO_DAYS = getTimeRangeString(MONTHLY, -3, 0, DAILY); - THREE_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -3, 0, MONTHLY); + THREE_MONTHS_RANGE_UPTO_DAYS = getTimeRangeString(DAILY, -90, 0, DAILY); + THREE_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -3, 0, DAILY); // calculate LAST_HOUR_TIME_RANGE LAST_HOUR_TIME_RANGE = getTimeRangeString(HOURLY, -1, 0); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java index 7ec332477..81c76ede5 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java @@ -371,21 +371,21 @@ public void testSingleFactSingleStorageWithMultipleTableDescriptions() throws Ex getValidFactTablesKey("testcube"), "testfact", FAIL_QUERY_ON_PARTIAL_DATA, false); - //If not beginning of month. Expecting this to pass at beginning of every month (example April 01 00:00) - if (!THREE_MONTHS_RANGE_UPTO_DAYS.equals(THREE_MONTHS_RANGE_UPTO_MONTH)) { - NoCandidateFactAvailableException e = getLensExceptionInRewrite("select count(msr4) from testCube where " - + THREE_MONTHS_RANGE_UPTO_DAYS, conf); - Set>> causes = - e.getBriefAndDetailedError().entrySet().stream().filter(x -> - x.getKey() instanceof StorageCandidate - && ((StorageCandidate)x.getKey()).getStorageTable().equalsIgnoreCase("c6_testfact")) - .collect(Collectors.toSet()); - assertEquals(causes.size(), 1); - List pruneCauses = causes.iterator().next().getValue(); - assertEquals(pruneCauses.size(), 1); - assertEquals(pruneCauses.get(0).getCause(), CandidateTablePruneCause. - CandidateTablePruneCode.STORAGE_NOT_AVAILABLE_IN_RANGE); - } +// //If not beginning of month. Expecting this to pass at beginning of every month (example April 01 00:00) +// if (!THREE_MONTHS_RANGE_UPTO_DAYS.equals(THREE_MONTHS_RANGE_UPTO_MONTH)) { +// NoCandidateFactAvailableException e = getLensExceptionInRewrite("select count(msr4) from testCube where " +// + THREE_MONTHS_RANGE_UPTO_DAYS, conf); +// Set>> causes = +// e.getBriefAndDetailedError().entrySet().stream().filter(x -> +// x.getKey() instanceof StorageCandidate +// && ((StorageCandidate)x.getKey()).getStorageTable().equalsIgnoreCase("c6_testfact")) +// .collect(Collectors.toSet()); +// assertEquals(causes.size(), 1); +// List pruneCauses = causes.iterator().next().getValue(); +// assertEquals(pruneCauses.size(), 1); +// assertEquals(pruneCauses.get(0).getCause(), CandidateTablePruneCause. +// CandidateTablePruneCode.STORAGE_NOT_AVAILABLE_IN_RANGE); +// } String hqlQuery2 = rewrite("select count(msr4) from testCube where " + THREE_MONTHS_RANGE_UPTO_MONTH, conf); System.out.println(hqlQuery2); From 904bac4256ab9551768062fa7bb8f9adbf9aadf3 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 1 Jun 2017 15:45:10 +0530 Subject: [PATCH 60/71] checkstyle fix --- .../test/java/org/apache/lens/cube/parse/TestUnionQueries.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java index 81c76ede5..871821648 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java @@ -28,10 +28,8 @@ import static org.testng.Assert.*; import java.util.*; -import java.util.stream.Collectors; import org.apache.lens.cube.error.LensCubeErrorCode; -import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.server.api.LensServerAPITestUtil; import org.apache.lens.server.api.error.LensException; From 515240717d7d5bff57fa60523b14eb9a775f2d14 Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 1 Jun 2017 16:40:22 +0530 Subject: [PATCH 61/71] test case fix --- .../lens/cli/TestLensSchemaCommands.java | 2 +- .../lens/cube/metadata/DateFactory.java | 4 +-- .../lens/cube/parse/TestUnionQueries.java | 32 ++++++++++--------- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java index ca6db2c69..4a235326e 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java @@ -37,7 +37,7 @@ public void testCreateSchema() throws Throwable { assertTrue(((String) execute("show databases")).contains(dbName)); execute("show storages", "local"); execute("show dimensions", "test_detail\ntest_dim"); - execute("show cubes", "sample_cube\ncube_with_no_weight_facts"); + execute("show cubes", "virtualcube\nsample_cube\ncube_with_no_weight_facts"); assertTrue(((String) execute("show dimtables")).contains("dim_table")); assertTrue(((String) execute("show facts")).contains("fact1")); execute("show segmentations", "seg1"); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java index 984d405d8..855f54a3a 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java @@ -219,8 +219,8 @@ public static boolean isZerothHour() { TWO_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -2, 0); TWO_MONTHS_RANGE_UPTO_DAYS = getTimeRangeString(MONTHLY, -2, 0, DAILY); TWO_MONTHS_RANGE_UPTO_HOURS = getTimeRangeString(MONTHLY, -2, 0, HOURLY); - THREE_MONTHS_RANGE_UPTO_DAYS = getTimeRangeString(DAILY, -90, 0, DAILY); - THREE_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -3, 0, DAILY); + THREE_MONTHS_RANGE_UPTO_DAYS = getTimeRangeString(MONTHLY, -3, 0, DAILY); + THREE_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -3, 0, MONTHLY); // calculate LAST_HOUR_TIME_RANGE LAST_HOUR_TIME_RANGE = getTimeRangeString(HOURLY, -1, 0); diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java index 871821648..90c6514e1 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java @@ -28,8 +28,10 @@ import static org.testng.Assert.*; import java.util.*; +import java.util.stream.Collectors; import org.apache.lens.cube.error.LensCubeErrorCode; +import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.server.api.LensServerAPITestUtil; import org.apache.lens.server.api.error.LensException; @@ -369,21 +371,21 @@ public void testSingleFactSingleStorageWithMultipleTableDescriptions() throws Ex getValidFactTablesKey("testcube"), "testfact", FAIL_QUERY_ON_PARTIAL_DATA, false); -// //If not beginning of month. Expecting this to pass at beginning of every month (example April 01 00:00) -// if (!THREE_MONTHS_RANGE_UPTO_DAYS.equals(THREE_MONTHS_RANGE_UPTO_MONTH)) { -// NoCandidateFactAvailableException e = getLensExceptionInRewrite("select count(msr4) from testCube where " -// + THREE_MONTHS_RANGE_UPTO_DAYS, conf); -// Set>> causes = -// e.getBriefAndDetailedError().entrySet().stream().filter(x -> -// x.getKey() instanceof StorageCandidate -// && ((StorageCandidate)x.getKey()).getStorageTable().equalsIgnoreCase("c6_testfact")) -// .collect(Collectors.toSet()); -// assertEquals(causes.size(), 1); -// List pruneCauses = causes.iterator().next().getValue(); -// assertEquals(pruneCauses.size(), 1); -// assertEquals(pruneCauses.get(0).getCause(), CandidateTablePruneCause. -// CandidateTablePruneCode.STORAGE_NOT_AVAILABLE_IN_RANGE); -// } + //If not beginning of month. Expecting this to pass at beginning of every month (example April 01 00:00) + if (Calendar.getInstance().get(Calendar.DAY_OF_MONTH) != 1) { + NoCandidateFactAvailableException e = getLensExceptionInRewrite("select count(msr4) from testCube where " + + THREE_MONTHS_RANGE_UPTO_DAYS, conf); + Set>> causes = + e.getBriefAndDetailedError().entrySet().stream().filter(x -> + x.getKey() instanceof StorageCandidate + && ((StorageCandidate)x.getKey()).getStorageTable().equalsIgnoreCase("c6_testfact")) + .collect(Collectors.toSet()); + assertEquals(causes.size(), 1); + List pruneCauses = causes.iterator().next().getValue(); + assertEquals(pruneCauses.size(), 1); + assertEquals(pruneCauses.get(0).getCause(), CandidateTablePruneCause. + CandidateTablePruneCode.STORAGE_NOT_AVAILABLE_IN_RANGE); + } String hqlQuery2 = rewrite("select count(msr4) from testCube where " + THREE_MONTHS_RANGE_UPTO_MONTH, conf); System.out.println(hqlQuery2); From cfefeb70c3a12523e61e3654462b9c1808aa34ee Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Thu, 1 Jun 2017 17:56:00 +0530 Subject: [PATCH 62/71] fixing tests --- .../lens/cube/parse/ExpressionResolver.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 45523b999..f38aa549c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -518,7 +518,8 @@ void pruneExpressions() { } // Remove expressions for which denormalized columns are no more reachable esc.getDeNormCtx().pruneReferences(cubeql); - if (esc.getDeNormCtx().getTableToRefCols().keySet().stream() + if (!esc.getDeNormCtx().getTableToRefCols().isEmpty() + && esc.getDeNormCtx().getTableToRefCols().keySet().stream() .map(esc.getDeNormCtx()::getNonReachableReferenceFields).noneMatch(Set::isEmpty)) { log.info("Removing expression {} as all tables have non reachable fields", esc); iterator.remove(); @@ -628,19 +629,15 @@ public void rewriteContext(CubeQueryContext cubeql) throws LensException { for (Map.Entry> ecEntry : exprCtx.allExprsQueried.entrySet()) { String expr = ecEntry.getKey(); Set ecSet = ecEntry.getValue(); - for (ExpressionContext ec : ecSet) { - if (ec.getSrcTable().getName().equals(cubeql.getCube().getName())) { - for (Iterator sItr = cubeql.getCandidates().iterator(); sItr.hasNext();) { - Candidate cand = sItr.next(); - if (!cand.isExpressionEvaluable(ec)) { - log.info("Not considering Candidate :{} as {} is not evaluable", cand, ec.exprCol.getName()); - sItr.remove(); - cubeql.addCandidatePruningMsg(cand, - CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName())); - } - } + cubeql.getCandidates().removeIf(x-> { + if (ecSet.stream().noneMatch(x::isExpressionEvaluable)) { + log.info("Not considering Candidate :{} as {} is not evaluable", x, expr); + cubeql.addCandidatePruningMsg(x, + CandidateTablePruneCause.expressionNotEvaluable(expr)); + return true; } - } + return false; + }); } } // prune candidate dims without any valid expressions From e6b974fddfe7b549247772ac03c5046bb6b86e4b Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 1 Jun 2017 19:25:09 +0530 Subject: [PATCH 63/71] checkstyle fix --- .../test/java/org/apache/lens/cli/TestLensFactCommands.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java index 265cd135c..05116869c 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java @@ -275,7 +275,8 @@ public static void updateVirtualFactTable() { String xmlContent = sb.toString(); xmlContent = xmlContent.replace("\n", - "" + "\n\n"); + "\n" + + "\n"); File newFile = new File("target/local-virtualfact.xml"); Writer writer = new OutputStreamWriter(new FileOutputStream(newFile)); From 069decd7e87cddee9337372f4eb0f53e8c809d0e Mon Sep 17 00:00:00 2001 From: "Rajitha.R" Date: Thu, 1 Jun 2017 21:20:47 +0530 Subject: [PATCH 64/71] checkstyle fix --- .../test/java/org/apache/lens/cli/TestLensFactCommands.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java index 05116869c..927b439d0 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java @@ -275,8 +275,8 @@ public static void updateVirtualFactTable() { String xmlContent = sb.toString(); xmlContent = xmlContent.replace("\n", - "\n" + - "\n"); + "\n" + + "\n"); File newFile = new File("target/local-virtualfact.xml"); Writer writer = new OutputStreamWriter(new FileOutputStream(newFile)); From c5e751dd1ebfd540a8ce4fe747d8f9bae15c841f Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Fri, 2 Jun 2017 14:44:40 +0530 Subject: [PATCH 65/71] findbugs --- .../java/org/apache/lens/cube/parse/ExpressionResolver.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index f38aa549c..794ad8cae 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -527,9 +527,6 @@ void pruneExpressions() { removed = true; break; } - if (removed) { - continue; - } //remove expressions which are not valid in the timerange queried // If an expression is defined as // ex = a + b // from t1 to t2; From 3d943e57a98afd3f281bdb90bc4882727a15cb04 Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Fri, 2 Jun 2017 19:29:46 +0530 Subject: [PATCH 66/71] vf+seg combo bugs --- .../lens/cube/parse/CubeQueryRewriter.java | 2 +- .../lens/cube/parse/LightestFactResolver.java | 28 +++++++++++-------- .../cube/parse/SegmentationCandidate.java | 7 +++-- .../parse/StorageCandidateHQLContext.java | 12 +++++++- .../lens/cube/parse/UnionQueryWriter.java | 17 ++++++----- 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java index d064cdb31..b7e7d631d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java @@ -206,7 +206,7 @@ private void setupRewriters(ImmutableList.Builder rewriters) { // Phase 2 of exprResolver : Prune candidate facts without any valid expressions rewriters.add(exprResolver); - if (!lightFactFirst) { + if (true || !lightFactFirst) { // Pick the least cost combination(s) (and prune others) out of a set of combinations produced // by CandidateCoveringSetsResolver rewriters.add(new LightestFactResolver()); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java index dd25f3e2b..a9f141a4c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java @@ -19,7 +19,11 @@ package org.apache.lens.cube.parse; -import java.util.*; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.server.api.error.LensException; @@ -35,21 +39,21 @@ public class LightestFactResolver implements ContextRewriter { @Override public void rewriteContext(CubeQueryContext cubeql) throws LensException { if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { - Map factWeightMap = new HashMap(); - - for (Candidate cand : cubeql.getCandidates()) { - factWeightMap.put(cand, cand.getCost()); - } + Map factWeightMap = cubeql.getCandidates().stream() + .filter(candidate -> candidate.getCost() > 0) + .collect(Collectors.toMap(Function.identity(), Candidate::getCost)); double minWeight = Collections.min(factWeightMap.values()); - for (Iterator i = cubeql.getCandidates().iterator(); i.hasNext();) { + for (Iterator i = cubeql.getCandidates().iterator(); i.hasNext(); ) { Candidate cand = i.next(); - if (factWeightMap.get(cand) > minWeight) { - log.info("Not considering candidate:{} from final candidates as it has more fact weight:{} minimum:{}", - cand, factWeightMap.get(cand), minWeight); - cubeql.addCandidatePruningMsg(cand, new CandidateTablePruneCause(CandidateTablePruneCode.MORE_WEIGHT)); - i.remove(); + if (factWeightMap.containsKey(cand)) { + if (factWeightMap.get(cand) > minWeight) { + log.info("Not considering candidate:{} from final candidates as it has more fact weight:{} minimum:{}", + cand, factWeightMap.get(cand), minWeight); + cubeql.addCandidatePruningMsg(cand, new CandidateTablePruneCause(CandidateTablePruneCode.MORE_WEIGHT)); + i.remove(); + } } } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java index a359d86d2..afc810dfc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java @@ -239,7 +239,10 @@ public Date getEndTime() { @Override public double getCost() { - return segmentation.weight(); + if (areCandidatesPicked()) { + return candidateStream().mapToDouble(Candidate::getCost).sum(); + } + return -1; } @Override @@ -281,7 +284,7 @@ public boolean isExpressionEvaluable(ExpressionResolver.ExpressionContext expr) // I can't ask my children to check this context for evaluability. return cubeStream() .map(cube -> cube.getExpressionByName(expr.getExprCol().getName())) - .allMatch(Predicate.isEqual(expr.getExprCol())); + .allMatch(Objects::nonNull); } private boolean areCandidatesPicked() { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java index c5351965a..d1101df4e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java @@ -34,6 +34,7 @@ import org.antlr.runtime.CommonToken; +import lombok.EqualsAndHashCode; import lombok.Getter; /** @@ -71,7 +72,7 @@ public void updateFromString() throws LensException { } } - CubeInterface getCube() { + public CubeInterface getCube() { return storageCandidate.getCubeQueryContext().getCube(); } @@ -150,4 +151,13 @@ protected void setMissingExpressions() throws LensException { } } } + + @Override + public int hashCode() { + final int PRIME = 59; + int result = 1; + result = result * PRIME + getStorageCandidate().hashCode(); + result = result * PRIME + getCube().hashCode(); + return result; + } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java index 9412f274d..cc0a2e597 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java @@ -48,7 +48,7 @@ public class UnionQueryWriter extends SimpleHQLContext { private Map innerToOuterSelectASTs = new HashMap<>(); private Map innerToOuterHavingASTs = new HashMap<>(); - private Map storageCandidateToSelectAstMap = new HashMap<>(); + private Map storageCandidateToSelectAstMap = new HashMap<>(); private CubeQueryContext cubeql; static final ASTNode DEFAULT_MEASURE_AST; private static final String DEFAULT_MEASURE = "0.0"; @@ -91,8 +91,7 @@ protected void setMissingExpressions() throws LensException { */ private void updateAsts() { for (StorageCandidateHQLContext sc : storageCandidates) { - storageCandidateToSelectAstMap.put(sc.getStorageCandidate().toString(), - new ASTNode(new CommonToken(TOK_SELECT, "TOK_SELECT"))); + storageCandidateToSelectAstMap.put(sc, new ASTNode(new CommonToken(TOK_SELECT, "TOK_SELECT"))); if (sc.getQueryAst().getHavingAST() != null) { cubeql.setHavingAST(sc.getQueryAst().getHavingAST()); } @@ -403,7 +402,7 @@ private void updateInnterSelectASTWithDefault() throws LensException { if (!phrase.hasMeasures(cubeql)) { for (StorageCandidateHQLContext sc : storageCandidates) { ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0); - storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()). + storageCandidateToSelectAstMap.get(sc). addChild(getSelectExpr(exprWithOutAlias, aliasNode, false)); } @@ -412,7 +411,7 @@ private void updateInnterSelectASTWithDefault() throws LensException { for (StorageCandidateHQLContext sc : storageCandidates) { if (sc.getStorageCandidate().getAnswerableMeasurePhraseIndices().contains(phrase.getPosition())) { ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0); - storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()). + storageCandidateToSelectAstMap.get(sc). addChild(getSelectExpr(exprWithOutAlias, aliasNode, false)); } else { ASTNode resolvedExprNode = getAggregateNodesExpression(i); @@ -421,7 +420,7 @@ private void updateInnterSelectASTWithDefault() throws LensException { } else { resolvedExprNode = getSelectExpr(null, null, true); } - storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()). + storageCandidateToSelectAstMap.get(sc). addChild(getSelectExpr(resolvedExprNode, aliasNode, false)); } } @@ -431,7 +430,7 @@ private void updateInnterSelectASTWithDefault() throws LensException { for (StorageCandidateHQLContext sc : storageCandidates) { if (sc.getStorageCandidate().getAnswerableMeasurePhraseIndices().contains(phrase.getPosition())) { ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0); - storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()). + storageCandidateToSelectAstMap.get(sc). addChild(getSelectExpr(exprWithOutAlias, aliasNode, false)); } else { ASTNode resolvedExprNode = getAggregateNodesExpression(i); @@ -440,7 +439,7 @@ private void updateInnterSelectASTWithDefault() throws LensException { } else { resolvedExprNode = getSelectExpr(null, null, true); } - storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()). + storageCandidateToSelectAstMap.get(sc). addChild(getSelectExpr(resolvedExprNode, aliasNode, false)); } } @@ -485,7 +484,7 @@ private void processSelectAndHavingAST() throws LensException { private void processSelectExpression(StorageCandidateHQLContext sc, ASTNode outerSelectAst, ASTNode innerSelectAST, AliasDecider aliasDecider) throws LensException { //ASTNode selectAST = sc.getQueryAst().getSelectAST(); - ASTNode selectAST = storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()); + ASTNode selectAST = storageCandidateToSelectAstMap.get(sc); if (selectAST == null) { return; } From 5f5aba47ed0ad8877d710d8278678412effe633b Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Mon, 5 Jun 2017 15:57:40 +0530 Subject: [PATCH 67/71] Dynamic segmentation weight --- .../org/apache/lens/cube/parse/Candidate.java | 2 +- .../lens/cube/parse/CubeQueryRewriter.java | 12 ++++----- .../apache/lens/cube/parse/JoinCandidate.java | 12 +++++++-- .../lens/cube/parse/LightestFactResolver.java | 26 +++++++++---------- .../cube/parse/SegmentationCandidate.java | 16 +++++++++--- .../lens/cube/parse/StorageCandidate.java | 5 ++-- .../parse/StorageCandidateHQLContext.java | 1 - .../lens/cube/parse/UnionCandidate.java | 15 ++++++++--- .../test/resources/schema/facts/b1fact1.xml | 2 +- 9 files changed, 58 insertions(+), 33 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java index 14fb2f9bb..dad806346 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java @@ -105,7 +105,7 @@ default boolean isPartiallyValidForTimeRange(TimeRange timeRange) { /** * @return the cost of this candidate */ - double getCost(); + OptionalDouble getCost(); /** * Returns true if this candidate contains the given candidate diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java index b7e7d631d..0ef41f37a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java @@ -149,6 +149,7 @@ private void setupRewriters(ImmutableList.Builder rewriters) { DenormalizationResolver denormResolver = new DenormalizationResolver(); CandidateTableResolver candidateTblResolver = new CandidateTableResolver(); StorageTableResolver storageTableResolver = new StorageTableResolver(conf); + LightestFactResolver lightestFactResolver = new LightestFactResolver(); // Phase 1 of exprResolver: Resolve expressions rewriters.add(exprResolver); @@ -185,7 +186,7 @@ private void setupRewriters(ImmutableList.Builder rewriters) { rewriters.add(exprResolver); // Pick the least cost combination(s) (and prune others) out of a set of combinations produced // by CandidateCoveringSetsResolver - rewriters.add(new LightestFactResolver()); + rewriters.add(lightestFactResolver); } // Phase 2 of storageTableResolver: resolve storage table partitions. @@ -206,11 +207,10 @@ private void setupRewriters(ImmutableList.Builder rewriters) { // Phase 2 of exprResolver : Prune candidate facts without any valid expressions rewriters.add(exprResolver); - if (true || !lightFactFirst) { - // Pick the least cost combination(s) (and prune others) out of a set of combinations produced - // by CandidateCoveringSetsResolver - rewriters.add(new LightestFactResolver()); - } + // Pick the least cost combination(s) (and prune others) out of a set of combinations produced + // by CandidateCoveringSetsResolver + rewriters.add(lightestFactResolver); + // if two combinations have the same least weight/cost, then the combination with least number of time partitions // queried will be picked. Rest of the combinations will be pruned rewriters.add(new LeastPartitionResolver()); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java index 3153f68b7..15bd162f2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java @@ -72,8 +72,16 @@ public Date getEndTime() { } @Override - public double getCost() { - return children.stream().mapToDouble(Candidate::getCost).sum(); + public OptionalDouble getCost() { + double cost = 0; + for (Candidate candidate : getChildren()) { + if (candidate.getCost().isPresent()) { + cost += candidate.getCost().getAsDouble(); + } else { + return OptionalDouble.empty(); + } + } + return OptionalDouble.of(cost); } @Override diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java index a9f141a4c..52e3632f2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java @@ -40,19 +40,19 @@ public class LightestFactResolver implements ContextRewriter { public void rewriteContext(CubeQueryContext cubeql) throws LensException { if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { Map factWeightMap = cubeql.getCandidates().stream() - .filter(candidate -> candidate.getCost() > 0) - .collect(Collectors.toMap(Function.identity(), Candidate::getCost)); - - double minWeight = Collections.min(factWeightMap.values()); - - for (Iterator i = cubeql.getCandidates().iterator(); i.hasNext(); ) { - Candidate cand = i.next(); - if (factWeightMap.containsKey(cand)) { - if (factWeightMap.get(cand) > minWeight) { - log.info("Not considering candidate:{} from final candidates as it has more fact weight:{} minimum:{}", - cand, factWeightMap.get(cand), minWeight); - cubeql.addCandidatePruningMsg(cand, new CandidateTablePruneCause(CandidateTablePruneCode.MORE_WEIGHT)); - i.remove(); + .filter(candidate -> candidate.getCost().isPresent()) + .collect(Collectors.toMap(Function.identity(), x -> x.getCost().getAsDouble())); + if (!factWeightMap.isEmpty()) { + double minWeight = Collections.min(factWeightMap.values()); + for (Iterator i = cubeql.getCandidates().iterator(); i.hasNext();) { + Candidate cand = i.next(); + if (factWeightMap.containsKey(cand)) { + if (factWeightMap.get(cand) > minWeight) { + log.info("Not considering candidate:{} from final candidates as it has more fact weight:{} minimum:{}", + cand, factWeightMap.get(cand), minWeight); + cubeql.addCandidatePruningMsg(cand, new CandidateTablePruneCause(CandidateTablePruneCode.MORE_WEIGHT)); + i.remove(); + } } } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java index afc810dfc..a2bd48500 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalDouble; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collector; @@ -238,11 +239,20 @@ public Date getEndTime() { } @Override - public double getCost() { + public OptionalDouble getCost() { if (areCandidatesPicked()) { - return candidateStream().mapToDouble(Candidate::getCost).sum(); + double cost = 0.0; + for (Candidate candidate : getChildren()) { + if (candidate.getCost().isPresent()) { + cost += candidate.getCost().getAsDouble(); + } else { + return OptionalDouble.empty(); + } + } + return OptionalDouble.of(cost); + } else { + return OptionalDouble.empty(); } - return -1; } @Override diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 902be17d7..d9d4b0f46 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -41,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalDouble; import java.util.Set; import java.util.TimeZone; import java.util.TreeSet; @@ -350,8 +351,8 @@ public Collection getColumns() { } @Override - public double getCost() { - return fact.weight(); + public OptionalDouble getCost() { + return OptionalDouble.of(fact.weight()); } @Override diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java index d1101df4e..f54645406 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java @@ -34,7 +34,6 @@ import org.antlr.runtime.CommonToken; -import lombok.EqualsAndHashCode; import lombok.Getter; /** diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java index 757a87703..510ea0c5b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java @@ -30,6 +30,7 @@ import java.util.ListIterator; import java.util.Map; import java.util.Optional; +import java.util.OptionalDouble; import java.util.Set; import org.apache.lens.cube.metadata.FactPartition; @@ -154,14 +155,18 @@ public Date getEndTime() { } @Override - public double getCost() { + public OptionalDouble getCost() { double cost = 0.0; for (TimeRange timeRange : getCubeQueryContext().getTimeRanges()) { for (Map.Entry entry : getTimeRangeSplit(timeRange).entrySet()) { - cost += entry.getKey().getCost() * entry.getValue().milliseconds() / timeRange.milliseconds(); + if (entry.getKey().getCost().isPresent()) { + cost += entry.getKey().getCost().getAsDouble() *entry.getValue().milliseconds() / timeRange.milliseconds(); + } else { + return OptionalDouble.empty(); + } } } - return cost; + return OptionalDouble.of(cost); } @Override @@ -256,7 +261,9 @@ private String getToString() { * @return */ private Map splitTimeRangeForChildren(TimeRange timeRange) { - children.sort(comparing(Candidate::getCost)); + if (children.stream().map(Candidate::getCost).allMatch(OptionalDouble::isPresent)) { + children.sort(comparing(x -> x.getCost().getAsDouble())); + } Map childrenTimeRangeMap = new HashMap<>(); // Sorted list based on the weights. Set ranges = new HashSet<>(); diff --git a/lens-cube/src/test/resources/schema/facts/b1fact1.xml b/lens-cube/src/test/resources/schema/facts/b1fact1.xml index b51a34ac5..150003702 100644 --- a/lens-cube/src/test/resources/schema/facts/b1fact1.xml +++ b/lens-cube/src/test/resources/schema/facts/b1fact1.xml @@ -19,7 +19,7 @@ under the License. --> - + From ed51d14dd55300c3bf9d52166236b7ff679324e9 Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Mon, 5 Jun 2017 17:22:10 +0530 Subject: [PATCH 68/71] VF bugs --- .../java/org/apache/lens/cube/parse/StorageCandidate.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index d9d4b0f46..f5b162452 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -265,7 +265,7 @@ private Set getValidStorageTableNames() throws LensException { Set uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : validUpdatePeriods) { uniqueStorageTables.add( - getCubeMetastoreClient().getStorageTableName(fact.getName(), storageName, updatePeriod) + getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, updatePeriod) ); } return uniqueStorageTables; @@ -884,7 +884,7 @@ private Date getStorageTableStartDate(UpdatePeriod interval) throws LensExceptio return this.startTime; } return getCubeMetastoreClient().getStorageTableStartDate( - getCubeMetastoreClient().getStorageTableName(fact.getName(), storageName, interval), fact.getName()); + getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact.getName()); } private Date getStorageTableEndDate(UpdatePeriod interval) throws LensException { @@ -893,7 +893,7 @@ private Date getStorageTableEndDate(UpdatePeriod interval) throws LensException return this.endTime; } return getCubeMetastoreClient().getStorageTableEndDate( - getCubeMetastoreClient().getStorageTableName(fact.getName(), storageName, interval), fact.getName()); + getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact.getName()); } From 300f2d1fb0a6daafb25388f666cb0c9d5bca26d5 Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Mon, 5 Jun 2017 18:22:51 +0530 Subject: [PATCH 69/71] VF bugs: iteration 2 --- .../java/org/apache/lens/cube/parse/StorageCandidate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index f5b162452..c6fd048c8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -769,9 +769,9 @@ public String getAliasForTable(String alias) { String database = SessionState.get().getCurrentDatabase(); String ret; if (alias == null || alias.isEmpty()) { - ret = getResolvedName(); + ret = getStorageTable(); } else { - ret = getResolvedName() + " " + alias; + ret = getStorageTable() + " " + alias; } if (StringUtils.isNotBlank(database) && !"default".equalsIgnoreCase(database)) { ret = database + "." + ret; From 2dbf754315c4a3e5297dc4423d7a06c129f81218 Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Mon, 5 Jun 2017 19:29:39 +0530 Subject: [PATCH 70/71] Sushil:LENS-1429 --- .../org/apache/lens/cube/parse/CandidateTableResolver.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index 93736cdef..f530650b2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -674,11 +674,13 @@ private static boolean checkForFactColumnExistsAndValidForRange(Candidate sc, if (colSet == null || colSet.isEmpty()) { return true; } + boolean isEvaluable = false; for (QueriedPhraseContext qur : colSet) { if (sc.isPhraseAnswerable(qur)) { - return true; + isEvaluable = true; + continue; } } - return false; + return isEvaluable; } } From 958e3f607925cb83c4adc0efd3a452cd07302b4c Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Tue, 6 Jun 2017 16:29:40 +0530 Subject: [PATCH 71/71] fixing broken tests --- .../java/org/apache/lens/cube/parse/StorageCandidate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index cee5163b2..e3b3e1bce 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -769,9 +769,9 @@ public String getAliasForTable(String alias) { String database = SessionState.get().getCurrentDatabase(); String ret; if (alias == null || alias.isEmpty()) { - ret = getStorageTable(); + ret = getResolvedName(); } else { - ret = getStorageTable() + " " + alias; + ret = getResolvedName() + " " + alias; } if (StringUtils.isNotBlank(database) && !"default".equalsIgnoreCase(database)) { ret = database + "." + ret;