diff --git a/checkstyle/pom.xml b/checkstyle/pom.xml index 2902c6b58..4c3777c7c 100644 --- a/checkstyle/pom.xml +++ b/checkstyle/pom.xml @@ -29,8 +29,7 @@ org.apache.lens checkstyle Lens Checkstyle Rules - 2.8.0-SNAPSHOT - + 2.7.8-inm-SNAPSHOT diff --git a/contrib/clients/pom.xml b/contrib/clients/pom.xml index d7677a33c..c57a27b63 100644 --- a/contrib/clients/pom.xml +++ b/contrib/clients/pom.xml @@ -25,7 +25,7 @@ lens-contrib org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-contrib-clients diff --git a/contrib/clients/python/pom.xml b/contrib/clients/python/pom.xml index ac842bd17..f866096ce 100644 --- a/contrib/clients/python/pom.xml +++ b/contrib/clients/python/pom.xml @@ -19,14 +19,13 @@ under the License. --> - + 4.0.0 Lens Python Client lens-contrib-clients org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-python-client diff --git a/contrib/pom.xml b/contrib/pom.xml index b36add094..9547d7263 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -25,7 +25,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-contrib diff --git a/lens-api/pom.xml b/lens-api/pom.xml index ece443034..5e920d633 100644 --- a/lens-api/pom.xml +++ b/lens-api/pom.xml @@ -25,7 +25,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-api diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml index 3640fdd70..955e6e8bc 100644 --- a/lens-cli/pom.xml +++ b/lens-cli/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT diff --git a/lens-client/pom.xml b/lens-client/pom.xml index 5b94800d1..4268e6c4e 100644 --- a/lens-client/pom.xml +++ b/lens-client/pom.xml @@ -24,7 +24,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml index deaeb3613..3d1d38941 100644 --- a/lens-cube/pom.xml +++ b/lens-cube/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-cube diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java index 0855ced92..93e489548 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java @@ -105,7 +105,7 @@ default boolean isPartiallyValidForTimeRange(TimeRange timeRange) { /** * @return the cost of this candidate */ - double getCost(); + OptionalDouble getCost(); /** * Returns true if this candidate contains the given candidate @@ -356,10 +356,10 @@ default QueryWriterContext toQueryWriterContext(Map dim throw new IllegalArgumentException("Candidate doesn't have children and no suitable implementation found"); } - default Set decideMeasurePhrasesToAnswer(Set measurePhraseIndices) throws LensException { + default Set decideMeasurePhrasesToAnswer(Set measureIndices) throws LensException { HashSet allCovered = Sets.newHashSet(); for (Candidate candidate : getChildren()) { - Set covered = candidate.decideMeasurePhrasesToAnswer(measurePhraseIndices); + Set covered = candidate.decideMeasurePhrasesToAnswer(measureIndices); allCovered.addAll(covered); } return allCovered; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java index 6ba46d69e..332f704f9 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java @@ -110,6 +110,7 @@ public static Set getStorageCandidates(Collection candidates, Collection storageCandidateSet) { for (Candidate candidate : candidates) { + getStorageCandidates(candidate, storageCandidateSet); } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java index d064cdb31..0ef41f37a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java @@ -149,6 +149,7 @@ private void setupRewriters(ImmutableList.Builder rewriters) { DenormalizationResolver denormResolver = new DenormalizationResolver(); CandidateTableResolver candidateTblResolver = new CandidateTableResolver(); StorageTableResolver storageTableResolver = new StorageTableResolver(conf); + LightestFactResolver lightestFactResolver = new LightestFactResolver(); // Phase 1 of exprResolver: Resolve expressions rewriters.add(exprResolver); @@ -185,7 +186,7 @@ private void setupRewriters(ImmutableList.Builder rewriters) { rewriters.add(exprResolver); // Pick the least cost combination(s) (and prune others) out of a set of combinations produced // by CandidateCoveringSetsResolver - rewriters.add(new LightestFactResolver()); + rewriters.add(lightestFactResolver); } // Phase 2 of storageTableResolver: resolve storage table partitions. @@ -206,11 +207,10 @@ private void setupRewriters(ImmutableList.Builder rewriters) { // Phase 2 of exprResolver : Prune candidate facts without any valid expressions rewriters.add(exprResolver); - if (!lightFactFirst) { - // Pick the least cost combination(s) (and prune others) out of a set of combinations produced - // by CandidateCoveringSetsResolver - rewriters.add(new LightestFactResolver()); - } + // Pick the least cost combination(s) (and prune others) out of a set of combinations produced + // by CandidateCoveringSetsResolver + rewriters.add(lightestFactResolver); + // if two combinations have the same least weight/cost, then the combination with least number of time partitions // queried will be picked. Rest of the combinations will be pruned rewriters.add(new LeastPartitionResolver()); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java index 43510dd69..1f951b8f2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java @@ -82,7 +82,7 @@ String genWhereClauseWithDimPartitions(String originalWhere) { return whereBuf.toString(); } - static void appendWhereClause(StringBuilder filterCondition, String whereClause, boolean hasMore) { + public static void appendWhereClause(StringBuilder filterCondition, String whereClause, boolean hasMore) { // Make sure we add AND only when there are already some conditions in where // clause if (hasMore && !filterCondition.toString().isEmpty() && !StringUtils.isBlank(whereClause)) { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index f38aa549c..794ad8cae 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -527,9 +527,6 @@ void pruneExpressions() { removed = true; break; } - if (removed) { - continue; - } //remove expressions which are not valid in the timerange queried // If an expression is defined as // ex = a + b // from t1 to t2; diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java index c4049cdcb..d9915f466 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java @@ -72,8 +72,16 @@ public Date getEndTime() { } @Override - public double getCost() { - return children.stream().mapToDouble(Candidate::getCost).sum(); + public OptionalDouble getCost() { + double cost = 0; + for (Candidate candidate : getChildren()) { + if (candidate.getCost().isPresent()) { + cost += candidate.getCost().getAsDouble(); + } else { + return OptionalDouble.empty(); + } + } + return OptionalDouble.of(cost); } @Override diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java index dd25f3e2b..52e3632f2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java @@ -19,7 +19,11 @@ package org.apache.lens.cube.parse; -import java.util.*; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.server.api.error.LensException; @@ -35,21 +39,21 @@ public class LightestFactResolver implements ContextRewriter { @Override public void rewriteContext(CubeQueryContext cubeql) throws LensException { if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { - Map factWeightMap = new HashMap(); - - for (Candidate cand : cubeql.getCandidates()) { - factWeightMap.put(cand, cand.getCost()); - } - - double minWeight = Collections.min(factWeightMap.values()); - - for (Iterator i = cubeql.getCandidates().iterator(); i.hasNext();) { - Candidate cand = i.next(); - if (factWeightMap.get(cand) > minWeight) { - log.info("Not considering candidate:{} from final candidates as it has more fact weight:{} minimum:{}", - cand, factWeightMap.get(cand), minWeight); - cubeql.addCandidatePruningMsg(cand, new CandidateTablePruneCause(CandidateTablePruneCode.MORE_WEIGHT)); - i.remove(); + Map factWeightMap = cubeql.getCandidates().stream() + .filter(candidate -> candidate.getCost().isPresent()) + .collect(Collectors.toMap(Function.identity(), x -> x.getCost().getAsDouble())); + if (!factWeightMap.isEmpty()) { + double minWeight = Collections.min(factWeightMap.values()); + for (Iterator i = cubeql.getCandidates().iterator(); i.hasNext();) { + Candidate cand = i.next(); + if (factWeightMap.containsKey(cand)) { + if (factWeightMap.get(cand) > minWeight) { + log.info("Not considering candidate:{} from final candidates as it has more fact weight:{} minimum:{}", + cand, factWeightMap.get(cand), minWeight); + cubeql.addCandidatePruningMsg(cand, new CandidateTablePruneCause(CandidateTablePruneCode.MORE_WEIGHT)); + i.remove(); + } + } } } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java index a359d86d2..a2bd48500 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalDouble; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collector; @@ -238,8 +239,20 @@ public Date getEndTime() { } @Override - public double getCost() { - return segmentation.weight(); + public OptionalDouble getCost() { + if (areCandidatesPicked()) { + double cost = 0.0; + for (Candidate candidate : getChildren()) { + if (candidate.getCost().isPresent()) { + cost += candidate.getCost().getAsDouble(); + } else { + return OptionalDouble.empty(); + } + } + return OptionalDouble.of(cost); + } else { + return OptionalDouble.empty(); + } } @Override @@ -281,7 +294,7 @@ public boolean isExpressionEvaluable(ExpressionResolver.ExpressionContext expr) // I can't ask my children to check this context for evaluability. return cubeStream() .map(cube -> cube.getExpressionByName(expr.getExprCol().getName())) - .allMatch(Predicate.isEqual(expr.getExprCol())); + .allMatch(Objects::nonNull); } private boolean areCandidatesPicked() { diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 3f5e2e56e..e3b3e1bce 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -41,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalDouble; import java.util.Set; import java.util.TimeZone; import java.util.TreeSet; @@ -264,7 +265,7 @@ private Set getValidStorageTableNames() throws LensException { Set uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : validUpdatePeriods) { uniqueStorageTables.add( - getCubeMetastoreClient().getStorageTableName(fact.getName(), storageName, updatePeriod) + getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, updatePeriod) ); } return uniqueStorageTables; @@ -307,6 +308,7 @@ public boolean isPhraseAnswerable(QueriedPhraseContext phrase) throws LensExcept return phrase.isEvaluable(this); } + @Override public AbstractCubeTable getTable() { return (AbstractCubeTable) fact; } @@ -350,8 +352,8 @@ public Collection getColumns() { } @Override - public double getCost() { - return fact.weight(); + public OptionalDouble getCost() { + return OptionalDouble.of(fact.weight()); } @Override @@ -367,7 +369,7 @@ public Collection getChildren() { private void updatePartitionStorage(FactPartition part) throws LensException { try { if (getCubeMetastoreClient().factPartitionExists(fact, part, storageTable)) { - part.getStorageTables().add(name); + part.getStorageTables().add(storageTable); part.setFound(true); } } catch (HiveException e) { @@ -417,10 +419,10 @@ private boolean getPartitions(Date fromDate, Date toDate, String partCol, Set getPeriodSpecificStorageCandidates() throws StorageCandidate updatePeriodSpecificSc; for (UpdatePeriod period : participatingUpdatePeriods) { updatePeriodSpecificSc = copy(); - updatePeriodSpecificSc.truncatePartitions(period); updatePeriodSpecificSc.setResolvedName(getCubeMetastoreClient().getStorageTableName(fact.getName(), storageName, period)); + updatePeriodSpecificSc.truncatePartitions(period); periodSpecificScList.add(updatePeriodSpecificSc); } periodSpecificStorageCandidates = periodSpecificScList; @@ -943,6 +944,10 @@ private void truncatePartitions(UpdatePeriod updatePeriod) { while (rangeItr.hasNext()) { Map.Entry> rangeEntry = rangeItr.next(); rangeEntry.getValue().removeIf(factPartition -> !factPartition.getPeriod().equals(updatePeriod)); + rangeEntry.getValue().forEach(factPartition -> { + factPartition.getStorageTables().remove(storageTable); + factPartition.getStorageTables().add(resolvedName); + }); if (rangeEntry.getValue().isEmpty()) { rangeItr.remove(); } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java index c5351965a..f54645406 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java @@ -71,7 +71,7 @@ public void updateFromString() throws LensException { } } - CubeInterface getCube() { + public CubeInterface getCube() { return storageCandidate.getCubeQueryContext().getCube(); } @@ -150,4 +150,13 @@ protected void setMissingExpressions() throws LensException { } } } + + @Override + public int hashCode() { + final int PRIME = 59; + int result = 1; + result = result * PRIME + getStorageCandidate().hashCode(); + result = result * PRIME + getCube().hashCode(); + return result; + } } diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java index 757a87703..510ea0c5b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java @@ -30,6 +30,7 @@ import java.util.ListIterator; import java.util.Map; import java.util.Optional; +import java.util.OptionalDouble; import java.util.Set; import org.apache.lens.cube.metadata.FactPartition; @@ -154,14 +155,18 @@ public Date getEndTime() { } @Override - public double getCost() { + public OptionalDouble getCost() { double cost = 0.0; for (TimeRange timeRange : getCubeQueryContext().getTimeRanges()) { for (Map.Entry entry : getTimeRangeSplit(timeRange).entrySet()) { - cost += entry.getKey().getCost() * entry.getValue().milliseconds() / timeRange.milliseconds(); + if (entry.getKey().getCost().isPresent()) { + cost += entry.getKey().getCost().getAsDouble() *entry.getValue().milliseconds() / timeRange.milliseconds(); + } else { + return OptionalDouble.empty(); + } } } - return cost; + return OptionalDouble.of(cost); } @Override @@ -256,7 +261,9 @@ private String getToString() { * @return */ private Map splitTimeRangeForChildren(TimeRange timeRange) { - children.sort(comparing(Candidate::getCost)); + if (children.stream().map(Candidate::getCost).allMatch(OptionalDouble::isPresent)) { + children.sort(comparing(x -> x.getCost().getAsDouble())); + } Map childrenTimeRangeMap = new HashMap<>(); // Sorted list based on the weights. Set ranges = new HashSet<>(); diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java index 9412f274d..cc0a2e597 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java @@ -48,7 +48,7 @@ public class UnionQueryWriter extends SimpleHQLContext { private Map innerToOuterSelectASTs = new HashMap<>(); private Map innerToOuterHavingASTs = new HashMap<>(); - private Map storageCandidateToSelectAstMap = new HashMap<>(); + private Map storageCandidateToSelectAstMap = new HashMap<>(); private CubeQueryContext cubeql; static final ASTNode DEFAULT_MEASURE_AST; private static final String DEFAULT_MEASURE = "0.0"; @@ -91,8 +91,7 @@ protected void setMissingExpressions() throws LensException { */ private void updateAsts() { for (StorageCandidateHQLContext sc : storageCandidates) { - storageCandidateToSelectAstMap.put(sc.getStorageCandidate().toString(), - new ASTNode(new CommonToken(TOK_SELECT, "TOK_SELECT"))); + storageCandidateToSelectAstMap.put(sc, new ASTNode(new CommonToken(TOK_SELECT, "TOK_SELECT"))); if (sc.getQueryAst().getHavingAST() != null) { cubeql.setHavingAST(sc.getQueryAst().getHavingAST()); } @@ -403,7 +402,7 @@ private void updateInnterSelectASTWithDefault() throws LensException { if (!phrase.hasMeasures(cubeql)) { for (StorageCandidateHQLContext sc : storageCandidates) { ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0); - storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()). + storageCandidateToSelectAstMap.get(sc). addChild(getSelectExpr(exprWithOutAlias, aliasNode, false)); } @@ -412,7 +411,7 @@ private void updateInnterSelectASTWithDefault() throws LensException { for (StorageCandidateHQLContext sc : storageCandidates) { if (sc.getStorageCandidate().getAnswerableMeasurePhraseIndices().contains(phrase.getPosition())) { ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0); - storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()). + storageCandidateToSelectAstMap.get(sc). addChild(getSelectExpr(exprWithOutAlias, aliasNode, false)); } else { ASTNode resolvedExprNode = getAggregateNodesExpression(i); @@ -421,7 +420,7 @@ private void updateInnterSelectASTWithDefault() throws LensException { } else { resolvedExprNode = getSelectExpr(null, null, true); } - storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()). + storageCandidateToSelectAstMap.get(sc). addChild(getSelectExpr(resolvedExprNode, aliasNode, false)); } } @@ -431,7 +430,7 @@ private void updateInnterSelectASTWithDefault() throws LensException { for (StorageCandidateHQLContext sc : storageCandidates) { if (sc.getStorageCandidate().getAnswerableMeasurePhraseIndices().contains(phrase.getPosition())) { ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0); - storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()). + storageCandidateToSelectAstMap.get(sc). addChild(getSelectExpr(exprWithOutAlias, aliasNode, false)); } else { ASTNode resolvedExprNode = getAggregateNodesExpression(i); @@ -440,7 +439,7 @@ private void updateInnterSelectASTWithDefault() throws LensException { } else { resolvedExprNode = getSelectExpr(null, null, true); } - storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()). + storageCandidateToSelectAstMap.get(sc). addChild(getSelectExpr(resolvedExprNode, aliasNode, false)); } } @@ -485,7 +484,7 @@ private void processSelectAndHavingAST() throws LensException { private void processSelectExpression(StorageCandidateHQLContext sc, ASTNode outerSelectAst, ASTNode innerSelectAST, AliasDecider aliasDecider) throws LensException { //ASTNode selectAST = sc.getQueryAst().getSelectAST(); - ASTNode selectAST = storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()); + ASTNode selectAST = storageCandidateToSelectAstMap.get(sc); if (selectAST == null) { return; } diff --git a/lens-cube/src/test/resources/schema/facts/b1fact1.xml b/lens-cube/src/test/resources/schema/facts/b1fact1.xml index b51a34ac5..150003702 100644 --- a/lens-cube/src/test/resources/schema/facts/b1fact1.xml +++ b/lens-cube/src/test/resources/schema/facts/b1fact1.xml @@ -19,7 +19,7 @@ under the License. --> - + diff --git a/lens-dist/pom.xml b/lens-dist/pom.xml index fb49931dc..c2b087626 100644 --- a/lens-dist/pom.xml +++ b/lens-dist/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-dist diff --git a/lens-dist/src/deb/control/postinst b/lens-dist/src/deb/control/postinst new file mode 100644 index 000000000..5d1f9766f --- /dev/null +++ b/lens-dist/src/deb/control/postinst @@ -0,0 +1,67 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Postinst script for lens + +manage_war_expansion () { + + # Lens Server war is expanded into lens server directory. + + WEB_APP_DIR=${mvn.lens.webapp.dir} + LENS_SERVER_DIR=${mvn.lens.server.dir} + + # 1. Remove lens server directory and its contents + rm -rf $LENS_SERVER_DIR + + # 2. Recreate lens server directory + mkdir -p $LENS_SERVER_DIR + + # 3. Expand the newly installed war in lens server directory + cd $LENS_SERVER_DIR + if [ $? -ne 0 ]; then + echo "Could not change directory to->"$LENS_SERVER_DIR + return 1 + fi + + jar -xf $WEB_APP_DIR/lens-server.war + + if [ $? -ne 0 ]; then + echo "War expansion failed" + return 1 + fi + + cd $OLDPWD + if [ $? -ne 0 ]; then + echo "Could not move back from the changed directory" + return 1 + fi + + return 0 + +} + +manage_war_expansion + +if [ $? -ne 0 ]; then + echo "Debian installation unsuccessful" + exit 1 +fi + +exit 0 diff --git a/lens-driver-es/pom.xml b/lens-driver-es/pom.xml index 7823a62d4..882328edc 100644 --- a/lens-driver-es/pom.xml +++ b/lens-driver-es/pom.xml @@ -24,7 +24,7 @@ org.apache.lens apache-lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-driver-es diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml index 51205e915..178a6f68c 100644 --- a/lens-driver-hive/pom.xml +++ b/lens-driver-hive/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT diff --git a/lens-driver-jdbc/pom.xml b/lens-driver-jdbc/pom.xml index 5acd5d257..ad0572390 100644 --- a/lens-driver-jdbc/pom.xml +++ b/lens-driver-jdbc/pom.xml @@ -25,7 +25,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-driver-jdbc diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java index 75153f669..fd6550727 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java @@ -1168,7 +1168,8 @@ protected void constructQuery(String selecttree, String wheretree, String groupb } else { finalJoinClause = factNameAndAlias; } - rewrittenQuery.append("select ").append(selecttree).append(" from "); + //Replace backquotes with double quotes + rewrittenQuery.append("select ").append(selecttree.replaceAll("`", "\"")).append(" from "); if (factInLineQuery.length() != 0) { finalJoinClause = finalJoinClause.substring(finalJoinClause.indexOf(" ")); rewrittenQuery.append(factInLineQuery); diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java index 8e042d1e5..9627673c8 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java @@ -1033,6 +1033,29 @@ public void testEscapeQuote() throws LensException { compareQueries(actual, expected); } + @Test + public void testReplaceBackQuote() throws LensException { + + String query = "select fact.time_key `time_key`,time_dim.day_of_week `day of week`," + + " sum(fact.item_sold) as `total item sold`, count(fact.item_sold) as `total_item_count` " + + "from db.sales_fact as fact " + + "inner join time_dim as time_dim on fact.time_key = time_dim.time_key "; + + SessionState.start(hconf); + + String actual = qtest.rewrite(query, conf, hconf); + String expected = "select ( sales_fact__db_sales_fact_fact . time_key ) as \"time_key\" , " + + "( time_dim___time_dim . day_of_week ) as \"day of week\" , " + + "sum(( sales_fact__db_sales_fact_fact . item_sold )) as \"total item sold\" , " + + "count(( sales_fact__db_sales_fact_fact . item_sold )) as \" total_item_count\" " + + "from db.sales_fact sales_fact__db_sales_fact_fact inner join " + + "(select time_key,day_of_week from time_dim) time_dim___time_dim on " + + "(( sales_fact__db_sales_fact_fact . time_key ) = " + + "( time_dim___time_dim . time_key ))"; + + compareQueries(expected, actual); + } + @Test public void testHavingClause() throws LensException { diff --git a/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml b/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml index ac5e43deb..3fe13f23b 100644 --- a/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml +++ b/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml @@ -68,4 +68,4 @@ interval$2 day), date_add\((.*?)\,\s*([0-9]+\s*)\)=date_add($1\, interval $2 day),positive=sleep Rewriting the HQL to optimized sql queries - \ No newline at end of file + diff --git a/lens-examples/pom.xml b/lens-examples/pom.xml index abf349fe3..9445cac69 100644 --- a/lens-examples/pom.xml +++ b/lens-examples/pom.xml @@ -25,7 +25,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-examples diff --git a/lens-examples/src/main/resources/cube11.xml b/lens-examples/src/main/resources/cube11.xml new file mode 100644 index 000000000..2283939e6 --- /dev/null +++ b/lens-examples/src/main/resources/cube11.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-examples/src/main/resources/cube22.xml b/lens-examples/src/main/resources/cube22.xml new file mode 100644 index 000000000..59b18283d --- /dev/null +++ b/lens-examples/src/main/resources/cube22.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-examples/src/main/resources/cube33.xml b/lens-examples/src/main/resources/cube33.xml new file mode 100644 index 000000000..47b9b1767 --- /dev/null +++ b/lens-examples/src/main/resources/cube33.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lens-ml-dist/pom.xml b/lens-ml-dist/pom.xml index c75e2ab91..1acbf25cc 100644 --- a/lens-ml-dist/pom.xml +++ b/lens-ml-dist/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-ml-dist diff --git a/lens-ml-lib/pom.xml b/lens-ml-lib/pom.xml index 959efe848..a1f18f931 100644 --- a/lens-ml-lib/pom.xml +++ b/lens-ml-lib/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java index ce3db4250..87ed84f5c 100644 --- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java +++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java @@ -37,7 +37,7 @@ import org.apache.lens.ml.impl.MLUtils; import org.apache.lens.ml.server.MLApp; import org.apache.lens.ml.server.MLServiceResource; -import org.apache.lens.server.LensJerseyTest; +//import org.apache.lens.server.LensJerseyTest; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.query.QueryServiceResource; import org.apache.lens.server.session.SessionResource; @@ -52,40 +52,44 @@ import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.media.multipart.MultiPartFeature; import org.testng.Assert; -import org.testng.annotations.AfterTest; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; +//import org.testng.annotations.AfterTest; +//import org.testng.annotations.BeforeMethod; +//import org.testng.annotations.BeforeTest; +//import org.testng.annotations.Test; import lombok.extern.slf4j.Slf4j; @Slf4j -@Test -public class TestMLResource extends LensJerseyTest { +//@Test +public class TestMLResource { //extends LensJerseyTest { private static final String TEST_DB = "default"; private WebTarget mlTarget; private LensMLClient mlClient; - @Override + protected int getTestPort() { + return 10058; + } + + //@Override protected Application configure() { return new MLApp(SessionResource.class, QueryServiceResource.class); } - @Override + //@Override protected void configureClient(ClientConfig config) { config.register(MultiPartFeature.class); } - @Override + //@Override protected URI getBaseUri() { return UriBuilder.fromUri("http://localhost/").port(getTestPort()).path("/lensapi").build(); } - @BeforeTest + //@BeforeTest public void setUp() throws Exception { - super.setUp(); + //super.setUp(); Hive hive = Hive.get(new HiveConf()); Database db = new Database(); db.setName(TEST_DB); @@ -98,9 +102,9 @@ public void setUp() throws Exception { mlClient = new LensMLClient(client); } - @AfterTest + //@AfterTest public void tearDown() throws Exception { - super.tearDown(); + //super.tearDown(); Hive hive = Hive.get(new HiveConf()); try { @@ -112,18 +116,18 @@ public void tearDown() throws Exception { mlClient.close(); } - @BeforeMethod + //@BeforeMethod public void setMLTarget() { - mlTarget = target().path("ml"); + //mlTarget = target().path("ml"); } - @Test + //@Test public void testMLResourceUp() throws Exception { String mlUpMsg = mlTarget.request().get(String.class); Assert.assertEquals(mlUpMsg, MLServiceResource.ML_UP_MESSAGE); } - @Test + //@Test public void testGetAlgos() throws Exception { List algoNames = mlClient.getAlgorithms(); Assert.assertNotNull(algoNames); @@ -144,7 +148,7 @@ public void testGetAlgos() throws Exception { MLUtils.getAlgoName(DecisionTreeAlgo.class)); } - @Test + //@Test public void testGetAlgoParams() throws Exception { Map params = mlClient.getAlgoParamDescription(MLUtils .getAlgoName(DecisionTreeAlgo.class)); @@ -156,7 +160,7 @@ public void testGetAlgoParams() throws Exception { } } - @Test + //@Test public void trainAndEval() throws Exception { log.info("Starting train & eval"); final String algoName = MLUtils.getAlgoName(NaiveBayesAlgo.class); diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java index ff47353d4..1176a03c1 100644 --- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java +++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java @@ -29,7 +29,7 @@ import org.apache.lens.ml.impl.MLRunner; import org.apache.lens.ml.impl.MLTask; import org.apache.lens.ml.server.MLApp; -import org.apache.lens.server.LensJerseyTest; +//import org.apache.lens.server.LensJerseyTest; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.metastore.MetastoreResource; import org.apache.lens.server.query.QueryServiceResource; @@ -40,32 +40,36 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.testng.Assert; -import org.testng.annotations.AfterTest; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; +//import org.testng.annotations.AfterTest; +//import org.testng.annotations.BeforeTest; +//import org.testng.annotations.Test; import lombok.extern.slf4j.Slf4j; -@Test +//@Test @Slf4j -public class TestMLRunner extends LensJerseyTest { +public class TestMLRunner { //extends LensJerseyTest { private static final String TEST_DB = TestMLRunner.class.getSimpleName(); private LensMLClient mlClient; - @Override + protected int getTestPort() { + return 10058; + } + + //@Override protected Application configure() { return new MLApp(SessionResource.class, QueryServiceResource.class, MetastoreResource.class); } - @Override + //@Override protected URI getBaseUri() { return UriBuilder.fromUri("http://localhost/").port(getTestPort()).path("/lensapi").build(); } - @BeforeTest + //@BeforeTest public void setUp() throws Exception { - super.setUp(); + //super.setUp(); Hive hive = Hive.get(new HiveConf()); Database db = new Database(); db.setName(TEST_DB); @@ -78,15 +82,15 @@ public void setUp() throws Exception { mlClient = new LensMLClient(client); } - @AfterTest + //@AfterTest public void tearDown() throws Exception { - super.tearDown(); + //super.tearDown(); Hive hive = Hive.get(new HiveConf()); hive.dropDatabase(TEST_DB); mlClient.close(); } - @Test + //@Test public void trainAndEval() throws Exception { log.info("Starting train & eval"); String algoName = "spark_naive_bayes"; @@ -110,7 +114,7 @@ public void trainAndEval() throws Exception { Assert.assertNotNull(reportID); } - @Test + //@Test public void trainAndEvalFromDir() throws Exception { log.info("Starting train & eval from Dir"); MLRunner runner = new MLRunner(); diff --git a/lens-query-lib/pom.xml b/lens-query-lib/pom.xml index c21e77f78..e743ad7de 100644 --- a/lens-query-lib/pom.xml +++ b/lens-query-lib/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-query-lib diff --git a/lens-regression/pom.xml b/lens-regression/pom.xml index a4f857d9b..30c52f2fd 100644 --- a/lens-regression/pom.xml +++ b/lens-regression/pom.xml @@ -25,7 +25,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-regression diff --git a/lens-server-api/pom.xml b/lens-server-api/pom.xml index 93f79fff4..7989b6e7f 100644 --- a/lens-server-api/pom.xml +++ b/lens-server-api/pom.xml @@ -24,7 +24,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-server-api diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java index 2667ebff2..efba22ad2 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java @@ -159,11 +159,15 @@ private static Priority[] priorities(Object... args) { public void testAllowsLaunchOfQuery(final int currentDriverLaunchedQueries, final boolean expectedCanLaunch) { QueryContext mockCandidateQuery = mock(QueryContext.class); + QueryContext mockCandidateQuery2 = mock(QueryContext.class); EstimatedImmutableQueryCollection mockLaunchedQueries = mock(EstimatedImmutableQueryCollection.class); LensDriver mockDriver = mock(LensDriver.class); + LensDriver mockDriver2 = mock(LensDriver.class); when(mockCandidateQuery.getSelectedDriver()).thenReturn(mockDriver); + when(mockCandidateQuery2.getSelectedDriver()).thenReturn(mockDriver2); when(mockLaunchedQueries.getQueriesCount(mockDriver)).thenReturn(currentDriverLaunchedQueries); + when(mockLaunchedQueries.getQueriesCount(mockDriver2)).thenReturn(1); String actualCanLaunch = constraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries); diff --git a/lens-server/pom.xml b/lens-server/pom.xml index 34a781953..698c9c95c 100644 --- a/lens-server/pom.xml +++ b/lens-server/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT diff --git a/lens-server/src/test/java/org/apache/lens/server/common/FormDataMultiPartFactory.java b/lens-server/src/test/java/org/apache/lens/server/common/FormDataMultiPartFactory.java index 9a1d79d56..13b17ceb2 100644 --- a/lens-server/src/test/java/org/apache/lens/server/common/FormDataMultiPartFactory.java +++ b/lens-server/src/test/java/org/apache/lens/server/common/FormDataMultiPartFactory.java @@ -24,7 +24,7 @@ import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.metastore.ObjectFactory; -import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XFact; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -83,11 +83,11 @@ public static FormDataMultiPart createFormDataMultiPartForSession( } public static FormDataMultiPart createFormDataMultiPartForFact(final LensSessionHandle sessionId, - final XFactTable xFactTable, MediaType mt) { + final XFact xFact, MediaType mt) { final FormDataMultiPart mp = new FormDataMultiPart(); mp.bodyPart(getSessionIdFormDataBodyPart(sessionId, mt)); - mp.bodyPart(getFormDataBodyPart("fact", "fact", cubeObjectFactory.createXFactTable(xFactTable), mt)); + mp.bodyPart(getFormDataBodyPart("fact", "fact", cubeObjectFactory.createXFact(xFact), mt)); return mp; } diff --git a/lens-ship-jars/pom.xml b/lens-ship-jars/pom.xml index 67867a420..1a37468e2 100644 --- a/lens-ship-jars/pom.xml +++ b/lens-ship-jars/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-ship-jars diff --git a/lens-storage-db/pom.xml b/lens-storage-db/pom.xml index 68c6e4677..7107a9a90 100644 --- a/lens-storage-db/pom.xml +++ b/lens-storage-db/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-storage-db diff --git a/lens-ui/pom.xml b/lens-ui/pom.xml index 1185be90a..2035ded58 100644 --- a/lens-ui/pom.xml +++ b/lens-ui/pom.xml @@ -25,7 +25,7 @@ apache-lens org.apache.lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT lens-ui diff --git a/pom.xml b/pom.xml index af7dafa45..c0b81e7dd 100644 --- a/pom.xml +++ b/pom.xml @@ -19,8 +19,7 @@ under the License. --> - + 4.0.0 @@ -128,7 +127,7 @@ org.apache.lens apache-lens - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT Lens pom Unified Analytics Platform @@ -157,9 +156,19 @@ + + InMobiReleases + InMobi release Repository + https://github.com/InMobi/mvn-repo/raw/master/releases + + + InMobiSnapshots + InMobi snapshot Repository + https://github.com/InMobi/mvn-repo/raw/master/snapshots + lens-site - http://lens.apache.org/ + https://inmobi.github.io/grill/ @@ -376,9 +385,9 @@ - scm:git:https://git-wip-us.apache.org/repos/asf/lens.git - scm:git:https://git-wip-us.apache.org/repos/asf/lens.git - https://git-wip-us.apache.org/repos/asf/lens.git + scm:git:https://github.com/InMobi/grill.git + https://github.com/InMobi/grill + scm:git:https://github.com/InMobi/grill.git HEAD @@ -577,6 +586,14 @@ maven-war-plugin ${war.plugin.version} + + org.apache.maven.plugins + maven-release-plugin + 2.5.1 + + ${arguments} + + org.codehaus.mojo findbugs-maven-plugin @@ -724,7 +741,7 @@ org.apache.lens checkstyle - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT @@ -1838,7 +1855,7 @@ org.apache.lens checkstyle - 2.8.0-SNAPSHOT + 2.7.8-inm-SNAPSHOT diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt index 1653e9f2b..de777957d 100644 --- a/src/site/apt/user/cli.apt +++ b/src/site/apt/user/cli.apt @@ -389,48 +389,48 @@ User CLI Commands *--+--+ |<>|<>| *--+--+ -|schema/create schema [--db] \ [--path/--file] \|Parses the specified resource file and executes commands for creation/updation of schema \ | -| |Expected\ structure\ is\ \ | -| |. \ | -| |\|--\ storages \ | -| |\|\ \ \|--\ storage1.xml \ | -| |\|\ \ \|--\ storage2.xml \ | -| |\| \ | -| |\|--\ dimensions \ | -| |\|\ \ \|--\ dim1.xml \ | -| |\|\ \ \|--\ dim2.xml \ | -| |\| \ | -| |\|--\ cubes \ | -| |\|\ \ \|--\ base \ | -| |\|\ \ \|\ \ \|--\ base_cube1.xml \ | -| |\|\ \ \|\ \ \|--\ base_cube2.xml \ | -| |\|\ \ \| \ | -| |\|\ \ \|--\ derived \ | -| |\|\ \ \|\ \ \|--\ derived_cube1.xml \ | -| |\|\ \ \|\ \ \|--\ derived_cube2.xml \ | -| |\|\ \ \| \ | -| |\|\ \ \|--\ independent_cube1.xml \ | -| |\|\ \ \|--\ independent_cube2.xml \ | -| |\| \ | -| |\|--\ dimensiontables \ | -| |\|\ \ \|--\ dimtable1.xml \ | -| |\|\ \ \|--\ dimtable2.xml \ | -| |\| \ | -| |\|--\ dimtables \ | -| |\|\ \ \|--\ dimtable3.xml \ | -| |\|\ \ \|--\ dimtable4.xml \ | -| |\| \ | -| |\|--\ facts \ | -| |\ \ \ \|--\ fact1.xml \ | -| |\ \ \ \|--\ fact2.xml \ | -| | \ | -| | \ | -| |If\ your\ cubes\ are\ divided\ between\ base\ and\ derived\ cubes, \ | -| |it\ makes\ sense\ to\ seperate\ into\ two\ directories,\ since\ derived\ cubes\ can't\ be\ created\ unless\ base\ cube\ exists.\ | -| |In\ the\ other\ case\ you\ can\ keep\ them\ in\ the\ cubes\ directory\ itself. \ | -| |For\ dimtables,\ you\ can\ keep\ your\ schema\ files\ in\ a\ directory\ named\ either\ dimtables\ or\ dimensiontables. \ | -| |Each\ of\ these\ directories\ is\ optional\ and\ the\ order\ of\ processing\ is\ top\ to\ bottom. \ | -| |CLI\ will\ let\ you\ know\ in\ case\ of\ any\ errors\ and\ proceed\ further\ without\ failing\ in\ between. \ | +|schema/create schema [--db] \ [--path/--file] \ [[--type] \] [[--name] \]|Parses the specified resource file and executes commands for creation/updation of schema. If <<>> is provided, only schema types matching that will be worked upon. If <<>> is provided, then only those files that contain the filter value will be worked upon. \ | +| |Expected\ directory\ structure\ is\ \ | +| |. \ | +| |\|--\ storages \ | +| |\|\ \ \|--\ storage1.xml \ | +| |\|\ \ \|--\ storage2.xml \ | +| |\| \ | +| |\|--\ dimensions \ | +| |\|\ \ \|--\ dim1.xml \ | +| |\|\ \ \|--\ dim2.xml \ | +| |\| \ | +| |\|--\ cubes \ | +| |\|\ \ \|--\ base \ | +| |\|\ \ \|\ \ \|--\ base_cube1.xml \ | +| |\|\ \ \|\ \ \|--\ base_cube2.xml \ | +| |\|\ \ \| \ | +| |\|\ \ \|--\ derived \ | +| |\|\ \ \|\ \ \|--\ derived_cube1.xml \ | +| |\|\ \ \|\ \ \|--\ derived_cube2.xml \ | +| |\|\ \ \| \ | +| |\|\ \ \|--\ independent_cube1.xml \ | +| |\|\ \ \|--\ independent_cube2.xml \ | +| |\| \ | +| |\|--\ dimensiontables \ | +| |\|\ \ \|--\ dimtable1.xml \ | +| |\|\ \ \|--\ dimtable2.xml \ | +| |\| \ | +| |\|--\ dimtables \ | +| |\|\ \ \|--\ dimtable3.xml \ | +| |\|\ \ \|--\ dimtable4.xml \ | +| |\| \ | +| |\|--\ facts \ | +| |\ \ \ \|--\ fact1.xml \ | +| |\ \ \ \|--\ fact2.xml \ | +| | \ | +| | \ | +| |If\ your\ cubes\ are\ divided\ between\ base\ and\ derived\ cubes, \ | +| |it\ makes\ sense\ to\ seperate\ into\ two\ directories,\ since\ derived\ cubes\ can't\ be\ created\ unless\ base\ cube\ exists. \ | +| |In\ the\ other\ case\ you\ can\ keep\ them\ in\ the\ cubes\ directory\ itself. \ | +| |For\ dimtables,\ you\ can\ keep\ your\ schema\ files\ in\ a\ directory\ named\ either\ dimtables\ or\ dimensiontables. \ | +| |Each\ of\ these\ directories\ is\ optional\ and\ the\ order\ of\ processing\ is\ top\ to\ bottom. \ | +| |CLI\ will\ let\ you\ know\ in\ case\ of\ any\ errors\ and\ proceed\ further\ without\ failing\ in\ between. \ | *--+--+ <>