diff --git a/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java b/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java index 807daa3cb7..23a86af37b 100644 --- a/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java +++ b/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java @@ -129,9 +129,9 @@ protected void runOneIteration() throws Exception { } for ( JobDescriptor jd : activeJobs ) { - logger.info( "Submitting work for {}", jd ); + logger.debug( "Submitting work for {}", jd ); submitWork( jd ); - logger.info( "Work submitted for {}", jd ); + logger.debug( "Work submitted for {}", jd ); } } } @@ -228,7 +228,7 @@ public Void call() throws Exception { // TODO wrap and throw specifically typed exception for onFailure, // needs jobId - logger.info( "Starting job {} with execution data {}", job, execution ); + logger.debug( "Starting job {} with execution data {}", job, execution ); job.execute( execution ); @@ -259,7 +259,7 @@ public void onSuccess( Void param ) { //TODO, refactor into the execution itself for checking if done if ( execution.getStatus() == Status.IN_PROGRESS ) { - logger.info( "Successful completion of bulkJob {}", execution ); + logger.debug( "Successful completion of bulkJob {}", execution ); execution.completed(); } diff --git a/stack/core/src/main/java/org/apache/usergrid/batch/service/SchedulerServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/batch/service/SchedulerServiceImpl.java index af933f6a8c..9b3714c8ee 100644 --- a/stack/core/src/main/java/org/apache/usergrid/batch/service/SchedulerServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/batch/service/SchedulerServiceImpl.java @@ -305,7 +305,7 @@ else if ( jobStatus == Status.DEAD ) { getEm().update( data ); } - logger.info( "Updating stats for job {}", data.getJobName() ); + logger.debug( "Updating stats for job {}", data.getJobName() ); getEm().update( stat ); } diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java index ef4bb048b0..af297f2c7f 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java @@ -20,6 +20,8 @@ import com.google.inject.assistedinject.FactoryModuleBuilder; import com.google.inject.multibindings.Multibinder; import org.apache.usergrid.corepersistence.asyncevents.*; +import org.apache.usergrid.corepersistence.export.ExportService; +import org.apache.usergrid.corepersistence.export.ExportServiceImpl; import org.apache.usergrid.corepersistence.index.*; import org.apache.usergrid.corepersistence.migration.CoreMigration; import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin; @@ -137,6 +139,8 @@ public void configureMigrationProvider() { bind( ReIndexService.class ).to( ReIndexServiceImpl.class ); + bind( ExportService.class ).to( ExportServiceImpl.class ); + install( new FactoryModuleBuilder().implement( AggregationService.class, AggregationServiceImpl.class ) .build( AggregationServiceFactory.class ) ); diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index 10718425fb..cdb4fc7a1f 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -3089,52 +3089,16 @@ public void addIndex(final String newIndexName,final int shards,final int replic managerCache.getEntityIndex(applicationScope).addIndex(newIndexName, shards, replicas, writeConsistency); } + @Override public void initializeIndex(){ managerCache.getEntityIndex(applicationScope).initialize(); } - /** - * TODO, these 3 methods are super janky. During refactoring we should clean this model up - */ - public EntityIndex.IndexRefreshCommandInfo refreshIndex() { - try { - long start = System.currentTimeMillis(); - // refresh special indexes without calling EntityManager refresh because stack overflow - Map map = new org.apache.usergrid.persistence.index.utils.MapUtils.HashMapBuilder<>(); - map.put("some prop", "test"); - boolean hasFinished = false; - Entity refreshEntity = create("refresh", map); - EntityIndex.IndexRefreshCommandInfo indexRefreshCommandInfo - = managerCache.getEntityIndex(applicationScope).refreshAsync().toBlocking().first(); - try { - for (int i = 0; i < 20; i++) { - if (searchCollection( - new SimpleEntityRef( - org.apache.usergrid.persistence.entities.Application.ENTITY_TYPE, getApplicationId()), - InflectionUtils.pluralize("refresh"), - Query.fromQL("select * where uuid='" + refreshEntity.getUuid() + "'") - ).size() > 0 - ) { - hasFinished = true; - break; - } - int sleepTime = 500; - logger.info("Sleeping {} ms during refreshIndex", sleepTime); - Thread.sleep(sleepTime); - indexRefreshCommandInfo - = managerCache.getEntityIndex(applicationScope).refreshAsync().toBlocking().first(); - } - if(!hasFinished){ - throw new RuntimeException("Did not find entity {} during refresh. uuid->"+refreshEntity.getUuid()); - } - }finally { - delete(refreshEntity); - } - Thread.sleep(100); - - return indexRefreshCommandInfo; + public EntityIndex.IndexRefreshCommandInfo refreshIndex() { + try { + return managerCache.getEntityIndex(applicationScope).refreshAsync().toBlocking().first(); } catch (Exception e) { throw new RuntimeException("refresh failed",e); } diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java index 4a759ac110..c02ca7deea 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java @@ -629,6 +629,7 @@ public Results searchCollection( String collectionName, Query query ) throws Exc final Query toExecute = adjustQuery( query ); final Optional queryString = query.isGraphSearch()? Optional.absent(): query.getQl(); final Id ownerId = headEntity.asId(); + final boolean analyzeOnly = query.getAnalyzeOnly(); if(query.getLevel() == Level.IDS ){ @@ -642,6 +643,8 @@ protected Observable> buildNewResultsPage( new CollectionSearch( applicationScope, ownerId, collectionName, collection.getType(), toExecute.getLimit(), queryString, cursor ); + search.setAnalyzeOnly(analyzeOnly); + return collectionService.searchCollectionIds( search ); } }.next(); @@ -658,6 +661,8 @@ protected Observable> buildNewResultsPage( final Opti final ConnectionSearch search = new ConnectionSearch( applicationScope, sourceId, entityType, connection, toExecute.getLimit(), queryString, cursor, isConnecting ); + search.setAnalyzeOnly(analyzeOnly); return connectionService.searchConnectionAsRefs( search ); } }.next(); @@ -966,6 +974,7 @@ protected Observable getApplicationScope(); + +} diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportRequestBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportRequestBuilderImpl.java new file mode 100644 index 0000000000..73fcec4f4a --- /dev/null +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportRequestBuilderImpl.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.usergrid.corepersistence.export; + + +import com.google.common.base.Optional; +import org.apache.usergrid.corepersistence.util.CpNamingUtils; +import org.apache.usergrid.persistence.core.scope.ApplicationScope; + +import java.util.UUID; +import java.util.concurrent.TimeUnit; + + +/** + * Index service request builder + */ +public class ExportRequestBuilderImpl implements ExportRequestBuilder { + + private Optional withApplicationId = Optional.absent(); + private Optional withCollectionName = Optional.absent(); + private Optional cursor = Optional.absent(); + private Optional updateTimestamp = Optional.absent(); + private Optional delayTimer = Optional.absent(); + private Optional timeUnitOptional = Optional.absent(); + + + /*** + * + * @param applicationId The application id + * @return + */ + @Override + public ExportRequestBuilder withApplicationId(final UUID applicationId ) { + this.withApplicationId = Optional.fromNullable( applicationId ); + return this; + } + + @Override + public Optional getApplicationScope() { + + if ( this.withApplicationId.isPresent() ) { + return Optional.of( CpNamingUtils.getApplicationScope( withApplicationId.get() ) ); + } + + return Optional.absent(); + } + +} diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportService.java new file mode 100644 index 0000000000..0c7789bc74 --- /dev/null +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportService.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.usergrid.corepersistence.export; + + +import java.io.IOException; +import java.io.OutputStream; + +/** + * An interface for exporting all entities within an application + */ +public interface ExportService { + + + /** + * Perform an application export into the provided OutputStream + * + * @param exportRequestBuilder The builder to build the request + */ + void export(final ExportRequestBuilder exportRequestBuilder, OutputStream stream) throws RuntimeException; + + + /** + * Generate a builder for the export request + */ + ExportRequestBuilder getBuilder(); + + + enum Status{ + STARTED, INPROGRESS, COMPLETE, UNKNOWN; + } +} diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java new file mode 100644 index 0000000000..e561ac67f1 --- /dev/null +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/export/ExportServiceImpl.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.usergrid.corepersistence.export; + + +import com.google.common.base.Optional; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.apache.usergrid.corepersistence.ManagerCache; +import org.apache.usergrid.corepersistence.index.*; +import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable; +import org.apache.usergrid.corepersistence.util.CpEntityMapUtils; +import org.apache.usergrid.corepersistence.util.CpNamingUtils; +import org.apache.usergrid.corepersistence.util.ObjectJsonSerializer; +import org.apache.usergrid.persistence.collection.EntityCollectionManager; +import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory; +import org.apache.usergrid.persistence.collection.EntitySet; +import org.apache.usergrid.persistence.collection.serialization.impl.EntitySetImpl; +import org.apache.usergrid.persistence.core.scope.ApplicationScope; +import org.apache.usergrid.persistence.graph.GraphManager; +import org.apache.usergrid.persistence.graph.SearchByEdgeType; +import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType; +import org.apache.usergrid.persistence.map.MapManager; +import org.apache.usergrid.persistence.map.MapManagerFactory; +import org.apache.usergrid.persistence.map.MapScope; +import org.apache.usergrid.persistence.map.impl.MapScopeImpl; +import org.apache.usergrid.persistence.model.entity.Entity; + +import org.apache.usergrid.persistence.model.entity.Id; +import org.apache.usergrid.utils.InflectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import rx.Observable; +import rx.schedulers.Schedulers; + + +import java.io.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + + +@Singleton +public class ExportServiceImpl implements ExportService { + + private static final Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class ); + + private final AllEntityIdsObservable allEntityIdsObservable; + private final EntityCollectionManagerFactory entityCollectionManagerFactory; + private final ManagerCache managerCache; + private final ObjectJsonSerializer jsonSerializer = ObjectJsonSerializer.INSTANCE; + private final int exportVersion = 1; + private final String keyTotalEntityCount = "__totalEntityCount__"; + + + + @Inject + public ExportServiceImpl(final AllEntityIdsObservable allEntityIdsObservable, + final ManagerCache managerCache, + final EntityCollectionManagerFactory entityCollectionManagerFactory) { + this.allEntityIdsObservable = allEntityIdsObservable; + this.entityCollectionManagerFactory = entityCollectionManagerFactory; + this.managerCache = managerCache; + } + + + @Override + public void export(final ExportRequestBuilder exportRequestBuilder, OutputStream stream) throws RuntimeException { + + final ZipOutputStream zipOutputStream = new ZipOutputStream(stream); + zipOutputStream.setLevel(9); // best compression to reduce the amount of data to stream over the wire + + final ApplicationScope appScope = exportRequestBuilder.getApplicationScope().get(); + final Observable applicationScopes = Observable.just(appScope); + + final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager(appScope); + + GraphManager gm = managerCache.getGraphManager( appScope ); + + final AtomicInteger entityFileCount = new AtomicInteger(); // entities are batched into files + final AtomicInteger connectionCount = new AtomicInteger(); + final Map collectionStats = new HashMap<>(); + collectionStats.put(keyTotalEntityCount, new AtomicInteger()); + final Map infoMap = new HashMap<>(); + infoMap.put("application", appScope.getApplication().getUuid().toString()); + infoMap.put("exportVersion", exportVersion); + infoMap.put("exportStarted", System.currentTimeMillis()); + + logger.info("Starting export of application: {}", appScope.getApplication().getUuid().toString()); + + allEntityIdsObservable.getEdgesToEntities( applicationScopes, Optional.absent(), Optional.absent() ) + .buffer(500) + .map( edgeScopes -> { + + List entityIds = new ArrayList<>(); + edgeScopes.forEach( edgeScope -> { + if (edgeScope.getEdge().getTargetNode() != null) { + logger.debug("adding entity to list: {}", edgeScope.getEdge().getTargetNode()); + entityIds.add(edgeScope.getEdge().getTargetNode()); + } + }); + + return entityIds; + + }) + .flatMap( entityIds -> { + + logger.debug("entityIds: {}", entityIds); + + // batch load the entities + EntitySet entitySet = ecm.load(entityIds).toBlocking().lastOrDefault(new EntitySetImpl(0)); + + final String filenameWithPath = "entities/entities." + entityFileCount.get() + ".json"; + + try { + + logger.debug("adding zip entry: {}", filenameWithPath); + zipOutputStream.putNextEntry(new ZipEntry(filenameWithPath)); + + entitySet.getEntities().forEach(mvccEntity -> { + + if (mvccEntity.getEntity().isPresent()) { + Map entityMap = CpEntityMapUtils.toMap(mvccEntity.getEntity().get()); + + try { + + collectionStats.putIfAbsent(mvccEntity.getId().getType(), new AtomicInteger()); + collectionStats.get(mvccEntity.getId().getType()).incrementAndGet(); + collectionStats.get(keyTotalEntityCount).incrementAndGet(); + + logger.debug("writing and flushing entity {} to zip stream for file: {}", mvccEntity.getId().getUuid().toString(), filenameWithPath); + zipOutputStream.write(jsonSerializer.toString(entityMap).getBytes()); + zipOutputStream.write("\n".getBytes()); + zipOutputStream.flush(); // entities can be large, flush after each + + + } catch (IOException e) { + logger.warn("unable to write entry in zip stream for entityId: {}", mvccEntity.getId()); + throw new RuntimeException("Unable to export data. Error writing to stream."); + + } + } else { + logger.warn("entityId {} did not have corresponding entity, not writing", mvccEntity.getId()); + } + }); + + zipOutputStream.closeEntry(); + entityFileCount.incrementAndGet(); + + }catch (IOException e){ + throw new RuntimeException("Unable to export data. Error writing to stream."); + } + + return Observable.from(entitySet.getEntities()); + + }) + .doOnNext( mvccEntity -> { + + gm.getEdgeTypesFromSource(CpNamingUtils.createConnectionTypeSearch(mvccEntity.getId())) + .flatMap(emittedEdgeType -> { + + logger.debug("loading edges of type {} from node {}", emittedEdgeType, mvccEntity.getId()); + return gm.loadEdgesFromSource(new SimpleSearchByEdgeType(mvccEntity.getId(), + emittedEdgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent())); + + }) + .doOnNext(markedEdge -> { + + if (!markedEdge.isDeleted() && !markedEdge.isTargetNodeDeleted() && markedEdge.getTargetNode() != null ) { + + // doing the load to just again make sure bad connections are not exported + Entity entity = ecm.load(markedEdge.getTargetNode()).toBlocking().lastOrDefault(null); + + if (entity != null) { + + try { + // since a single stream is being written, and connecitons are loaded per entity, + // it cannot easily be batched eventlyinto files so write them separately + final String filenameWithPath = "connections/" + + markedEdge.getSourceNode().getUuid().toString()+"_" + + CpNamingUtils.getConnectionNameFromEdgeName(markedEdge.getType()) + "_" + + markedEdge.getTargetNode().getUuid().toString() + ".json"; + + logger.debug("adding zip entry: {}", filenameWithPath); + zipOutputStream.putNextEntry(new ZipEntry(filenameWithPath)); + + + final Map connectionMap = new HashMap(1){{ + put("sourceNodeUUID", markedEdge.getSourceNode().getUuid().toString() ); + put("relationship", CpNamingUtils.getConnectionNameFromEdgeName(markedEdge.getType()) ); + put("targetNodeUUID", markedEdge.getTargetNode().getUuid().toString()); + }}; + + logger.debug("writing and flushing connection to zip stream: {}", jsonSerializer.toString(connectionMap).getBytes()); + zipOutputStream.write(jsonSerializer.toString(connectionMap).getBytes()); + + zipOutputStream.closeEntry(); + zipOutputStream.flush(); + + connectionCount.incrementAndGet(); + + + } catch (IOException e) { + logger.warn("Unable to create entry in zip export for edge {}", markedEdge.toString()); + throw new RuntimeException("Unable to export data. Error writing to stream."); + } + } else { + logger.warn("Exported connection has a missing target node, not creating connection in export. Edge: {}", markedEdge); + } + } + + }).toBlocking().lastOrDefault(null); + }) + .doOnCompleted(() -> { + + infoMap.put("exportFinished", System.currentTimeMillis()); + + + try { + + zipOutputStream.putNextEntry(new ZipEntry("metadata.json")); + zipOutputStream.write(jsonSerializer.toString(infoMap).getBytes()); + zipOutputStream.closeEntry(); + + zipOutputStream.putNextEntry(new ZipEntry("stats.json")); + Map stats = new HashMap<>(); + stats.put("totalEntities", collectionStats.get(keyTotalEntityCount).get()); + stats.put("totalConnections", connectionCount.get()); + collectionStats.remove(keyTotalEntityCount); + stats.put("collectionCounts", new HashMap(collectionStats.size()){{ + collectionStats.forEach( (collection,count) -> { + put(InflectionUtils.pluralize(collection),count.get()); + }); + }}); + zipOutputStream.write(jsonSerializer.toString(stats).getBytes()); + zipOutputStream.closeEntry(); + + logger.debug("closing zip stream"); + zipOutputStream.close(); + + logger.info("Finished export of application: {}", appScope.getApplication().getUuid().toString()); + + + } catch (IOException e) { + throw new RuntimeException("Unable to export data due to inability to close zip stream."); + } + + }) + .subscribeOn( Schedulers.io() ).toBlocking().lastOrDefault(null); + } + + + @Override + public ExportRequestBuilder getBuilder() { + return new ExportRequestBuilderImpl(); + } + +} + + diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java index 85e9069996..a7f9ad92a0 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java @@ -117,12 +117,13 @@ public IdBuilder traverseConnection( final String connectionName, final Optional * @param collectionName The name of the collection * @param ql The user's query to execute * @param entityType The type of the entity + * @param analyzeOnly * @return Candidate results */ - public CandidateBuilder searchCollection( final String collectionName, final String ql, final String entityType ) { + public CandidateBuilder searchCollection(final String collectionName, final String ql, final String entityType, boolean analyzeOnly) { final Pipeline> newFilter = pipeline.withFilter( filterFactory.searchCollectionFilter( - ql, collectionName, entityType ) ); + ql, collectionName, entityType, analyzeOnly ) ); return new CandidateBuilder( newFilter, filterFactory ); } @@ -133,13 +134,14 @@ public CandidateBuilder searchCollection( final String collectionName, final Str * @param connectionName The connection name to search * @param ql The query to execute * @param entityType The optional type of entity. If this is absent, all entity types in the connection will be searched + * @param analyzeOnly * @return Candidate results */ - public CandidateBuilder searchConnection( final String connectionName, final String ql , final Optional entityType) { + public CandidateBuilder searchConnection(final String connectionName, final String ql, final Optional entityType, boolean analyzeOnly) { final Pipeline> newFilter = pipeline.withFilter( filterFactory.searchConnectionFilter( - ql, connectionName, entityType ) ); + ql, connectionName, entityType,analyzeOnly ) ); return new CandidateBuilder( newFilter, filterFactory ); } diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java index 4b615d840c..7b61b3d794 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java @@ -96,7 +96,8 @@ ReadGraphConnectionByTypeFilter readGraphConnectionByTypeFilter( */ SearchCollectionFilter searchCollectionFilter( @Assisted( "query" ) final String query, @Assisted( "collectionName" ) final String collectionName, - @Assisted( "entityType" ) final String entityType ); + @Assisted( "entityType" ) final String entityType, + @Assisted( "analyzeOnly") final boolean analyzeOnly); /** @@ -108,8 +109,8 @@ SearchCollectionFilter searchCollectionFilter( @Assisted( "query" ) final String */ SearchConnectionFilter searchConnectionFilter( @Assisted( "query" ) final String query, @Assisted( "connectionName" ) final String connectionName, - @Assisted( "connectedEntityType" ) - final Optional connectedEntityType ); + @Assisted( "connectedEntityType" ) final Optional connectedEntityType, + @Assisted( "analyzeOnly") final boolean analyzeOnly); /** diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java index 4bf723ecf5..9f6be2abd1 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java @@ -23,6 +23,8 @@ import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory; import org.apache.usergrid.persistence.Schema; import org.apache.usergrid.persistence.index.*; +import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerEnforcementException; +import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,19 +56,21 @@ public abstract class AbstractElasticSearchFilter extends AbstractPathFilter> call( final Observable fieldMappingCollection = candidateResults.getGetFieldMappings(); @@ -156,8 +160,11 @@ public Observable> call( final Observable connectedEntityType ) { - super( entityIndexFactory, metricsFactory, indexLocationStrategyFactory, query ); + @Assisted( "connectedEntityType" ) final Optional connectedEntityType, + @Assisted( "analyzeOnly") final boolean analyzeOnly) { + super( entityIndexFactory, metricsFactory, indexLocationStrategyFactory, query, analyzeOnly); this.connectionName = connectionName; this.connectedEntityType = connectedEntityType; diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java index 44010dbe28..6588486ca5 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java @@ -73,7 +73,7 @@ public long getApplicationSize(ApplicationScope applicationScope) { MathObservable.sumLong( graphManager.getEdgeTypesFromSource(new SimpleSearchEdgeType(applicationScope.getApplication(), CpNamingUtils.EDGE_COLL_PREFIX, Optional.absent())) .map(type -> CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), type)) - .map(edge -> entityIndex.getEntitySize(edge)) + .map(edge -> entityIndex.getTotalEntitySizeInBytes(edge)) ), sumTimer).toBlocking().last(); return sum.longValue(); @@ -90,7 +90,7 @@ public Map getEachCollectionSize(ApplicationScope applicationScope { SearchEdge edge = CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), type); final String collectionName = CpNamingUtils.getCollectionNameFromEdgeName(type); - long sumType = entityIndex.getEntitySize(edge); + long sumType = entityIndex.getTotalEntitySizeInBytes(edge); map.put(collectionName,sumType); }) ) @@ -103,7 +103,7 @@ public Map getEachCollectionSize(ApplicationScope applicationScope public long getSize(ApplicationScope applicationScope, SearchEdge edge) { final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope); EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy); - return entityIndex.getEntitySize(edge); + return entityIndex.getTotalEntitySizeInBytes(edge); } @Override diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java index 602a5b60b6..6240028390 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java @@ -41,6 +41,7 @@ public enum Level { private final Optional query; private final Optional cursor; private Level level = Level.ALL; + private boolean analyzeOnly; public CollectionSearch( final ApplicationScope applicationScope, final Id collectionOwnerId, final String @@ -53,6 +54,7 @@ public CollectionSearch( final ApplicationScope applicationScope, final Id colle this.limit = limit; this.query = query; this.cursor = cursor; + this.analyzeOnly = false; } @@ -93,4 +95,12 @@ public Id getCollectionOwnerId() { public void setResultsLevel(Level level){ this.level = level; } public Level getResultsLevel(){ return level; } + + public boolean getAnalyzeOnly() { + return analyzeOnly; + } + + public void setAnalyzeOnly(final boolean analyzeOnly){ + this.analyzeOnly = analyzeOnly; + } } diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java index 9244315e50..7684050ed8 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java @@ -68,7 +68,7 @@ public Observable> searchCollection( final CollectionSearch results = pipelineBuilder.traverseCollection( collectionName ).loadEntities(); } else { - results = pipelineBuilder.searchCollection( collectionName, query.get(),search.getEntityType()).loadEntities(); + results = pipelineBuilder.searchCollection( collectionName, query.get(),search.getEntityType(), search.getAnalyzeOnly()).loadEntities(); } diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionSearch.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionSearch.java index 8ad57fbdcd..98f36bea43 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionSearch.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionSearch.java @@ -37,6 +37,8 @@ public class ConnectionSearch { private final Optional query; private final Optional cursor; private final boolean isConnecting; + private boolean analyzeOnly; + public ConnectionSearch( final ApplicationScope applicationScope, final Id sourceNodeId, final Optional entityType, @@ -50,6 +52,8 @@ public ConnectionSearch( final ApplicationScope applicationScope, final Id sourc this.query = query; this.cursor = cursor; this.isConnecting = isConnecting; + this.analyzeOnly = false; + } @@ -90,4 +94,12 @@ public Optional getEntityType() { public boolean getIsConnecting(){ return isConnecting; } + + public boolean getAnalyzeOnly() { + return analyzeOnly; + } + + public void setAnalyzeOnly(final boolean analyzeOnly){ + this.analyzeOnly = analyzeOnly; + } } diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java index 926c67627e..7c70a3d4f9 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImpl.java @@ -106,7 +106,7 @@ public Observable> searchConnection( final ConnectionSearch else { results = - pipelineBuilder.searchConnection( search.getConnectionName(), query.get(), search.getEntityType() ) + pipelineBuilder.searchConnection( search.getConnectionName(), query.get(), search.getEntityType(), search.getAnalyzeOnly() ) .loadEntities(); } @@ -135,7 +135,7 @@ public Observable> searchConnectionAsRefs( final Conn } else { traversedIds = - pipelineBuilder.searchConnection( connectionName, query.get(), search.getEntityType() ).loadIds(); + pipelineBuilder.searchConnection( connectionName, query.get(), search.getEntityType(), search.getAnalyzeOnly() ).loadIds(); } //create connection refs diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/ObjectJsonSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/ObjectJsonSerializer.java index 4e5873a443..b704afac0c 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/ObjectJsonSerializer.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/ObjectJsonSerializer.java @@ -77,6 +77,23 @@ public String toString( final T toSerialize ) { return stringValue; } + public String toString( final T toSerialize ) { + + Preconditions.checkNotNull( toSerialize, "toSerialize must not be null" ); + final String stringValue; + //mark this version as empty + + //Convert to internal entity map + try { + stringValue = MAPPER.writeValueAsString( toSerialize ); + } + catch ( JsonProcessingException jpe ) { + throw new RuntimeException( "Unable to serialize entity", jpe ); + } + + return stringValue; + } + public T fromString( final String value, final Class toSerialize ) { diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java index d68c085345..900bda51ea 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java @@ -78,6 +78,7 @@ public enum Level { private String collection; private String ql; private Collection selectFields; + private boolean analyzeOnly = false; private static ObjectMapper mapper = new ObjectMapper(); @@ -120,6 +121,7 @@ public Query( Query q ) { ? new ArrayList<>( q.counterFilters ) : null; collection = q.collection; level = q.level; + analyzeOnly = q.analyzeOnly; } @@ -235,6 +237,7 @@ public static Query fromQueryParams( Map> params ) } if ( ql != null ) { + ql = ql.replace("+", "%2b"); // ql string supports literal + symbol, encode so it will decode correctly later q = Query.fromQL( decode( ql ) ); } @@ -481,6 +484,13 @@ public void setPermissions( List permissions ) { this.permissions = permissions; } + public void setAnalyzeOnly(final boolean analyzeOnly){ + this.analyzeOnly = analyzeOnly; + } + + public boolean getAnalyzeOnly(){ + return analyzeOnly; + } public boolean isMergeSelectResults() { return mergeSelectResults; diff --git a/stack/core/src/main/java/org/apache/usergrid/utils/JsonUtils.java b/stack/core/src/main/java/org/apache/usergrid/utils/JsonUtils.java index 3bb96f9c0c..6c1875b099 100644 --- a/stack/core/src/main/java/org/apache/usergrid/utils/JsonUtils.java +++ b/stack/core/src/main/java/org/apache/usergrid/utils/JsonUtils.java @@ -83,7 +83,7 @@ public static String mapToJsonString( Object obj ) { /** Converts object to JSON string, throws runtime exception JsonWriteException on failure. */ public static String mapToFormattedJsonString( Object obj ) { try { - return indentObjectMapper.writeValueAsString( obj ); + return indentObjectMapper.writerWithDefaultPrettyPrinter().writeValueAsString( obj ); } catch ( Throwable t ) { if (logger.isDebugEnabled()) { diff --git a/stack/core/src/test/java/org/apache/usergrid/Application.java b/stack/core/src/test/java/org/apache/usergrid/Application.java index 378a4f7a63..102ee9cbe1 100644 --- a/stack/core/src/test/java/org/apache/usergrid/Application.java +++ b/stack/core/src/test/java/org/apache/usergrid/Application.java @@ -152,7 +152,9 @@ public interface Application extends TestRule { public void remove( EntityRef entityRef ) throws Exception; - public void refreshIndex(); + public void waitForQueueDrainAndRefreshIndex(int waitTimeMillis); + + public void waitForQueueDrainAndRefreshIndex(); /** * Get the entity manager diff --git a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java b/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java index 9046f021e2..f505eadc36 100644 --- a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java +++ b/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java @@ -181,8 +181,8 @@ public void createApplication(final String orgName, final String appName) throws logger.info( "Created new application {} in organization {}", appName, orgName ); -// //wait for the index before proceeding -// em.refreshIndex(); + //wait for the index before proceeding + waitForQueueDrainAndRefreshIndex(500); } @@ -223,19 +223,21 @@ public Entity get( UUID id, String type ) throws Exception { return em.get( new SimpleEntityRef( type, id ) ); } - @Override - public synchronized void refreshIndex() { - //Insert test entity and find it - setup.getEmf().refreshIndex(CpNamingUtils.getManagementApplicationId().getUuid()); - - if (!em.getApplicationId().equals(CpNamingUtils.getManagementApplicationId().getUuid())) { - setup.getEmf().refreshIndex(em.getApplicationId()); + public synchronized void waitForQueueDrainAndRefreshIndex(int waitTimeMillis) { + try{ + Thread.sleep(waitTimeMillis); + } catch (InterruptedException e ){ + //noop } - em.refreshIndex(); } + @Override + public synchronized void waitForQueueDrainAndRefreshIndex() { + waitForQueueDrainAndRefreshIndex(750); + } + @Override public EntityManager getEntityManager() { diff --git a/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java b/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java index 64b001c634..bd6ae3ead9 100644 --- a/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java +++ b/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java @@ -156,16 +156,28 @@ public TestEntityIndex getEntityIndex(){ @Override public void refresh(UUID appId){ try { - Thread.sleep(50); - } catch (InterruptedException ie){ + Thread.sleep(125); + + } catch (InterruptedException ie){ + //noop } + emf.refreshIndex(appId); + } + + @Override + public void waitForQueueDrainAndRefresh(UUID appId, int waitTimeMillis){ try { - Thread.sleep(50); - } catch (InterruptedException ie){ + Thread.sleep(waitTimeMillis); + + } catch (InterruptedException ie){ + //noop } + + emf.refreshIndex(appId); + } } diff --git a/stack/core/src/test/java/org/apache/usergrid/TestEntityIndex.java b/stack/core/src/test/java/org/apache/usergrid/TestEntityIndex.java index 7da187a234..e5e979edc3 100644 --- a/stack/core/src/test/java/org/apache/usergrid/TestEntityIndex.java +++ b/stack/core/src/test/java/org/apache/usergrid/TestEntityIndex.java @@ -26,4 +26,5 @@ */ public interface TestEntityIndex { void refresh(UUID appId); + void waitForQueueDrainAndRefresh(UUID appId, int waitTimeMillis); } diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/AggregationServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/AggregationServiceTest.java index 9f1c9a4f75..55ce26ed18 100644 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/AggregationServiceTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/AggregationServiceTest.java @@ -48,8 +48,8 @@ public void testEntitySize() throws Exception { props.put("name", "myname"); Entity entity1 = this.app.getEntityManager().create("test", props); Entity entity2 = this.app.getEntityManager().create("test2", props); - this.app.refreshIndex(); - Thread.sleep(500); + + this.app.waitForQueueDrainAndRefreshIndex(500); long sum = aggregationService.getApplicationSize(applicationScope); @@ -57,23 +57,24 @@ public void testEntitySize() throws Exception { Assert.assertTrue(sum > (entity1.getSize() + entity2.getSize())); long sum1 = aggregationService.getSize(applicationScope, CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), "tests")); - Assert.assertEquals(sum1, entity1.getSize()); + Assert.assertEquals(entity1.getSize(), sum1); long sum2 = aggregationService.getSize(applicationScope, CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), "test2s")); - Assert.assertEquals(sum2, entity2.getSize()); + Assert.assertEquals(entity2.getSize(), sum2); props = new HashMap<>(); props.put("test", 1234); props.put("name", "myname2"); Entity entity3 = this.app.getEntityManager().create("test", props); - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(500); + long sum3 = aggregationService.getSize(applicationScope, CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), "tests")); - Assert.assertEquals(sum3, entity1.getSize() + entity3.getSize()); + Assert.assertEquals(entity1.getSize() + entity3.getSize(), sum3); Map sumEach = aggregationService.getEachCollectionSize(applicationScope); Assert.assertTrue(sumEach.containsKey("tests") && sumEach.containsKey("test2s")); - Assert.assertEquals(sum3, (long) sumEach.get("tests")); + Assert.assertEquals((long) sumEach.get("tests"), sum3); } } diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java index 65c373fb4b..0abd7a2c9a 100644 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java @@ -20,7 +20,6 @@ import java.util.HashMap; import java.util.List; import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -105,7 +104,7 @@ public void testUpdateVersioning() throws Exception { Entity thing = em.create("thing", new HashMap() {{ put("name", "thing1"); }}); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Thread.sleep(1000); assertEquals(1, queryCollectionCp("things", "thing", "select *").size()); @@ -116,7 +115,7 @@ public void testUpdateVersioning() throws Exception { em.updateProperties(thing, new HashMap() {{ put("stuff", "widget"); }}); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Thread.sleep(1000); org.apache.usergrid.persistence.model.entity.Entity cpUpdated = getCpEntity(thing); @@ -161,7 +160,7 @@ public void testCleanupOnDelete() throws Exception { }})); Thread.sleep( writeDelayMs ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); CandidateResults crs = queryCollectionCp( "things", "thing", "select *"); Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() ); @@ -210,7 +209,7 @@ public void testCleanupOnDelete() throws Exception { Thread.sleep(250); // delete happens asynchronously, wait for some time //refresh the app index - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Thread.sleep(250); // refresh happens asynchronously, wait for some time @@ -231,7 +230,7 @@ public void testCleanupOnDelete() throws Exception { } }); //refresh the app index - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); crs = queryCollectionCp("things", "thing", "select *"); @@ -265,7 +264,7 @@ public void testCleanupOnUpdate() throws Exception { put("name", dogName); }})); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); CandidateResults crs = queryCollectionCp( "dogs", "dog", "select *"); Assert.assertEquals("Expect no stale candidates yet", numEntities, crs.size()); @@ -288,7 +287,7 @@ public void testCleanupOnUpdate() throws Exception { } } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // wait for indexes to be cleared for the deleted entities count = 0; @@ -296,7 +295,7 @@ public void testCleanupOnUpdate() throws Exception { do { //trigger the repair queryCollectionEm("dogs", "select * order by created"); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); crs = queryCollectionCp("dogs", "dog", "select *"); } while ( crs.size() != numEntities && count++ < 15 ); @@ -344,7 +343,7 @@ private CandidateResults queryCollectionCp( SearchEdge is = CpNamingUtils.createCollectionSearchEdge( rootId, collName ); - return ei.search( is, SearchTypes.fromTypes( type ), query, 1000, 0 ); + return ei.search( is, SearchTypes.fromTypes( type ), query, 1000, 0, false ); } /** diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/export/ExportServiceIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/export/ExportServiceIT.java new file mode 100644 index 0000000000..7412ef0acd --- /dev/null +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/export/ExportServiceIT.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.usergrid.corepersistence.export; + +import com.google.inject.Injector; +import org.apache.usergrid.AbstractCoreIT; +import org.apache.usergrid.cassandra.SpringResource; +import org.apache.usergrid.corepersistence.EntityWriteHelper; +import org.apache.usergrid.persistence.EntityManager; +import org.apache.usergrid.persistence.SimpleEntityRef; + +import org.apache.usergrid.persistence.model.entity.Id; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.HashSet; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + + +import static org.junit.Assert.assertTrue; + + +public class ExportServiceIT extends AbstractCoreIT { + + + @Test + public void testExport() throws Exception { + + Injector injector = SpringResource.getInstance().getBean(Injector.class); + + ExportService exportService = injector.getInstance(ExportService.class); + + final EntityManager em = app.getEntityManager(); + + // create two types of entities + + final String type1 = "type1thing"; + final String type2 = "type2thing"; + final int size = 1; + + final Set type1Identities = EntityWriteHelper.createTypes( em, type1, size ); + final Set type2Identities = EntityWriteHelper.createTypes( em, type2, size ); + + // connect the first type1 entity to all type2 entities + + final Id source = type1Identities.iterator().next(); + final Set connections = new HashSet<>(); + + for ( Id target : type2Identities ) { + em.createConnection( SimpleEntityRef.fromId( source ), + "likes", SimpleEntityRef.fromId( target ) ); + connections.add( target ); + } + + ExportRequestBuilder builder = new ExportRequestBuilderImpl().withApplicationId(app.getId()); + + // fill the output stream + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + exportService.export(builder, stream); + + // convert the output stream to an input stream and read it as a zip + InputStream inputStream = new ByteArrayInputStream(stream.toByteArray()); + ZipInputStream zip = new ZipInputStream(inputStream); + + + boolean entityEntryExists = false; + boolean connectionEntryExists = false; + boolean statsEntryExists = false; + boolean metaEntryExists = false; + + final String entityFile = "entities/entities.0.json"; + final String connectionFile = "connections/"+source.getUuid().toString()+"_"+"likes"+"_"+connections.iterator().next().getUuid().toString()+".json"; + final String statsFile = "stats.json"; + final String metaFile = "metadata.json"; + + ZipEntry zipEntry; + while ( (zipEntry = zip.getNextEntry()) != null ) { + + final String name = zipEntry.getName(); + + if (name.equals(entityFile)) { + entityEntryExists = true; + } + if(name.equals(connectionFile)){ + connectionEntryExists = true; + } + if(name.equals(statsFile)){ + statsEntryExists = true; + } + if(name.equals(metaFile)){ + metaEntryExists = true; + } + + } + + assertTrue("Expected zip entries are missing", entityEntryExists && connectionEntryExists && statsEntryExists && metaEntryExists); + + } + + +} diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java index 366ab3438f..cecc3b24ba 100644 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java @@ -182,7 +182,7 @@ private CandidateResults getResults( final EntityIndex entityIndex, for ( int i = 0; i < attempts; i++ ) { final CandidateResults candidateResults = - entityIndex.search( searchEdge, searchTypes, "select *", 100, 0 ); + entityIndex.search( searchEdge, searchTypes, "select *", 100, 0, false ); if ( candidateResults.size() == expectedSize ) { return candidateResults; diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java index ecc2b46389..f47afe6de9 100644 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java @@ -353,7 +353,7 @@ public void testDeleteSingleConnectingEdge() throws InterruptedException { //ensure that no edges remain CandidateResults connectionResultsEmpty = EntityIndex.search( connectionSearchEdge, - SearchTypes.fromTypes( "thing" ),"select *",10,0 ); + SearchTypes.fromTypes( "thing" ),"select *",10,0, false ); assertEquals(1,connectionResultsEmpty.size()); @@ -375,7 +375,7 @@ public void testDeleteSingleConnectingEdge() throws InterruptedException { //ensure that no edges remain connectionResultsEmpty = EntityIndex.search( connectionSearchEdge, - SearchTypes.fromTypes( "thing" ),"select *",10,0 ); + SearchTypes.fromTypes( "thing" ),"select *",10,0, false ); assertEquals(0,connectionResultsEmpty.size()); @@ -434,7 +434,7 @@ public void testDeleteMultipleConnectingEdges() throws InterruptedException { //ensure that no edges remain final CandidateResults connectionResultsEmpty = EntityIndex.search( connectionSearchEdge, - SearchTypes.fromTypes( "things" ),"select *",10,0 ); + SearchTypes.fromTypes( "things" ),"select *",10,0, false ); assertEquals(0,connectionResultsEmpty.size()); } @@ -535,7 +535,7 @@ private CandidateResults getResults( final EntityIndex EntityIndex, String ql = "select *"; for ( int i = 0; i < attempts; i++ ) { final CandidateResults candidateResults = - EntityIndex.search( searchEdge, searchTypes, ql , 100, 0 ); + EntityIndex.search( searchEdge, searchTypes, ql , 100, 0, false ); if ( candidateResults.size() == expectedSize ) { return candidateResults; diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java index 9ad90ebb96..547691fb31 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java @@ -23,7 +23,6 @@ import com.google.inject.Injector; import org.apache.usergrid.AbstractCoreIT; import org.apache.usergrid.cassandra.SpringResource; -import org.apache.usergrid.corepersistence.service.AggregationService; import org.apache.usergrid.corepersistence.service.AggregationServiceFactory; import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.persistence.core.scope.ApplicationScope; @@ -32,7 +31,6 @@ import org.apache.usergrid.persistence.graph.GraphManagerFactory; import org.apache.usergrid.persistence.graph.MarkedEdge; import org.apache.usergrid.persistence.graph.SearchByEdgeType; -import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge; import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType; import org.apache.usergrid.persistence.model.entity.Id; import org.junit.Assert; @@ -65,7 +63,7 @@ public void testDeletes() throws Exception{ map.put("somekey", UUID.randomUUID()); Entity entity = entityManager.create("tests", map); } - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); Thread.sleep(500); ApplicationScope appScope = CpNamingUtils.getApplicationScope(entityManager.getApplicationId()); Observable ids = @@ -76,7 +74,7 @@ public void testDeletes() throws Exception{ this.app.getApplicationService().deleteAllEntities(appScope, 5); count = ids.count().toBlocking().last(); Assert.assertEquals(count, 5); - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); Injector injector = SpringResource.getInstance().getBean(Injector.class); GraphManagerFactory factory = injector.getInstance(GraphManagerFactory.class); GraphManager graphManager = factory.createEdgeManager(appScope); diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java index 3305e0e92e..f484f4fab3 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java @@ -132,7 +132,7 @@ public void testCollection() throws Exception { activity3 = app.get( activity3.getUuid(), activity3.getType() ); app.addToCollection( user, "activities", activity3 ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // empty query Query query = new Query(); @@ -259,7 +259,7 @@ public void containsTest() throws Exception { activity3 = app.get(activity3.getUuid(), activity3.getType()); app.addToCollection(user, "activities", activity3); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // empty query Query query = new Query(); @@ -295,7 +295,7 @@ public void userFirstNameSearch() throws Exception { Entity user = em.create( "user", properties ); assertNotNull( user ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // EntityRef Query query = Query.fromQL( "firstname = '" + firstName + "'" ); @@ -315,7 +315,7 @@ public void userFirstNameSearch() throws Exception { em.update( user ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // search with the old username, should be no results query = Query.fromQL( "firstname = '" + firstName + "'" ); @@ -354,7 +354,7 @@ public void userMiddleNameSearch() throws Exception { Entity user = em.create( "user", properties ); assertNotNull( user ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // EntityRef final Query query = Query.fromQL( "middlename = '" + middleName + "'" ); @@ -386,7 +386,7 @@ public void userLastNameSearch() throws Exception { Entity user = em.create( "user", properties ); assertNotNull( user ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // EntityRef final Query query = Query.fromQL( "lastname = '" + lastName + "'" ); @@ -434,7 +434,7 @@ public void testGroups() throws Exception { properties.put("nickname", "ed"); em.updateProperties(user1, properties); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Thread.sleep(1000); final Query query = Query.fromQL( "nickname = 'ed'" ); @@ -469,7 +469,7 @@ public void groupNameSearch() throws Exception { Entity group = em.create( "group", properties ); assertNotNull( group ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // EntityRef final Query query = Query.fromQL( "name = '" + groupName + "'" ); @@ -501,7 +501,7 @@ public void groupTitleSearch() throws Exception { Entity group = em.create( "group", properties ); assertNotNull( group ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // EntityRef @@ -559,7 +559,7 @@ public void testSubkeys() throws Exception { em.addToCollection( user, "activities", em.create( "activity", properties ) ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); final Query query = Query.fromQL( "verb = 'post'" ); @@ -593,7 +593,7 @@ public void emptyQuery() throws Exception { Entity user2 = em.create( "user", properties ); assertNotNull( user2 ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // EntityRef Query query = new Query(); @@ -636,7 +636,7 @@ public void emptyQueryReverse() throws Exception { Entity user2 = em.create( "user", properties ); assertNotNull( user2 ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // EntityRef Query query = new Query(); @@ -677,7 +677,7 @@ public void orQuery() throws Exception { Entity game2 = em.create( "orquerygame", properties ); assertNotNull( game2 ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // EntityRef Query query = Query @@ -756,7 +756,7 @@ public void andQuery() throws Exception { Entity game2 = em.create( "game", properties ); assertNotNull( game2 ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // overlap Query query = Query.fromQL( @@ -817,7 +817,7 @@ public void notQuery() throws Exception { Entity game2 = em.create( "game", properties ); assertNotNull( game2 ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // simple not Query query = Query.fromQL( "select * where NOT keywords contains 'game'" ); @@ -893,7 +893,7 @@ public void notSubObjectQuery() throws Exception { Entity entity2 = em.create( "game", properties ); assertNotNull( entity2 ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // search for games without sub-field Foo should returned zero entities @@ -949,7 +949,7 @@ public void testKeywordsOrQuery() throws Exception { properties.put("keywords", "Action, New"); em.create( "game", properties ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query query = Query.fromQL( "select * where keywords contains 'hot' or title contains 'hot'" ); Results r = em.searchCollection( em.getApplicationRef(), "games", query ); @@ -980,7 +980,7 @@ public void testKeywordsAndQuery() throws Exception { properties.put( "keywords", "Action, New" ); Entity thirdGame = em.create( "game", properties ); - app.refreshIndex();//need to track all batches then resolve promises + app.waitForQueueDrainAndRefreshIndex();//need to track all batches then resolve promises Query query = Query.fromQL( "select * where keywords contains 'new' and title contains 'extreme'" ); Results r = em.searchCollection( em.getApplicationRef(), "games", query ); @@ -1011,7 +1011,7 @@ public void pagingAfterDelete() throws Exception { entityIds.add( created.getUuid() ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query query = new Query(); query.setLimit( 50 ); @@ -1039,7 +1039,7 @@ public void pagingAfterDelete() throws Exception { numDeleted++; } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // wait for indexes to be cleared Thread.sleep(1000); //TODO find why we have to wait. This is a bug @@ -1096,7 +1096,7 @@ public void pagingLessThanWithCriteria() throws Exception { int pageSize = 10; - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); final Query query = Query.fromQL( "index < " + size * 2 + " order by index asc" ); Results r = null; @@ -1147,7 +1147,7 @@ public void pagingGreaterThanWithCriteria() throws Exception { int pageSize = 10; - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query query = Query.fromQL( "select * where index >= " + size / 2 + " sort by index asc" ); query.setLimit( pageSize ); @@ -1201,7 +1201,7 @@ public void pagingWithBoundsCriteria() throws Exception { entityIds.add( created.getUuid() ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); int pageSize = 10; @@ -1254,7 +1254,7 @@ public void testPagingWithGetNextResults() throws Exception { entityIds.add( created.getUuid() ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); int pageSize = 5; @@ -1310,7 +1310,7 @@ public void subpropertyQuerying() throws Exception { Entity saved = em.create( "test", root ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query query = Query.fromQL( "rootprop1 = 'simpleprop'" ); Entity entity; @@ -1357,7 +1357,7 @@ public void arrayQuerying() throws Exception { Entity saved = em.create( "test", jsonData ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query query = Query.fromQL( "intprop = 10" ); @@ -1416,7 +1416,7 @@ public void stringWithSpaces() throws Exception { Entity saved = em.create( "test", props ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query query = Query.fromQL( "myString = 'My simple string'" ); @@ -1441,7 +1441,7 @@ public void testSelectTerms() throws Exception { em.create( "user", properties ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); String s = "select username, email where username = 'edanuff'"; Query query = Query.fromQL( s ); @@ -1471,7 +1471,7 @@ public void testRedefineTerms() throws Exception { em.create( "user", properties ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); String s = "select {name: username, email: email} where username = 'edanuff'"; Query query = Query.fromQL( s ); @@ -1503,7 +1503,7 @@ public void testSelectEmailViaConnection() throws Exception { final Entity entity = em.create( "user", properties ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); String s = "select * where username = 'ed@anuff.com'"; Query query = Query.fromQL( s ); @@ -1525,7 +1525,7 @@ public void testSelectEmailViaConnection() throws Exception { em.createConnection( foo, "testconnection", entity ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // now query via the testConnection, this should work @@ -1569,7 +1569,7 @@ public void testNotQueryAnd() throws Exception { em.create( "loveobject", properties ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); location = new LinkedHashMap(); location.put( "Place", "Via Pietro Maroncelli, 48, 62012 Santa Maria Apparente Province of Macerata, Italy" ); @@ -1587,7 +1587,7 @@ public void testNotQueryAnd() throws Exception { em.create( "loveobject", properties ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // String s = "select * where Flag = 'requested'"; // String s = "select * where Flag = 'requested' and NOT Recipient.Username = @@ -1632,7 +1632,7 @@ public void runtimeTypeCorrect() throws Exception { createdEntities.add( created ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Results r = em.getCollection( em.getApplicationRef(), "users", null, 50, Level.ALL_PROPERTIES, false ); @@ -1729,7 +1729,7 @@ public void uuidIdentifierTest() throws Exception { Entity game2 = em.create( "game", properties ); assertNotNull( game2 ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // overlap Query query = new Query(); @@ -1763,7 +1763,7 @@ public void nameIdentifierTest() throws Exception { Entity game2 = em.create( "game", properties ); assertNotNull( game2 ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // overlap Query query = new Query(); @@ -1797,7 +1797,7 @@ public void emailIdentifierTest() throws Exception { Entity createUser2 = em.create( user2 ); assertNotNull( createUser2 ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // overlap Query query = new Query(); diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CountingMutatorIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CountingMutatorIT.java index 63c7cb801d..596ec7cb68 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/CountingMutatorIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CountingMutatorIT.java @@ -74,7 +74,7 @@ public void testFlushingMutatorOnConnections() throws Exception { properties.put( "username", "testuser" ); properties.put( "email", "test@foo.bar" ); Entity created = em.create( "user", properties ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Entity returned = em.get( created.getUuid() ); @@ -89,7 +89,7 @@ public void testFlushingMutatorOnConnections() throws Exception { Entity connectedEntity = em.create( "user", connectedProps ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // Connect from our new entity to our root one so it's updated when paging em.createConnection( connectedEntity, "following", returned ); diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java index 296bf53c91..e1e24c41b8 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java @@ -64,7 +64,7 @@ public void testEntityConnectionsSimple() throws Exception { assertEquals( 1, connectionTypes.size()); assertEquals("likes", connectionTypes.iterator().next()); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Results r = em.getTargetEntities(firstUserEntity, "likes", null, Level.IDS); @@ -128,7 +128,7 @@ public void testEntityConnections() throws Exception { logger.info( "\n\nConnecting " + awardA.getUuid() + " \"awarded\" " + catB.getUuid() + "\n" ); em.createConnection( awardA, "awarded", catB ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // List forward connections for cat A @@ -149,7 +149,7 @@ public void testEntityConnections() throws Exception { logger.info( "\n\nConnecting " + awardA.getUuid() + " \"awarded\" " + catA.getUuid() + "\n" ); em.createConnection( awardA, "awarded", catA ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // List forward connections for cat A // Not valid with current usages @@ -256,7 +256,7 @@ public void testEntityConnectionsMembership() throws Exception { em.createConnection( secondUserEntity, "likes", arrogantbutcher ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Results r = em.getTargetEntities(firstUserEntity, "likes", "restaurant", Level.IDS); @@ -310,7 +310,7 @@ public void testGetConnectingEntities() throws Exception { em.createConnection( fredEntity, "likes", wilmaEntity ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // // search for "likes" edges from fred // assertEquals( 1, @@ -363,7 +363,7 @@ public void testGetConnectingEntitiesCursor() throws Exception { em.createConnection( fredEntity, "likes", JohnEntity ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // now query via the testConnection, this should work @@ -410,7 +410,7 @@ public void testConnectionsIterable() throws Exception { } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Results r = em.getTargetEntities(firstUserEntity, "likes", null, Level.ALL_PROPERTIES) ; @@ -453,7 +453,7 @@ public void testConnectionsIterable() throws Exception { // // em.createConnection( fredEntity, "likes", wilmaEntity ); // -// app.refreshIndex(); +// app.waitForQueueDrainAndRefreshIndex(); // //// // search for "likes" edges from fred //// assertEquals( 1, diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java index cb3a728158..e1e4a0534e 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java @@ -75,7 +75,7 @@ public void testEntityManager() throws Exception { assertEquals( "user.username not expected value", "edanuff", user.getProperty( "username" ) ); assertEquals( "user.email not expected value", "ed@anuff.com", user.getProperty( "email" ) ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); EntityRef userRef = em.getAlias( new SimpleEntityRef( "application", applicationId ), "users", "edanuff" ); @@ -274,13 +274,13 @@ public void testCreateAndDelete() throws Exception { Entity thing = em.create( "thing", properties ); logger.info( "Entity created" ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); logger.info( "Starting entity delete" ); em.delete( thing ); logger.info( "Entity deleted" ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // now search by username, no results should be returned @@ -310,13 +310,13 @@ public void testCreateAndDeleteUser() throws Exception { Entity user = em.create( "user", properties ); logger.info( "Entity created" ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); logger.info( "Starting entity delete" ); em.delete( user ); logger.info( "Entity deleted" ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // now search by username, no results should be returned @@ -335,7 +335,7 @@ public void testCreateAndDeleteUser() throws Exception { user = em.create( "user", properties ); logger.info( "Entity created" ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); final Query userNameQuery = Query.fromQL( "username = '" + name + "'" ); @@ -456,7 +456,7 @@ public void testCreateAndList() throws Exception { EntityRef appRef = em.get( new SimpleEntityRef( "application", app.getId() ) ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Results r = em.getCollection( appRef, "things", null, 10, Level.ALL_PROPERTIES, false ); @@ -548,7 +548,7 @@ public void ownershipScopeCorrect() throws Exception { Entity createdDevice = em.createItemInCollection( createdUser, "devices", "device", device ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Entity returnedDevice = em.get( new SimpleEntityRef( "device", createdDevice.getUuid() ) ); @@ -580,7 +580,7 @@ public void testDeprecatedGet() throws Exception { Entity user = em.create( "robot", properties ); assertNotNull( user ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); assertNotNull( em.get( user.getUuid() ) ); } @@ -608,7 +608,7 @@ public void testFilteringOfDuplicateEdges() throws Exception { em.addToCollection(appRef, "fluffies", entityRef); em.addToCollection(appRef, "fluffies", entityRef); - //app.refreshIndex(); + //app.waitForQueueDrainAndRefreshIndex(); Results results = em.getCollection(appRef, "fluffies", null, 10, Level.ALL_PROPERTIES, true); diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java index b7d708ef90..df7708432e 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java @@ -96,7 +96,7 @@ public void testRoundingGeolocationIssue() throws Exception { assertNotNull(hotel); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //2. Query with a globally large distance to verify location @@ -142,7 +142,7 @@ public void testRemovedLocationQuery() throws Exception { }}; Entity user = em.create("user", properties); assertNotNull(user); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //2. Query with a globally large distance to verify location Query query = Query.fromQL("select * where location within " + Integer.MAX_VALUE + " of 0, 0"); @@ -154,7 +154,7 @@ public void testRemovedLocationQuery() throws Exception { user.getDynamicProperties().remove("location"); em.updateProperties(user, properties); em.update(user); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //4. Repeat the query, expecting no results listResults = em.searchCollection(em.getApplicationRef(), "users", query); @@ -188,7 +188,7 @@ public void testMovingTarget() throws Exception { }}; Entity user = em.create("user", properties); assertNotNull(user); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); final double lat = 37.776753; final double lon = -122.407846; @@ -234,7 +234,7 @@ public void validateDistanceQueryExists() throws Exception { }}; Entity user = em.create("user", properties); assertNotNull(user); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); final double lat = 37.776753; final double lon = -122.407846; @@ -284,12 +284,12 @@ public void testGeoDistanceOfConnection() throws Exception { Entity user = em.create("user", userProperties); assertNotNull(user); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //3. Create a connection between the user and the entity em.createConnection(user, "likes", restaurant); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //4. Test that the user is within 2000m of the entity Results emSearchResults = em.searchTargetEntities(user, Query.fromQL("location within 5000 of " @@ -326,7 +326,7 @@ public void testGeolocationEntities() throws Exception { assertNotNull(entity); logger.debug("Entity {} created", entity.getProperty("name")); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //2. validate the size of the result Query query = new Query(); Results listResults = em.searchCollection(em.getApplicationRef(), "stores", query); @@ -367,7 +367,7 @@ public void testNonGeolocationEntities() throws Exception { assertNotNull(entity); logger.debug("Entity {} created", entity.getProperty("name")); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //2. validate the size of the result Query query = new Query(); Results listResults = em.searchCollection(em.getApplicationRef(), "stores", query); @@ -540,7 +540,7 @@ public void testPointPaging() throws Exception { em.create("store", data); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // earth's circumference is 40,075 kilometers. Up it to 50,000kilometers // just to be save @@ -596,7 +596,7 @@ public void testSamePointPaging() throws Exception { em.create("store", data); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Thread.sleep(2000); // earth's circumference is 40,075 kilometers. Up it to 50,000kilometers @@ -669,7 +669,7 @@ public void testDistanceByLimit() throws Exception { em.create("store", data); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // earth's circumference is 40,075 kilometers. Up it to 50,000kilometers // just to be save @@ -729,7 +729,7 @@ public void testGeoWithIntersection() throws Exception { created.add(e); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); int startDelta = size - min; @@ -794,7 +794,7 @@ public void testDenseSearch() throws Exception { em.create("store", data); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //do a direct geo iterator test. We need to make sure that we short circuit on the correct tile. @@ -838,7 +838,7 @@ private EntityManager loadGeolocationTestEntities() throws Exception { assertNotNull(entity); } //3. refresh the index - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //4. return the entity manager return em; } @@ -857,7 +857,7 @@ public void updatePos(EntityManager em, EntityRef entity, double latitude, doubl latlong.put("longitude", longitude); em.setProperty(entity, "location", latlong); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); } diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoQueryBooleanTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoQueryBooleanTest.java index 9a3f5a6736..609f977232 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoQueryBooleanTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoQueryBooleanTest.java @@ -79,7 +79,7 @@ public void testGeoQueryWithOr() throws Exception { Entity user2 = em.create( "user", properties ); assertNotNull( user2 ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // define center point about 300m from that location final double lat = 37.774277; @@ -158,7 +158,7 @@ public void testGeoQueryWithNot() throws Exception { Entity userFred = em.create( "user", properties ); assertNotNull( userFred ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // define center point about 300m from that location final double lat = 37.774277; diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java index d62f88ef2a..5933b57c5c 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java @@ -60,7 +60,7 @@ public void testCollectionOrdering() throws Exception { em.create( "item", properties ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); int i = 0; @@ -133,7 +133,7 @@ public void testCollectionFilters() throws Exception { em.create( "item", properties ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query query = Query.fromQL( "name < 'delta' order by name asc" ); Results r = em.searchCollection( em.getApplicationRef(), "items", query ); @@ -261,7 +261,7 @@ public void testSecondarySorts() throws Exception { em.create( "item", properties ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query query = Query.fromQL( "group = 1 order by name desc" ); Results r = em.searchCollection( em.getApplicationRef(), "items", query ); @@ -290,7 +290,7 @@ public void testEntityReduction() throws Exception { em.create("names", entity1); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //should return valid values Query query = Query.fromQL("select status where status = 'pickled'"); @@ -338,7 +338,7 @@ public void testPropertyUpdateWithConnection() throws Exception { em.createConnection( entity2Ref, "connecting", entity1Ref ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //should return valid values Query query = Query.fromQL( "select * where status = 'pickled'" ); @@ -357,7 +357,7 @@ public void testPropertyUpdateWithConnection() throws Exception { em.update( entity1Ref ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //query and check the status has been updated, shouldn't return results query = Query.fromQL( "select * where status = 'pickled'" ); @@ -413,7 +413,7 @@ public void testPropertyUpdateWithConnectionEntityIndexEntryAudit() throws Excep em.createConnection( entity2Ref, "connecting", entity1Ref ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //should return valid values Query query = Query.fromQL( "select * where status = 'pickled'" ); @@ -432,7 +432,7 @@ public void testPropertyUpdateWithConnectionEntityIndexEntryAudit() throws Excep em.update( entity1Ref ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); //query and check the status has been updated, shouldn't return results query = Query.fromQL( "select * where status = 'pickled'" ); @@ -500,7 +500,7 @@ public void testSelectMappings() throws Exception { }}; em.create("names", entity2); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // simple single-field select mapping { diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java index e6ecf97697..329a5be7d0 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java @@ -28,7 +28,6 @@ import org.junit.Test; import org.apache.usergrid.AbstractCoreIT; -import org.apache.usergrid.persistence.Query; import org.apache.usergrid.persistence.Query.Level; import org.apache.usergrid.persistence.model.util.UUIDGenerator; @@ -63,7 +62,7 @@ public void testUserDevicePathQuery() throws Exception { } } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Thread.sleep(1000); @@ -135,7 +134,7 @@ public void testGroupUserDevicePathQuery() throws Exception { } } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // pick an arbitrary group, ensure it has 7 users Results ru = em.getCollection( groups.get( 2 ), "users", null, 20, Level.IDS, false ); @@ -152,7 +151,7 @@ public void testGroupUserDevicePathQuery() throws Exception { } } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); // pick an arbitrary user, ensure it has 7 devices Results rd = em.getCollection( users.get( 6 ), "devices", null, 20, Level.IDS, false ); diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PermissionsIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PermissionsIT.java index 11f0692c23..1072d291b7 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/PermissionsIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PermissionsIT.java @@ -28,8 +28,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.commons.lang3.RandomStringUtils; - import org.apache.usergrid.AbstractCoreIT; import org.apache.usergrid.persistence.entities.Role; import org.apache.usergrid.persistence.Query.Level; @@ -147,7 +145,7 @@ public void testPermissions() throws Exception { dump( "group roles", roles ); em.deleteGroupRole( group.getUuid(), "author" ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Thread.sleep(1000); roles = em.getGroupRoles( group.getUuid() ); @@ -156,7 +154,7 @@ public void testPermissions() throws Exception { em.addUserToGroupRole( user.getUuid(), group.getUuid(), "admin" ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Results r = em.getUsersInGroupRole( group.getUuid(), "admin", Level.ALL_PROPERTIES ); dump( "entities", r.getEntities() ); assertEquals( "proper number of users in group role not set", 1, r.size() ); diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java new file mode 100644 index 0000000000..38584dd3e4 --- /dev/null +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.persistence; + + +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class QueryTest { + + @Test + public void testQueryParamsWithPlus(){ + + String qlString = "select * where email='test+value@usergrid.com'"; + + Map> queryParams = new HashMap<>(); + queryParams.put("ql", Collections.singletonList(qlString) ); + Query query = Query.fromQueryParams(queryParams); + + assertEquals(qlString, query.getQl().get()); + + } + + @Test + public void testQueryParamsWithUrlEncodedPlus(){ + + String qlString = "select * where email='test+value@usergrid.com'"; + Map> queryParams = new HashMap<>(); + queryParams.put("ql", Collections.singletonList(qlString.replace("+", "%2b"))); + Query query = Query.fromQueryParams(queryParams); + + assertEquals(qlString, query.getQl().get()); + + } + +} diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java index 383d6201b9..a7759ded40 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java @@ -125,7 +125,7 @@ public void rebuildOneCollectionIndex() throws Exception { } logger.info( "Created {} entities", ENTITIES_TO_INDEX ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(5000); // ----------------- test that we can read them, should work fine @@ -163,6 +163,7 @@ public void rebuildOneCollectionIndex() throws Exception { waitForRebuild( status, reIndexService ); + app.waitForQueueDrainAndRefreshIndex(5000); // ----------------- test that we can read the catherder collection and not the catshepard @@ -233,7 +234,7 @@ public void rebuildIndex() throws Exception { } logger.info( "Created {} entities", ENTITIES_TO_INDEX ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(15000); // ----------------- test that we can read them, should work fine @@ -247,6 +248,8 @@ public void rebuildIndex() throws Exception { deleteIndex( em.getApplicationId() ); + app.waitForQueueDrainAndRefreshIndex(); + // ----------------- test that we can read them, should fail // deleting sytem app index will interfere with other concurrently running tests @@ -283,7 +286,6 @@ public void rebuildIndex() throws Exception { logger.info( "Rebuilt index" ); - app.refreshIndex(); } catch ( Exception ex ) { logger.error( "Error rebuilding index", ex ); @@ -292,7 +294,7 @@ public void rebuildIndex() throws Exception { // ----------------- test that we can read them - Thread.sleep( 2000 ); + app.waitForQueueDrainAndRefreshIndex(15000); readData( em, collectionName, ENTITIES_TO_INDEX, 3 ); } @@ -343,7 +345,7 @@ public void rebuildIndexGeo() throws Exception { logger.info( "Created {} entities", ENTITIES_TO_INDEX ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(5000); // ----------------- test that we can read them, should work fine @@ -392,7 +394,6 @@ public void rebuildIndexGeo() throws Exception { logger.info( "Rebuilt index" ); - app.refreshIndex(); } catch ( Exception ex ) { logger.error( "Error rebuilding index", ex ); @@ -401,7 +402,7 @@ public void rebuildIndexGeo() throws Exception { // ----------------- test that we can read them - Thread.sleep( 2000 ); + app.waitForQueueDrainAndRefreshIndex(5000); results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 3 ); assertEquals(results.size(),3); q = Query.fromQL("select * where location within 100 of "+lat+", "+lon); @@ -435,7 +436,7 @@ public void rebuildUpdatedSince() throws Exception { final Entity secondEntity = em.create( "thing", entityData); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(5000); // ----------------- test that we can read them, should work fine @@ -493,7 +494,6 @@ public void rebuildUpdatedSince() throws Exception { logger.info( "Rebuilt index" ); - app.refreshIndex(); } catch ( Exception ex ) { logger.error( "Error rebuilding index", ex ); @@ -502,7 +502,7 @@ public void rebuildUpdatedSince() throws Exception { // ----------------- test that we can read them - Thread.sleep( 2000 ); + app.waitForQueueDrainAndRefreshIndex(5000); countEntities( em, collectionName, 1 ); } @@ -547,14 +547,14 @@ private void deleteIndex( UUID appUuid ) { ); ei.deleteApplication().toBlocking().lastOrDefault( null ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); } private int readData( EntityManager em, String collectionName, int expectedEntities, int expectedConnections ) throws Exception { - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query q = Query.fromQL( "select * where key1=1000" ).withLimit( 1000 ); Results results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, expectedEntities ); @@ -593,7 +593,7 @@ private int readData( EntityManager em, String collectionName, int expectedEntit private int countEntities( EntityManager em, String collectionName, int expectedEntities) throws Exception { - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query q = Query.fromQL( "select * where key1=1000" ).withLimit( 1000 ); Results results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, expectedEntities ); diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java index d287d7e7d9..3652b6f4da 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java @@ -26,11 +26,8 @@ import org.apache.usergrid.Application; import org.apache.usergrid.CoreApplication; import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder; -import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilderImpl; import org.apache.usergrid.corepersistence.index.ReIndexService; -import org.apache.usergrid.corepersistence.index.ReIndexServiceImpl; import org.apache.usergrid.persistence.*; -import org.apache.usergrid.utils.UUIDUtils; import org.junit.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,8 +40,6 @@ import org.apache.usergrid.persistence.cassandra.util.TraceTagReporter; import org.apache.usergrid.persistence.model.util.UUIDGenerator; import org.apache.usergrid.setup.ConcurrentProcessSingleton; -import rx.functions.Func0; -import rx.functions.Func1; import rx.functions.Func2; import javax.annotation.concurrent.NotThreadSafe; @@ -140,7 +135,7 @@ public void testDeleteApplication() throws Exception { Thread.sleep( 500 ); } - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); // wait for it to appear in delete apps list @@ -164,7 +159,7 @@ public void testDeleteApplication() throws Exception { // delete the application setup.getEmf().deleteApplication(deletedAppId); - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); found = findApps.call( deletedAppId, emf.getDeletedApplications() ); @@ -196,14 +191,14 @@ public void testDeleteApplication() throws Exception { } }while (status.getStatus()!= ReIndexService.Status.COMPLETE); - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); // test to see that app now works and is happy // it should not be found in the deleted apps collection found = findApps.call( deletedAppId, emf.getDeletedApplications()); assertFalse("Restored app found in deleted apps collection", found); - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); apps = setup.getEmf().getApplications(); found = findApps.call(deletedAppId, apps); diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ConnectionHelper.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ConnectionHelper.java index e5c84f8359..1f53c0aa5a 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ConnectionHelper.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ConnectionHelper.java @@ -76,7 +76,7 @@ public Entity writeEntity( Map entity ) throws Exception { @Override public Results getResults( Query query ) throws Exception { - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); query.setConnectionType( CONNECTION ); query.setEntityType( "test" ); diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionTransitivePagingIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionTransitivePagingIT.java index cea3b35219..dcc8eae588 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionTransitivePagingIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionTransitivePagingIT.java @@ -131,7 +131,7 @@ private List performSetup( final IoHelper io ) throws Exception { } - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); Thread.sleep(1000); return expected; diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionUnionPagingIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionUnionPagingIT.java index 4d6016490a..3403dc80e8 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionUnionPagingIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionUnionPagingIT.java @@ -135,7 +135,7 @@ private Set performSetup( final IoHelper io ) throws Exception { } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); logger.info( "Writes took {} ms", stop - start ); diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java index b2003fe169..dac3f68c52 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java @@ -298,7 +298,7 @@ public void singleOrderByMaxLimit( IoHelper io ) throws Exception { //we have to sleep, or we kill embedded cassandra } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Thread.sleep(1000); long stop = System.currentTimeMillis(); @@ -367,7 +367,7 @@ protected void singleOrderByIntersection( IoHelper io ) throws Exception { expected.add( name ); } } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); @@ -438,7 +438,7 @@ protected void singleOrderByComplexIntersection( IoHelper io ) throws Exception } } - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); logger.info( "Writes took {} ms", stop - start ); @@ -551,7 +551,7 @@ protected void singleOrderByComplexUnion( IoHelper io ) throws Exception { expectedResults.add( name ); } } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); @@ -623,7 +623,7 @@ protected void singleOrderByNot( IoHelper io ) throws Exception { } } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); logger.info( "Writes took {} ms", stop - start ); @@ -689,7 +689,7 @@ protected void singleOrderByLessThanLimit( IoHelper io ) throws Exception { } } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); logger.info( "Writes took {} ms", stop - start ); @@ -742,7 +742,7 @@ protected void singleOrderBySameRangeScanLessThanEqual( IoHelper io ) throws Exc io.writeEntity( entity ); expected.add( name ); } - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); @@ -803,7 +803,7 @@ protected void singleOrderBySameRangeScanLessEqual( IoHelper io ) throws Excepti io.writeEntity( entity ); expected.add( name ); } - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); @@ -865,7 +865,7 @@ protected void singleOrderBySameRangeScanGreaterThanEqual( IoHelper io ) throws expected.add( name ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); logger.info( "Writes took {} ms", stop - start ); @@ -924,7 +924,7 @@ protected void singleOrderBySameRangeScanGreater( IoHelper io ) throws Exception io.writeEntity( entity ); expected.add( name ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Thread.sleep(500); long stop = System.currentTimeMillis(); @@ -987,7 +987,7 @@ protected void singleOrderByBoundRangeScanDesc( IoHelper io ) throws Exception { expected.add( name ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); logger.info( "Writes took {} ms", stop - start ); @@ -1050,7 +1050,7 @@ protected void singleOrderByBoundRangeScanAsc( IoHelper io ) throws Exception { io.writeEntity( entity ); expected.add( name ); } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); @@ -1110,7 +1110,7 @@ protected void allIn( IoHelper io ) throws Exception { io.writeEntity( entity ); } - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); @@ -1216,7 +1216,7 @@ else if ( o2Index > o1Index ) { logger.info( "Writes took {} ms", stop - start ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query query = Query.fromQL( "select * order by boolean desc, index asc" ); query.setLimit( queryLimit ); @@ -1322,7 +1322,7 @@ else if ( o2Index > o1Index ) { logger.info( "Writes took {} ms", stop - start ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); Query query = Query.fromQL( "select * where intersect = true OR intersect2 = true order by created, intersect desc" ); @@ -1384,7 +1384,7 @@ protected void notOrderBy( IoHelper io ) throws Exception { io.writeEntity( entity ); } - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); long stop = System.currentTimeMillis(); diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/NotSubPropertyIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/NotSubPropertyIT.java index f7308da60d..3f5573ff83 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/NotSubPropertyIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/NotSubPropertyIT.java @@ -132,7 +132,7 @@ private List performSetup( final IoHelper io ) throws Exception { logger.info( "Writes took {} ms", stop - start ); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); return expected; } diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ParenthesisProblemIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ParenthesisProblemIT.java index 60c16224dd..89641a8d09 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/ParenthesisProblemIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/ParenthesisProblemIT.java @@ -72,7 +72,7 @@ private void parenthesisProblem( int expect, String query ) throws Exception { put("age",1); }}); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); final Results entities = em.searchCollection( em.getApplicationRef(), "cats", Query.fromQL(query)); diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties index c54496739a..8f9058d26c 100644 --- a/stack/core/src/test/resources/usergrid-custom-test.properties +++ b/stack/core/src/test/resources/usergrid-custom-test.properties @@ -49,6 +49,9 @@ collection.uniquevalues.authoritative.region=us-east # Queueing Test Settings # Reduce the long polling time for the tests queue.long.polling.time.millis=50 +elasticsearch.worker_count=8 +elasticsearch.worker_count_utility=8 +queue.get.timeout.seconds=8 # --- End: Usergrid cluster/actor system settings diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java index 54702a844b..438f731147 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java @@ -146,11 +146,13 @@ public EntitySet load( final ApplicationScope applicationScope, final Collection for ( final Id entityId : entityIds ) { - final ScopedRowKey rowKey = - ScopedRowKey.fromKey( applicationId, entityId ); + if ( entityId != null ) { // the size of entityIds is checked as preconditions, but the values are not + final ScopedRowKey rowKey = + ScopedRowKey.fromKey(applicationId, entityId); - rowKeys.add( rowKey ); + rowKeys.add(rowKey); + } } /** diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml index 527e439627..8a4532333c 100644 --- a/stack/corepersistence/pom.xml +++ b/stack/corepersistence/pom.xml @@ -67,6 +67,7 @@ limitations under the License. + UTF-8 1.8 1.8 diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java index ee12ec82ac..14020a9f01 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java @@ -76,7 +76,7 @@ void addIndex( * @param edge * @return */ - long getEntitySize(final SearchEdge edge); + long getTotalEntitySizeInBytes(final SearchEdge edge); /** * Initialize the index if necessary. This is an idempotent operation and should not create an index @@ -98,10 +98,12 @@ void addIndex( * @param query The query to execute * @param limit The limit of values to return * @param offset The offset to query on + * @param analyzeOnly This optional param will instruct the query processing to only analyze the query and + * provide info but not actually execute the query. * @return */ CandidateResults search(final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, - final int limit, final int offset); + final int limit, final int offset, final boolean analyzeOnly); /** * Search on every document in the specified search edge. Also search by the types if specified @@ -113,10 +115,13 @@ CandidateResults search(final SearchEdge searchEdge, final SearchTypes searchTyp * @param offset The offset to query on * @param fieldsWithType An optional param that allows the caller to provide schema related info which might * relate to data in the query, such as sort predicate types + * @param analyzeOnly This optional param will instruct the query processing to only analyze the query and + * provide info but not actually execute the query. * @return */ CandidateResults search(final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, - final int limit, final int offset, final Map fieldsWithType); + final int limit, final int offset, final Map fieldsWithType, + final boolean analyzeOnly); /** diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java index c4596a2319..9b42da3670 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java @@ -62,6 +62,19 @@ public interface IndexFig extends GuicyFig { String ELASTICSEARCH_VERSION_QUERY_LIMIT = "elasticsearch.version_query_limit"; + String USERGRID_QUERYANALYZER_OPERAND_COUNT = "usergrid.queryanalyzer.operand_count"; + + String USERGRID_QUERYANALYZER_SORTPREDICATE_COUNT = "usergrid.queryanalyzer.sortpredicate_count"; + + String USERGRID_QUERYANALYZER_COLLECTIONSIZE = "usergrid.queryanalyzer.collectionsize_bytes"; + + String USERGRID_QUERYANALYZER_INDEXSIZE = "usergrid.queryanalyzer.indexsize_bytes"; + + String USERGRID_QUERYANALYZER_ENFORCE = "usergrid.queryanalyzer.enforce"; + + + + /** * Comma-separated list of Elasticsearch hosts. @@ -196,7 +209,33 @@ public interface IndexFig extends GuicyFig { @Key( "elasticsearch_queue_error_sleep_ms" ) long getSleepTimeForQueueError(); + @Default("100") @Key( ELASTICSEARCH_VERSION_QUERY_LIMIT ) int getVersionQueryLimit(); + + + @Default("8") + @Key( USERGRID_QUERYANALYZER_OPERAND_COUNT ) + int getQueryBreakerErrorOperandCount(); + + + @Default("8") + @Key( USERGRID_QUERYANALYZER_SORTPREDICATE_COUNT ) + int getQueryBreakerErrorSortPredicateCount(); + + + @Default("500000000") // 500 MB + @Key(USERGRID_QUERYANALYZER_COLLECTIONSIZE) + long getQueryBreakerErrorCollectionSizeBytes(); + + + @Default("10000000000") // 10 GB + @Key( USERGRID_QUERYANALYZER_INDEXSIZE ) + long getQueryBreakerErrorIndexSizeBytes(); + + + @Default("false") + @Key( USERGRID_QUERYANALYZER_ENFORCE ) + boolean enforceQueryBreaker(); } diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java new file mode 100644 index 0000000000..6d3cd92f4a --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/QueryAnalyzer.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. The ASF licenses this file to You + * under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. For additional information regarding + * copyright in this work, please see the NOTICE file in the top level + * directory of this distribution. + */ +package org.apache.usergrid.persistence.index; + +import org.apache.usergrid.persistence.index.query.ParsedQuery; +import org.apache.usergrid.persistence.index.query.tree.Operand; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class QueryAnalyzer { + + public static final String v_predicate_count = "sort_predicate_count_exceeded"; + public static final String v_operand_count = "operand_count_exceeded"; + public static final String v_large_collection = "large_collection_size_bytes"; + public static final String v_large_index = "large_index_size_bytes"; + public static final String v_full_collection_sort = "full_collection_sort"; + + public static final String k_violation = "violation"; + public static final String k_limit = "limit"; + public static final String k_actual = "actual"; + + + public static List> analyze(final ParsedQuery parsedQuery, final long collectionSizeInBytes, + final long indexSizeInBytes, final IndexFig indexFig ) { + + List> violations = new ArrayList<>(); + + // get configured breaker values + final int errorPredicateCount = indexFig.getQueryBreakerErrorSortPredicateCount(); + final int errorOperandCount = indexFig.getQueryBreakerErrorOperandCount(); + final long errorCollectionSizeBytes = indexFig.getQueryBreakerErrorCollectionSizeBytes(); + final long errorIndexSizeBytes = indexFig.getQueryBreakerErrorIndexSizeBytes(); + + + // get the actual values to compare against the configured enforcement values + int queryPredicatesSize = parsedQuery.getSortPredicates().size(); + int queryOperandCount = getTotalChildCount(parsedQuery.getRootOperand()); + + // large indexes can cause issues, this is never returned from the API and only logged + if( indexSizeInBytes > errorIndexSizeBytes ){ + violations.add(new HashMap(3){{ + put(k_violation, v_large_index); + put(k_limit, errorIndexSizeBytes); + put(k_actual, indexSizeInBytes); + }}); + } + + // large collections mean that sorts and other complex queries can impact the query service (Elasticsearch) + if (collectionSizeInBytes > errorCollectionSizeBytes ){ + violations.add(new HashMap(3){{ + put(k_violation, v_large_collection); + put(k_limit, errorCollectionSizeBytes); + put(k_actual, collectionSizeInBytes); + }}); + + // query like "select * order by created asc" + if(parsedQuery.getSelectFieldMappings().size() < 1 && + !parsedQuery.getOriginalQuery().toLowerCase().contains("where") && + parsedQuery.getSortPredicates().size() > 0 ){ + + violations.add(new HashMap(3){{ + put(k_violation, v_full_collection_sort); + put(k_limit, null); + put(k_actual, null); + }}); + } + + } + + // complex queries can be determined from the # of operands and sort predicates + if ( queryPredicatesSize > errorPredicateCount){ + violations.add(new HashMap(3){{ + put(k_violation, v_predicate_count); + put(k_limit, errorPredicateCount); + put(k_actual, queryPredicatesSize); + }}); + } + if (queryOperandCount > errorOperandCount){ + violations.add(new HashMap(3){{ + put(k_violation, v_operand_count); + put(k_limit, errorOperandCount); + put(k_actual, queryOperandCount); + }}); + } + + return violations; + + } + + public static String violationsAsString(List> violations, String originalQuery){ + + final StringBuilder logMessage = new StringBuilder(); + logMessage.append( "QueryAnalyzer Violations Detected [").append(violations.size()).append("]: [" ); + violations.forEach(violation -> { + + final StringBuilder violationMessage = new StringBuilder(); + violation.forEach((k,v) -> { + violationMessage.append(k).append(":").append(v).append(","); + + }); + violationMessage.deleteCharAt(violationMessage.length()-1); + logMessage.append(" (").append(violationMessage).append(") "); + }); + logMessage.append("]"); + logMessage.append(" [Original Query: ").append(originalQuery).append("]"); + return logMessage.toString(); + + } + + private static int getTotalChildCount(Operand rootOperand){ + int count = 0; + if( rootOperand != null) { + count ++; + if (rootOperand.getChildren() != null) { + for (Object child : rootOperand.getChildren()) { + if (child instanceof Operand) { + count += getTotalChildCount((Operand) child); + } + } + } + } + return count; + } + +} diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerEnforcementException.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerEnforcementException.java new file mode 100644 index 0000000000..454fbe13cf --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerEnforcementException.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. The ASF licenses this file to You + * under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. For additional information regarding + * copyright in this work, please see the NOTICE file in the top level + * directory of this distribution. + */ +package org.apache.usergrid.persistence.index.exceptions; + + +import org.apache.usergrid.persistence.index.QueryAnalyzer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class QueryAnalyzerEnforcementException extends RuntimeException { + + private List> violations = new ArrayList<>(); + private String originalQuery; + + public QueryAnalyzerEnforcementException(final List> violations, final String originalQuery) { + super("Query Analyzer Enforced"); + this.violations = violations; + this.originalQuery = originalQuery; + } + + public List> getViolations(){ + return violations; + } + + public String getErrorMessage() { + return QueryAnalyzer.violationsAsString(violations, originalQuery); + } +} diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerException.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerException.java new file mode 100644 index 0000000000..c39282271d --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/exceptions/QueryAnalyzerException.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. The ASF licenses this file to You + * under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. For additional information regarding + * copyright in this work, please see the NOTICE file in the top level + * directory of this distribution. + */ +package org.apache.usergrid.persistence.index.exceptions; + + +import org.apache.usergrid.persistence.index.QueryAnalyzer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public class QueryAnalyzerException extends RuntimeException { + + private List> violations = new ArrayList<>(); + private String originalQuery; + private UUID applicationUUID; + + public QueryAnalyzerException(final List> violations, final String originalQuery, + final UUID applicationUUID) { + super("Query Analyzer"); + this.violations = violations; + this.originalQuery = originalQuery; + this.applicationUUID = applicationUUID; + } + + public List> getViolations(){ + return violations; + } + + public String getErrorMessage() { + return QueryAnalyzer.violationsAsString(violations, originalQuery); + } + + public String getOriginalQuery(){ + return originalQuery; + } + + public UUID getApplicationUUID(){ + return applicationUUID; + } +} diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 3615d867c8..a35921c20c 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -21,6 +21,8 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; import com.google.common.base.*; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableMap; import com.google.common.io.Resources; import com.google.inject.Inject; @@ -36,6 +38,8 @@ import org.apache.usergrid.persistence.index.*; import org.apache.usergrid.persistence.index.ElasticSearchQueryBuilder.SearchRequestBuilderStrategyV2; import org.apache.usergrid.persistence.index.exceptions.IndexException; +import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerException; +import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerEnforcementException; import org.apache.usergrid.persistence.index.migration.IndexDataVersions; import org.apache.usergrid.persistence.index.query.ParsedQuery; import org.apache.usergrid.persistence.index.query.ParsedQueryBuilder; @@ -54,6 +58,8 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -78,6 +84,7 @@ import java.io.IOException; import java.net.URL; import java.util.*; +import java.util.concurrent.TimeUnit; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.APPLICATION_ID_FIELDNAME; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.applicationId; @@ -129,6 +136,10 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData { private Meter refreshIndexMeter; + private Cache sizeCache = + CacheBuilder.newBuilder().maximumSize( 1000 ).expireAfterWrite(5, TimeUnit.MINUTES).build(); + + @Inject public EsEntityIndexImpl( final EsProvider provider, final IndexCache indexCache, @@ -412,12 +423,13 @@ public EntityIndexBatch createBatch() { } public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, - final int limit, final int offset ) { - return search(searchEdge, searchTypes, query, limit, offset, new HashMap<>(0)); + final int limit, final int offset, final boolean analyzeOnly ) { + return search(searchEdge, searchTypes, query, limit, offset, new HashMap<>(0), analyzeOnly); } public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query, - final int limit, final int offset, final Map fieldsWithType ) { + final int limit, final int offset, final Map fieldsWithType, + final boolean analyzeOnly ) { IndexValidationUtils.validateSearchEdge(searchEdge); Preconditions.checkNotNull(searchTypes, "searchTypes cannot be null"); @@ -442,6 +454,36 @@ public CandidateResults search( final SearchEdge searchEdge, final SearchTypes s } + final String cacheKey = applicationScope.getApplication().getUuid().toString()+"_"+searchEdge.getEdgeName(); + final Object totalEdgeSizeFromCache = sizeCache.getIfPresent(cacheKey); + long totalEdgeSizeInBytes; + if (totalEdgeSizeFromCache == null){ + totalEdgeSizeInBytes = getTotalEntitySizeInBytes(searchEdge); + sizeCache.put(cacheKey, totalEdgeSizeInBytes); + }else{ + totalEdgeSizeInBytes = (long) totalEdgeSizeFromCache; + } + + final Object totalIndexSizeFromCache = sizeCache.getIfPresent(indexLocationStrategy.getIndexRootName()); + long totalIndexSizeInBytes; + if (totalIndexSizeFromCache == null){ + totalIndexSizeInBytes = getIndexSize(); + sizeCache.put(indexLocationStrategy.getIndexRootName(), totalIndexSizeInBytes); + }else{ + totalIndexSizeInBytes = (long) totalIndexSizeFromCache; + } + + List> violations = QueryAnalyzer.analyze(parsedQuery, totalEdgeSizeInBytes, totalIndexSizeInBytes, indexFig); + if(indexFig.enforceQueryBreaker() && violations.size() > 0){ + throw new QueryAnalyzerEnforcementException(violations, parsedQuery.getOriginalQuery()); + }else if (violations.size() > 0){ + logger.warn( QueryAnalyzer.violationsAsString(violations, parsedQuery.getOriginalQuery()) ); + } + + if(analyzeOnly){ + throw new QueryAnalyzerException(violations, parsedQuery.getOriginalQuery(), applicationScope.getApplication().getUuid()); + } + final SearchRequestBuilder srb = searchRequest .getBuilder( searchEdge, searchTypes, visitor, limit, offset, parsedQuery.getSortPredicates(), fieldsWithType ) .setTimeout(TimeValue.timeValueMillis(queryTimeout)); @@ -824,9 +866,20 @@ public Health getIndexHealth() { return Health.RED; } + private long getIndexSize(){ + final IndicesStatsResponse statsResponse = esProvider.getClient() + .admin() + .indices() + .prepareStats(indexLocationStrategy.getIndexInitialName()) + .all() + .execute() + .actionGet(); + final CommonStats indexStats = statsResponse.getIndex(indexLocationStrategy.getIndexInitialName()).getTotal(); + return indexStats.getStore().getSizeInBytes(); + } @Override - public long getEntitySize(final SearchEdge edge){ + public long getTotalEntitySizeInBytes(final SearchEdge edge){ //"term":{"edgeName":"zzzcollzzz|roles"} SearchRequestBuilder builder = searchRequestBuilderStrategyV2.getBuilder(); builder.setQuery(new TermQueryBuilder("edgeSearch",IndexingUtils.createContextName(applicationScope,edge))); diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java index 7ee89616cd..4dd0d247e2 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java @@ -363,7 +363,7 @@ public void visit( Equal op ) throws NoIndexException { //special case so we support our '*' char with wildcard, also should work for uuids if ( value instanceof String || value instanceof UUID ) { - final String stringValue = ((value instanceof String) ? (String)value : value.toString()).toLowerCase().trim(); + String stringValue = ((value instanceof String) ? (String)value : value.toString()).toLowerCase().trim(); // or field is just a string that does need a prefix us a query if ( stringValue.contains( "*" ) ) { @@ -377,6 +377,11 @@ public void visit( Equal op ) throws NoIndexException { return; } + // Usergrid query parser allows single quotes to be escaped in values + if ( stringValue.contains("\\'")) { + stringValue = stringValue.replace("\\'", "'"); + } + //it's an exact match, use a filter final TermFilterBuilder termFilter = FilterBuilders.termFilter( IndexingUtils.FIELD_STRING_NESTED_UNANALYZED, stringValue ); diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java index f98ccc703d..84a28f0422 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java @@ -31,9 +31,9 @@ public class Identifier implements Serializable { - public static final String UUID_REX = + public static final String UUID_REX = "[A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}"; - public static final String EMAIL_REX = "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"; + public static final String EMAIL_REX = "[a-zA-Z0-9._%'+\\-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"; public enum Type { UUID, NAME, EMAIL @@ -46,7 +46,7 @@ public enum Type { static Pattern emailRegEx = Pattern.compile( EMAIL_REX ); // "Pattern nameRegEx" below used to be [a-zA-Z0-9_\\-./], changed it to contain a 'space' to a // ddress https://issues.apache.org/jira/browse/USERGRID-94 - static Pattern nameRegEx = Pattern.compile( "[a-zA-Z0-9_\\-./ ]*" ); + static Pattern nameRegEx = Pattern.compile( "[a-zA-Z0-9_\\-./'+ ]*" ); private Identifier( Type type, Object value ) { diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index e64db83208..ac7d10d2bd 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -160,7 +160,7 @@ public void testIndexVariations() throws IOException { StopWatch timer = new StopWatch(); timer.start(); CandidateResults candidateResults = - entityIndex.search( indexEdge, searchTypes, "select * where testfield = 'test' order by ordinal", 100, 0 ); + entityIndex.search( indexEdge, searchTypes, "select * where testfield = 'test' order by ordinal", 100, 0, false ); timer.stop(); @@ -182,11 +182,11 @@ public void testIndexVariations() throws IOException { //make sure we can query uuids out as strings and not wrapped candidateResults = - entityIndex.search( indexEdge, searchTypes, "select * where testuuid = '"+uuid+"'", 100, 0 ); + entityIndex.search( indexEdge, searchTypes, "select * where testuuid = '"+uuid+"'", 100, 0, false ); assertEquals(entity1.getId(),candidateResults.get(0).getId()); candidateResults = - entityIndex.search( indexEdge, searchTypes, "select * where testuuid = "+uuid, 100, 0 ); + entityIndex.search( indexEdge, searchTypes, "select * where testuuid = "+uuid, 100, 0, false); assertEquals(entity1.getId(),candidateResults.get(0).getId()); } @@ -369,7 +369,7 @@ public void testDeindex() { entityIndex.refreshAsync().toBlocking().first(); CandidateResults candidateResults = entityIndex - .search( searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0 ); + .search( searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0, false ); assertEquals( 1, candidateResults.size() ); EntityIndexBatch batch = entityIndex.createBatch(); @@ -378,7 +378,7 @@ public void testDeindex() { entityIndex.refreshAsync().toBlocking().first(); candidateResults = entityIndex - .search(searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0 ); + .search(searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0, false ); assertEquals(0, candidateResults.size()); } @@ -390,7 +390,7 @@ private CandidateResults testQuery( final SearchEdge scope, final SearchTypes se StopWatch timer = new StopWatch(); timer.start(); - CandidateResults candidateResults = entityIndex.search( scope, searchTypes, queryString, 1000, 0 ); + CandidateResults candidateResults = entityIndex.search( scope, searchTypes, queryString, 1000, 0, false ); timer.stop(); @@ -518,7 +518,7 @@ public void deleteVerification() throws Throwable { final String query = "where username = 'edanuff'"; - CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0); + CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false); assertEquals( user.getId(), r.get( 0 ).getId()); batch.deindex( indexSCope, user.getId(), user.getVersion() ); @@ -528,7 +528,7 @@ public void deleteVerification() throws Throwable { // EntityRef - r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0 ); + r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false ); assertFalse( r.iterator().hasNext() ); } @@ -589,16 +589,16 @@ public void multiValuedTypes() { final SearchTypes searchTypes = SearchTypes.fromTypes( "user" ); - CandidateResults r = entityIndex.search( indexScope, searchTypes, "where username = 'bill'", 10, 0); + CandidateResults r = entityIndex.search( indexScope, searchTypes, "where username = 'bill'", 10, 0, false); assertEquals( bill.getId(), r.get( 0 ).getId() ); - r = entityIndex.search( indexScope, searchTypes, "where username = 'fred'", 10, 0); + r = entityIndex.search( indexScope, searchTypes, "where username = 'fred'", 10, 0, false); assertEquals(fred.getId(), r.get(0).getId()); - r = entityIndex.search( indexScope, searchTypes, "where age = 41", 10, 0); + r = entityIndex.search( indexScope, searchTypes, "where age = 41", 10, 0, false); assertEquals(fred.getId(), r.get(0).getId()); - r = entityIndex.search( indexScope, searchTypes, "where age = 'thirtysomething'", 10, 0); + r = entityIndex.search( indexScope, searchTypes, "where age = 'thirtysomething'", 10, 0, false); assertEquals(bill.getId(), r.get(0).getId()); } @@ -677,8 +677,8 @@ public void testCursorFormat() throws Exception { int i ; for ( i=0; i < expectedPages; i++ ) { final CandidateResults results = !offset.isPresent() - ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit, 0 ) - : entityIndex.search(indexEdge, SearchTypes.allTypes(), query, limit, i * limit); + ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit, 0, false ) + : entityIndex.search(indexEdge, SearchTypes.allTypes(), query, limit, i * limit, false); assertEquals(limit, results.size()); @@ -690,7 +690,7 @@ public void testCursorFormat() throws Exception { } //get our next page, we shouldn't get a cursor - final CandidateResults results = entityIndex.search(indexEdge, SearchTypes.allTypes(), query, limit, i * limit); + final CandidateResults results = entityIndex.search(indexEdge, SearchTypes.allTypes(), query, limit, i * limit, false); assertEquals( 0, results.size() ); assertFalse(results.hasOffset()); @@ -733,7 +733,7 @@ public void queryByUUID() throws Throwable { final String query = "where searchUUID = " + searchUUID; final CandidateResults r = - entityIndex.search( indexSCope, SearchTypes.fromTypes(entityId.getType()), query, 10, 0); + entityIndex.search( indexSCope, SearchTypes.fromTypes(entityId.getType()), query, 10, 0, false); assertEquals(user.getId(), r.get(0).getId()); } @@ -772,7 +772,7 @@ public void queryByStringWildCardSpaces() throws Throwable { final String query = "where string = 'I am*'"; final CandidateResults r = - entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), query, 10, 0); + entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), query, 10, 0, false); assertEquals(user.getId(), r.get(0).getId()); @@ -780,7 +780,7 @@ public void queryByStringWildCardSpaces() throws Throwable { final String queryNoWildCard = "where string = 'I am'"; final CandidateResults noWildCardResults = - entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), queryNoWildCard, 10, 0 ); + entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), queryNoWildCard, 10, 0, false ); assertEquals( 0, noWildCardResults.size() ); } @@ -830,7 +830,7 @@ public void sortyByString() throws Throwable { final String ascQuery = "order by string"; final CandidateResults ascResults = - entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), ascQuery, 10 , 0); + entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), ascQuery, 10 , 0, false); assertEquals( first.getId(), ascResults.get( 0).getId() ); @@ -841,7 +841,7 @@ public void sortyByString() throws Throwable { final String descQuery = "order by string desc"; final CandidateResults descResults = - entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), descQuery, 10 , 0); + entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), descQuery, 10 , 0, false); assertEquals( second.getId(), descResults.get( 0).getId() ); @@ -895,7 +895,7 @@ public void unionString() throws Throwable { final String singleMatchQuery = "string contains 'alpha' OR string contains 'foo'"; final CandidateResults singleResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), singleMatchQuery, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), singleMatchQuery, 10, 0, false ); assertEquals(1, singleResults.size()); @@ -906,7 +906,7 @@ public void unionString() throws Throwable { final String bothKeywordsMatch = "string contains 'alpha' OR string contains 'bravo'"; final CandidateResults singleKeywordUnion = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), bothKeywordsMatch, 10 , 0); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), bothKeywordsMatch, 10 , 0, false); assertEquals( 2, singleKeywordUnion.size() ); @@ -917,7 +917,7 @@ public void unionString() throws Throwable { final String twoKeywordMatches = "string contains 'alpha' OR string contains 'long'"; final CandidateResults towMatchResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), twoKeywordMatches, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), twoKeywordMatches, 10, 0, false ); assertEquals( 2, towMatchResults.size() ); @@ -976,7 +976,7 @@ public void notRootOperandFilter() throws Throwable { final String notFirst = "NOT int = 1"; final CandidateResults notFirstResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirst, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirst, 10, 0, false ); assertEquals( 1, notFirstResults.size() ); @@ -987,7 +987,7 @@ public void notRootOperandFilter() throws Throwable { final String notSecond = "NOT int = 2"; final CandidateResults notSecondUnion = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecond, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecond, 10, 0, false ); assertEquals( 1, notSecondUnion.size() ); @@ -997,7 +997,7 @@ public void notRootOperandFilter() throws Throwable { final String notBothReturn = "NOT int = 3"; final CandidateResults notBothReturnResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notBothReturn, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notBothReturn, 10, 0, false ); assertEquals( 2, notBothReturnResults.size() ); @@ -1008,7 +1008,7 @@ public void notRootOperandFilter() throws Throwable { final String notFilterBoth = "(NOT int = 1) AND (NOT int = 2) "; final CandidateResults filterBoth = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFilterBoth, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFilterBoth, 10, 0, false ); assertEquals( 0, filterBoth.size() ); @@ -1016,7 +1016,7 @@ public void notRootOperandFilter() throws Throwable { final String noMatchesAnd = "(NOT int = 3) AND (NOT int = 4)"; final CandidateResults noMatchesAndResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesAnd, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesAnd, 10, 0, false ); assertEquals( 2, noMatchesAndResults.size() ); @@ -1027,7 +1027,7 @@ public void notRootOperandFilter() throws Throwable { final String noMatchesOr = "(NOT int = 3) AND (NOT int = 4)"; final CandidateResults noMatchesOrResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesOr, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesOr, 10, 0, false ); assertEquals( 2, noMatchesOrResults.size() ); @@ -1086,7 +1086,7 @@ public void notRootOperandQuery() throws Throwable { final String notFirst = "NOT string = 'I ate a sammich'"; final CandidateResults notFirstResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirst, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirst, 10, 0, false ); assertEquals( 1, notFirstResults.size() ); @@ -1096,7 +1096,7 @@ public void notRootOperandQuery() throws Throwable { final String notFirstWildCard = "NOT string = 'I ate*'"; final CandidateResults notFirstWildCardResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirstWildCard, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirstWildCard, 10, 0, false ); assertEquals( 1, notFirstWildCardResults.size() ); @@ -1106,7 +1106,7 @@ public void notRootOperandQuery() throws Throwable { final String notFirstContains = "NOT string contains 'sammich'"; final CandidateResults notFirstContainsResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirstContains, 10 , 0); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFirstContains, 10 , 0, false); assertEquals( 1, notFirstContainsResults.size() ); @@ -1117,7 +1117,7 @@ public void notRootOperandQuery() throws Throwable { final String notSecond = "NOT string = 'I drank a beer'"; final CandidateResults notSecondUnion = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecond, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecond, 10, 0, false ); assertEquals( 1, notSecondUnion.size() ); @@ -1127,7 +1127,7 @@ public void notRootOperandQuery() throws Throwable { final String notSecondWildcard = "NOT string = 'I drank*'"; final CandidateResults notSecondWildcardUnion = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecondWildcard, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecondWildcard, 10, 0, false ); assertEquals( 1, notSecondWildcardUnion.size() ); @@ -1137,7 +1137,7 @@ public void notRootOperandQuery() throws Throwable { final String notSecondContains = "NOT string contains 'beer'"; final CandidateResults notSecondContainsUnion = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecondContains, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notSecondContains, 10, 0, false ); assertEquals( 1, notSecondContainsUnion.size() ); @@ -1147,7 +1147,7 @@ public void notRootOperandQuery() throws Throwable { final String notBothReturn = "NOT string = 'I'm a foodie'"; final CandidateResults notBothReturnResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notBothReturn, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notBothReturn, 10, 0, false ); assertEquals( 2, notBothReturnResults.size() ); @@ -1158,7 +1158,7 @@ public void notRootOperandQuery() throws Throwable { final String notFilterBoth = "(NOT string = 'I ate a sammich') AND (NOT string = 'I drank a beer') "; final CandidateResults filterBoth = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFilterBoth, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), notFilterBoth, 10, 0, false ); assertEquals( 0, filterBoth.size() ); @@ -1166,7 +1166,7 @@ public void notRootOperandQuery() throws Throwable { final String noMatchesAnd = "(NOT string = 'I ate*') AND (NOT string = 'I drank*')"; final CandidateResults noMatchesAndResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesAnd, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesAnd, 10, 0, false ); assertEquals( 0, noMatchesAndResults.size() ); @@ -1174,7 +1174,7 @@ public void notRootOperandQuery() throws Throwable { final String noMatchesContainsAnd = "(NOT string contains 'ate') AND (NOT string contains 'drank')"; final CandidateResults noMatchesContainsAndResults = entityIndex - .search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesContainsAnd, 10, 0 ); + .search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesContainsAnd, 10, 0, false ); assertEquals( 0, noMatchesContainsAndResults.size() ); @@ -1183,7 +1183,7 @@ public void notRootOperandQuery() throws Throwable { final String noMatchesOr = "(NOT string = 'I ate*') AND (NOT string = 'I drank*')"; final CandidateResults noMatchesOrResults = - entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesOr, 10, 0 ); + entityIndex.search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesOr, 10, 0, false ); assertEquals( 0, noMatchesOrResults.size() ); @@ -1191,7 +1191,7 @@ public void notRootOperandQuery() throws Throwable { final String noMatchesContainsOr = "(NOT string contains 'ate') AND (NOT string contains 'drank')"; final CandidateResults noMatchesContainsOrResults = entityIndex - .search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesContainsOr, 10, 0 ); + .search(indexScope1, SearchTypes.fromTypes( first.getId().getType() ), noMatchesContainsOr, 10, 0, false ); assertEquals( 0, noMatchesContainsOrResults.size() ); @@ -1236,7 +1236,7 @@ public void testSizeByEdge(){ indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); - long size = entityIndex.getEntitySize(new SearchEdgeImpl(ownerId,type, SearchEdge.NodeType.SOURCE)); + long size = entityIndex.getTotalEntitySizeInBytes(new SearchEdgeImpl(ownerId,type, SearchEdge.NodeType.SOURCE)); assertTrue( size == 100 ); } @@ -1289,7 +1289,7 @@ public void testNullFields() throws IOException { StopWatch timer = new StopWatch(); timer.start(); CandidateResults candidateResults = - entityIndex.search( indexEdge, searchTypes, "select * where uuid = '"+uuid+"'", 100, 0 ); + entityIndex.search( indexEdge, searchTypes, "select * where uuid = '"+uuid+"'", 100, 0, false ); timer.stop(); diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java index 3d68fe1e31..ba3303070a 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java @@ -138,7 +138,7 @@ public void groupQueriesWithDistanceOrderedResults() throws Exception { final String query = "select * where location within 1500000 of 37, -75" ; final CandidateResults - candidates = entityIndex.search( edge, SearchTypes.fromTypes( "cat" ), query, 100, 0 ); + candidates = entityIndex.search( edge, SearchTypes.fromTypes( "cat" ), query, 100, 0, false ); assertNotNull( candidates ); diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java index afaebb7bb1..a5c3b538e4 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java @@ -223,7 +223,7 @@ public void testHeavyLoadValidate() { .search( searchEdge, SearchTypes.fromTypes( searchEdge.getEdgeName() ), "select * where " + FIELD_WORKER_INDEX + " = " + workerIndex + " AND " + FIELD_ORDINAL + " = " + ordinal + " AND " + FIELD_UNIQUE_IDENTIFIER + " = '" + uniqueIdentifier - + "'" , 100 , 0); + + "'" , 100 , 0, false); queryTps.mark(); queryTimerContext.stop(); diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/QueryAnalyzerTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/QueryAnalyzerTest.java new file mode 100644 index 0000000000..8c319c04c5 --- /dev/null +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/QueryAnalyzerTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. The ASF licenses this file to You + * under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. For additional information regarding + * copyright in this work, please see the NOTICE file in the top level + * directory of this distribution. + */ +package org.apache.usergrid.persistence.index.impl; + + +import java.util.List; +import java.util.Map; + +import org.apache.usergrid.persistence.index.IndexFig; +import org.apache.usergrid.persistence.index.QueryAnalyzer; +import org.apache.usergrid.persistence.index.query.ParsedQuery; +import org.apache.usergrid.persistence.index.query.ParsedQueryBuilder; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.*; + +public class QueryAnalyzerTest extends BaseIT { + + private static final Logger logger = LoggerFactory.getLogger(QueryAnalyzerTest.class); + + private IndexFig fig; + + @Before + public void setup() { + + // the mock will return 0/empty values for all fig items + fig = mock ( IndexFig.class ); + + } + + @Test + public void testNoViolations() throws Throwable { + + when ( fig.getQueryBreakerErrorSortPredicateCount() ).thenReturn( 5 ); + when ( fig.getQueryBreakerErrorOperandCount() ).thenReturn( 5 ); + when ( fig.getQueryBreakerErrorIndexSizeBytes() ).thenReturn( 5L ); + when ( fig.getQueryBreakerErrorCollectionSizeBytes() ).thenReturn( 5L ); + + ParsedQuery parsedQuery; + List> violations; + + + parsedQuery = ParsedQueryBuilder.build("select * order by created asc"); + violations = QueryAnalyzer.analyze(parsedQuery, 1, 1, fig ); + assertEquals(0, violations.size()); + + parsedQuery = ParsedQueryBuilder.build("select name='value' order by created asc"); + violations = QueryAnalyzer.analyze(parsedQuery, 1, 1, fig ); + assertEquals(0, violations.size()); + + parsedQuery = ParsedQueryBuilder.build("where name='value'"); + violations = QueryAnalyzer.analyze(parsedQuery, 1, 1, fig ); + assertEquals(0, violations.size()); + + + } + + + @Test + public void IndexSizeViolation() throws Throwable { + + + ParsedQuery parsedQuery = ParsedQueryBuilder.build("select created order by created asc"); + List> violations; + violations = QueryAnalyzer.analyze(parsedQuery, 0, 1, fig ); + boolean violationExists = violationExists(violations, QueryAnalyzer.v_large_index); + + if(!violationExists){ + fail("Index Size Violation should be present"); + } + + } + + @Test + public void collectionSizeViolation() throws Throwable { + + // the sort violation is only tripped when the collection size warning is tripped + when ( fig.getQueryBreakerErrorCollectionSizeBytes() ).thenReturn( 0L ); + + ParsedQuery parsedQuery = ParsedQueryBuilder.build("select created order by created asc"); + List> violations; + violations = QueryAnalyzer.analyze(parsedQuery, 1, 1, fig ); + boolean violationExists = violationExists(violations, QueryAnalyzer.v_large_collection); + + if(!violationExists){ + fail("Collection Size Violation should be present"); + } + + } + + + @Test + public void fullSortViolation() throws Throwable { + + ParsedQuery parsedQuery = ParsedQueryBuilder.build("select * order by created asc"); + List> violations; + violations = QueryAnalyzer.analyze(parsedQuery, 1, 1, fig ); + boolean violationExists = violationExists(violations, QueryAnalyzer.v_full_collection_sort); + + if(!violationExists){ + fail("Full Collection Sort Violation should be present"); + } + + } + + @Test + public void operandCountViolation() throws Throwable { + + ParsedQuery parsedQuery = ParsedQueryBuilder.build("where name='value'"); + List> violations; + violations = QueryAnalyzer.analyze(parsedQuery, 0, 0, fig ); + boolean violationExists = violationExists(violations, QueryAnalyzer.v_operand_count); + + if(!violationExists){ + fail("Operand Count Violation should be present"); + } + + } + + @Test + public void predicateCountViolation() throws Throwable { + + ParsedQuery parsedQuery = ParsedQueryBuilder.build("where name='value'"); + List> violations; + violations = QueryAnalyzer.analyze(parsedQuery, 0, 0, fig ); + boolean violationExists = violationExists(violations, QueryAnalyzer.v_operand_count); + + if(!violationExists){ + fail("Operand Count Violation should be present"); + } + + } + + + + private boolean violationExists(final List> violations, final String expectedViolation){ + for ( Map violation : violations ){ + if (violation.get(QueryAnalyzer.k_violation) == expectedViolation){ + return true; + } + + } + return false; + } + +} + + + diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/QakkaFig.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/QakkaFig.java index 061807b60e..778274e0dd 100644 --- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/QakkaFig.java +++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/QakkaFig.java @@ -165,7 +165,7 @@ public interface QakkaFig extends GuicyFig, Serializable { long getMaxShardSize(); @Key(QUEUE_LONG_POLL_TIME_MILLIS) - @Default("5000") + @Default("1000") long getLongPollTimeMillis(); /** Max time-to-live for queue message and payload data */ @@ -174,7 +174,7 @@ public interface QakkaFig extends GuicyFig, Serializable { int getMaxTtlSeconds(); @Key(QUEUE_IN_MEMORY) - @Default("true") + @Default("false") // in memory not ready yet; leave this to false else msgs could be processed more than once boolean getInMemoryCache(); @Key(QUEUE_IN_MEMORY_REFRESH_ASYNC) diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/core/impl/InMemoryQueue.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/core/impl/InMemoryQueue.java index 09bb8de722..fa5ee0b704 100644 --- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/core/impl/InMemoryQueue.java +++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/core/impl/InMemoryQueue.java @@ -59,7 +59,7 @@ private Queue getQueue( String queueName ) { } } - public void add( String queueName, DatabaseQueueMessage databaseQueueMessage ) { + synchronized public void add( String queueName, DatabaseQueueMessage databaseQueueMessage ) { UUID newest = newestByQueueName.get( queueName ); if ( newest == null ) { @@ -76,7 +76,7 @@ public void add( String queueName, DatabaseQueueMessage databaseQueueMessage ) { getQueue( queueName ).add( databaseQueueMessage ); } - public UUID getNewest( String queueName ) { + synchronized public UUID getNewest( String queueName ) { if ( getQueue( queueName ).isEmpty() ) { newestByQueueName.remove( queueName ); } diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/core/impl/QueueMessageManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/core/impl/QueueMessageManagerImpl.java index ac2857f262..fd4257b1f2 100644 --- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/core/impl/QueueMessageManagerImpl.java +++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/core/impl/QueueMessageManagerImpl.java @@ -44,6 +44,7 @@ import java.util.Collection; import java.util.List; import java.util.UUID; +import java.util.concurrent.TimeUnit; @Singleton @@ -188,7 +189,7 @@ private List joinMessages( String queueName, Collection getMessagesFromStorage(String queueName } } - newestFetchedUuid.put( queueName, since ); + updateUUIDPointer(queueName, since); // Shard currentShard = multiShardIterator.getCurrentShard(); // if ( currentShard != null ) { @@ -279,7 +280,7 @@ DistributedQueueService.Status ackQueueMessage(String queueName, UUID queueMessa } - void queueRefresh( String queueName ) { + synchronized void queueRefresh( String queueName ) { Timer.Context timer = metricsService.getMetricRegistry().timer( MetricsService.REFRESH_TIME).time(); @@ -327,7 +328,7 @@ void queueRefresh( String queueName ) { startingShards.put( shardKey, shardId ); - lastRefreshTimeMillis.put( queueName, System.currentTimeMillis() ); + updateLastRefreshedTime(queueName); if ( count > 0 ) { Object shard = shardIdOptional.isPresent() ? shardIdOptional.get() : "null"; @@ -346,4 +347,12 @@ private String createShardKey(String queueName, Shard.Type type, String region ) return queueName + "_" + type + region; } + private synchronized void updateUUIDPointer(String queueName, UUID newUUIDPointer){ + newestFetchedUuid.put( queueName, newUUIDPointer ); + } + + private synchronized void updateLastRefreshedTime(String queueName){ + lastRefreshTimeMillis.put( queueName, System.currentTimeMillis() ); + } + } diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/actors/QueueActorRouter.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/actors/QueueActorRouter.java index 1ff8502020..cbc7245169 100644 --- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/actors/QueueActorRouter.java +++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/actors/QueueActorRouter.java @@ -133,7 +133,7 @@ private void initIfNeeded( String queueName ) { getContext().dispatcher(), getSelf() ); shardAllocationSchedulersByQueueName.put( queueName, scheduler ); - logger.debug( "Created shard allocater for queue {}", queueName ); + logger.debug( "Created shard allocator for queue {}", queueName ); } } } diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocator.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocator.java index 19059e6cc1..75c1c22fe9 100644 --- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocator.java +++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocator.java @@ -139,8 +139,8 @@ private void checkLatestShard( String queueName, Shard.Type type ) { shardSerialization.createShard( newShard ); shardCounterSerialization.incrementCounter( queueName, type, newShard.getShardId(), 0 ); - logger.info("{} Created new shard for queue {} shardId {} timestamp {} counterValue {}", - this.hashCode(), queueName, shard.getShardId(), futureUUID.timestamp(), counterValue ); + logger.info("Allocated new shard for queue, newShardID: {}, queueName: {}, shardMessageCount: {}, usedPercent: {}%", + newShard.getShardId(), queueName, counterValue, (long)((double)counterValue/(double)qakkaFig.getMaxShardSize()*100) ); } else { // logger.debug("No new shard for queue {} counterValue {} of max {}", diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/impl/DistributedQueueServiceImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/impl/DistributedQueueServiceImpl.java index 98e055a52e..b1f72aaaf2 100644 --- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/impl/DistributedQueueServiceImpl.java +++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/distributed/impl/DistributedQueueServiceImpl.java @@ -20,6 +20,8 @@ package org.apache.usergrid.persistence.qakka.distributed.impl; import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import akka.dispatch.OnFailure; import akka.pattern.Patterns; import akka.util.Timeout; import com.codahale.metrics.*; @@ -42,7 +44,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.concurrent.Await; +import scala.concurrent.ExecutionContext; import scala.concurrent.Future; +import scala.concurrent.Promise; import java.lang.reflect.Method; import java.util.*; @@ -235,20 +239,21 @@ public Collection getNextMessages( String queueName, int c public Collection getNextMessagesInternal( String queueName, int count ) { if ( actorSystemManager.getClientActor() == null || !actorSystemManager.isReady() ) { - logger.error("Akka Actor System is not ready yet for requests."); - return Collections.EMPTY_LIST; + logger.warn("Akka Actor System is not ready yet for requests."); + return Collections.emptyList(); } int maxRetries = qakkaFig.getMaxGetRetries(); int tries = 0; + boolean interrupted = false; + QueueGetRequest request = new QueueGetRequest( queueName, count ); while ( ++tries < maxRetries ) { try { Timeout t = new Timeout( qakkaFig.getGetTimeoutSeconds(), TimeUnit.SECONDS ); // ask ClientActor and wait (up to timeout) for response - Future fut = Patterns.ask( actorSystemManager.getClientActor(), request, t ); Object responseObject = Await.result( fut, t.duration() ); @@ -259,8 +264,8 @@ public Collection getNextMessagesInternal( String queueNam if ( response != null && response instanceof QueueGetResponse) { QueueGetResponse qprm = (QueueGetResponse)response; if ( qprm.isSuccess() ) { - if (tries > 1) { - logger.warn( "getNextMessage {} SUCCESS after {} tries", queueName, tries ); + if (tries > 1 && !interrupted) { + logger.warn( "getNextMessage for queue {} SUCCESS after {} tries", queueName, tries ); } } return qprm.getQueueMessages(); @@ -284,10 +289,13 @@ public Collection getNextMessagesInternal( String queueNam } } catch ( TimeoutException e ) { - logger.trace("TIMEOUT popping to queue " + queueName + " retrying " + tries, e ); - - } catch ( Exception e ) { - logger.debug("ERROR popping to queue " + queueName + " retrying " + tries, e ); + logger.warn("TIMEOUT popping queue " + queueName + ", attempt: " + tries, e ); + } catch(InterruptedException e){ + interrupted = true; + // this might happen, retry the ask again + logger.trace("Thread was marked interrupted so unable to wait for the result, attempt: {}", tries); + }catch ( Exception e ) { + logger.error("ERROR popping queue " + queueName + ", attempt: " + tries, e ); } } diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocatorTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocatorTest.java index 11f3d0877a..4745cb1a94 100644 --- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocatorTest.java +++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/distributed/actors/ShardAllocatorTest.java @@ -181,6 +181,9 @@ public void testBasicOperationWithMessages() throws InterruptedException { distributedQueueService.refresh(); + // the shard allocator kicks in when messages are first received + distributedQueueService.getNextMessages(queueName,10); + try { // Create number of messages diff --git a/stack/corepersistence/queue/src/test/resources/qakka.properties b/stack/corepersistence/queue/src/test/resources/qakka.properties index 94bfeffb97..d77e7e88a3 100644 --- a/stack/corepersistence/queue/src/test/resources/qakka.properties +++ b/stack/corepersistence/queue/src/test/resources/qakka.properties @@ -36,7 +36,8 @@ usergrid.cluster.seeds=us-east:localhost # Port used for cluster communications. usergrid.cluster.port=3545 -queue.inmemory.cache=true +# In-Memory Queueing Not Ready Yet; Leave this to false else, messages are potentially processed more than once +queue.inmemory.cache=false queue.num.actors=50 queue.sender.num.actors=100 @@ -47,7 +48,7 @@ queue.get.timeout.seconds=5 # set shard size and times low for testing purposes queue.shard.max.size=10 -queue.shard.allocation.check.frequency.millis=1000 +queue.shard.allocation.check.frequency.millis=500 queue.shard.allocation.advance.time.millis=200 # set low for testing purposes diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java index 9836f1c9b9..7479a9035f 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java @@ -25,7 +25,13 @@ import org.apache.amber.oauth2.common.message.types.GrantType; import org.apache.shiro.authz.UnauthorizedException; import org.apache.shiro.codec.Base64; +import org.apache.usergrid.corepersistence.export.ExportRequestBuilder; +import org.apache.usergrid.corepersistence.export.ExportRequestBuilderImpl; +import org.apache.usergrid.corepersistence.export.ExportService; +import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder; +import org.apache.usergrid.corepersistence.index.ReIndexService; import org.apache.usergrid.management.ApplicationInfo; +import org.apache.usergrid.management.OrganizationInfo; import org.apache.usergrid.management.exceptions.DisabledAdminUserException; import org.apache.usergrid.management.exceptions.DisabledAppUserException; import org.apache.usergrid.management.exceptions.UnactivatedAdminUserException; @@ -37,6 +43,7 @@ import org.apache.usergrid.persistence.entities.User; import org.apache.usergrid.persistence.exceptions.EntityNotFoundException; import org.apache.usergrid.persistence.index.query.Identifier; +import org.apache.usergrid.persistence.index.utils.UUIDUtils; import org.apache.usergrid.rest.AbstractContextResource; import org.apache.usergrid.rest.ApiResponse; import org.apache.usergrid.rest.applications.assets.AssetsResource; @@ -58,6 +65,8 @@ import javax.ws.rs.*; import javax.ws.rs.core.*; +import java.io.IOException; +import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.Map; @@ -676,4 +685,45 @@ public String setAPMConfig( @Context UriInfo ui, } + private ExportService getExportService() { + return injector.getInstance( ExportService.class ); + } + + + @GET + @Path("export") + @RequireApplicationAccess + @Produces({"application/zip"}) + public Response getExport( @Context UriInfo ui, + @QueryParam("callback") @DefaultValue("callback") String callback ) + throws Exception { + + if (logger.isTraceEnabled()) { + logger.trace("ApplicationResource.getExport"); + } + + if ( !isApplicationAdmin( Identifier.fromUUID( applicationId ) ) ) { + throw new UnauthorizedException(); + } + + ApplicationInfo appInfo = management.getApplicationInfo(applicationId); + + final ExportRequestBuilder request = new ExportRequestBuilderImpl().withApplicationId( applicationId ); + StreamingOutput stream = new StreamingOutput() { + @Override + public void write(OutputStream outputStream) throws IOException, WebApplicationException { + getExportService().export(request,outputStream); + } + }; + return Response + .ok(stream) + .header("Content-Disposition", "attachment; filename=\""+appInfo.getName().replace("/","_")+"_"+System.currentTimeMillis()+".zip\"") + .build(); + } + + + + + + } diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/AuthResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/AuthResource.java index e5464dee8a..fe498ea0cf 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/AuthResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/AuthResource.java @@ -57,6 +57,7 @@ import static org.apache.usergrid.rest.utils.JSONPUtils.wrapWithCallback; +/** @noinspection Duplicates*/ @Component @Scope("prototype") @Produces({ @@ -211,6 +212,42 @@ public Response authFB( @Context UriInfo ui, @QueryParam("fb_access_token") Stri } } + @CheckPermissionsForPath + @GET + @Path("google") + public Response authGoogle( @Context UriInfo ui, @QueryParam("google_access_token") String access_token, + @QueryParam("ttl") long ttl, @QueryParam("callback") @DefaultValue("") String callback ) + throws Exception { + + if (logger.isTraceEnabled()) { + logger.trace("AuthResource.authGoogle"); + } + + try { + if ( StringUtils.isEmpty( access_token ) ) { + return missingTokenFail( callback ); + } + SignInAsProvider facebookProvider = signInProviderFactory.google( services.getApplication() ); + User user = facebookProvider.createOrAuthenticate( access_token ); + + if ( user == null ) { + return findAndCreateFail( callback ); + } + + String token = management.getAccessTokenForAppUser( services.getApplicationId(), user.getUuid(), ttl ); + + AccessInfo access_info = + new AccessInfo().withExpiresIn( tokens.getMaxTokenAgeInSeconds( token ) ).withAccessToken( token ) + .withProperty( "user", user ); + + return Response.status( SC_OK ).type( jsonMediaType( callback ) ) + .entity( wrapWithCallback( access_info, callback ) ).build(); + } + catch ( Exception e ) { + return generalAuthError( callback, e ); + } + } + @CheckPermissionsForPath @POST diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java index 898eb3bc09..b8c1caa5d6 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java @@ -18,16 +18,10 @@ package org.apache.usergrid.rest.applications; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; -import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -35,7 +29,6 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.PathSegment; -import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import org.springframework.context.annotation.Scope; @@ -44,25 +37,15 @@ import org.apache.commons.lang.NullArgumentException; import org.apache.commons.lang.StringUtils; -import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder; -import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilderImpl; -import org.apache.usergrid.corepersistence.index.ReIndexService; import org.apache.usergrid.persistence.Query; -import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException; -import org.apache.usergrid.persistence.index.utils.UUIDUtils; -import org.apache.usergrid.rest.AbstractContextResource; import org.apache.usergrid.rest.ApiResponse; -import org.apache.usergrid.rest.RootResource; -import org.apache.usergrid.rest.exceptions.RequiredPropertyNotFoundExceptionMapper; import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess; import org.apache.usergrid.rest.security.annotations.RequireSystemAccess; import org.apache.usergrid.rest.system.IndexResource; -import org.apache.usergrid.services.AbstractCollectionService; import org.apache.usergrid.services.ServiceAction; import org.apache.usergrid.services.ServiceParameter; import org.apache.usergrid.services.ServicePayload; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.jaxrs.json.annotation.JSONP; diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java index 91c7db9f62..3835b75530 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java @@ -32,7 +32,6 @@ import org.apache.usergrid.rest.RootResource; import org.apache.usergrid.rest.applications.assets.AssetsResource; import org.apache.usergrid.rest.security.annotations.CheckPermissionsForPath; -import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess; import org.apache.usergrid.security.oauth.AccessInfo; import org.apache.usergrid.services.*; import org.apache.usergrid.services.assets.data.AssetUtils; @@ -262,7 +261,7 @@ public ServiceResults executeServiceGetRequestForSettings(UriInfo ui, ApiRespons addQueryParams( getServiceParameters(), ui ); ServiceRequest r = services.newRequest( action, tree, getServiceParameters(), payload, - returnInboundConnections, returnOutboundConnections ); + returnInboundConnections, returnOutboundConnections, false); response.setServiceRequest( r ); @@ -316,7 +315,7 @@ public ServiceResults executeServicePostRequestForSettings(UriInfo ui, ApiRespon addQueryParams( getServiceParameters(), ui ); ServiceRequest r = services.newRequest( action, tree, getServiceParameters(), payload, - returnInboundConnections, returnOutboundConnections ); + returnInboundConnections, returnOutboundConnections, false); response.setServiceRequest( r ); @@ -400,13 +399,15 @@ public ServiceResults executeServiceRequest( UriInfo ui, ApiResponse response, S } } + boolean analyzeQueryOnly = Boolean.valueOf(ui.getQueryParameters().getFirst("analyzeOnly")); + boolean collectionGet = false; if ( action == ServiceAction.GET ) { collectionGet = getServiceParameters().size() == 1; } addQueryParams( getServiceParameters(), ui ); ServiceRequest r = services.newRequest( action, tree, getServiceParameters(), payload, - returnInboundConnections, returnOutboundConnections ); + returnInboundConnections, returnOutboundConnections, analyzeQueryOnly); response.setServiceRequest( r ); ServiceResults results = r.execute(); if ( results != null ) { diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AbstractExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AbstractExceptionMapper.java index 4a4b8b0c2a..8319ff5985 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AbstractExceptionMapper.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AbstractExceptionMapper.java @@ -72,10 +72,9 @@ public Response toResponse( int status, E e ) { if ( status >= 500 ) { // only log real errors as errors logger.error( "{} 5XX Uncaught Exception ({})", e.getClass().getCanonicalName(), status, e ); - } else { if (logger.isDebugEnabled()) { - logger.debug( "{} Following Exception Thrown ({})", e.getClass().getCanonicalName(), status, e ); + logger.debug( "{} Following Exception Thrown ({}), {}", e.getClass().getCanonicalName(), status, e ); } } @@ -98,12 +97,19 @@ public Response toResponse( int status, E e ) { public Response toResponse( Status status, String jsonResponse ) { - return toResponse( status.getStatusCode(), jsonResponse ); + return toResponse( status.getStatusCode(), jsonResponse, false ); } + public Response toResponse( Status status, String jsonResponse, boolean skipLogging ) { + return toResponse( status.getStatusCode(), jsonResponse, skipLogging ); + } - protected Response toResponse( int status, String jsonResponse ) { - if ( status >= 500 ) { + public Response toResponse( int statusCode, String jsonResponse ) { + return toResponse( statusCode, jsonResponse, false ); + } + + protected Response toResponse( int status, String jsonResponse, boolean skipLogging ) { + if ( status >= 500 && !skipLogging) { // only log real errors as errors logger.error( "Server Error ({}):\n{}", status, jsonResponse ); } else if ( logger.isDebugEnabled() ) { diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerEnforcementExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerEnforcementExceptionMapper.java new file mode 100644 index 0000000000..fed97d1049 --- /dev/null +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerEnforcementExceptionMapper.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.rest.exceptions; + + +import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerEnforcementException; +import org.apache.usergrid.rest.ApiResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.Provider; + +import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; +import static org.apache.usergrid.utils.JsonUtils.mapToJsonString; + + +@Provider +public class QueryAnalyzerEnforcementExceptionMapper extends AbstractExceptionMapper { + + private static final Logger logger = LoggerFactory.getLogger( QueryAnalyzerEnforcementExceptionMapper.class ); + + @Override + public Response toResponse( QueryAnalyzerEnforcementException e ) { + + // build a proper ApiResponse object + ApiResponse apiResponse = new ApiResponse(); + apiResponse.setError("query_analyzer_violations_enforced"); + apiResponse.setErrorDescription(e.getErrorMessage()); + + logger.warn(e.getErrorMessage()); + + // give toResponse() the json string value of the ApiResponse + // skip logging because we use a 5XX but it's not technically an error for the logs + return toResponse( SERVICE_UNAVAILABLE, mapToJsonString(apiResponse), true ); + } +} diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerExceptionMapper.java new file mode 100644 index 0000000000..bd5db1455c --- /dev/null +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/QueryAnalyzerExceptionMapper.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.rest.exceptions; + + +import org.apache.usergrid.persistence.index.QueryAnalyzer; +import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerException; +import org.apache.usergrid.rest.ApiResponse; + +import javax.ws.rs.core.MultivaluedHashMap; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.Provider; + +import java.util.Collections; +import java.util.HashMap; + +import static javax.ws.rs.core.Response.Status.OK; +import static org.apache.usergrid.utils.JsonUtils.mapToFormattedJsonString; + + +@Provider +public class QueryAnalyzerExceptionMapper extends AbstractExceptionMapper { + + // This mapper is used to short-circuit the query process on any collection query. Therefore, it does its best + // job to build a normal ApiResponse format with 200 OK status. + + @Override + public Response toResponse( QueryAnalyzerException e ) { + + + MultivaluedMap params = new MultivaluedHashMap<>(); + params.add("ql", e.getOriginalQuery()); + params.add("analyzeOnly", "true"); + + // build a proper ApiResponse object + ApiResponse apiResponse = new ApiResponse(); + apiResponse.setParams(params); + apiResponse.setSuccess(); + + // remove large_index warnings because indexes are shared buckets and not specific for an app + for( int i=0; i < e.getViolations().size(); i++) { + if (e.getViolations().get(i).get(QueryAnalyzer.k_violation) == QueryAnalyzer.v_large_index) { + e.getViolations().remove(i); + } + } + + apiResponse.setMetadata(new HashMap(){{put("queryWarnings", e.getViolations());}}); + apiResponse.setEntities(Collections.emptyList()); + apiResponse.setAction("query analysis only"); + + // give toResponse() the json string value of the ApiResponse + return toResponse( OK, mapToFormattedJsonString(apiResponse) ); + } +} diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java index a12e4846a9..7411ea4304 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java @@ -66,8 +66,8 @@ public class OrganizationResource extends AbstractContextResource { private static final Logger logger = LoggerFactory.getLogger( OrganizationsResource.class ); - @Autowired - protected ExportService exportService; + //@Autowired + //protected ExportService exportService; OrganizationInfo organization; @@ -281,99 +281,99 @@ public ApiResponse executePut( @Context UriInfo ui, Map json, return response; } - @POST - @Path("export") - @Consumes(APPLICATION_JSON) - @RequireOrganizationAccess - public Response exportPostJson( @Context UriInfo ui,Map json, - @QueryParam("callback") @DefaultValue("") String callback ) - throws OAuthSystemException { - - if (logger.isTraceEnabled()) { - logger.trace("executePostJson"); - } - - Map uuidRet = new HashMap<>(); - - try { - Object propertiesObj = json.get("properties"); - if (propertiesObj == null) { - throw new NullArgumentException("Could not find 'properties'"); - } - if (!(propertiesObj instanceof Map)) { - throw new IllegalArgumentException("'properties' not a map"); - } - - @SuppressWarnings("unchecked") - Map properties = (Map)propertiesObj; - - String storage_provider = ( String ) properties.get( "storage_provider" ); - if(storage_provider == null) { - throw new NullArgumentException( "Could not find field 'storage_provider'" ); - } - - Object storageInfoObj = properties.get("storage_info"); - if(storageInfoObj == null) { - throw new NullArgumentException( "Could not find field 'storage_info'" ); - } - @SuppressWarnings("unchecked") - Map storage_info = (Map)storageInfoObj; - - String bucketName = ( String ) storage_info.get( "bucket_location" ); - String accessId = ( String ) storage_info.get( "s3_access_id" ); - String secretKey = ( String ) storage_info.get( "s3_key" ); - - if ( bucketName == null ) { - throw new NullArgumentException( "Could not find field 'bucketName'" ); - } - if ( accessId == null ) { - throw new NullArgumentException( "Could not find field 's3_access_id'" ); - } - if ( secretKey == null ) { - - throw new NullArgumentException( "Could not find field 's3_key'" ); - } - - json.put( "organizationId",organization.getUuid()); - - UUID jobUUID = exportService.schedule( json ); - uuidRet.put( "Export Entity", jobUUID.toString() ); - } - catch ( NullArgumentException e ) { - return Response.status( SC_BAD_REQUEST ).type( JSONPUtils.jsonMediaType( callback ) ) - .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build(); - } - catch ( Exception e ) { - //TODO:throw descriptive error message and or include on in the response - //TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse. - return Response.status( SC_INTERNAL_SERVER_ERROR ).type( JSONPUtils.jsonMediaType( callback ) ) - .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build(); - } - return Response.status( SC_ACCEPTED ).entity( uuidRet ).build(); - } - - @GET - @RequireOrganizationAccess - @Path("export/{exportEntity: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}") - public Response exportGetJson( @Context UriInfo ui, @PathParam("exportEntity") UUID exportEntityUUIDStr, - @QueryParam("callback") @DefaultValue("") String callback ) throws Exception { - - Export entity; - try { - entity = smf.getServiceManager( emf.getManagementAppId() ).getEntityManager() - .get( exportEntityUUIDStr, Export.class ); - } - catch ( Exception e ) { //this might not be a bad request and needs better error checking - return Response.status( SC_BAD_REQUEST ).type( JSONPUtils.jsonMediaType( callback ) ) - .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build(); - } - - if ( entity == null ) { - return Response.status( SC_BAD_REQUEST ).build(); - } - - return Response.status( SC_OK ).entity( entity).build(); - } +// @POST +// @Path("export") +// @Consumes(APPLICATION_JSON) +// @RequireOrganizationAccess +// public Response exportPostJson( @Context UriInfo ui,Map json, +// @QueryParam("callback") @DefaultValue("") String callback ) +// throws OAuthSystemException { +// +// if (logger.isTraceEnabled()) { +// logger.trace("executePostJson"); +// } +// +// Map uuidRet = new HashMap<>(); +// +// try { +// Object propertiesObj = json.get("properties"); +// if (propertiesObj == null) { +// throw new NullArgumentException("Could not find 'properties'"); +// } +// if (!(propertiesObj instanceof Map)) { +// throw new IllegalArgumentException("'properties' not a map"); +// } +// +// @SuppressWarnings("unchecked") +// Map properties = (Map)propertiesObj; +// +// String storage_provider = ( String ) properties.get( "storage_provider" ); +// if(storage_provider == null) { +// throw new NullArgumentException( "Could not find field 'storage_provider'" ); +// } +// +// Object storageInfoObj = properties.get("storage_info"); +// if(storageInfoObj == null) { +// throw new NullArgumentException( "Could not find field 'storage_info'" ); +// } +// @SuppressWarnings("unchecked") +// Map storage_info = (Map)storageInfoObj; +// +// String bucketName = ( String ) storage_info.get( "bucket_location" ); +// String accessId = ( String ) storage_info.get( "s3_access_id" ); +// String secretKey = ( String ) storage_info.get( "s3_key" ); +// +// if ( bucketName == null ) { +// throw new NullArgumentException( "Could not find field 'bucketName'" ); +// } +// if ( accessId == null ) { +// throw new NullArgumentException( "Could not find field 's3_access_id'" ); +// } +// if ( secretKey == null ) { +// +// throw new NullArgumentException( "Could not find field 's3_key'" ); +// } +// +// json.put( "organizationId",organization.getUuid()); +// +// UUID jobUUID = exportService.schedule( json ); +// uuidRet.put( "Export Entity", jobUUID.toString() ); +// } +// catch ( NullArgumentException e ) { +// return Response.status( SC_BAD_REQUEST ).type( JSONPUtils.jsonMediaType( callback ) ) +// .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build(); +// } +// catch ( Exception e ) { +// //TODO:throw descriptive error message and or include on in the response +// //TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse. +// return Response.status( SC_INTERNAL_SERVER_ERROR ).type( JSONPUtils.jsonMediaType( callback ) ) +// .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build(); +// } +// return Response.status( SC_ACCEPTED ).entity( uuidRet ).build(); +// } + +// @GET +// @RequireOrganizationAccess +// @Path("export/{exportEntity: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}") +// public Response exportGetJson( @Context UriInfo ui, @PathParam("exportEntity") UUID exportEntityUUIDStr, +// @QueryParam("callback") @DefaultValue("") String callback ) throws Exception { +// +// Export entity; +// try { +// entity = smf.getServiceManager( emf.getManagementAppId() ).getEntityManager() +// .get( exportEntityUUIDStr, Export.class ); +// } +// catch ( Exception e ) { //this might not be a bad request and needs better error checking +// return Response.status( SC_BAD_REQUEST ).type( JSONPUtils.jsonMediaType( callback ) ) +// .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build(); +// } +// +// if ( entity == null ) { +// return Response.status( SC_BAD_REQUEST ).build(); +// } +// +// return Response.status( SC_OK ).entity( entity).build(); +// } protected Set getSetFromCommaSeparatedString(String input) { diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java index a8ed8dc454..79973c3145 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java @@ -74,8 +74,8 @@ public class ApplicationResource extends AbstractContextResource { public static final String CONFIRM_APPLICATION_IDENTIFIER = "confirm_application_identifier"; - @Autowired - protected ExportService exportService; + //@Autowired + //protected ExportService exportService; OrganizationInfo organization; UUID applicationId; @@ -267,161 +267,161 @@ else if ( StringUtils.equalsIgnoreCase( siaProvider, "foursquare" ) ) { return response; } - @POST - @Path("export") - @Consumes(APPLICATION_JSON) - @RequireOrganizationAccess - public Response exportPostJson( @Context UriInfo ui,Map json, - @QueryParam("callback") @DefaultValue("") String callback ) - throws OAuthSystemException { - - UsergridAwsCredentials uac = new UsergridAwsCredentials(); - - UUID jobUUID = null; - Map uuidRet = new HashMap(); - - Map properties; - Map storage_info; - - try { - if((properties = ( Map ) json.get( "properties" )) == null){ - throw new NullArgumentException("Could not find 'properties'"); - } - storage_info = ( Map ) properties.get( "storage_info" ); - String storage_provider = ( String ) properties.get( "storage_provider" ); - if(storage_provider == null) { - throw new NullArgumentException( "Could not find field 'storage_provider'" ); - } - if(storage_info == null) { - throw new NullArgumentException( "Could not find field 'storage_info'" ); - } - - - String bucketName = ( String ) storage_info.get( "bucket_location" ); - String accessId = ( String ) storage_info.get( "s3_access_id" ); - String secretKey = ( String ) storage_info.get( "s3_key" ); - - if ( bucketName == null ) { - throw new NullArgumentException( "Could not find field 'bucketName'" ); - } - if ( accessId == null ) { - throw new NullArgumentException( "Could not find field 's3_access_id'" ); - } - if ( secretKey == null ) { - - throw new NullArgumentException( "Could not find field 's3_key'" ); - } - - json.put("organizationId", organization.getUuid()); - json.put( "applicationId",applicationId); - - jobUUID = exportService.schedule( json ); - uuidRet.put( "Export Entity", jobUUID.toString() ); - } - catch ( NullArgumentException e ) { - return Response.status( SC_BAD_REQUEST ) - .type( JSONPUtils.jsonMediaType( callback ) ) - .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build(); - } - catch ( Exception e ) { - // TODO: throw descriptive error message and or include on in the response - // TODO: fix below, it doesn't work if there is an exception. - // Make it look like the OauthResponse. - return Response.status( SC_INTERNAL_SERVER_ERROR ) - .type( JSONPUtils.jsonMediaType( callback ) ) - .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build(); - } - - return Response.status( SC_ACCEPTED ).entity( uuidRet ).build(); - } - - @POST - @Path("collection/{collection_name}/export") - @Consumes(APPLICATION_JSON) - @RequireOrganizationAccess - public Response exportPostJson( @Context UriInfo ui, - @PathParam( "collection_name" ) String collection_name ,Map json, - @QueryParam("callback") @DefaultValue("") String callback ) - throws OAuthSystemException { - - UsergridAwsCredentials uac = new UsergridAwsCredentials(); - UUID jobUUID = null; - String colExport = collection_name; - Map uuidRet = new HashMap(); - - Map properties; - Map storage_info; - - try { - //checkJsonExportProperties(json); - if((properties = ( Map ) json.get( "properties" )) == null){ - throw new NullArgumentException("Could not find 'properties'"); - } - storage_info = ( Map ) properties.get( "storage_info" ); - String storage_provider = ( String ) properties.get( "storage_provider" ); - if(storage_provider == null) { - throw new NullArgumentException( "Could not find field 'storage_provider'" ); - } - if(storage_info == null) { - throw new NullArgumentException( "Could not find field 'storage_info'" ); - } - - String bucketName = ( String ) storage_info.get( "bucket_location" ); - String accessId = ( String ) storage_info.get( "s3_access_id" ); - String secretKey = ( String ) storage_info.get( "s3_key" ); - - if ( accessId == null ) { - throw new NullArgumentException( "Could not find field 's3_access_id'" ); - } - if ( secretKey == null ) { - throw new NullArgumentException( "Could not find field 's3_key'" ); - } - - if(bucketName == null) { - throw new NullArgumentException( "Could not find field 'bucketName'" ); - } - - json.put( "organizationId",organization.getUuid() ); - json.put( "applicationId", applicationId); - json.put( "collectionName", colExport); - - jobUUID = exportService.schedule( json ); - uuidRet.put( "Export Entity", jobUUID.toString() ); - } - catch ( NullArgumentException e ) { - return Response.status( SC_BAD_REQUEST ) - .type( JSONPUtils.jsonMediaType( callback ) ) - .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ) - .build(); - } - catch ( Exception e ) { - - // TODO: throw descriptive error message and or include on in the response - // TODO: fix below, it doesn't work if there is an exception. - // Make it look like the OauthResponse. - - OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_INTERNAL_SERVER_ERROR ) - .setErrorDescription( e.getMessage() ) - .buildJSONMessage(); - - return Response.status( errorMsg.getResponseStatus() ) - .type( JSONPUtils.jsonMediaType( callback ) ) - .entity( ServiceResource.wrapWithCallback( errorMsg.getBody(), callback ) ) - .build(); - } - - return Response.status( SC_ACCEPTED ).entity( uuidRet ).build(); - } - - - @Path( "imports" ) - public ImportsResource importGetJson( @Context UriInfo ui, - @QueryParam( "callback" ) @DefaultValue( "" ) String callback ) - throws Exception { - - - return getSubResource( ImportsResource.class ).init( organization, application ); - } +// @POST +// @Path("export") +// @Consumes(APPLICATION_JSON) +// @RequireOrganizationAccess +// public Response exportPostJson( @Context UriInfo ui,Map json, +// @QueryParam("callback") @DefaultValue("") String callback ) +// throws OAuthSystemException { +// +// UsergridAwsCredentials uac = new UsergridAwsCredentials(); +// +// UUID jobUUID = null; +// Map uuidRet = new HashMap(); +// +// Map properties; +// Map storage_info; +// +// try { +// if((properties = ( Map ) json.get( "properties" )) == null){ +// throw new NullArgumentException("Could not find 'properties'"); +// } +// storage_info = ( Map ) properties.get( "storage_info" ); +// String storage_provider = ( String ) properties.get( "storage_provider" ); +// if(storage_provider == null) { +// throw new NullArgumentException( "Could not find field 'storage_provider'" ); +// } +// if(storage_info == null) { +// throw new NullArgumentException( "Could not find field 'storage_info'" ); +// } +// +// +// String bucketName = ( String ) storage_info.get( "bucket_location" ); +// String accessId = ( String ) storage_info.get( "s3_access_id" ); +// String secretKey = ( String ) storage_info.get( "s3_key" ); +// +// if ( bucketName == null ) { +// throw new NullArgumentException( "Could not find field 'bucketName'" ); +// } +// if ( accessId == null ) { +// throw new NullArgumentException( "Could not find field 's3_access_id'" ); +// } +// if ( secretKey == null ) { +// +// throw new NullArgumentException( "Could not find field 's3_key'" ); +// } +// +// json.put("organizationId", organization.getUuid()); +// json.put( "applicationId",applicationId); +// +// jobUUID = exportService.schedule( json ); +// uuidRet.put( "Export Entity", jobUUID.toString() ); +// } +// catch ( NullArgumentException e ) { +// return Response.status( SC_BAD_REQUEST ) +// .type( JSONPUtils.jsonMediaType( callback ) ) +// .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build(); +// } +// catch ( Exception e ) { +// // TODO: throw descriptive error message and or include on in the response +// // TODO: fix below, it doesn't work if there is an exception. +// // Make it look like the OauthResponse. +// return Response.status( SC_INTERNAL_SERVER_ERROR ) +// .type( JSONPUtils.jsonMediaType( callback ) ) +// .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build(); +// } +// +// return Response.status( SC_ACCEPTED ).entity( uuidRet ).build(); +// } +// +// @POST +// @Path("collection/{collection_name}/export") +// @Consumes(APPLICATION_JSON) +// @RequireOrganizationAccess +// public Response exportPostJson( @Context UriInfo ui, +// @PathParam( "collection_name" ) String collection_name ,Map json, +// @QueryParam("callback") @DefaultValue("") String callback ) +// throws OAuthSystemException { +// +// UsergridAwsCredentials uac = new UsergridAwsCredentials(); +// UUID jobUUID = null; +// String colExport = collection_name; +// Map uuidRet = new HashMap(); +// +// Map properties; +// Map storage_info; +// +// try { +// //checkJsonExportProperties(json); +// if((properties = ( Map ) json.get( "properties" )) == null){ +// throw new NullArgumentException("Could not find 'properties'"); +// } +// storage_info = ( Map ) properties.get( "storage_info" ); +// String storage_provider = ( String ) properties.get( "storage_provider" ); +// if(storage_provider == null) { +// throw new NullArgumentException( "Could not find field 'storage_provider'" ); +// } +// if(storage_info == null) { +// throw new NullArgumentException( "Could not find field 'storage_info'" ); +// } +// +// String bucketName = ( String ) storage_info.get( "bucket_location" ); +// String accessId = ( String ) storage_info.get( "s3_access_id" ); +// String secretKey = ( String ) storage_info.get( "s3_key" ); +// +// if ( accessId == null ) { +// throw new NullArgumentException( "Could not find field 's3_access_id'" ); +// } +// if ( secretKey == null ) { +// throw new NullArgumentException( "Could not find field 's3_key'" ); +// } +// +// if(bucketName == null) { +// throw new NullArgumentException( "Could not find field 'bucketName'" ); +// } +// +// json.put( "organizationId",organization.getUuid() ); +// json.put( "applicationId", applicationId); +// json.put( "collectionName", colExport); +// +// jobUUID = exportService.schedule( json ); +// uuidRet.put( "Export Entity", jobUUID.toString() ); +// } +// catch ( NullArgumentException e ) { +// return Response.status( SC_BAD_REQUEST ) +// .type( JSONPUtils.jsonMediaType( callback ) ) +// .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ) +// .build(); +// } +// catch ( Exception e ) { +// +// // TODO: throw descriptive error message and or include on in the response +// // TODO: fix below, it doesn't work if there is an exception. +// // Make it look like the OauthResponse. +// +// OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_INTERNAL_SERVER_ERROR ) +// .setErrorDescription( e.getMessage() ) +// .buildJSONMessage(); +// +// return Response.status( errorMsg.getResponseStatus() ) +// .type( JSONPUtils.jsonMediaType( callback ) ) +// .entity( ServiceResource.wrapWithCallback( errorMsg.getBody(), callback ) ) +// .build(); +// } +// +// return Response.status( SC_ACCEPTED ).entity( uuidRet ).build(); +// } +// +// +// @Path( "imports" ) +// public ImportsResource importGetJson( @Context UriInfo ui, +// @QueryParam( "callback" ) @DefaultValue( "" ) String callback ) +// throws Exception { +// +// +// return getSubResource( ImportsResource.class ).init( organization, application ); +// } @GET @Path("/status") diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java index 905c522677..997f56c039 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java @@ -21,6 +21,7 @@ import org.apache.usergrid.rest.test.resource.AbstractRestIT; import org.apache.usergrid.rest.test.resource.model.ApiResponse; import org.apache.usergrid.rest.test.resource.model.Entity; +import org.apache.usergrid.rest.test.resource.model.QueryParameters; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,6 +70,38 @@ public void testGenericCollectionEntityNameUuid() throws Exception { assertNotNull( returnedUser ); } + @Test + public void testEntityWithSingleQuoteName() throws Exception { + + Entity payload = new Entity(); + payload.put( "name", "test'value" ); + + //Create a user with the payload + Entity returnedUser = this.app().collection( "suspects" ).post(payload); + assertNotNull( returnedUser ); + + + returnedUser = this.app().collection( "suspects" ).entity("test'value").get(); + + assertNotNull( returnedUser ); + } + + @Test + public void testEntityWithPlusName() throws Exception { + + Entity payload = new Entity(); + payload.put( "name", "test+value" ); + + //Create a user with the payload + Entity returnedUser = this.app().collection( "suspects" ).post(payload); + assertNotNull( returnedUser ); + + + returnedUser = this.app().collection( "suspects" ).entity("test+value").get(); + + assertNotNull( returnedUser ); + } + @Test public void serviceResourceNotFoundReturns404() throws Exception { diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/CollectionMetadataIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/CollectionMetadataIT.java index 4e5893596e..a96d725235 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/CollectionMetadataIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/CollectionMetadataIT.java @@ -63,7 +63,7 @@ public void testCollectionQueryParameter() throws Exception { e3 = this.app().collection(collectionName).post(e3); assertNotNull(e3); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // create connections // e1 hates e3 @@ -73,7 +73,7 @@ public void testCollectionQueryParameter() throws Exception { // e3 has one in (hates) connection this.app().collection(collectionName).entity(e1).connection("hates").entity(e3).post(); this.app().collection(collectionName).entity(e2).connection("likes").entity(e1).post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // no query param, "all", and invalid param all the same checkMetadata(e1, null, "hates", "likes"); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/NotificationsIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/NotificationsIT.java index fa68350454..922c678133 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/NotificationsIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/NotificationsIT.java @@ -18,17 +18,15 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Slf4jReporter; -import com.fasterxml.jackson.databind.JsonNode; + import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import javax.ws.rs.core.MediaType; + import org.apache.commons.lang3.time.StopWatch; -import org.apache.usergrid.rest.test.resource.*; -import org.apache.usergrid.rest.test.resource.endpoints.NamedResource; import org.apache.usergrid.rest.test.resource.model.*; import org.apache.usergrid.rest.test.resource.model.ApiResponse; import org.junit.After; @@ -85,7 +83,7 @@ public void testPaging() throws Exception { String unIndexedCollectionName = "notifications"; app().collection( unIndexedCollectionName ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // create notifier Entity notifier = new Entity().chainPut("name", "mynotifier").chainPut("provider", "noop"); @@ -103,7 +101,7 @@ public void testPaging() throws Exception { Token token = this.app().token().post(new Token("ed", "sesame")); this.clientSetup.getRestClient().token().setToken(token); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); // create devices int devicesCount = 0; @@ -129,7 +127,7 @@ public void testPaging() throws Exception { devicesCount++; } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); String postMeterName = getClass().getSimpleName() + ".postNotifications"; Meter postMeter = registry.meter( postMeterName ); @@ -168,7 +166,7 @@ public void testPaging() throws Exception { } registry.remove( postMeterName ); - refreshIndex( ); + waitForQueueDrainAndRefreshIndex( ); logger.info("Waiting for all notifications to be sent"); StopWatch sw = new StopWatch(); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java index 10673659be..9b295f0702 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/PartialUpdateTest.java @@ -61,7 +61,7 @@ public void testPartialUpdate() throws IOException { String uuid = userNode.get("uuid").toString(); assertNotNull(uuid); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Map updateProperties = new LinkedHashMap(); // update user bart passing only an update to a property @@ -81,7 +81,7 @@ public void testPartialUpdate() throws IOException { fail("Update failed due to: " + uie.getResponse().readEntity(String.class)); } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // retrieve the user from the backend userNode = this.app().collection("users").entity(userNode).get(); @@ -123,7 +123,7 @@ public void testPartialUpdate() throws IOException { } catch (ClientErrorException uie) { fail("Update failed due to: " + uie.getResponse().readEntity(String.class)); } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); userNode = this.app().collection("users").entity(userNode).get(); assertNotNull(userNode); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/SystemResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/SystemResourceIT.java index dd2a7330de..383c0460ad 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/SystemResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/SystemResourceIT.java @@ -24,7 +24,6 @@ import java.util.LinkedHashMap; import java.util.Map; -import java.util.UUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -57,7 +56,7 @@ public void testDeleteAllApplicationEntities() throws Exception{ for(int i =0; i fileMetadata = (Map)getResponse.getEntities().get(0).get("file-metadata"); assertNotNull( fileMetadata ); @@ -403,7 +403,7 @@ public void fileTooLargeShouldResultInError() throws Exception { @Test public void deleteConnectionToAsset() throws IOException { - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); // create the entity that will be the asset, an image @@ -427,7 +427,7 @@ public void deleteConnectionToAsset() throws IOException { ApiResponse connectResponse = pathResource( getOrgAppPath( "imagegalleries/" + imageGalleryId + "/contains/" + uuid ) ).post( ApiResponse.class ); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); // verify connection from imagegallery to asset @@ -438,7 +438,7 @@ public void deleteConnectionToAsset() throws IOException { // delete the connection pathResource( getOrgAppPath( "imagegalleries/" + imageGalleryId + "/contains/" + uuid ) ).delete(); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); // verify that connection is gone diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/BrowserCompatibilityTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/BrowserCompatibilityTest.java index b453ed293e..b63400a42b 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/BrowserCompatibilityTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/BrowserCompatibilityTest.java @@ -69,7 +69,7 @@ private void testBrowserAccept( String acceptHeader ) throws IOException { Entity entity = this.app().collection("things").post(payload); assertEquals(entity.get("name"), name); String uuid = entity.getAsString("uuid"); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //now get this new entity with "text/html" in the accept header Entity returnedEntity = this.app().collection("things").withAcceptHeader(acceptHeader).entity(entity).get(); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java index d72054a381..bf06c211ad 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java @@ -136,7 +136,7 @@ public void postToCollectionSchemaUsingOrgAppCreds(){ fail("This should return a success."); } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Collection collection = this.app().collection( "testCollections" ).collection( "_settings" ).get(); @@ -159,7 +159,7 @@ public void deleteCollectionSchema() throws Exception { //Post index to the collection metadata Entity thing = this.app().collection( "testCollections" ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //The above verifies the test case. @@ -172,7 +172,7 @@ public void deleteCollectionSchema() throws Exception { //Post entity. Entity postedEntity = this.app().collection( "testCollections" ).post( testEntity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Do a query to see if you can find the indexed query. String query = "two ='query'"; @@ -198,11 +198,11 @@ public void deleteCollectionSchema() throws Exception { //next part is to delete the schema then reindex it and it should work. this.app().collection( "testCollections" ).collection( "_settings" ).delete(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); this.app().collection( "testCollections" ).collection( "_reindex" ) .post(true,clientSetup.getSuperuserToken(),ApiResponse.class,null,null,false); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Do a query to see if you can find the indexed query. @@ -233,14 +233,14 @@ public void postCollectionSchemaWithWildcardIndexAll() throws Exception { Entity payload = new Entity(); payload.put( "fields", "all"); app().collection( "testCollection" ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // post entity with two fields Entity testEntity = new Entity(); testEntity.put( "one", "helper" ); testEntity.put( "two","query" ); app().collection( "testCollection" ).post( testEntity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // verify it can be queried on both fields @@ -288,7 +288,7 @@ public void postToCollectionSchemaUpdateExistingCollection() throws Exception { //Post index to the collection metadata Entity thing = this.app().collection( "testCollection" ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Reindex and verify that the entity only has field one index. @@ -339,7 +339,7 @@ public void postToCollectionSchemaAndVerifyFieldsAreUpdated() throws Exception { //Post index to the collection metadata Entity thing = this.app().collection( "testCollection" ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Collection collection = this.app().collection( "testCollection" ).collection( "_settings" ).get(); @@ -419,7 +419,7 @@ public void postToCollectionSchemaWithSchemaFirst() throws Exception { //Post index to the collection metadata this.app().collection( "testCollection" ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Create test collection with a test entity that is partially indexed. Entity testEntity = new Entity(); @@ -428,7 +428,7 @@ public void postToCollectionSchemaWithSchemaFirst() throws Exception { //Post entity. this.app().collection( "testCollection" ).post( testEntity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Do a query to see if you can find the indexed query. String query = "two ='query'"; @@ -461,7 +461,7 @@ public void postToCollectionArraySchemaWithSchemaFirst() throws Exception { //Post index to the collection metadata this.app().collection( "testCollection" ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Map arrayFieldsForTesting = new HashMap<>(); @@ -475,7 +475,7 @@ public void postToCollectionArraySchemaWithSchemaFirst() throws Exception { //Post entity. this.app().collection( "testCollection" ).post( testEntity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Do a query to see if you can find the indexed query. String query = "one.key = 'value'"; @@ -511,7 +511,7 @@ public void postToCollectionSchemaArrayWithTopLevelIndexing() throws Exception { //Post index to the collection metadata this.app().collection( "testCollection" ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Map arrayFieldsForTesting = new HashMap<>(); @@ -525,7 +525,7 @@ public void postToCollectionSchemaArrayWithTopLevelIndexing() throws Exception { //Post entity. this.app().collection( "testCollection" ).post( testEntity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Do a query to see if you can find the indexed query. String query = "one.key = 'value'"; @@ -554,7 +554,7 @@ public void postToCollectionSchemaArrayWithSelectiveTopLevelIndexing() throws Ex //Post index to the collection metadata this.app().collection( "testCollection" ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Map arrayFieldsForTestingSelectiveIndexing = new HashMap<>(); @@ -573,7 +573,7 @@ public void postToCollectionSchemaArrayWithSelectiveTopLevelIndexing() throws Ex //Post entity. this.app().collection( "testCollection" ).post( testEntity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Do a query to see if you can find the indexed query. String query = "one.key.wowMoreKeys = 'value'"; @@ -609,7 +609,7 @@ public void postToCollectionSchemaArrayWithSelectiveTopLevelIndexingAddingDefaul //Post index to the collection metadata this.app().collection( "testCollection" ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Map arrayFieldsForTestingSelectiveIndexing = new HashMap<>(); @@ -629,7 +629,7 @@ public void postToCollectionSchemaArrayWithSelectiveTopLevelIndexingAddingDefaul //Post entity. this.app().collection( "testCollection" ).post( testEntity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Do a query to see if you can find the indexed query. String query = "name = 'howdy'"; @@ -660,7 +660,7 @@ public void putToCollectionSchema() throws Exception { //Post index to the collection metadata this.app().collection( "testCollection" ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Create test collection with a test entity that is partially indexed. Entity testEntity = new Entity(); @@ -669,11 +669,11 @@ public void putToCollectionSchema() throws Exception { //Post entity. Entity postedEntity = this.app().collection( "testCollection" ).post( testEntity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); testEntity.put( "one","three" ); this.app().collection( "testCollection" ).entity( postedEntity.getUuid() ).put( testEntity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Do a query to see if you can find the indexed query. String query = "one = 'three'"; @@ -715,7 +715,7 @@ public void permissionWithMeInString() throws Exception { Entity user = this.app().collection("users").post(payload); assertEquals(user.get("username"), username); assertEquals(user.get("email"), email); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); String collectionName = "nestprofiles"; //create a permission with the path "me" in it @@ -743,7 +743,7 @@ public void permissionWithMeInString() throws Exception { Entity nestProfile = this.app().collection(collectionName).post(payload); assertEquals(nestProfile.get("name"), profileName); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); Entity nestprofileReturned = this.app().collection(collectionName).entity(nestProfile).get(); assertEquals(nestprofileReturned.get("name"), profileName); @@ -766,7 +766,7 @@ public void stringWithSpaces() throws IOException { assertEquals( calendarlistOne.get( "summaryOverview" ), summaryOverview ); assertEquals(calendarlistOne.get("caltype"), calType); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //post a second entity payload = new Entity(); @@ -819,9 +819,9 @@ public void testNewlyCreatedCollectionReturnsWhenEmpty(){ assertNotSame( null, ((LinkedHashMap)(collectionHashMap.get( "collections" ))).get( collectionName.toLowerCase() )); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); this.app().collection( collectionName ).entity( testEntity.getEntity().getUuid() ).delete(); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //Verify that the collection still exists despite deleting its only entity.) @@ -850,7 +850,7 @@ public void testNoDuplicateFields() throws Exception { payload.put("name", name); Entity user = this.app().collection("app_users").post(payload); assertEquals(user.get("name"), name); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); Entity user2 = this.app().collection("app_users").entity(user).get(); @@ -880,7 +880,7 @@ public void postCollectionSchemaWithWildcardIndexNone() throws Exception { String randomizer = RandomStringUtils.randomAlphanumeric(10); String collectionName = "col_" + randomizer; app().collection( collectionName ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // was the no-index wildcard saved and others ignored? Collection collection = app().collection( collectionName ).collection( "_settings" ).get(); @@ -923,7 +923,7 @@ public void testIndexedEntityToUnindexedEntityConnections() { String randomizer = RandomStringUtils.randomAlphanumeric(10); String unIndexedCollectionName = "col_" + randomizer; app().collection( unIndexedCollectionName ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); String entityName1 = "unindexed1"; Entity unindexed1 = this.app().collection( unIndexedCollectionName ) @@ -982,7 +982,7 @@ public void testUnindexedEntityToIndexedEntityConnections() { String unIndexedCollectionName = "col_" + randomizer; app().collection( unIndexedCollectionName ).collection( "_settings" ).post( payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); String entityName1 = "unindexed1"; Entity unindexed1 = this.app().collection( unIndexedCollectionName ) @@ -1018,7 +1018,7 @@ public void testCollectionAuthoritativeRegion() { app().collection( collectionName ).collection( "_settings" ) .post( new Entity().chainPut( "fields", "all" ) ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // get collection settings, should see no region @@ -1051,7 +1051,7 @@ public void testCollectionAuthoritativeRegion() { app().collection( collectionName ).collection( "_settings" ) .post( new Entity().chainPut( REGION_SETTING, "" ) ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // get collection settings, should see no region @@ -1091,14 +1091,14 @@ public void testBeingAbleToRetreiveMigratedValues() throws Exception { this.app().collection("notifications/"+ UUIDUtils.newTimeUUID()).post(payload ); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); Collection user2 = this.app().collection("notifications").get(); assertEquals(1,user2.getNumOfEntities()); this.app().collection("notifications/"+ UUIDUtils.newTimeUUID()).put(null,payload ); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); user2 = this.app().collection("notifications").get(); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java index 0776705855..7e1c5a5955 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java @@ -41,7 +41,7 @@ public void duplicateNamePrevention() { entity.put("name", "enzo"); //Create an entity named "enzo" in the "things" collection entity = this.app().collection(collectionName).post(entity); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); try { // Try to create a second entity in "things" with the name "enzo". this.app().collection(collectionName).post(entity); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/ActivityResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/ActivityResourceIT.java index c7f39b264f..a9f5feef09 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/ActivityResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/ActivityResourceIT.java @@ -59,7 +59,7 @@ public void setup() { this.activityDesc = "testActivity" ; this.activity = new ActivityEntity().putActor(current).chainPut("title", activityTitle).chainPut("content", activityDesc).chainPut("category", "testCategory").chainPut("verb", "POST"); this.groupActivityResource = groupsResource.entity(entity).activities(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); } @@ -87,7 +87,7 @@ public void postGroupActivity() { { throw e; } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Collection results = groupActivityResource.get(); @@ -111,7 +111,7 @@ public void postUserActivity() { usersResource.entity(current).activities().post(activity); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Collection results = usersResource.entity(current).activities().get(); @@ -136,7 +136,7 @@ public void postActivity() { this.app().collection("activities").post(activity); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Collection results = this.app().collection("activities").get(); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/PutTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/PutTest.java index d61d363b72..4ba897730f 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/PutTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/activities/PutTest.java @@ -17,7 +17,6 @@ package org.apache.usergrid.rest.applications.collection.activities; -import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -28,7 +27,6 @@ import org.apache.usergrid.rest.test.resource.endpoints.CollectionEndpoint; import org.apache.usergrid.rest.test.resource.model.Collection; import org.apache.usergrid.rest.test.resource.model.Entity; -import org.junit.Rule; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +58,7 @@ public void putMassUpdateTest() throws IOException { Entity activity = activities.post(new Entity(props)); } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); String query = "select * "; @@ -72,7 +70,7 @@ public void putMassUpdateTest() throws IOException { props.put( "actor", newActor ); Entity activity = activities.post(new Entity(props)); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); collection = activities.get( ); assertEquals( 6, collection.getResponse().getEntities().size() ); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/devices/DevicesResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/devices/DevicesResourceIT.java index 67cf19fc0d..b73bcbd945 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/devices/DevicesResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/devices/DevicesResourceIT.java @@ -17,11 +17,8 @@ package org.apache.usergrid.rest.applications.collection.devices; -import java.util.HashMap; -import java.util.Map; import java.util.UUID; -import com.fasterxml.jackson.databind.JsonNode; import org.apache.usergrid.persistence.model.util.UUIDGenerator; import org.apache.usergrid.rest.test.resource.AbstractRestIT; import org.apache.usergrid.rest.test.resource.endpoints.CollectionEndpoint; @@ -35,7 +32,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; -import org.junit.Ignore; import javax.ws.rs.ClientErrorException; @@ -51,7 +47,7 @@ public void putWithUUIDShouldCreateAfterDelete() throws IOException { CollectionEndpoint devicesResource =this.app().collection("devices"); Entity entity = devicesResource.entity(uuid).put(payload); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // create assertNotNull( entity ); @@ -62,7 +58,7 @@ public void putWithUUIDShouldCreateAfterDelete() throws IOException { ApiResponse deleteResponse =devicesResource.entity(uuid).delete(); assertNotNull(deleteResponse.getEntities().get(0)); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // check deleted try { @@ -72,7 +68,7 @@ public void putWithUUIDShouldCreateAfterDelete() throws IOException { catch ( ClientErrorException e ) { assertEquals( 404, e.getResponse().getStatus() ); } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // create again entity = devicesResource.entity(uuid).put(payload); @@ -80,7 +76,7 @@ public void putWithUUIDShouldCreateAfterDelete() throws IOException { assertNotNull( entity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // check existence entity = devicesResource.entity(uuid).get(); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/groups/GroupResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/groups/GroupResourceIT.java index 769852dd5f..b94050b341 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/groups/GroupResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/groups/GroupResourceIT.java @@ -21,7 +21,6 @@ import org.apache.usergrid.rest.test.resource.AbstractRestIT; import org.apache.usergrid.rest.test.resource.model.Collection; import org.apache.usergrid.rest.test.resource.model.Entity; -import org.junit.Ignore; import org.junit.Test; import javax.ws.rs.ClientErrorException; @@ -58,7 +57,7 @@ private Entity createGroup(String groupName, String groupPath, String groupTitle Entity entity = this.app().collection("groups").post(payload); assertEquals(entity.get("name"), groupName); assertEquals(entity.get("path"), groupPath); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); return entity; } @@ -74,7 +73,7 @@ private Entity createRole(String roleName, String roleTitle) throws IOException{ Entity entity = this.app().collection("roles").post(payload); assertEquals(entity.get("name"), roleName); assertEquals(entity.get("title"), roleTitle); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); return entity; } @@ -91,7 +90,7 @@ private Entity createUser(String username, String email, String password) throws Entity entity = this.app().collection("users").post(payload); assertEquals(entity.get("username"), username); assertEquals(entity.get("email"), email); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); return entity; } @@ -180,7 +179,7 @@ public void groupCRUDOperations() throws IOException { group.put("path", newGroupPath); Entity groupResponse = this.app().collection("groups").entity(group).put(group); assertEquals(groupResponse.get("path"), newGroupPath); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //4. do a GET to verify the property really was set groupResponseGET = this.app().collection("groups").entity(group).get(); @@ -223,7 +222,7 @@ public void addRemoveUserGroup() throws IOException { // 3. add the user to the group Entity response = this.app().collection("users").entity(user).connection().collection("groups").entity(group).post(); assertEquals(response.get("name"), groupName); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); // 4. make sure the user is in the group Collection collection = this.app().collection("groups").entity(group).connection().collection("users").get(); @@ -237,7 +236,7 @@ public void addRemoveUserGroup() throws IOException { //6. remove the user from the group this.app().collection("group").entity(group).connection().collection("users").entity(user).delete(); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //6. make sure the connection no longer exists collection = this.app().collection("group").entity(group).connection().collection("users").get(); @@ -266,12 +265,12 @@ public void addRemoveRoleGroup() throws Exception { String roleName = "tester"; String roleTitle = "tester"; Entity role = this.createRole(roleName, roleTitle); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //3. add role to the group Entity response = this.app().collection("roles").entity(role).connection().collection("groups").entity(group).post(); assertEquals(response.get("name"), groupName); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //4. make sure the role is in the group Collection collection = this.app().collection("groups").entity(group).connection().collection("roles").get(); @@ -280,7 +279,7 @@ public void addRemoveRoleGroup() throws Exception { //5. remove Role from the group (should only delete the connection) this.app().collection("groups").entity(group).connection().collection("roles").entity(role).delete(); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //6. make sure the connection no longer exists collection = this.app().collection("groups").entity(group).connection().collection("roles").get(); @@ -294,7 +293,7 @@ public void addRemoveRoleGroup() throws Exception { //8. delete the role this.app().collection("role").entity(role).delete(); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); Thread.sleep(5000); //9. do a GET to make sure the role was deleted @@ -359,7 +358,7 @@ public void addRolePermissionToGroupVerifyPermission() throws IOException { payload.put("name", catName); Entity fluffy = this.app().collection("cats").post(payload); assertEquals(fluffy.get("name"), catName); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //10. get the cat - permissions should allow this fluffy = this.app().collection("cats").uniqueID(catName).get(); @@ -436,7 +435,7 @@ public void postPrivateGroupActivity() throws IOException { //7. get all the users in the groups - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); Collection usersInGroup = this.app().collection("groups").uniqueID(groupName).connection("users").get(); assertEquals(usersInGroup.getResponse().getEntityCount(), 2); @@ -444,7 +443,7 @@ public void postPrivateGroupActivity() throws IOException { this.app().collection("role").uniqueID("Default").delete(); Entity data = new Entity().chainPut("name", "group1role"); this.app().collection("roles").post(data); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); Entity perms = new Entity(); String permission = "get,post,put,delete:/groups/" + group.getUuid() + "/**"; @@ -452,18 +451,18 @@ public void postPrivateGroupActivity() throws IOException { this.app().collection("roles").uniqueID("group1role").connection("permissions").post(perms); this.app().collection("roles").uniqueID("group1role").connection("users").uniqueID( user1Username ).post(); this.app().collection("roles").uniqueID("group1role").connection("users").uniqueID( user2Username ).post(); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //7b. everybody gets access to /activities perms = new Entity(); permission = "get:/activities/**"; perms.put("permission",permission); this.app().collection("roles").uniqueID("Guest").connection("permissions").post(perms); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); this.app().collection("roles").uniqueID("Guest").connection("users").uniqueID( user1Username ).post(); this.app().collection("roles").uniqueID("Guest").connection("users").uniqueID( user2Username ).post(); this.app().collection("roles").uniqueID("Guest").connection("users").uniqueID( user3Username ).post(); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //8. post an activity to the group @@ -482,7 +481,7 @@ public void postPrivateGroupActivity() throws IOException { Entity activityResponse = this.app().collection("groups") .uniqueID(groupName).connection("activities").post(activity); assertEquals(activityResponse.get("content"), content); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //11. log user1 in, should then be using the app user's token not the admin token this.getAppUserToken(user1Username, password); @@ -565,7 +564,7 @@ public void updateGroupWithSameNameAsApp() throws IOException { group.put("title", newTitle); Entity groupResponse = this.app().collection("groups").entity(group).put(group); assertEquals(groupResponse.get("title"), newTitle); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); // update that group by giving it a new title and using UUID in URL String evenNewerTitle = "Even New Title"; @@ -573,6 +572,6 @@ public void updateGroupWithSameNameAsApp() throws IOException { String uuid = group.getAsString("uuid"); groupResponse = this.app().collection("groups").uniqueID(uuid).put(group); assertEquals(groupResponse.get("title"), evenNewerTitle); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); } } diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/paging/PagingResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/paging/PagingResourceIT.java index 4ca46b166b..da15b2e038 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/paging/PagingResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/paging/PagingResourceIT.java @@ -130,7 +130,7 @@ public void deleteByPage(int deletePageSize,int totalPages,String collectionName ApiResponse response = this.app().collection( collectionName ).delete( queryParameters ); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); if(validate) assertEquals("Entities should have been deleted", deletePageSize,response.getEntityCount() ); @@ -268,7 +268,7 @@ public void pageThroughConnectedEntities() throws IOException { entityPayload.put( "name", created ); Entity entity = new Entity( entityPayload ); entity = this.app().collection( collectionName ).post( entity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); if(created == 1){ connectedEntity = entity; } @@ -277,7 +277,7 @@ else if (created > 0){ } } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); QueryParameters qp = new QueryParameters(); qp.setQuery("select * order by created asc"); @@ -323,7 +323,7 @@ public void pagingQueryReturnCorrectResults() throws Exception { this.app().collection( collectionName ).post( entity ); } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Creates query looking for entities with the very stop. String query = "select * where verb = 'stop'"; @@ -454,7 +454,7 @@ public List createEntities(String collectionName ,int numOfEntities ){ } } - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); return entities; } diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java index 6202c6a391..7315cad9bf 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/ConnectionResourceTest.java @@ -63,7 +63,7 @@ public void connectionsQueryTest() throws IOException { Entity objectOfDesire = new Entity(); objectOfDesire.put( "codingmunchies", "doritoes" ); objectOfDesire = this.app().collection( "snacks" ).post( objectOfDesire ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Entity toddWant = this.app().collection( "users" ).entity( todd ).collection( "likes" ).collection( "snacks" ) .entity( objectOfDesire ).post(); @@ -93,11 +93,11 @@ public void connectionsLoopbackTest() throws IOException { thing2.put( "name", "thing2" ); thing2 = this.app().collection( "things" ).post( thing2 ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //create the connection: thing1 likes thing2 this.app().collection( "things" ).entity( thing1 ) .connection("likes").collection( "things" ).entity( thing2 ).post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //test we have the "likes" in our connection meta data response thing1 = this.app().collection( "things" ).entity( thing1 ).get(); @@ -150,14 +150,14 @@ public void connectionsDeleteSecondEntityInConnectionTest() throws IOException { thing2.put( "name", "thing2" ); thing2 = this.app().collection( "things" ).post( thing2 ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //create the connection: thing1 likes thing2 this.app().collection( "things" ).entity( thing1 ) .connection("likes").collection( "things" ).entity( thing2 ).post(); //delete thing2 this.app().collection( "things" ).entity( thing2 ).delete(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); try { //attempt to retrieve thing1 @@ -185,14 +185,14 @@ public void connectionsDeleteFirstEntityInConnectionTest() throws IOException { thing2.put( "name", "thing2" ); thing2 = this.app().collection( "things" ).post( thing2 ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //create the connection: thing1 likes thing2 this.app().collection( "things" ).entity( thing1 ) .connection("likes").collection( "things" ).entity( thing2 ).post(); //delete thing1 this.app().collection( "things" ).entity( thing1 ).delete(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); try { //attempt to retrieve thing1 @@ -236,7 +236,7 @@ public void testRePostOrder() { //connect thing1 -> thing3 connectionEndpoint.entity( thing3 ).post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //now do a GET, we should see thing2 then thing3 @@ -257,7 +257,7 @@ public void testRePostOrder() { //now re-post thing 2 it should appear second connectionEndpoint.entity( thing2 ).post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); final ApiResponse order2 = connectionEndpoint.get().getResponse(); @@ -304,7 +304,7 @@ public void testRePutOrder() { //connect thing1 -> thing3 connectionEndpoint.entity( thing3 ).put( thing3 ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //now do a GET, we should see thing2 then thing3 @@ -325,7 +325,7 @@ public void testRePutOrder() { //now re-post thing 2 it should appear second connectionEndpoint.entity( thing2 ).put( thing2 ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); final ApiResponse order2 = connectionEndpoint.get().getResponse(); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/OwnershipResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/OwnershipResourceIT.java index cfd08e5fa0..3393582785 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/OwnershipResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/OwnershipResourceIT.java @@ -17,19 +17,15 @@ package org.apache.usergrid.rest.applications.collection.users; -import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; import org.apache.usergrid.rest.test.resource.AbstractRestIT; import org.apache.usergrid.rest.test.resource.endpoints.CollectionEndpoint; import org.apache.usergrid.rest.test.resource.model.*; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.apache.usergrid.utils.MapUtils; - import javax.ws.rs.ClientErrorException; import static org.junit.Assert.assertEquals; @@ -65,7 +61,7 @@ public void setup(){ user1 = new User(this.usersResource.post(user1)); user2 = new User(this.usersResource.post(user2)); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); } @@ -95,7 +91,7 @@ public void meVerify() throws Exception { //Revoke the user1 token usersResource.entity(user1).connection("revoketokens").post(new Entity().chainPut("token", token.getAccessToken())); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //See if we can still access the me entity after revoking its token try { @@ -127,7 +123,7 @@ public void contextualPathOwnership() throws IOException { // create device 1 on user1 devices usersResource.entity("me").collection("devices") .post(new Entity( ).chainPut("name", "device1").chainPut("number", "5551112222")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Clear the current user token this.app().token().clearToken(); @@ -137,7 +133,7 @@ public void contextualPathOwnership() throws IOException { usersResource.entity("me").collection("devices") .post(new Entity( ).chainPut("name", "device2").chainPut("number", "5552223333")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Check that we can get back device1 on user1 token = this.app().token().post(new Token(user1.getUsername(),"password")); @@ -236,13 +232,13 @@ public void contextualConnectionOwnership() throws IOException { // create a 4peaks restaurant Entity data = this.app().collection("restaurants").post(new Entity().chainPut("name", "4peaks")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Create a restaurant and link it to user1/me Entity fourPeaksData = usersResource.entity("me") .connection("likes").collection( "restaurants" ).entity( "4peaks" ).post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // anonymous user this.app().token().clearToken(); @@ -252,11 +248,11 @@ public void contextualConnectionOwnership() throws IOException { data = this.app().collection("restaurants") .post(new Entity().chainPut("name", "arrogantbutcher")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); data = usersResource.entity("me").connection( "likes" ).collection( "restaurants" ) .entity( "arrogantbutcher" ).post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); String arrogantButcherId = data.getUuid().toString(); @@ -390,7 +386,7 @@ public void contextualConnectionOwnershipGuestAccess() throws IOException { //Sets up the cities collection with the city tempe Entity city = this.app().collection("cities").post(new Entity().chainPut("name", "tempe")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // create a 4peaks restaurant that is connected by a like to tempe. Entity data = this.app().collection("cities").entity( "tempe" ).connection( "likes" ) @@ -410,7 +406,7 @@ public void contextualConnectionOwnershipGuestAccess() throws IOException { CollectionEndpoint likeRestaurants = this.app().collection("cities").entity( "tempe" ).connection( "likes" ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // check we can get the resturant entities back via uuid without a collection name data = likeRestaurants.entity( peaksId ).get(); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java index aff952b7f2..2dddcf669f 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java @@ -66,7 +66,7 @@ public void setup(){ user = new User(USER,USER,USER+"@apigee.com","password"); user = new User( this.app().collection("users").post(user)); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); } @@ -86,13 +86,13 @@ public void deleteUserFromRole() throws IOException { assertEquals( ROLE, node.get("name").toString() ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Post the user with a specific role into the users collection node = this.app().collection("roles").entity(node).collection("users").entity(USER).post(); assertNull( node.get( "error" ) ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // now check the user has the role node = this.app().collection("users").entity(USER).collection("roles").entity(ROLE).get(); @@ -104,7 +104,7 @@ public void deleteUserFromRole() throws IOException { // now delete the role this.app().collection("users").entity(USER).collection("roles").entity(ROLE).delete(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // check if the role was deleted @@ -136,14 +136,14 @@ public void deleteUserGroup() throws IOException { assertNull( node.get( "error" ) ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Create a user that is in the group. node = this.app().collection("groups").entity(groupPath).collection("users").entity(user).post(); assertNull( node.get( "error" ) ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Get the user and make sure that they are part of the group Collection groups = this.app().collection("users").entity(user).collection("groups").get(); @@ -157,7 +157,7 @@ public void deleteUserGroup() throws IOException { assertNull( response.getError() ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Check that the user no longer exists in the group int status = 0; @@ -193,7 +193,7 @@ public void dictionaryPermissions() throws Exception { assertNull( entity.getError() ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // now try to add permission as the user, this should work addPermission( "usercreatedrole", "get,put,post:/foo/**" ); @@ -247,13 +247,13 @@ public void applicationPermissions() throws Exception { assertNull( node.getError() ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // delete the default role to test permissions later ApiResponse response = this.app().collection("roles").entity("default").delete(); assertNull( response.getError() ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // Grants a permission to GET, POST, and PUT the reviews url for the reviewer role addPermission( "reviewer", "get,put,post:/reviews/**" ); @@ -266,22 +266,22 @@ public void applicationPermissions() throws Exception { this.app().collection("groups").post(group); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // Adds the reviewer to the reviewerGroup this.app().collection("groups").entity("reviewergroup").collection("roles").entity("reviewer").post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // Adds reviewer2 user to the reviewergroup this.app().collection("users").entity("reviewer2").collection("groups").entity("reviewergroup").post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // Adds reviewer1 to the reviewer role this.app().collection("users").entity("reviewer1").collection("roles").entity("reviewer").post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // Set the current context to reviewer1 this.app().token().post(new Token("reviewer1","password")); @@ -295,7 +295,7 @@ public void applicationPermissions() throws Exception { .chainPut ("rating", "4").chainPut( "name", "4peaks").chainPut("review", "Huge beer selection" ); this.app().collection("reviews").post(review); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // get the reviews and assert they were created QueryParameters queryParameters = new QueryParameters(); @@ -330,7 +330,7 @@ public void applicationPermissions() throws Exception { assertEquals( Response.Status.UNAUTHORIZED.getStatusCode(), status ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //TODO: maybe make this into two different tests? @@ -346,7 +346,7 @@ public void applicationPermissions() throws Exception { .chainPut( "rating", "4" ).chainPut("name", "currycorner").chainPut( "review", "Authentic" ); this.app().collection("reviews").post(review); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // get all reviews as reviewer2 queryParameters = new QueryParameters(); @@ -372,7 +372,7 @@ public void applicationPermissions() throws Exception { assertEquals( Response.Status.UNAUTHORIZED.getStatusCode(), status ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); status = 0; @@ -409,7 +409,7 @@ public void wildcardMiddlePermission() throws Exception { Entity data = new Entity().chainPut("name", "reviewer"); this.app().collection("roles").post(data); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // allow access to reviews excluding delete addPermission( "reviewer", @@ -433,13 +433,13 @@ public void wildcardMiddlePermission() throws Exception { "wildcardpermusertwo@apigee.com" ); assertNotNull( userTwoId ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Add user1 to the reviewer role this.app().collection("users").entity(userOneId).collection("roles").entity("reviewer").post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Add a book to the books collection Entity book = new Entity().chainPut( "title", "Ready Player One" ).chainPut("author", "Earnest Cline"); @@ -449,7 +449,7 @@ public void wildcardMiddlePermission() throws Exception { assertEquals( "Ready Player One", book.get("title").toString() ); String bookId = book.get("uuid").toString(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Switch the contex to be that of user1 this.app().token().post(new Token("wildcardpermuserone","password")); @@ -461,7 +461,7 @@ public void wildcardMiddlePermission() throws Exception { review = this.app().collection("reviews").post(review); String reviewId = review.get("uuid").toString(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // POST https://api.usergrid.com/my-org/my-app/users/me/wrote/review/${reviewId} this.app().collection("users").entity("me").connection("wrote").collection("review").entity(reviewId).post(); @@ -469,13 +469,13 @@ public void wildcardMiddlePermission() throws Exception { // POST https://api.usergrid.com/my-org/my-app/users/me/reviewed/review/${reviewId} this.app().collection("users").entity("me").connection("reviewed").collection("books").entity(bookId).post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // POST https://api.usergrid.com/my-org/my-app/books/${bookId}/review/${reviewId} this.app().collection("books").entity(bookId).collection("review").entity(reviewId).post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // now try to post the same thing to books to verify as userOne does not have correct permissions int status = 0; @@ -522,7 +522,7 @@ public void wildcardFollowingPermission() throws Exception { //allow patients to add doctors as their followers addPermission( "patient", "delete,post:/users/*/following/users/${user}" ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // create examplepatient UUID patientId = createRoleUser( "examplepatient", "examplepatient@apigee.com" ); @@ -531,12 +531,12 @@ public void wildcardFollowingPermission() throws Exception { // create exampledoctor UUID doctorId = createRoleUser( "exampledoctor", "exampledoctor@apigee.com" ); assertNotNull( doctorId ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // assign examplepatient the patient role this.app().collection("users").entity(patientId).collection("roles").entity("patient").post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); this.app().token().post(new Token("examplepatient","password")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //not working yet, used to be ignored // this.app().collection("users").entity("exampledoctor").connection("following") // .collection("users").entity("examplepatient").post(); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/RetrieveUsersTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/RetrieveUsersTest.java index d5f7163019..ddb1557905 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/RetrieveUsersTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/RetrieveUsersTest.java @@ -17,19 +17,15 @@ package org.apache.usergrid.rest.applications.collection.users; -import java.util.HashMap; import java.util.Map; -import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; import org.apache.usergrid.rest.test.resource.AbstractRestIT; import org.apache.usergrid.rest.test.resource.endpoints.CollectionEndpoint; -import org.apache.usergrid.rest.test.resource.endpoints.EntityEndpoint; import org.apache.usergrid.rest.test.resource.model.Entity; import org.apache.usergrid.rest.test.resource.model.QueryParameters; import org.junit.Assert; -import org.junit.Rule; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,7 +51,7 @@ public void queryForUsername() throws IOException { - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); String query = "select *"; String incorrectQuery = "select * where username = 'Alica'"; @@ -72,7 +68,7 @@ public void userEntityDictionaryHasRoles() throws IOException { props.put( "username", "Nina" ); Entity entity = users.post(props); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Map metadata = (Map)entity.get( "metadata" ); Map sets = (Map)metadata.get( "sets" ); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java index c9dc0d8a31..af87ca5c46 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java @@ -75,7 +75,7 @@ public void setup() { usersResource = this.app().collection("users"); userResource = this.app().collection("user"); - clientSetup.refreshIndex(); + waitForQueueDrainAndRefreshIndex(); } @Test @@ -249,14 +249,14 @@ public void userActivitiesDefaultOrder() throws IOException { // same as above, but with actor partially filled out Entity entity = usersResource.entity(userId.toString()).activities().post(activity); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); UUID firstActivityId = entity.getUuid(); activity = new ActivityEntity("rod@rodsimpson.com", "POST", "activity 2"); entity = usersResource.entity(userId.toString()).activities().post(activity); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); UUID secondActivityId = entity.getUuid(); @@ -284,7 +284,7 @@ public void getUserWIthEmailUsername() throws IOException { map.put("email", email); Entity userEntity = usersResource.post(new Entity(map)); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // get the user with username property that has an email value Entity testUser = usersResource.entity(username).get(); @@ -303,6 +303,54 @@ public void getUserWIthEmailUsername() throws IOException { } + @Test + public void getUserWithEmailContainingSingleQuote() throws IOException { + UUID id = UUIDUtils.newTimeUUID(); + + String username = "username-email" + "@usergrid.org"; + String name = "name" + id; + String email = "e'mail" + id + "@usergrid.org"; + + User map = new User(username, name, email, null); + map.put("email", email); + + usersResource.post(new Entity(map)); + waitForQueueDrainAndRefreshIndex(); + + // get the user with email property value + // get the user with username property that has an email value + Entity testUser = usersResource.entity(email).get(); + + assertEquals(username, testUser.get("username").toString()); + assertEquals(name, testUser.get("name").toString()); + assertEquals(email, testUser.get("email").toString()); + + } + + @Test + public void getUserWithEmailContainingPlus() throws IOException { + UUID id = UUIDUtils.newTimeUUID(); + + String username = "username-email" + "@usergrid.org"; + String name = "name" + id; + String email = "e+mail" + id + "@usergrid.org"; + + User map = new User(username, name, email, null); + map.put("email", email); + + usersResource.post(new Entity(map)); + waitForQueueDrainAndRefreshIndex(); + + // get the user with email property value + // get the user with username property that has an email value + Entity testUser = usersResource.entity(email).get(); + + assertEquals(username, testUser.get("username").toString()); + assertEquals(name, testUser.get("name").toString()); + assertEquals(email, testUser.get("email").toString()); + + } + /** * Tests that when querying all users, we get the same result size when using "order by" @@ -361,7 +409,7 @@ public void clientNameQuery() { Entity entity = usersResource.post(user); UUID createdId = entity.getUuid(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Collection results = usersResource.get(new QueryParameters().setQuery(String.format("name = '%s'", name))); entity = new User(results.getResponse().getEntities().get(0)); assertEquals(createdId, entity.getUuid()); @@ -381,13 +429,13 @@ public void deleteUser() throws IOException { UUID createdId = entity.getUuid(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Entity newEntity = usersResource.entity(createdId.toString()).get(); userResource.entity(newEntity).delete(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Collection results = usersResource.get( new QueryParameters().setQuery(String.format("username = '%s'", username))); @@ -412,7 +460,7 @@ public void singularCollectionName() throws IOException { User entity = new User(username, name, email, "password"); entity = new User(usersResource.post(entity)); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); UUID firstCreatedId = entity.getUuid(); username = "username2"; @@ -422,7 +470,7 @@ public void singularCollectionName() throws IOException { entity = new User(username, name, email, "password"); entity = new User(usersResource.post(entity)); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); UUID secondCreatedId = entity.getUuid(); @@ -436,7 +484,7 @@ public void singularCollectionName() throws IOException { assertEquals(secondCreatedId.toString(), conn1.getUuid().toString()); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Entity conn2 = usersResource.entity( @@ -444,7 +492,7 @@ public void singularCollectionName() throws IOException { assertEquals(secondCreatedId.toString(), conn2.getUuid().toString()); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Collection conn1Connections = usersResource.entity(firstCreatedId.toString()).connection("conn1").get(); @@ -494,7 +542,7 @@ public void connectionByNameAndType() throws IOException { // now create a connection of "likes" between the first user and the // second using pluralized form - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // named entity in collection name Entity conn1 = usersResource.entity(firstCreatedId.toString()).connection("conn1", "users") @@ -547,6 +595,7 @@ public void connectionQuerybyEmail() throws IOException { role = new Entity(); role.put("name", "connectionQuerybyEmail2"); role = this.app().collection("roles").post(role); + waitForQueueDrainAndRefreshIndex(); UUID roleId2 = role.getUuid(); @@ -557,24 +606,24 @@ public void connectionQuerybyEmail() throws IOException { perms.put("permission", "get:/stuff/**"); Entity perms2 = this.app().collection("roles").entity(roleId2.toString()).connection("permissions") .post(new Entity(perms)); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); + //connect the entities where role is the root Entity perms3 = this.app().collection("roles").entity(roleId1.toString()).connection("users") .entity(userId.toString()).post(); - - // now create a connection of "likes" between the first user and the - // second using pluralized form - assertEquals(userId.toString(), perms3.getUuid().toString()); + waitForQueueDrainAndRefreshIndex(); + // now create a connection of "likes" between the first user and the + // second using pluralized form //connect the second role Entity perms4 = this.app().collection("roles").entity(roleId2).connection("users").entity(userId).post(); - assertEquals(userId.toString(), perms4.getUuid().toString()); + waitForQueueDrainAndRefreshIndex(); + - refreshIndex(); //query the second role, it should work Collection userRoles = this.app().collection("roles").entity(roleId2).connection("users") .get(new QueryParameters().setQuery("select%20*%20where%20username%20=%20'" + email + "'")); @@ -630,7 +679,7 @@ public void connectionByNameAndDynamicType() throws IOException { Entity pizzaEntity = this.app().collection("pizzas").post(pizza); UUID secondCreatedId = pizzaEntity.getUuid(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // now create a connection of "likes" between the first user and the // second using pluralized form @@ -660,7 +709,7 @@ public void nameUpdate() throws IOException { Entity userEntity = usersResource.post(entity); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // attempt to log in Token token = this.app().token().post(new Token(username, "password")); @@ -680,7 +729,7 @@ public void nameUpdate() throws IOException { userEntity = usersResource.entity(username).put(userEntity); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // now see if we've updated @@ -734,7 +783,7 @@ public void deactivateUser() throws IOException { .chainPut("pin", "1234"); usersResource.post(entity); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Collection response = usersResource.get(); // disable the user @@ -750,7 +799,7 @@ public void deactivateUser() throws IOException { public void test_PUT_password_fail() { Entity entity = usersResource.post(new User("edanuff", "edanuff", "edanuff@email.com", "sesame")); this.app().token().post(new Token("edanuff", "sesame")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); boolean fail = false; try { Entity changeResponse = usersResource.entity("edanuff").collection("password") @@ -781,17 +830,17 @@ public void test_GET_user_ok() throws InterruptedException, IOException { @Test public void test_PUT_password_ok() { Entity entity = usersResource.post(new User("edanuff", "edanuff", "edanuff@email.com", "sesame")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); usersResource.entity(entity).collection("password").post(new ChangePasswordEntity("sesame", "sesame1")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); this.app().token().post(new Token("edanuff", "sesame1")); // if this was successful, we need to re-set the password for other // tests Entity changeResponse = usersResource.entity("edanuff").collection("password") .post(new ChangePasswordEntity("sesame1", "sesame")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); assertNotNull(changeResponse); } @@ -801,14 +850,14 @@ public void test_PUT_password_ok() { public void setUserPasswordAsAdmin() throws IOException { usersResource.post(new User("edanuff", "edanuff", "edanuff@email.com", "sesame")); String newPassword = "foofoo"; - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // change the password as admin. The old password isn't required Entity node = usersResource.entity("edanuff").connection("password") .post(new ChangePasswordEntity(newPassword)); assertNotNull(node); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Token response = this.app().token().post(new Token("edanuff", newPassword)); assertNotNull(response); } @@ -851,7 +900,7 @@ public void createAppUserWithInvalidPassword() { public void testChangePassordToInvalidValue() { Entity entity = usersResource.post(new User("edanuff", "edanuff", "edanuff@email.com", "sesame")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); try { usersResource.entity(entity).collection("password").post(new ChangePasswordEntity("sesame", "abc")); @@ -882,12 +931,12 @@ public void addRemoveRole() throws IOException { this.app().collection("roles").post(role); // check it - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // add Role role = usersResource.entity(createdId).collection("roles").entity(roleName).post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // check it assertNotNull(role); assertNotNull(role.get("name")); @@ -918,7 +967,7 @@ public void addRemoveRole() throws IOException { public void revokeToken() throws Exception { this.app().collection("users").post(new User("edanuff", "edanuff", "edanuff@email.com", "sesame")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Token token1 = this.app().token().post(new Token("edanuff", "sesame")); Token token2 = this.app().token().post(new Token("edanuff", "sesame")); @@ -936,7 +985,7 @@ public void revokeToken() throws Exception { this.app().token().setToken(adminToken); usersResource.entity("edanuff").connection("revoketokens").post(new Entity().chainPut("token", token1)); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // the tokens shouldn't work int status = 0; @@ -984,7 +1033,7 @@ public void revokeToken() throws Exception { // now revoke the tokens this.app().token().setToken(adminToken); usersResource.entity("edanuff").connection("revoketokens").post(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // the token3 shouldn't work @@ -1022,7 +1071,7 @@ public void getToken() throws Exception { usersResource.post(new User("test_1", "Test1 User", "test_1@test.com", "test123")); // client.setApiUrl(apiUrl); usersResource.post(new User("test_2", "Test2 User", "test_2@test.com", "test123")); // client.setApiUrl(apiUrl); usersResource.post(new User("test_3", "Test3 User", "test_3@test.com", "test123")); // client.setApiUrl(apiUrl); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Entity appInfo = this.app().get().getResponse().getEntities().get(0); @@ -1032,7 +1081,7 @@ public void getToken() throws Exception { assertNotNull(token.getAccessToken()); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); int status = 0; @@ -1075,7 +1124,7 @@ public void getToken() throws Exception { Entity entityConn = usersResource.entity(userId).connection("deactivate").post(new Entity()); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); try { this.app().token().post(new Token("test_1", "test123")); @@ -1093,7 +1142,7 @@ public void delegatePutOnNotFound() throws Exception { String randomName = "user1_" + UUIDUtils.newTimeUUID().toString(); User user = new User(randomName, randomName, randomName + "@apigee.com", "password"); usersResource.post(user); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // should update a field Entity response = usersResource.entity(randomName).get(); @@ -1107,7 +1156,7 @@ public void delegatePutOnNotFound() throws Exception { response = usersResource.post(user2); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Entity response2 = usersResource.entity(randomName).get(); @@ -1160,7 +1209,7 @@ public void queryForUserUuids() throws Exception { assertNotNull(userId); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); ql = "uuid = " + userId; @@ -1175,7 +1224,7 @@ public void queryForUserUuids() throws Exception { public void testCredentialsTransfer() throws Exception { usersResource.post(new User("test_1", "Test1 User", "test_1@test.com", "test123")); // client.setApiUrl(apiUrl); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Entity appInfo = this.app().get().getResponse().getEntities().get(0); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/events/EventsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/events/EventsResourceIT.java index 965105b4bb..315f1f5231 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/events/EventsResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/events/EventsResourceIT.java @@ -57,7 +57,7 @@ public void testEventPostandGet() throws IOException { assertNotNull(node.getEntities()); String advertising = node.getEntity().get("uuid").toString(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); payload = new LinkedHashMap(); payload.put( "timestamp", 0 ); @@ -74,7 +74,7 @@ public void testEventPostandGet() throws IOException { assertNotNull(node.getEntities()); String sales = node.getEntity().get("uuid").toString(); - refreshIndex( ); + waitForQueueDrainAndRefreshIndex( ); payload = new LinkedHashMap(); payload.put( "timestamp", 0 ); @@ -91,7 +91,7 @@ public void testEventPostandGet() throws IOException { assertNotNull(node.getEntities()); String marketing = node.getEntity().get( "uuid" ).toString(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); String lastId = null; diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BasicGeoTests.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BasicGeoTests.java index f47fba0c47..74ad38b86e 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BasicGeoTests.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BasicGeoTests.java @@ -109,7 +109,7 @@ public void updateEntityWithGeoLocationPoint() throws IOException { assertEquals( lat.toString(), entity.getMap("location").get("latitude").toString() ); assertEquals( lon.toString(), entity.getMap("location").get("longitude").toString() ); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //2. read back that entity make sure it is accurate /* @@ -144,7 +144,7 @@ public void updateEntityWithGeoLocationPoint() throws IOException { assertEquals( newLon.toString(), entity.get( "location" ).get("longitude").asText() ); */ - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //4. read back the updated entity, make sure it is accurate diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java index 91ccf38358..9a1cb3ccfc 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/GeoPagingTest.java @@ -91,7 +91,7 @@ public void groupQueriesWithGeoPaging() throws IOException { } } - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); // 2. Query the groups from a nearby location, restricting the search // by creation time to a single entity where created[i-1] < created[i] < created[i+1] //since this geo location is contained by an actor it needs to be actor.location. @@ -150,7 +150,7 @@ public void testFarAwayLocationFromCenter() throws IOException { .map("latitude", -33.889058) .map("longitude", 151.124024)); this.app().collection(collectionType).post(props2); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); Collection collection = this.app().collection(collectionType).get(); assertEquals("Should return both entities", 2, collection.getResponse().getEntityCount()); @@ -182,7 +182,7 @@ public void groupQueriesWithConsistentResults() throws IOException { cats[i] = cat; this.app().collection("cats").post(cat); } - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); QueryParameters params = new QueryParameters(); for (int consistent = 0; consistent < 20; consistent++) { @@ -229,7 +229,7 @@ public void geoQuerywithDistanceSort() throws IOException { cats[i] = cat; this.app().collection("cats").post(cat); } - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); final QueryParameters params = new QueryParameters(); for (int consistent = 0; consistent < 20; consistent++) { @@ -279,6 +279,7 @@ public void testFarAwayLocationWithOneResultCloser() throws IOException { .map("latitude", -33.746369) .map("longitude", 150.952183)); this.app().collection(collectionType).post(props); + this.waitForQueueDrainAndRefreshIndex(); Entity props2 = new Entity(); props2.put("name", "usergrid2"); @@ -286,7 +287,7 @@ public void testFarAwayLocationWithOneResultCloser() throws IOException { .map("latitude", -33.889058) .map("longitude", 151.124024)); this.app().collection(collectionType).post(props2); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); // 2. Query from the center point to ensure that one is returned Collection collection = this.app().collection(collectionType).get(queryClose); @@ -326,7 +327,7 @@ public void createHugeMatrixOfCoordinates() throws IOException { .map("longitude", 150.952183)); this.app().collection("users").post(props); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); // 2. Create a list of geo points List points = new ArrayList<>(); points.add(new double []{33.746369, -89});//Woodland, MS diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/MatrixQueryTests.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/MatrixQueryTests.java index 7525d51bab..02a54daaf7 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/MatrixQueryTests.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/MatrixQueryTests.java @@ -21,7 +21,6 @@ import org.apache.usergrid.rest.test.resource.model.Collection; import org.apache.usergrid.rest.test.resource.model.Entity; import org.apache.usergrid.rest.test.resource.model.QueryParameters; -import org.junit.Ignore; import org.junit.Test; import java.util.UUID; @@ -78,7 +77,7 @@ public void connectionsTest() throws Exception { restaurant2 = this.app().collection("restaurants").post(restaurant2); restaurant3 = this.app().collection("restaurants").post(restaurant3); restaurant4 = this.app().collection("restaurants").post(restaurant4); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //3. Create "likes" connections between users and restaurants //user 1 likes old major @@ -91,7 +90,7 @@ public void connectionsTest() throws Exception { //user 3 likes Lola (it shouldn't appear in the results) this.app().collection("users").entity(user3).connection("likes").collection("restaurants").entity(restaurant4).post(); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //4. Retrieve "likes" connections per user and ensure the correct restaurants are returned Collection user1likes = this.app().collection("users").entity(user1).connection("likes").get(); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java index a1905261b1..659171339f 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java @@ -246,7 +246,7 @@ public void orderByShouldNotAffectResults() throws IOException { } } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //2. Query without 'order by' String query = "select * where created > " + created; QueryParameters params = new QueryParameters().setQuery(query); @@ -289,7 +289,7 @@ public void orderByComesBeforeLimitResult() throws IOException { this.app().collection("activity").post(props); } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //2. Query a subset of the entities, specifying order and limit String query = "select * where created > " + 1 + " order by created desc"; QueryParameters params = new QueryParameters().setQuery(query).setLimit(5); @@ -334,7 +334,7 @@ public void orderByReturnCorrectResults() throws IOException { logger.info(String.valueOf(Long.parseLong(activities[0].get("created").toString()))); } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(750); ArrayUtils.reverse(activities); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java index f7eb3fe4b6..0adafefd47 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java @@ -65,7 +65,7 @@ protected Entity[] generateTestEntities(int numberOfEntities, String collectionN logger.info(entities[i].entrySet().toString()); } //refresh the index so that they are immediately searchable - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); return entities; } diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java index fd33c15424..acf51c1359 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java @@ -83,7 +83,7 @@ public void testMixedCaseDupField() throws Exception { .withProp( "testProp", value ) .withProp( "TESTPROP", otherValue); app().collection( collectionName ).post( entity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // testProp and TESTPROP should now have otherValue @@ -98,6 +98,55 @@ public void testMixedCaseDupField() throws Exception { assertEquals( 1, things.getNumOfEntities() ); } + @Test + public void testStringWithSingleQuote() throws Exception { + + String collectionName = "things"; + + String value = "test'value"; + String escapedValue = "test\\'value"; + + // create entity with testProp=value + Entity entity = new Entity() + .withProp( "testprop", value ); + app().collection( collectionName ).post( entity ); + waitForQueueDrainAndRefreshIndex(); + + // testProp and TESTPROP should now have otherValue + + QueryParameters params = new QueryParameters() + .setQuery( "select * where testprop='" + escapedValue + "'" ); + Collection things = this.app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + } + + @Test + public void testStringWithPlus() throws Exception { + + String collectionName = "things"; + String value = "ed+test@usergrid.com"; + + // create entity with value containing a plus symbol + Entity entity = new Entity() + .withProp( "testprop", value ); + app().collection( collectionName ).post( entity ); + waitForQueueDrainAndRefreshIndex(); + + // now query this without encoding the plus symbol + QueryParameters params = new QueryParameters() + .setQuery( "select * where testprop='" + value + "'" ); + Collection things = this.app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + + // again query with the plus symbol url encoded + String escapedValue = "ed%2Btest@usergrid.com"; + params = new QueryParameters() + .setQuery( "select * where testprop='" + escapedValue + "'" ); + things = this.app().collection( "things" ).get( params ); + assertEquals( 1, things.getNumOfEntities() ); + + } + /** * Field named testProp can be over-written by field named TESTPROP. @@ -111,13 +160,13 @@ public void testFieldOverride1() throws Exception { String value = RandomStringUtils.randomAlphabetic( 20 ); Entity entity = new Entity().withProp( "testProp", value ); app().collection( collectionName ).post( entity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // override with TESTPROP=newValue String newValue = RandomStringUtils.randomAlphabetic( 20 ); entity = new Entity().withProp( "TESTPROP", newValue ); app().collection( collectionName ).post( entity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // testProp and TESTPROP should new be queryable by new value @@ -144,13 +193,13 @@ public void testFieldOverride2() throws Exception { String value = RandomStringUtils.randomAlphabetic( 20 ); Entity entity = new Entity().withProp( "TESTPROP", value ); app().collection( collectionName ).post( entity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // override with testProp=newValue String newValue = RandomStringUtils.randomAlphabetic( 20 ); entity = new Entity().withProp( "testProp", newValue ); app().collection( collectionName ).post( entity ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // testProp and TESTPROP should new be queryable by new value diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AccessTokenIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AccessTokenIT.java index 431d224353..841ac1da7b 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AccessTokenIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AccessTokenIT.java @@ -153,7 +153,7 @@ public void meTokenPost() throws Exception { assertNotNull( token.getAccessToken() ); management().token().setToken( token ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); assertNotNull( management().me().get( Token.class ) ); @@ -177,7 +177,7 @@ public void meTokenPostForm() throws IOException { assertNotNull( adminToken ); assertNotNull( adminToken.getAccessToken() ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); assertNotNull( management().me().get( Token.class ) ); @@ -237,7 +237,7 @@ public void revokeTokens() throws Exception { management().token().setToken( clientSetup.getSuperuserToken() ); management().users().user( clientSetup.getUsername() ).revokeTokens().post(true , ApiResponse.class, null,null); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //test that token 1 doesn't work @@ -278,7 +278,7 @@ public void revokeSingleToken() throws Exception { management().token().setToken( clientSetup.getSuperuserToken() ); management().users().user( clientSetup.getUsername() ).revokeToken().post( false, ApiResponse.class,null,queryParameters ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //test that token 1 doesn't work diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java index f80f131bcc..829f561842 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java @@ -17,7 +17,6 @@ package org.apache.usergrid.rest.management; -import com.sun.jersey.api.client.UniformInterfaceException; import net.jcip.annotations.NotThreadSafe; import org.apache.commons.lang.RandomStringUtils; import org.apache.usergrid.management.MockImapClient; @@ -77,7 +76,7 @@ public void setSelfAdminPasswordAsAdmin() throws IOException { // change the password as admin. The old password isn't required management.users().user( username ).password().post(Entity.class,passwordPayload); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //Get the token using the new password Token adminToken = management.token().post( false, Token.class, new Token( username, "testPassword" ) ,null ); @@ -159,7 +158,7 @@ public void passwordMismatchErrorAdmin() { // change the password as admin. The old password isn't required management.users().user( username ).password().post(Entity.class, passwordPayload ); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //Get the token using the new password @@ -197,7 +196,7 @@ public void setAdminPasswordAsSysAdmin() throws IOException { management.token().setToken( clientSetup.getSuperuserToken()); management.users().user( username ).password().post( passwordPayload ); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); assertNotNull( management.token().post( false, Token.class, new Token(username, "testPassword"), null )); @@ -260,7 +259,7 @@ public void testUnconfirmedAdminLogin() throws Exception{ //Send rest call to the /testProperties endpoint to persist property changes clientSetup.getRestClient().testPropertiesResource().post( testPropertiesPayload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Create organization for the admin user to be confirmed Organization organization = createOrgPayload( "testUnconfirmedAdminLogin", null ); @@ -342,7 +341,7 @@ public void testSystemAdminNeedsNoConfirmation() throws Exception{ //Send rest call to the /testProperties endpoint to persist property changes clientSetup.getRestClient().testPropertiesResource().post( testPropertiesPayload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Token superuserToken = management.token().post( Token.class, new Token( clientSetup.getSuperuserName(), clientSetup.getSuperuserPassword() ) ); @@ -379,7 +378,7 @@ public void testTestUserNeedsNoConfirmation() throws Exception{ //Send rest call to the /testProperties endpoint to persist property changes clientSetup.getRestClient().testPropertiesResource().post( testPropertiesPayload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Token testToken = management().token().post(Token.class, new Token( originalTestProperties.getAsString( PROPERTIES_TEST_ACCOUNT_ADMIN_USER_EMAIL ), @@ -586,7 +585,7 @@ public void testAdminRemovalFromOrg() throws Exception { public void reactivateTest() throws Exception { //call reactivate endpoint on default user clientSetup.getRestClient().management().users().user( clientSetup.getUsername() ).reactivate().get(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Create mocked inbox and check to see if you recieved an email in the users inbox. List inbox = Mailbox.get( clientSetup.getEmail()); @@ -599,7 +598,7 @@ public void checkFormPasswordReset() throws Exception { // initiate password reset management().users().user( clientSetup.getUsername() ).resetpw().post(new Form()); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // create mocked inbox, get password reset email and extract token List inbox = Mailbox.get( clientSetup.getEmail() ); @@ -630,7 +629,7 @@ public void checkFormPasswordReset() throws Exception { assertTrue( html.contains( "password set" ) ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); html = management().users().user( clientSetup.getUsername() ).resetpw().post( formData ); @@ -644,7 +643,7 @@ public void passwordResetIncorrectUserName() throws Exception { // initiate password reset management().users().user( clientSetup.getUsername() ).resetpw().post(new Form()); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // create mocked inbox, get password reset email and extract token List inbox = Mailbox.get( clientSetup.getEmail() ); @@ -725,7 +724,7 @@ public void checkPasswordHistoryConflict() throws Exception { payload.put( "newpassword", passwords[1] ); management().users().user( clientSetup.getUsername() ).password().post( Entity.class,payload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); payload.put( "newpassword", passwords[0] ); payload.put( "oldpassword", passwords[1] ); @@ -747,7 +746,7 @@ public void checkPasswordChangeTime() throws Exception { // request password reset management().users().user( clientSetup.getUsername() ).resetpw().post(new Form()); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // get resetpw token from email @@ -774,7 +773,7 @@ public void checkPasswordChangeTime() throws Exception { String html = management().users().user( clientSetup.getUsername() ).resetpw().getTarget().request() .post( javax.ws.rs.client.Entity.form(formData), String.class ); assertTrue( html.contains( "password set" ) ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // login with new password and get token @@ -797,7 +796,7 @@ public void checkPasswordChangeTime() throws Exception { put("newpassword", "test"); }}; management().users().user( clientSetup.getUsername() ).password().post( false, payload, null ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); // get password and check password change time again @@ -851,7 +850,7 @@ public void listOrgUsersByName() { //Create admin user management().orgs().org( clientSetup.getOrganizationName() ).users().post(ApiResponse.class ,adminUserPayload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Retrieves the admin users ApiResponse adminUsers = management().orgs().org( clientSetup.getOrganizationName() ).users().get(ApiResponse.class); @@ -905,7 +904,7 @@ public void testProperties(){ //Send rest call to the /testProperties endpoint to persist property changes clientSetup.getRestClient().testPropertiesResource().post( testPropertiesPayload ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); //Retrieve properties and ensure that they are set correctly. ApiResponse apiResponse = clientSetup.getRestClient().testPropertiesResource().get(); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java index 81ff2d34a4..1b649d2ef0 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java @@ -141,7 +141,7 @@ public void exportGetApplicationJobStatTest() throws Exception { assertNotNull( uuid ); exportEntity = null; - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); try { exportEntity = management().orgs().org( clientSetup.getOrganizationName() ) diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java index c390393cc2..9c35a6cec1 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java @@ -193,7 +193,7 @@ public void importTokenAuthorizationTest() throws Exception { Organization orgPayload = new Organization( newOrgName, newOrgUsername, newOrgEmail, newOrgName, newOrgPassword, null); Organization orgCreatedResponse = clientSetup.getRestClient().management().orgs().post(orgPayload); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); assertNotNull(orgCreatedResponse); @@ -391,8 +391,8 @@ public void importPostApplicationNullPointerStorageVerification() throws Excepti // for ( org.apache.usergrid.persistence.Entity importedThing : importedThings ) { // emApp1.delete( importedThing ); // } -// emApp1.refreshIndex(); -// emApp2.refreshIndex(); +// emApp1.waitForQueueDrainAndRefreshIndex(); +// emApp2.waitForQueueDrainAndRefreshIndex(); // // importedThings = emApp2.getCollection( // appId2, "things", null, Query.Level.ALL_PROPERTIES).getEntities(); @@ -438,7 +438,7 @@ public void testImportGoodJson() throws Exception { .addToPath(importEntity.getUuid().toString()) .get(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Entity importGetIncludes = this.management().orgs().org( org ).app() .addToPath(app) @@ -671,7 +671,7 @@ private Entity importCollection() throws Exception { Thread.sleep(1000); } - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); return importEntity; } @@ -697,7 +697,7 @@ private void createTestEntities() throws Exception { } - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); // // first two things are related to each other // em.createConnection(new SimpleEntityRef(type, created.get(0).getUuid()), @@ -705,7 +705,7 @@ private void createTestEntities() throws Exception { // em.createConnection(new SimpleEntityRef(type, created.get(1).getUuid()), // "related", new SimpleEntityRef(type, created.get(0).getUuid())); // -// em.refreshIndex(); +// em.waitForQueueDrainAndRefreshIndex(); } /** diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java index 635368e46c..0a80d73a87 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java @@ -213,7 +213,7 @@ public void mgmtFollowsUserFeed() throws Exception { users1.add( "follower" + Integer.toString( i ) ); } - refreshIndex( ); + waitForQueueDrainAndRefreshIndex( ); checkFeed( "leader1", users1 ); //try with 11 @@ -230,20 +230,20 @@ private void checkFeed( String leader, List followers ) throws IOExcepti //create user createUser( leader ); - refreshIndex( ); + waitForQueueDrainAndRefreshIndex( ); String preFollowContent = leader + ": pre-something to look for " + UUID.randomUUID().toString(); addActivity( leader, leader + " " + leader + "son", preFollowContent ); - refreshIndex( ); + waitForQueueDrainAndRefreshIndex( ); String lastUser = followers.get( followers.size() - 1 ); int i = 0; for ( String user : followers ) { createUser( user ); - refreshIndex( ); + waitForQueueDrainAndRefreshIndex( ); follow( user, leader ); - refreshIndex( ); + waitForQueueDrainAndRefreshIndex( ); } userFeed = getUserFeed( lastUser ); assertTrue( userFeed.size() == 1 ); @@ -254,7 +254,7 @@ private void checkFeed( String leader, List followers ) throws IOExcepti String postFollowContent = leader + ": something to look for " + UUID.randomUUID().toString(); addActivity( leader, leader + " " + leader + "son", postFollowContent ); - refreshIndex( ); + waitForQueueDrainAndRefreshIndex( ); //check feed userFeed = getUserFeed( lastUser ); @@ -321,7 +321,7 @@ public void mgmtCreateAndGetApplication() throws Exception { .post( new Application( "mgmt-org-app" ) ); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Entity appdata = apiResponse.getEntities().get(0); assertEquals((clientSetup.getOrganizationName() + "/mgmt-org-app") @@ -336,7 +336,7 @@ public void mgmtCreateAndGetApplication() throws Exception { assertEquals("Roles", roles.get("title").toString()); assertEquals(4, roles.size()); - refreshIndex( ); + waitForQueueDrainAndRefreshIndex( ); // GET /applications/mgmt-org-app @@ -361,7 +361,7 @@ public void mgmtCreateAndGetApplication() throws Exception { public void checkSizes() throws Exception { final String appname = clientSetup.getAppName(); this.app().collection("testCollection").post(new Entity().chainPut("name","test")); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Entity size = management().orgs().org( clientSetup.getOrganizationName() ).app().addToPath(appname).addToPath("_size").get(); Entity rolesSize = management().orgs().org(clientSetup.getOrganizationName()).app().addToPath(appname).addToPath("roles/_size").get(); Entity collectionsSize = management().orgs().org(clientSetup.getOrganizationName()).app().addToPath(appname).addToPath("collections/_size").get(); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java index 29841a059c..ad204ae442 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java @@ -68,7 +68,7 @@ public void createOrgAndOwner() throws Exception { assertNotNull( organizationResponse ); // Thread.sleep( 1000 ); -// this.refreshIndex(); +// this.waitForQueueDrainAndRefreshIndex(); //Creates token Token token = @@ -78,7 +78,7 @@ public void createOrgAndOwner() throws Exception { assertNotNull( token ); - //this.refreshIndex(); + //this.waitForQueueDrainAndRefreshIndex(); //Assert that the get returns the correct org and owner. Organization returnedOrg = clientSetup.getRestClient().management().orgs().org( organization.getOrganization() ).get(); @@ -92,6 +92,38 @@ public void createOrgAndOwner() throws Exception { assertEquals( "Apigee", returnedUser.getProperties().get( "company" ) ); } + // Admin users are allowed to be created with an email address only, in which a password reset flow is invoked + @Test + public void createNewOrgUserNoPassword() throws Exception { + + //User property to see if owner properties exist when created. + Map userProperties = new HashMap(); + userProperties.put( "company", "Usergrid" ); + + //Create organization + Organization organization = createOrgPayload( "createNewOrgUserNoPassword", userProperties ); + + //Get back organization response + Organization organizationResponse = clientSetup.getRestClient().management().orgs().post( organization ); + + assertNotNull( organizationResponse ); + + + //Creates token + Token token = + clientSetup.getRestClient().management().token().post(false,Token.class, new Token( "password", + organization.getUsername(), organization.getPassword() ) ,null); + this.management().token().setToken(token); + + assertNotNull( token ); + + + Map newUser = new HashMap(){{put("email", "ed@usergrid.com");}}; + clientSetup.getRestClient().management().orgs().org( organization.getOrganization() ).users().post(newUser); + + // the test will fail if any exception is thrown + } + /** * Creates a organization with an owner, then attempts to create an organization with the same name ( making sure it @@ -104,7 +136,7 @@ public void testCreateDuplicateOrgName() throws Exception { // Create organization Organization organization = createOrgPayload( "testCreateDuplicateOrgName", null ); Organization orgCreatedResponse = clientSetup.getRestClient().management().orgs().post( organization ); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); assertNotNull( orgCreatedResponse ); @@ -161,7 +193,7 @@ public void testCreateDuplicateOrgEmail() throws Exception { //create the org/owner Organization orgCreatedResponse = clientSetup.getRestClient().management().orgs().post( organization ); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); assertNotNull( orgCreatedResponse ); @@ -362,7 +394,7 @@ public void testOrganizationUpdate() throws Exception { //update the organization. management().orgs().org( clientSetup.getOrganizationName() ).put(orgPayload); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //retrieve the organization Organization orgResponse = management().orgs().org( clientSetup.getOrganizationName() ).get(); @@ -376,7 +408,7 @@ public void testOrganizationUpdate() throws Exception { //update the organization. management().orgs().org( clientSetup.getOrganizationName() ).put(orgPayload); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); orgResponse = management().orgs().org( clientSetup.getOrganizationName() ).get(); @@ -401,7 +433,7 @@ public void testOrganizationNoDuplicateConnectionsPUT() throws Exception { Organization orgResponse = management().orgs().org( clientSetup.getOrganizationName() ).get(); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //attempt to post duplicate connection Entity userPostResponse = management().orgs().org( clientSetup.getOrganizationName() ).users().user( clientSetup.getEmail() ).put( entity ); @@ -429,7 +461,7 @@ public void testOrganizationNoDuplicateConnectionsPOST() throws Exception { Organization orgResponse = management().orgs().org( clientSetup.getOrganizationName() ).get(); - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); //attempt to post duplicate connection try { Entity userPostResponse = management().orgs().org( clientSetup.getOrganizationName() ).users().post( Entity.class, entity ); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/RegistrationIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/RegistrationIT.java index 8404632676..90a6919e0a 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/RegistrationIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/RegistrationIT.java @@ -170,7 +170,7 @@ public void addNewAdminUserWithNoPwdToOrganization() throws Exception { "changeme"); UUID userId = node.getUuid(); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); String subject = "Password Reset"; @@ -239,7 +239,7 @@ public void addExistingAdminUserToOrganization() throws Exception { //Disgusting data manipulation to parse the form response. Map adminUserPostResponse = (management().users().post( User.class, userForm )); - refreshIndex(); + waitForQueueDrainAndRefreshIndex(); Map adminDataMap = ( Map ) adminUserPostResponse.get( "data" ); diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/organizations/AdminEmailEncodingIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/organizations/AdminEmailEncodingIT.java index b188aba90d..0f1a794665 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/organizations/AdminEmailEncodingIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/organizations/AdminEmailEncodingIT.java @@ -77,8 +77,6 @@ public void getTokenDash() throws Exception { * @throws Exception */ @Test - @Ignore("Pending https://issues.apache.org/jira/browse/USERGRID-1117") - // This fails. I'm not sure if it is by design, but a single quote is valid in an email address public void getTokenQuote() throws Exception { doTest("'"); } diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/AbstractRestIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/AbstractRestIT.java index 6e5e4f9782..4799b0c89e 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/AbstractRestIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/AbstractRestIT.java @@ -173,10 +173,12 @@ protected Token getAppUserToken(String username, String password){ return this.app().token().post( new Token( username, password ) ); } - public void refreshIndex() { - //TODO see how we can refresh index (not async) for tests so sleep may not be needed + public void waitForQueueDrainAndRefreshIndex(int waitTimeMillis) { + // indexing is async, tests will need to wait for stuff to be processed. + // this sleep is slightly longer becasue distributed queueing on top of Cassandra can be used without and in-mem + // copy. see Qakka in the persistence module try { - Thread.sleep(250); //ensure index docs are finished being sent to Elasticsearch by Usergrid before refresh + Thread.sleep(waitTimeMillis); clientSetup.refreshIndex(); } catch (InterruptedException e) { System.out.println("Error refreshing index"); @@ -184,6 +186,10 @@ public void refreshIndex() { } } + public void waitForQueueDrainAndRefreshIndex() { + waitForQueueDrainAndRefreshIndex(750); + } + /** * Takes in the expectedStatus message and the expectedErrorMessage then compares it to the ClientErrorException diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties index 98c5640ccd..615bedd62e 100644 --- a/stack/rest/src/test/resources/usergrid-custom-test.properties +++ b/stack/rest/src/test/resources/usergrid-custom-test.properties @@ -72,6 +72,11 @@ elasticsearch.queue_impl=DISTRIBUTED # Queueing Test Settings # Reduce the long polling time for the tests queue.long.polling.time.millis=50 -queue.num.actors=50 +queue.num.actors=100 queue.sender.num.actors=100 queue.writer.num.actors=100 +elasticsearch.worker_count=12 +elasticsearch.worker_count_utility=4 +queue.get.timeout.seconds=10 +queue.send.timeout.seconds=10 +usergrid.push.worker_count=8 diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java index c0915e43ad..2ba9bdeee0 100644 --- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java +++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java @@ -949,7 +949,7 @@ public UserInfo createAdminFromPrexistingPassword( UUID organizationId, User use @Override public UserInfo createAdminFrom( UUID organizationId, User user, String password ) throws Exception { - Collection policyVioliations = passwordPolicy.policyCheck( password, false ); + Collection policyVioliations = passwordPolicy.policyCheck( password, true ); if ( !policyVioliations.isEmpty() ) { throw new PasswordPolicyViolationException( passwordPolicy.getDescription( true ), policyVioliations ); } @@ -1011,11 +1011,10 @@ protected UserInfo createAdminUserInternal( UUID organizationId, String username throws Exception { - logger.info( "createAdminUserInternal: {}", username ); + logger.debug( "createAdminUserInternal - username: {}, email: {}, name: {}", username, email, name ); - Collection policyVioliations = passwordPolicy.policyCheck( password, true ); - if ( !policyVioliations.isEmpty() ) { - throw new PasswordPolicyViolationException( passwordPolicy.getDescription( true ), policyVioliations ); + if ( isBlank( password ) ) { + password = encodeBase64URLSafeString( bytes( UUID.randomUUID() ) ); } if ( username == null ) { diff --git a/stack/services/src/main/java/org/apache/usergrid/security/providers/GoogleProvider.java b/stack/services/src/main/java/org/apache/usergrid/security/providers/GoogleProvider.java new file mode 100644 index 0000000000..e2b4a00ce9 --- /dev/null +++ b/stack/services/src/main/java/org/apache/usergrid/security/providers/GoogleProvider.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.security.providers; + + +import org.apache.usergrid.management.ManagementService; +import org.apache.usergrid.persistence.EntityManager; +import org.apache.usergrid.persistence.Query; +import org.apache.usergrid.persistence.Results; +import org.apache.usergrid.persistence.entities.User; +import org.apache.usergrid.persistence.index.query.Identifier; +import org.apache.usergrid.security.tokens.exceptions.BadTokenException; +import org.apache.usergrid.utils.JsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.MediaType; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.apache.usergrid.persistence.Schema.PROPERTY_MODIFIED; +import static org.apache.usergrid.utils.ListUtils.anyNull; + + +/** + * Provider implementation for sign-in-as with facebook + * + * @author zznate + */ +@SuppressWarnings("Duplicates") +public class GoogleProvider extends AbstractProvider { + private static final String DEF_API_URL = "https://www.googleapis.com/userinfo/v2/me"; + + private static final Logger logger = LoggerFactory.getLogger(GoogleProvider.class); + + private String apiUrl = DEF_API_URL; + + + GoogleProvider(EntityManager entityManager, ManagementService managementService) { + super(entityManager, managementService); + } + + + @Override + void configure() { + try { + Map config = loadConfigurationFor("googleProvider"); + if (config != null) { + String foundApiUrl = (String) config.get("google_api_url"); + if (foundApiUrl != null) { + apiUrl = foundApiUrl; + } + } + } catch (Exception ex) { + logger.error("Error in configure()", ex); + } + } + + + @Override + public Map loadConfigurationFor() { + return loadConfigurationFor("googleProvider"); + } + + + /** + * Configuration parameters we look for:
  • api_url
  • pic_url
+ */ + @Override + public void saveToConfiguration(Map config) { + saveToConfiguration("googleProvider", config); + } + + + @SuppressWarnings("unchecked") + @Override + Map userFromResource(String externalToken) { + + return client.target(apiUrl) + .queryParam("access_token", externalToken) + .request() + .accept(MediaType.APPLICATION_JSON) + .get(Map.class); + } + + + @Override + public User createOrAuthenticate(String externalToken) throws BadTokenException { + + Map google_user = userFromResource(externalToken); + + String id = (String) google_user.get("id"); + String user_name = (String) google_user.get("name"); + String user_email = (String) google_user.get("email"); + String user_picture = (String) google_user.get("picture"); + + if (logger.isDebugEnabled()) { + logger.debug("FacebookProvider.createOrAuthenticate: {}", JsonUtils.mapToFormattedJsonString(google_user)); + } + + User user = null; + + if ((google_user != null) && !anyNull(id, user_name)) { + + Results r = null; + try { + final Query query = Query.fromEquals("facebook.id", id); + r = entityManager.searchCollection(entityManager.getApplicationRef(), "users", query); + } catch (Exception ex) { + throw new BadTokenException("Could not lookup user for that Facebook ID", ex); + } + if (r.size() > 1) { + logger.error("Multiple users for FB ID: {}", id); + throw new BadTokenException("multiple users with same Facebook ID"); + } + + if (r.size() < 1) { + Map properties = new LinkedHashMap(); + + properties.put("google_plus", google_user); + properties.put("username", "fb_" + id); + properties.put("name", user_name); + properties.put("picture", String.format(user_picture, id)); + + if (user_email != null) { + try { + user = managementService.getAppUserByIdentifier(entityManager.getApplication().getUuid(), + Identifier.fromEmail(user_email)); + } catch (Exception ex) { + throw new BadTokenException( + "Could not find existing user for this applicaiton for email: " + user_email, ex); + } + // if we found the user by email, unbind the properties from above + // that will conflict + // then update the user + if (user != null) { + properties.remove("username"); + properties.remove("name"); + try { + entityManager.updateProperties(user, properties); + } catch (Exception ex) { + throw new BadTokenException("Could not update user with new credentials", ex); + } + user.setProperty(PROPERTY_MODIFIED, properties.get(PROPERTY_MODIFIED)); + } else { + properties.put("email", user_email); + } + } + if (user == null) { + properties.put("activated", true); + try { + user = entityManager.create("user", User.class, properties); + } catch (Exception ex) { + throw new BadTokenException("Could not create user for that token", ex); + } + } + } else { + user = (User) r.getEntity().toTypedEntity(); + Map properties = new LinkedHashMap(); + + properties.put("facebook", google_user); + properties.put("picture", String.format(user_picture, id)); + try { + entityManager.updateProperties(user, properties); + user.setProperty(PROPERTY_MODIFIED, properties.get(PROPERTY_MODIFIED)); + user.setProperty("facebook", google_user); + user.setProperty("picture", String.format(user_picture, id)); + } catch (Exception ex) { + throw new BadTokenException("Could not update user properties", ex); + } + } + } else { + throw new BadTokenException("Unable to confirm Facebook access token"); + } + + return user; + } +} diff --git a/stack/services/src/main/java/org/apache/usergrid/security/providers/SignInProviderFactory.java b/stack/services/src/main/java/org/apache/usergrid/security/providers/SignInProviderFactory.java index 56c5f7efc1..6988d81635 100644 --- a/stack/services/src/main/java/org/apache/usergrid/security/providers/SignInProviderFactory.java +++ b/stack/services/src/main/java/org/apache/usergrid/security/providers/SignInProviderFactory.java @@ -49,6 +49,12 @@ public SignInAsProvider facebook( Application application ) { return facebookProvider; } + public SignInAsProvider google( Application application ) { + GoogleProvider googleProvider = + new GoogleProvider( emf.getEntityManager( application.getUuid() ), managementService ); + googleProvider.configure(); + return googleProvider; + } public SignInAsProvider foursquare( Application application ) { FoursquareProvider foursquareProvider = diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java index 673689465b..f60bfddba5 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java @@ -29,6 +29,7 @@ import org.apache.usergrid.persistence.core.metrics.MetricsFactory; import org.apache.usergrid.persistence.core.metrics.ObservableTimer; import org.apache.usergrid.persistence.core.rx.RxTaskScheduler; +import org.apache.usergrid.persistence.index.exceptions.QueryAnalyzerException; import org.apache.usergrid.security.shiro.utils.LocalShiroCache; import org.apache.usergrid.security.shiro.utils.SubjectUtils; import org.apache.usergrid.services.ServiceParameter.IdParameter; @@ -686,6 +687,7 @@ else if ( first_parameter instanceof NameParameter ) { Query.fromIdentifier( name ), parameters, payload ); } else if ( query != null ) { + query.setAnalyzeOnly(request.isAnalyzeQueryOnly()); return new ServiceContext( this, action, request, previousResults, owner, collectionName, query, parameters, payload ); } diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java index 04e00e024f..075278f96f 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java @@ -31,7 +31,6 @@ import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; import org.apache.usergrid.persistence.EntityRef; -import org.apache.usergrid.persistence.cassandra.CassandraService; import org.apache.usergrid.persistence.entities.Application; import org.apache.usergrid.services.ServiceParameter.IdParameter; import org.apache.usergrid.services.applications.ApplicationsService; @@ -358,13 +357,13 @@ private Service getServiceInstance( ServiceInfo info ) { public ServiceRequest newRequest( ServiceAction action, List parameters ) throws Exception { - return newRequest( action, false, parameters, null, true, true ); + return newRequest( action, false, parameters, null, true, true, false); } public ServiceRequest newRequest( ServiceAction action, List parameters, ServicePayload payload ) throws Exception { - return newRequest( action, false, parameters, payload, true, true ); + return newRequest( action, false, parameters, payload, true, true, false); } @@ -381,9 +380,9 @@ private ServiceRequest getApplicationRequest( ServiceAction action, boolean retu static ApplicationsService appService = new ApplicationsService(); - public ServiceRequest newRequest( ServiceAction action, boolean returnsTree, List parameters, - ServicePayload payload, boolean returnsInboundConnections, - boolean returnsOutboundConnections ) throws Exception { + public ServiceRequest newRequest(ServiceAction action, boolean returnsTree, List parameters, + ServicePayload payload, boolean returnsInboundConnections, + boolean returnsOutboundConnections, boolean analyzeQueryOnly) throws Exception { if ( em != null ) { if ( action != null ) { @@ -414,12 +413,12 @@ public ServiceRequest newRequest( ServiceAction action, boolean returnsTree, Lis String serviceName = pluralize( ServiceParameter.dequeueParameter( parameters ).getName() ); return new ServiceRequest( this, action, serviceName, parameters, payload, returnsTree, - returnsInboundConnections, returnsOutboundConnections ); + returnsInboundConnections, returnsOutboundConnections, analyzeQueryOnly); } public ServiceRequest newRequest( ServiceAction action, boolean returnsTree, List parameters, ServicePayload payload ) throws Exception { - return newRequest( action, returnsTree, parameters, payload, true, true ); + return newRequest( action, returnsTree, parameters, payload, true, true, false); } public void notifyExecutionEventListeners( ServiceAction action, ServiceRequest request, ServiceResults results, diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceRequest.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceRequest.java index f8a5abb05c..89f4976043 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceRequest.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceRequest.java @@ -59,13 +59,14 @@ public class ServiceRequest { private final boolean returnsOutboundConnections; private final ServicePayload payload; private final List originalParameters; + private final boolean analyzeQueryOnly; // return results_set, result_entity, new_service, param_list, properties - public ServiceRequest( ServiceManager services, ServiceAction action, String serviceName, - List parameters, ServicePayload payload, boolean returnsTree, - boolean returnsInboundConnections, boolean returnsOutboundConnections ) { + public ServiceRequest(ServiceManager services, ServiceAction action, String serviceName, + List parameters, ServicePayload payload, boolean returnsTree, + boolean returnsInboundConnections, boolean returnsOutboundConnections, boolean analyzeQueryOnly) { this.services = services; this.action = action; parent = null; @@ -78,6 +79,7 @@ public ServiceRequest( ServiceManager services, ServiceAction action, String ser this.returnsTree = returnsTree; this.returnsInboundConnections = returnsInboundConnections; this.returnsOutboundConnections = returnsOutboundConnections; + this.analyzeQueryOnly = analyzeQueryOnly; if ( payload == null ) { payload = new ServicePayload(); } @@ -87,13 +89,13 @@ public ServiceRequest( ServiceManager services, ServiceAction action, String ser public ServiceRequest( ServiceManager services, ServiceAction action, String serviceName, List parameters, ServicePayload payload, boolean returnsTree) { - this( services, action, serviceName, parameters, payload, returnsTree, true, true); + this( services, action, serviceName, parameters, payload, returnsTree, true, true, false); } public ServiceRequest( ServiceManager services, ServiceAction action, String serviceName, List parameters, ServicePayload payload ) { - this( services, action, serviceName, parameters, payload, false, true, true ); + this( services, action, serviceName, parameters, payload, false, true, true, false); } @@ -103,6 +105,7 @@ public ServiceRequest( ServiceRequest parent, EntityRef owner, String path, Stri this.returnsTree = parent.returnsTree; this.returnsInboundConnections = parent.returnsInboundConnections; this.returnsOutboundConnections = parent.returnsOutboundConnections; + this.analyzeQueryOnly = parent.analyzeQueryOnly; this.action = parent.action; this.payload = parent.payload; this.parent = parent; @@ -121,7 +124,7 @@ public ServiceRequest( ServiceRequest parent, EntityRef owner, String path, Stri public ServiceRequest( ServiceManager services, ServiceAction action, ServiceRequest parent, EntityRef owner, String path, String childPath, String serviceName, List parameters, ServicePayload payload, boolean returnsTree, boolean returnsInboundConnections, - boolean returnsOutboundConnections ) { + boolean returnsOutboundConnections, boolean analyzeQueryOnly ) { this.services = services; this.action = action; this.parent = parent; @@ -134,25 +137,26 @@ public ServiceRequest( ServiceManager services, ServiceAction action, ServiceReq this.returnsTree = returnsTree; this.returnsInboundConnections = returnsInboundConnections; this.returnsOutboundConnections = returnsOutboundConnections; + this.analyzeQueryOnly = analyzeQueryOnly; this.payload = payload; } public ServiceRequest( ServiceManager services, ServiceAction action, ServiceRequest parent, EntityRef owner, String path, String childPath, String serviceName, List parameters, ServicePayload payload, boolean returnsTree ) { - this(services, action, parent, owner, path, childPath, serviceName, parameters, payload, returnsTree, true, true); + this(services, action, parent, owner, path, childPath, serviceName, parameters, payload, returnsTree, true, true, false); } public static ServiceRequest withPath( ServiceRequest r, String path ) { return new ServiceRequest( r.services, r.action, r.parent, r.owner, path, r.childPath, r.serviceName, - r.parameters, r.payload, r.returnsTree, r.returnsInboundConnections, r.returnsOutboundConnections ); + r.parameters, r.payload, r.returnsTree, r.returnsInboundConnections, r.returnsOutboundConnections, r.analyzeQueryOnly ); } public static ServiceRequest withChildPath( ServiceRequest r, String childPath ) { return new ServiceRequest( r.services, r.action, r.parent, r.owner, r.path, childPath, r.serviceName, - r.parameters, r.payload, r.returnsTree, r.returnsInboundConnections, r.returnsOutboundConnections ); + r.parameters, r.payload, r.returnsTree, r.returnsInboundConnections, r.returnsOutboundConnections, r.analyzeQueryOnly ); } @@ -390,4 +394,8 @@ public String getChildPath() { public List getOriginalParameters() { return originalParameters; } + + public boolean isAnalyzeQueryOnly(){ + return analyzeQueryOnly; + } } diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManagerCache.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManagerCache.java index 3b2eff3598..8fbe69d405 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManagerCache.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManagerCache.java @@ -62,7 +62,7 @@ public ApplicationQueueManagerCache(){ this.cache = CacheBuilder.newBuilder() .maximumSize(Math.min(1000,configuredMaxSize)) - .expireAfterWrite(ttl, TimeUnit.MINUTES) + .expireAfterAccess(ttl, TimeUnit.MINUTES) .removalListener(new RemovalListener() { @Override public void onRemoval( diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java index b43594aaa8..d8c314e4db 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java @@ -134,7 +134,14 @@ public ServiceResults postCollection(ServiceContext context) throws Exception { postMeter.mark(); try { - validate(null, context.getPayload()); + validatePayload(null, context.getPayload()); + + final Object deliverValue = context.getProperties().get("deliver"); + if ( deliverValue != null && deliverValue instanceof Number && (long)context.getProperties().get("deliver") < System.currentTimeMillis() ){ + throw new IllegalArgumentException("Property 'deliver' cannot have a value in the past. It is expected to be a unix timestamp in milliseconds"); + } else if ( deliverValue != null && !(deliverValue instanceof Number) ){ + throw new IllegalArgumentException("Property 'deliver' must be a number. It is expected to be a unix timestamp in milliseconds"); + } // perform some input validates on useGraph payload property vs. ql= path query final List parameters = context.getRequest().getOriginalParameters(); @@ -227,7 +234,7 @@ public ServiceResults postItemsByQuery(ServiceContext context, Query query) thro public Entity updateEntity(ServiceRequest request, EntityRef ref, ServicePayload payload) throws Exception { - validate(ref, payload); + validatePayload(ref, payload); Notification notification = em.get(ref, Notification.class); @@ -273,7 +280,7 @@ protected boolean isDeleteAllowed(ServiceContext context, Entity entity) { } // validate payloads - private void validate(EntityRef ref, ServicePayload servicePayload) + private void validatePayload(EntityRef ref, ServicePayload servicePayload) throws Exception { Object obj_payloads = servicePayload.getProperty("payloads"); if (obj_payloads == null && ref == null) { diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java index 7d02360689..22d73446ac 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java @@ -20,6 +20,7 @@ import com.codahale.metrics.Timer; import com.google.inject.Injector; +import org.apache.commons.lang.RandomStringUtils; import org.apache.usergrid.persistence.EntityManagerFactory; import org.apache.usergrid.persistence.core.metrics.MetricsFactory; @@ -148,7 +149,7 @@ private void execute(int threadNumber){ Thread.currentThread().setDaemon(true); } - Thread.currentThread().setName(getClass().getSimpleName()+"_PushNotifications-"+threadNumber); + Thread.currentThread().setName(getClass().getSimpleName()+"_Push-"+ RandomStringUtils.randomAlphanumeric(4)+"-"+threadNumber); final AtomicInteger consecutiveExceptions = new AtomicInteger(); @@ -268,12 +269,16 @@ private void execute(int threadNumber){ if (logger.isTraceEnabled()) { logger.trace("no messages...sleep...{}", sleepWhenNoneFound); } - Thread.sleep(sleepWhenNoneFound); + try { + Thread.sleep(sleepWhenNoneFound); + } catch (InterruptedException e){ + // noop + } } timerContext.stop(); //send to the providers consecutiveExceptions.set(0); - }catch (Exception ex){ + } catch (Exception ex){ logger.error("failed to dequeue",ex); // clear the queue name cache b/c tests might have wiped the keyspace @@ -286,7 +291,7 @@ private void execute(int threadNumber){ Thread.sleep(sleeptime); }catch (InterruptedException ie){ if (logger.isTraceEnabled()) { - logger.info("sleep interrupted"); + logger.trace("sleep interrupted"); } } } @@ -306,7 +311,7 @@ public void stop(){ return; } for(Future future : futures){ - future.cancel(true); + future.cancel(false); } pool.shutdownNow(); diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java index 870cae9906..678b88a0bd 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java @@ -112,7 +112,6 @@ public void failed(Notifier notifier, Receipt receipt, UUID deviceUUID, Object c this.saveReceipt( notification, new SimpleEntityRef( Device.ENTITY_TYPE, deviceUUID ), receipt, true ); } - completed(notifier, deviceUUID); finishedBatch(); } catch (Exception e){ diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java index 6b619b7b18..44b0139892 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java @@ -16,7 +16,6 @@ */ package org.apache.usergrid.services.notifications.gcm; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.android.gcm.server.*; import org.apache.usergrid.persistence.entities.Notification; @@ -99,7 +98,7 @@ public void sendNotification(String providerId, Object payload, Notification not if(!map.containsKey(priorityKey) && notification.getPriority() != null){ map.put(priorityKey, notification.getPriority()); } - Batch batch = getBatch( map); + Batch batch = getBatch( map ); batch.add(providerId, tracker); } diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceApplication.java b/stack/services/src/test/java/org/apache/usergrid/ServiceApplication.java index 89ff272660..74c5c92f4c 100644 --- a/stack/services/src/test/java/org/apache/usergrid/ServiceApplication.java +++ b/stack/services/src/test/java/org/apache/usergrid/ServiceApplication.java @@ -73,7 +73,7 @@ public ServiceResults testRequest( ServiceAction action, int expectedCount, Obje ServiceResults testRequest = testRequest( action, expectedCount, true, params ); if ( !action.equals( ServiceAction.GET )) { - this.refreshIndex(); + this.waitForQueueDrainAndRefreshIndex(); } return testRequest; diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java index 6cdb0eaa9d..ee5313e9f1 100644 --- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java +++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java @@ -17,6 +17,7 @@ package org.apache.usergrid; +import org.apache.usergrid.batch.service.JobSchedulerService; import org.apache.usergrid.management.ApplicationCreator; import org.apache.usergrid.management.ManagementService; import org.apache.usergrid.management.export.ExportService; @@ -44,6 +45,8 @@ public interface ServiceITSetup extends CoreITSetup { ImportService getImportService(); + JobSchedulerService getJobSchedulerService(); + void refreshIndex(UUID appid); /** diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java index ac40ae3059..0308e1563d 100644 --- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java +++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java @@ -19,6 +19,7 @@ import java.util.Properties; +import org.apache.usergrid.batch.service.JobSchedulerService; import org.apache.usergrid.corepersistence.GuiceFactory; import org.apache.usergrid.management.AppInfoMigrationPlugin; import org.junit.runner.Description; @@ -36,13 +37,7 @@ import org.apache.usergrid.security.providers.SignInProviderFactory; import org.apache.usergrid.security.tokens.TokenService; import org.apache.usergrid.services.ServiceManagerFactory; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.config.PropertiesFactoryBean; -import java.util.Properties; import java.util.UUID; @@ -59,6 +54,7 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu private ExportService exportService; private ImportService importService; private AppInfoMigrationPlugin appInfoMigrationPlugin; + private JobSchedulerService jobSchedulerService; public ServiceITSetupImpl() { @@ -72,6 +68,8 @@ public ServiceITSetupImpl() { smf = springResource.getBean( ServiceManagerFactory.class ); exportService = springResource.getBean( ExportService.class ); importService = springResource.getBean( ImportService.class ); + jobSchedulerService = springResource.getBean(JobSchedulerService.class); + try { appInfoMigrationPlugin = springResource.getBean(GuiceFactory.class) @@ -125,12 +123,19 @@ public ManagementService getMgmtSvc() { return managementService; } + + @Override public ExportService getExportService() { return exportService; } @Override public ImportService getImportService() { return importService; } + @Override + public JobSchedulerService getJobSchedulerService() { + return jobSchedulerService; + } + public ServiceManagerFactory getSmf() { if ( smf == null ) { diff --git a/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java b/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java index a46bd603e4..60a5ba0e66 100644 --- a/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java @@ -235,7 +235,7 @@ public void testAppUserActivationResetpwdMail() throws Exception { assertNotNull( orgOwner ); ApplicationInfo app = setup.getMgmtSvc().createApplication( orgOwner.getOrganization().getUuid(), appName ); - this.app.refreshIndex(); + this.app.waitForQueueDrainAndRefreshIndex(); //turn on app admin approval for app users enableAdminApproval(app.getId()); diff --git a/stack/services/src/test/java/org/apache/usergrid/management/RoleIT.java b/stack/services/src/test/java/org/apache/usergrid/management/RoleIT.java index 075ee03023..20d12aba0f 100644 --- a/stack/services/src/test/java/org/apache/usergrid/management/RoleIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/management/RoleIT.java @@ -62,7 +62,7 @@ public void testRoleInactivity() throws Exception { UUID applicationId = setup.getMgmtSvc().createApplication( organization.getUuid(), "test-app" ).getId(); EntityManager em = setup.getEmf().getEntityManager( applicationId ); - setup.getEntityIndex().refresh(em.getApplicationId()); + setup.getEntityIndex().waitForQueueDrainAndRefresh(em.getApplicationId(), 500); Map properties = new LinkedHashMap(); properties.put( "username", "edanuff5" ); @@ -71,8 +71,7 @@ public void testRoleInactivity() throws Exception { User user = em.create( User.ENTITY_TYPE, User.class, properties ); em.createRole( "logged-in", "Logged In", 2000 ); - setup.getEntityIndex().refresh(em.getApplicationId()); - setup.getEntityIndex().refresh(em.getApplicationId()); + setup.getEntityIndex().waitForQueueDrainAndRefresh(em.getApplicationId(), 500); em.addUserToRole( user.getUuid(), "logged-in" ); String accessToken = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, user.getUuid(), 0 ); diff --git a/stack/services/src/test/java/org/apache/usergrid/services/CollectionServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/CollectionServiceIT.java index c071d1ff97..62818c2f36 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/CollectionServiceIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/CollectionServiceIT.java @@ -232,7 +232,7 @@ public void testGenericEntityCollectionWithIdName() throws Exception { // ok } - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(); try { // try DELETE on cats with dogs name app.testRequest( ServiceAction.DELETE, 0, "cats", "Danny" ); diff --git a/stack/services/src/test/java/org/apache/usergrid/services/GroupServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/GroupServiceIT.java index d3c243647d..d37bb10702 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/GroupServiceIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/GroupServiceIT.java @@ -68,13 +68,14 @@ public void testPermissions() throws Exception { app.createGroupRole( group.getUuid(), "admin", 0 ); app.createGroupRole( group.getUuid(), "author", 0 ); - setup.getEntityIndex().refresh(app.getId()); + app.waitForQueueDrainAndRefreshIndex(500); app.grantGroupRolePermission( group.getUuid(), "admin", "users:access:*" ); app.grantGroupRolePermission( group.getUuid(), "admin", "groups:access:*" ); app.grantGroupRolePermission( group.getUuid(), "author", "assets:access:*" ); - setup.getEntityIndex().refresh(app.getId()); + + app.waitForQueueDrainAndRefreshIndex(500); app.testDataRequest( ServiceAction.GET, "groups", group.getUuid(), "rolenames" ); app.testDataRequest( ServiceAction.GET, "groups", group.getUuid(), "roles", "admin", "permissions" ); diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java index 8c2be2cac2..81dced1e58 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java @@ -82,6 +82,8 @@ public void testServices() throws Exception { app.testRequest( ServiceAction.POST, 1, null, "users", "edanuff", "likes", cat.getUuid() ); + app.waitForQueueDrainAndRefreshIndex(250); + Entity restaurant = app.doCreate( "restaurant", "Brickhouse" ); app.createConnection( user, "likes", restaurant ); @@ -92,6 +94,8 @@ public void testServices() throws Exception { app.testRequest( ServiceAction.POST, 1, "users", user.getUuid(), "connections", "likes", restaurant.getUuid() ); + app.waitForQueueDrainAndRefreshIndex(250); + app.testRequest( ServiceAction.GET, 1, "users", "edanuff", "likes", "cats" ); app.testRequest( ServiceAction.GET, 3, "users", "edanuff", "likes" ); @@ -104,7 +108,7 @@ public void testServices() throws Exception { app.testRequest( ServiceAction.GET, 1, "users", "edanuff", "likes", Query.fromQL( "select * where name='axis*'" ) ); -// TODO, we don't allow this at the RESt level, why is this a test? +// TODO, we don't allow this at the REST level, why is this a test? // app.testRequest( ServiceAction.GET, 3, null, "users", "edanuff", "connections" ); app.put( "color", "blacknwhite" ); diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java index 5ea815f990..c035192303 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java @@ -28,7 +28,6 @@ import org.apache.usergrid.persistence.entities.Notification; import org.apache.usergrid.persistence.entities.Receipt; import org.apache.usergrid.services.AbstractServiceIT; -import org.apache.usergrid.services.notifications.gcm.NotificationsServiceIT; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,8 +51,7 @@ protected Notification notificationWaitForComplete(Notification notification) throws Exception { long timeout = System.currentTimeMillis() + 60000; while (System.currentTimeMillis() < timeout) { - Thread.sleep(200); - app.refreshIndex(); + app.waitForQueueDrainAndRefreshIndex(200); notification = app.getEntityManager().get(notification.getUuid(), Notification.class); if (notification.getFinished() != null) { return notification; @@ -95,10 +93,9 @@ protected void checkReceipts(Notification notification, int expected) } } - //assertEquals(expected, receipts.size()); - if( expected != receipts.size()){ - logger.warn("Expected receipt count {} does not match actual count {}", expected, receipts.size()); - } + + assertEquals(expected, receipts.size()); + for (EntityRef receipt : receipts) { diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java index bcfd654f2d..4fb9addf5c 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java @@ -16,6 +16,7 @@ */ package org.apache.usergrid.services.notifications.apns; +import com.google.common.util.concurrent.Service; import com.relayrides.pushy.apns.util.*; import net.jcip.annotations.NotThreadSafe; import org.apache.commons.io.IOUtils; @@ -33,12 +34,8 @@ import org.apache.usergrid.services.ServiceAction; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; - import static org.apache.usergrid.services.notifications.impl.ApplicationQueueManagerImpl.NOTIFIER_ID_POSTFIX; +import static org.junit.Assert.*; @NotThreadSafe @@ -121,8 +118,12 @@ public void before() throws Exception { setup.getEntityIndex().refresh(app.getId()); listener = new QueueListener(ns.getServiceManagerFactory(),ns.getEntityManagerFactory(), new Properties()); - listener.DEFAULT_SLEEP = 200; listener.start(); + + if ( !setup.getJobSchedulerService().isRunning()) { + setup.getJobSchedulerService().startAsync(); + setup.getJobSchedulerService().awaitRunning(); + } } @After @@ -238,24 +239,24 @@ public void pushWithNoValidDevicesShouldComplete() throws Exception { ); - // verify Query for CREATED state + notificationWaitForComplete(notification); + app.waitForQueueDrainAndRefreshIndex(250); + + // verify Query for FINISHED state and that the devices processed is 0 Query query = Query.fromEquals( "state", Notification.State.FINISHED.toString() ); Results results = app.getEntityManager().searchCollection(app.getEntityManager().getApplicationRef(), "notifications", query); - Entity entity = results.getEntitiesMap().get(notification.getUuid()); - assertNotNull(entity); - - notificationWaitForComplete(notification); + notification = (Notification)results.getEntitiesMap().get(notification.getUuid()).toTypedEntity(); + assertEquals(0, notification.getDeviceProcessedCount()); // perform push // - //ns.getQueueManager().processBatchAndReschedule(notification, null); notification = app.getEntityManager().get(e.getUuid(), Notification.class); // verify Query for FINISHED state query = Query.fromEquals("state", Notification.State.FINISHED.toString()); results = app.getEntityManager().searchCollection(app.getEntityManager().getApplicationRef(), "notifications", query); - entity = results.getEntitiesMap().get(notification.getUuid()); + Entity entity = results.getEntitiesMap().get(notification.getUuid()); assertNotNull(entity); notification = (Notification) entity.toTypedEntity(); @@ -295,14 +296,41 @@ public void scheduledNotification() throws Exception { notification = app.getEntityManager().get(e.getUuid(), Notification.class); assertEquals(Notification.State.SCHEDULED, notification.getState()); + } + + @Test + public void scheduledNotificationAndEnsureSend() throws Exception { + + // create push notification // + app.clear(); + String payload = getPayload(); + Map payloads = new HashMap(1); + payloads.put(notifier.getUuid().toString(), payload); + app.put("payloads", payloads); + app.put("deliver", System.currentTimeMillis() + 2000); + app.put("debug",true); + + Entity e = app.testRequest(ServiceAction.POST, 1,"devices",device1.getUuid(), "notifications") + .getEntity(); + app.testRequest(ServiceAction.GET, 1, "notifications", e.getUuid()); + + + Notification notification = app.getEntityManager().get(e.getUuid(), + Notification.class); + assertEquals( + notification.getPayloads().get(notifier.getUuid().toString()), + payload); + + + // delay until the scheduler has time to run + logger.info("Sleeping while the scheduler does its work"); + Thread.sleep(5000); + + + notification = app.getEntityManager().get(e.getUuid(), Notification.class); + + assertEquals(Notification.State.FINISHED, notification.getState()); -// try { -// e = app.testRequest(ServiceAction.DELETE, 1, "notifications", -// e.getUuid()).getEntity(); -// }catch (Exception deleteException){ -// LOG.error("Couldn't delete",deleteException); -// } -// app.getEntityManager().get(e.getUuid(), Notification.class); } @Test @@ -505,7 +533,7 @@ public void twoDevicesTwoNotifiers() throws Exception { @Test public void oneDeviceTwoNotifiers() throws Exception { - // This test should configure 2 notifiers on a device and ensure that we can send to one of them + // This test should configure 2 notifiers on device1 and ensure that we can send to one of them // create a 2nd notifier // Object notifierName1 = "apNs2"; @@ -529,17 +557,19 @@ public void oneDeviceTwoNotifiers() throws Exception { assertEquals(notifier2.getProvider(), PROVIDER); assertEquals(notifier2.getEnvironment(), environment1); + + // Add a device token for the 2nd notifier + app.clear(); String key2 = notifier2.getName() + NOTIFIER_ID_POSTFIX; - device1.setProperty(key2, PUSH_TOKEN); - app.getEntityManager().update(device1); - setup.getEntityIndex().refresh(app.getId()); // need to refresh the index after an update + app.put(key2, PUSH_TOKEN); + app.testRequest(ServiceAction.PUT, 1, "devices", device1).getEntity(); // create push notification // app.clear(); String payload = getPayload(); Map payloads = new HashMap(1); - payloads.put(notifier.getUuid().toString(), payload); + payloads.put(notifierName, payload); app.put("payloads", payloads); app.put("debug",true); @@ -547,14 +577,14 @@ public void oneDeviceTwoNotifiers() throws Exception { "notifications").getEntity(); app.testRequest(ServiceAction.GET, 1, "notifications", notificationEntity.getUuid()); - Notification notification = app.getEntityManager().get(notificationEntity.getUuid(), - Notification.class); - assertEquals( - notification.getPayloads().get(notifier.getUuid().toString()), - payload); + Notification notification = app.getEntityManager().get(notificationEntity.getUuid(), Notification.class); + assertEquals(payload, notification.getPayloads().get(notifierName)); // perform push // notification = notificationWaitForComplete(notification); + + app.waitForQueueDrainAndRefreshIndex(2500); + checkReceipts(notification, 1); } @@ -696,6 +726,9 @@ public void singleUserMultipleDevices() throws Exception{ // perform push // notification = notificationWaitForComplete(notification); + + app.waitForQueueDrainAndRefreshIndex(250); + checkReceipts(notification, 2); // Statistics are not accurate. See - https://issues.apache.org/jira/browse/USERGRID-1207 @@ -809,6 +842,49 @@ public void testDeleteNotification() throws Exception { } } + @Test + public void pushNotificationBadDeliverProperty() throws Exception { + + // try with timestamp in the past + app.clear(); + String payload = getPayload(); + Map payloads = new HashMap(1); + payloads.put(notifier.getName(), payload); + app.put("payloads", payloads); + app.put("deliver", System.currentTimeMillis() - 1000); // some time in the past + app.put("debug",true); + + // post notification to + try { + app.testRequest(ServiceAction.POST, 1,"devices",device1.getUuid(), "notifications").getEntity(); + } catch (Exception e) { + // should be bad request due to time in the past + assertEquals(e.getClass(), IllegalArgumentException.class); + assertTrue(e.getMessage().contains("Property 'deliver' cannot have a value in the past")); + } + + + // try with invalid type of a string for deliver, should be a number + app.clear(); + payload = getPayload(); + payloads = new HashMap(1); + payloads.put(notifier.getName(), payload); + app.put("payloads", payloads); + app.put("deliver", "notatime"); // some time in the past + app.put("debug",true); + + // post notification + try { + app.testRequest(ServiceAction.POST, 1,"devices",device1.getUuid(), "notifications").getEntity(); + } catch (Exception e) { + // should be bad request due invalid deliver value + assertEquals(e.getClass(), IllegalArgumentException.class); + assertTrue(e.getMessage().contains("Property 'deliver' must be a number")); + } + + + } + private String getPayload(){ ApnsPayloadBuilder builder = new ApnsPayloadBuilder(); builder.setAlertBody("Hello, World!"); diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java index 1a9f4f7b30..8360009108 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java @@ -62,13 +62,6 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT { private NotificationsService ns; private QueueListener listener; - - @BeforeClass - public static void setup() { - - - } - @Before public void before() throws Exception { @@ -103,7 +96,6 @@ public void before() throws Exception { ns = getNotificationService(); listener = new QueueListener(ns.getServiceManagerFactory(), ns.getEntityManagerFactory(), new Properties()); - listener.DEFAULT_SLEEP = 200; listener.start(); } @@ -569,6 +561,9 @@ public void badToken() throws Exception { // wait for notification to be marked finished notification = notificationWaitForComplete(notification); + // receipts are created and queried, wait a bit longer for this to happen as indexing + app.waitForQueueDrainAndRefreshIndex(500); + // get the receipts entity IDs List receipts = getNotificationReceipts(notification); assertEquals(1, receipts.size()); @@ -635,6 +630,10 @@ public void sendNotificationWithBadAPIKey() throws Exception { notification = notificationWaitForComplete(notification); app.testRequest(ServiceAction.GET, 1, "notifications", e.getUuid()); + + // receipts are created and queried, wait a bit longer for this to happen as indexing + app.waitForQueueDrainAndRefreshIndex(500); + // get the receipts entity IDs List receipts = getNotificationReceipts(notification); assertEquals(1, receipts.size()); diff --git a/stack/services/src/test/resources/usergrid-custom-test.properties b/stack/services/src/test/resources/usergrid-custom-test.properties index 49f8b5dcf3..bcc8b8edc2 100644 --- a/stack/services/src/test/resources/usergrid-custom-test.properties +++ b/stack/services/src/test/resources/usergrid-custom-test.properties @@ -38,13 +38,14 @@ elasticsearch.queue_impl.resolution=true elasticsearch.queue_impl=DISTRIBUTED # Queueing Test Settings -# Reduce the long polling time for the tests -queue.long.polling.time.millis=150 -queue.num.actors=5 -queue.sender.num.actors=5 -queue.writer.num.actors=5 -elasticsearch.worker_count=2 -usergrid.push.worker_count=2 +queue.long.polling.time.millis=50 +queue.num.actors=50 +queue.sender.num.actors=50 +queue.writer.num.actors=50 +queue.get.timeout.seconds=10 +elasticsearch.worker_count=8 +elasticsearch.worker_count_utility=8 +usergrid.push.worker_count=8 # This property is required to be set and cannot be defaulted anywhere