Merge pull request #1211 from alliance-genome/SCRUM-4496

Added code to pull shard count into a configuration variable

Author: oblodgett

agr_java_core/src/main/java/org/alliancegenome/core/config/ConfigHelper.java

@@ -19,6 +19,7 @@
 import static org.alliancegenome.core.config.Constants.ES_INDEX_PREFIX;
 import static org.alliancegenome.core.config.Constants.ES_INDEX_SUFFIX;
 import static org.alliancegenome.core.config.Constants.ES_PORT;
+import static org.alliancegenome.core.config.Constants.ES_SHARD_COUNT;
 import static org.alliancegenome.core.config.Constants.EXTRACTOR_OUTPUTDIR;
 import static org.alliancegenome.core.config.Constants.FMS_URL;
 import static org.alliancegenome.core.config.Constants.GO_TERM_LIST;
@@ -80,6 +81,7 @@ public static void init() {
 		// If both are used then the resulting index name is: {ES_INDEX_PREFIX}_{ES_INDEX}_{ES_INDEX_SUFFIX}_{TIMESTAMP}"
 		defaults.put(ES_HOST, "localhost");
 		defaults.put(ES_PORT, "9200");
+		defaults.put(ES_SHARD_COUNT, "4");
 
 		// ES Bulk Processing defaults
 		defaults.put(ES_BULK_ACTION_SIZE, "400");
@@ -239,6 +241,17 @@ public static String getEsHost() {
 		}
 		return config.get(ES_HOST);
 	}
+	
+	public static int getEsShardCount() {
+		if (!init) {
+			init();
+		}
+		try {
+			return Integer.parseInt(config.get(ES_SHARD_COUNT));
+		} catch (NumberFormatException e) {
+			return 0;
+		}
+	}
 
 	public static Multimap<String, Integer> getEsHostMap() {
 

agr_java_core/src/main/java/org/alliancegenome/core/config/Constants.java

@@ -13,6 +13,7 @@ private Constants() { }
 	public static final String ES_INDEX_SUFFIX = "ES_INDEX_SUFFIX";
 	public static final String ES_HOST = "ES_HOST";
 	public static final String ES_PORT = "ES_PORT";
+	public static final String ES_SHARD_COUNT = "ES_SHARD_COUNT";
 
 	public static final String ES_BULK_ACTION_SIZE = "ES_BULK_ACTION_SIZE";
 	public static final String ES_BULK_REQUEST_SIZE = "ES_BULK_REQUEST_SIZE";

agr_java_core/src/main/java/org/alliancegenome/es/index/site/schema/settings/SiteIndexSettings.java

@@ -6,18 +6,22 @@
 
 public class SiteIndexSettings extends Settings {
 
-	public SiteIndexSettings(Boolean pretty) {
+	private Integer shardCount;
+	
+	public SiteIndexSettings(Boolean pretty, Integer shardCount) {
 		super(pretty);
+		this.shardCount = shardCount;
 	}
 
 	// Used for the settings for site_index
+	@Override
 	public void buildSettings() throws IOException {
 		builder.startObject();
 			builder.startObject("index")
 				.field("max_result_window", "250000")
 				.field("mapping.total_fields.limit", "25000")
 				.field("number_of_replicas", "0")
-				.field("number_of_shards", "16");
+				.field("number_of_shards", shardCount);
 				buildAnalysis(false);
 			builder.endObject();
 		builder.endObject();

agr_java_core/src/main/java/org/alliancegenome/es/util/IndexManager.java

@@ -67,7 +67,7 @@ public IndexManager(Settings settings) {
 	}
 
 	public IndexManager() {
-		this(new SiteIndexSettings(true), new Mapping(true));
+		this(new SiteIndexSettings(true, ConfigHelper.getEsShardCount()), new Mapping(true));
 	}
 
 	public void createAlias(String alias, String index) { // ES Util
@@ -325,7 +325,7 @@ private String createRepo(String repoName) {
 		if (repoName != null && repoName.length() > 0) {
 			try {
 
-				SiteIndexSettings settings = new SiteIndexSettings(true);
+				SiteIndexSettings settings = new SiteIndexSettings(true, 1);
 				settings.buildRepositorySettings("agr-es-backup-" + repoName);
 
 				PutRepositoryRequest request = new PutRepositoryRequest();

agr_java_core/src/main/java/org/alliancegenome/neo4j/repository/DataExtractorRepository.java

@@ -15,8 +15,8 @@ public Result getAllGenes() {
 		query += " OPTIONAL MATCH p2=(g:Gene)-[:ANNOTATED_TO]-(so:SOTerm)";
 		query += " OPTIONAL MATCH p3=(g:Gene)-[:ALSO_KNOWN_AS]-(syn:Synonym)";
 		query += " OPTIONAL MATCH p4=(g:Gene)-[:CROSS_REFERENCE]-(cr:CrossReference)";
-		query += " RETURN g.primaryKey, g.modLocalId, collect(distinct replace(syn.name, ',', '')) as synonyms, collect(distinct cr.name) as crossrefs, g.name, g.symbol,  g.geneSynopsis, g.automatedGeneSynopsis, s.primaryKey, gl.chromosome, gl.start, gl.end, gl.strand, so.name";
-		//query += " RETURN g.primaryKey, g.modLocalId, collect(distinct replace(syn.name, ',', '')) as synonyms, collect(distinct cr.name) as crossrefs, g.name, g.symbol,  replace(g.geneSynopsis, '\n', ''), replace(g.automatedGeneSynopsis, '\n', ''), s.primaryKey, gl.chromosome, gl.start, gl.end, gl.strand, so.name";
+		query += " RETURN g.primaryKey, g.modLocalId, collect(distinct replace(syn.name, ',', '')) as synonyms, collect(distinct cr.name) as crossrefs, g.name, g.symbol, g.geneSynopsis, g.automatedGeneSynopsis, s.primaryKey, gl.chromosome, gl.start, gl.end, gl.strand, so.name";
+		//query += " RETURN g.primaryKey, g.modLocalId, collect(distinct replace(syn.name, ',', '')) as synonyms, collect(distinct cr.name) as crossrefs, g.name, g.symbol, replace(g.geneSynopsis, '\n', ''), replace(g.automatedGeneSynopsis, '\n', ''), s.primaryKey, gl.chromosome, gl.start, gl.end, gl.strand, so.name";
 
 		return queryForResult(query);