Initial geocoder status API

ols-geocoder-core/pom.xml

@@ -24,6 +24,31 @@
 				<filtering>true</filtering>
 			</resource>
 		</resources>
+		<plugins>
+			<plugin>
+				<groupId>io.github.git-commit-id</groupId>
+				<artifactId>git-commit-id-maven-plugin</artifactId>
+				<version>5.0.0</version>
+				<executions>
+					<execution>
+						<id>get-the-git-infos</id>
+						<goals>
+							<goal>revision</goal>
+						</goals>
+						<phase>initialize</phase>
+					</execution>
+				</executions>
+				<configuration>
+					<generateGitPropertiesFile>true</generateGitPropertiesFile>
+					<generateGitPropertiesFilename>
+						${project.build.outputDirectory}/git.properties</generateGitPropertiesFilename>
+					<includeOnlyProperties>
+						<includeOnlyProperty>^git.commit.id.(abbrev|full)$</includeOnlyProperty>
+					</includeOnlyProperties>
+					<commitIdGenerationMode>full</commitIdGenerationMode>
+				</configuration>
+			</plugin>
+		</plugins>
 	</build>
 
 	<dependencies>
@@ -56,7 +81,6 @@
 		<dependency>
 			<groupId>ch.qos.logback</groupId>
 			<artifactId>logback-classic</artifactId>
-			<version>1.4.8</version>
 			<scope>test</scope>
 		</dependency>
 		<dependency>

ols-geocoder-core/src/main/java/ca/bc/gov/ols/geocoder/DummyGeocoder.java

@@ -30,6 +30,7 @@
 import ca.bc.gov.ols.geocoder.data.enumTypes.LocationDescriptor;
 import ca.bc.gov.ols.geocoder.data.enumTypes.MatchPrecision;
 import ca.bc.gov.ols.geocoder.data.enumTypes.PositionalAccuracy;
+import ca.bc.gov.ols.geocoder.status.SystemStatus;
 import ca.bc.gov.ols.geocoder.util.GeocoderUtil;
 
 /**
@@ -90,5 +91,10 @@ public static SearchResults getDummyResults(GeocodeQuery query, GeometryFactory
 	public GeocoderConfig getConfig() {
 		return new GeocoderConfig();
 	}
+
+	@Override
+	public SystemStatus getStatus() {
+		return new SystemStatus();
+	}
 
 }

ols-geocoder-core/src/main/java/ca/bc/gov/ols/geocoder/Geocoder.java

@@ -72,6 +72,7 @@
 import ca.bc.gov.ols.geocoder.parser.generator.RuleOperator;
 import ca.bc.gov.ols.geocoder.parser.generator.RuleSequence;
 import ca.bc.gov.ols.geocoder.parser.generator.RuleTerm;
+import ca.bc.gov.ols.geocoder.status.SystemStatus;
 import ca.bc.gov.ols.geocoder.util.GeocoderUtil;
 import ca.bc.gov.ols.rowreader.DateType;
 import ca.bc.gov.ols.util.PairedList;
@@ -91,13 +92,15 @@ public class Geocoder implements IGeocoder {
 			+ Geocoder.class.getCanonicalName());
 
 	private GeocoderDataStore datastore;
+	private SystemStatus status;
 	private AddressParser parser;
 	//private DateFormatter dateFormat = new SimpleDateFormat("yyyy-MM-dd");
 	private Lexer lexer;
 	private SiteAddress fallbackSiteAddress;
 
 	public Geocoder(GeocoderDataStore datastore) {
 		this.datastore = datastore;
+		this.status = datastore.getStatus();
 		lexer = new Lexer(new DraLexicalRules(), datastore.getWordMap());
 		parser = createParser(lexer);
 		fallbackSiteAddress = geocodeFallbackAddress(datastore.getConfig().getFallbackAddress());
@@ -333,6 +336,7 @@ public SearchResults geocode(GeocodeQuery query) {
 			gm.resolve(datastore);
 		}
 		query.stopTimer();
+		//status.slowQueries.offer(query);
 		return new SearchResults(query, limitedMatches, datastore.getDate(DateType.PROCESSING_DATE));
 	}
 
@@ -1968,6 +1972,11 @@ public GeocoderDataStore getDatastore() {
 	public AddressParser getParser() {
 		return parser;
 	}
+
+	@Override
+	public SystemStatus getStatus() {
+		return status;
+	}
 
 }
 

ols-geocoder-core/src/main/java/ca/bc/gov/ols/geocoder/GeocoderDataStore.java

@@ -122,6 +122,8 @@
 import ca.bc.gov.ols.geocoder.filters.OnlyCivicSiteFilter;
 import ca.bc.gov.ols.geocoder.filters.PointLocationBboxFilter;
 import ca.bc.gov.ols.geocoder.filters.StreetIntersectionDegreeFilter;
+import ca.bc.gov.ols.geocoder.status.BasicStatus;
+import ca.bc.gov.ols.geocoder.status.SystemStatus;
 import ca.bc.gov.ols.geocoder.util.GeocoderUtil;
 import ca.bc.gov.ols.rowreader.DateType;
 import ca.bc.gov.ols.rowreader.RowReader;
@@ -149,6 +151,7 @@ public class GeocoderDataStore {
 			GeocoderDataStore.class.getCanonicalName());
 
 	private GeocoderDataSource dataSource;
+	private SystemStatus status;
 
 	/* This is the global geometry factory, all geometries are created using it */
 	private static GeometryFactory geometryFactory;
@@ -249,6 +252,8 @@ public GeocoderDataStore(Properties bootstrapConfig, GeometryFactory gf, Geometr
 		logger.info("GeocoderDataStore() constructor called");
 		geometryFactory = gf;
 		this.reprojector = reprojector;
+		status = new SystemStatus();
+		status.startTimestamp = ZonedDateTime.now().toString();
 		EnumSet<GeocoderFeature> features = GeocoderFeature.fromStringList(bootstrapConfig.getProperty("features"));
 		if(features != null) {
 			featureSet = features;
@@ -429,6 +434,7 @@ private void loadData() {
 				"Memory in use after loading(Megs): {}",
 				((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1000000));
 		dates = dataSource.getDates();
+		status.setDates(dates);
 		if(dates == null) {
 			logger.error("Data file dates are inconsistent; Errors may result! Check that all files were processed and copied correctly.");
 		} else {
@@ -610,6 +616,7 @@ private TIntObjectHashMap<StreetIntersection> buildIntersectionIdMap() {
 			intMap.put(id, intersection);
 		}
 		logger.debug("Intersection count: {}", count);
+		status.counts.put("intersections", count);
 		rr.close();
 		return intMap;
 	}
@@ -633,6 +640,7 @@ private Map<String, StateProvTerr> buildStateProvTerrMap(
 			stateProvTerrById.put(id, spt);
 		}
 		logger.debug("State/Prov/Terr count: {}", count);
+		status.counts.put("state_prov_terrs", count);
 		rr.close();
 
 		return sptMap;
@@ -674,6 +682,7 @@ private Map<String, Set<LocalityMapTarget>> buildLocalityNameTrie(
 			names.add(new LocalityMapTarget(100, locality));
 		}
 		logger.debug("Locality count: {}", count);
+		status.counts.put("localities", count);
 		rr.close();
 
 		Stream<LocalityMapping> locMaps = dataSource.getLocalityMappings();
@@ -712,6 +721,7 @@ private Map<String, Set<LocalityMapTarget>> buildLocalityNameTrie(
 			}
 		};
 		logger.debug("Locality Mapping count: {}", count);
+		status.counts.put("locality_mappings", count);
 
 		// minimize the arrays used for the locality mappings sets
 		for(Set<LocalityMapTarget> set : localityMappings.values()) {
@@ -733,6 +743,7 @@ private TIntObjectHashMap<String> buildElectoralAreaIdMap() {
 			eaMap.put(id, name);
 		}
 		logger.debug("Electoral Area count: {}", count);
+		status.counts.put("electorial_areas", count);
 		rr.close();
 		return eaMap;
 	}
@@ -836,7 +847,9 @@ private TIntObjectHashMap<List<AccessPoint>> buildAccessPointAndSiteMaps(WordMap
 
 		}
 		logger.debug("Site Count: " + siteCount);
+		status.counts.put("sites", siteCount);
 		logger.debug("Access Point Count: " + apCount);
+		status.counts.put("access_points", apCount);
 		rr.close();
 
 		// loop over all the Sites in the siteIdMap and resolve their parents
@@ -1067,7 +1080,9 @@ private TIntObjectHashMap<List<StreetSegment>> buildStreetSegmentNameMap(
 		rr.close();
 
 		logger.debug("Street Segment count: {}", count);
+		status.counts.put("street_segments", count);
 		logger.debug("Block Face count: {}", faceCount);
+		status.counts.put("block_faces", faceCount);
 
 		TIntObjectHashMap<List<StreetSegment>> nameIdToSegmentsMap = new TIntObjectHashMap<List<StreetSegment>>();
 		rr = dataSource.getStreetNameOnSegments();
@@ -1124,6 +1139,7 @@ private TIntObjectHashMap<List<StreetSegment>> buildStreetSegmentNameMap(
 		rr.close();
 
 		logger.debug("Street Name on Segment count: {}", count);
+		status.counts.put("street_name_on_segments", count);
 		return nameIdToSegmentsMap;
 	}
 
@@ -1219,6 +1235,7 @@ private NameLookupTrie<StreetNameBody> buildStreetNameTrie(
 		rr.close();
 
 		logger.debug("Street Name count: {}", count);
+		status.counts.put("street_names", count);
 		// loop over the PrimaryNameId -> IntersectionList map
 		for(TIntObjectIterator<List<StreetIntersection>> it = primaryNameIdToIntersectionsMap.iterator(); it.hasNext(); ){
 			it.advance();
@@ -1285,6 +1302,7 @@ public int compare(StreetName sn1, StreetName sn2) {
 		rr.close();
 
 		logger.debug("Street Locality Centroid count: {}", count);
+		status.counts.put("street_locality_centroids", count);
 
 		// loop over all streetNames and compact them
 		for(StreetName sn : streetNameIdMap.valueCollection()) {
@@ -1380,6 +1398,7 @@ private Map<String, BusinessCategory> buildBusinessCategories() {
 		}
 		rr.close();
 		logger.info("Business Category Count: {}", count);
+		status.counts.put("business_categories", count);
 		return cats;
 	}
 
@@ -1423,6 +1442,7 @@ private Map<UUID, IOccupant> buildOccupantMap(WordMapBuilder wordMapBuilder,
 		}
 		rr.close();
 		logger.info("Occupants Loaded: {}", count);
+		status.counts.put("occupants", count);
 
 		// build the occupantNameIndex
 		InvertedIndexBuilder<IOccupant> builder = new InvertedIndexBuilder<IOccupant>();
@@ -1610,6 +1630,7 @@ private Map<String, String> buildAbbreviationMappings(
 		rr.close();
 
 		logger.debug("Abbreviation Mapping count: {}", count);
+		status.counts.put("abbreviation_mappings", count);
 
 		Map<String, String> abbrMappings = new THashMap<String, String>(count);
 
@@ -1981,4 +2002,8 @@ public ZonedDateTime getDate(DateType source) {
 		}
 		return null;
 	}
+
+	public SystemStatus getStatus() {
+		return status;
+	}
 }

ols-geocoder-core/src/main/java/ca/bc/gov/ols/geocoder/IGeocoder.java

@@ -18,12 +18,15 @@
 import ca.bc.gov.ols.geocoder.api.GeocodeQuery;
 import ca.bc.gov.ols.geocoder.api.data.SearchResults;
 import ca.bc.gov.ols.geocoder.config.GeocoderConfig;
+import ca.bc.gov.ols.geocoder.status.SystemStatus;
 
 public interface IGeocoder {
 	public SearchResults geocode(GeocodeQuery query);
 
 	public GeocoderDataStore getDatastore();
 
 	public GeocoderConfig getConfig();
+
+	public SystemStatus getStatus();
 
 }

ols-geocoder-core/src/main/java/ca/bc/gov/ols/geocoder/config/GeocoderConfig.java

@@ -44,6 +44,7 @@
 
 public class GeocoderConfig {
 	public static final String VERSION;
+	public static final String GIT_COMMIT_ID;
 	public static final String LOGGER_PREFIX = "BGEO.";
 	public static final PrecisionModel BASE_PRECISION_MODEL = new PrecisionModel(1000);
 	private static final Logger logger = LoggerFactory.getLogger(LOGGER_PREFIX
@@ -98,6 +99,13 @@ public class GeocoderConfig {
 		} catch(IOException ioe) {
 			throw new RuntimeException(ioe);
 		}
+		try (InputStream input = Thread.currentThread().getContextClassLoader().getResourceAsStream("git.properties")) {
+            Properties props = new Properties();
+            props.load(input);
+            GIT_COMMIT_ID = props.getProperty("git.commit.id.abbrev");
+		} catch(IOException ioe) {
+			throw new RuntimeException(ioe);
+		}
 	}
 
 	public GeocoderConfig() {

ols-geocoder-core/src/main/java/ca/bc/gov/ols/geocoder/status/BasicStatus.java

@@ -0,0 +1,17 @@
+package ca.bc.gov.ols.geocoder.status;
+
+import java.util.Map;
+
+public record BasicStatus(
+		String version,
+		String gitCommitId,
+		String dataProcessingTimestamp,
+		String roadNetworkTimestamp,
+		String startTimestamp,
+		Map<String, Integer> counts) {
+
+	public BasicStatus(SystemStatus status) {
+		this(status.version, status.gitCommitId, status.dataProcessingTimestamp, 
+				status.roadNetworkTimestamp, status.startTimestamp, status.counts);
+	}
+}

ols-geocoder-core/src/main/java/ca/bc/gov/ols/geocoder/status/QueryLog.java

@@ -0,0 +1,19 @@
+package ca.bc.gov.ols.geocoder.status;
+
+import ca.bc.gov.ols.geocoder.api.GeocodeQuery;
+
+public class QueryLog implements Comparable<QueryLog>{
+
+	public final String addressString;
+	public final long executionTimeNanos;
+
+	public QueryLog(GeocodeQuery query) {
+		addressString = query.getQueryAddress();
+		executionTimeNanos = query.getExecutionTimeNanos();
+	}
+
+	public int compareTo(QueryLog other) {
+		return Long.compare(executionTimeNanos, other.executionTimeNanos);
+	}
+
+}

ols-geocoder-core/src/main/java/ca/bc/gov/ols/geocoder/status/SlowQueryList.java

@@ -0,0 +1,32 @@
+package ca.bc.gov.ols.geocoder.status;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.PriorityQueue;
+
+import ca.bc.gov.ols.geocoder.api.GeocodeQuery;
+
+public class SlowQueryList {
+	private PriorityQueue<QueryLog> q;
+	private int size = 10;
+
+	public SlowQueryList(int size) {
+		this.size = size;
+		q = new PriorityQueue<QueryLog>(size + 1);
+	}
+
+	public synchronized void offer(GeocodeQuery query) {
+		if(q.size() == size && query.getExecutionTimeNanos() > q.peek().executionTimeNanos) {
+            q.poll(); // remove least slow query, new one is slower
+		}
+        if (q.size() < size) {
+            q.add(new QueryLog(query)); // we removed one or are not full, so add
+        }
+	}
+
+	public synchronized QueryLog[] get() {
+		QueryLog[] logs = (QueryLog[]) q.toArray(new QueryLog[q.size()]);
+		Arrays.sort(logs, Collections.reverseOrder());
+		return logs;
+	}
+}

ols-geocoder-core/src/main/java/ca/bc/gov/ols/geocoder/status/SystemStatus.java

@@ -0,0 +1,30 @@
+package ca.bc.gov.ols.geocoder.status;
+
+import java.time.ZonedDateTime;
+import java.util.HashMap;
+import java.util.Map;
+
+import ca.bc.gov.ols.geocoder.config.GeocoderConfig;
+import ca.bc.gov.ols.rowreader.DateType;
+
+public class SystemStatus {
+
+	public final String version = GeocoderConfig.VERSION;
+	public final String gitCommitId = GeocoderConfig.GIT_COMMIT_ID;
+	public String dataProcessingTimestamp;
+	public String roadNetworkTimestamp;
+
+	public String startTimestamp = "";
+
+	public Map<String, Integer> counts = new HashMap<>();
+
+	//public SlowQueryList slowQueries = new SlowQueryList(10);
+
+	public void setDates(Map<DateType, ZonedDateTime> dates) {
+		if(dates != null) {
+			dataProcessingTimestamp = String.valueOf(dates.get(DateType.PROCESSING_DATE));
+			roadNetworkTimestamp = String.valueOf(dates.get(DateType.ITN_VINTAGE_DATE));
+		}
+	}
+
+}