refactor!: Redo parsing to improve perf and reduce mem usage

x2-data-explorer/docs/user-guide.adoc

@@ -75,9 +75,8 @@ For expressions on frames, `f` is the candidate frame. It is a `com.github.rcd47
 
 For a given frame or object to match the expression, the expression must return a result that is https://groovy-lang.org/semantics.html#the-groovy-truth[truthy as defined by Groovy]. This means that the result does not have to be a boolean. E.g. to find any object that contains a `m_template` field, you could simply search for `o.m_template`.
 
-IMPORTANT: When writing expressions to filter on fields within a game state object, remember that each field in the tree is a `com.github.rcd47.x2data.explorer.file.GameStateObjectField`, and the value must be accessed explicitly. E.g. to find a soldier whose first name is Ben, your filter would need to be `o.strFirstName.value == 'Ben'`, not just `o.strFirstName == 'Ben'`.
 
-IMPORTANT: When writing expressions to match fields that are Unreal names, remember that the name is a `com.github.rcd47.x2data.lib.unreal.mappings.UnrealName`, and the value must be accessed through the `original` or `normalized` fields. E.g. to find the eastern US region, your filter would need to be `o.m_TemplateName.value.original == 'WorldRegion_EastNA'`, not just `o.m_TemplateName.value == 'WorldRegion_EastNA'`.
+IMPORTANT: When writing expressions to match fields that are Unreal names, remember that the name is a `com.github.rcd47.x2data.lib.unreal.mappings.UnrealName`, and the value must be accessed through the `original` or `normalized` fields. E.g. to find the eastern US region, your filter would need to be `o.m_TemplateName.original == 'WorldRegion_EastNA'`, not just `o.m_TemplateName == 'WorldRegion_EastNA'`.
 
 ==== Context and Object Summaries
 

x2-data-explorer/pom.xml

@@ -35,6 +35,14 @@
 			<groupId>commons-codec</groupId>
 			<artifactId>commons-codec</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>it.unimi.dsi</groupId>
+			<artifactId>fastutil</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-lang3</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>org.apache.groovy</groupId>
 			<artifactId>groovy</artifactId>
@@ -138,6 +146,11 @@
 								<addModule>java.desktop</addModule>
 								<addModule>java.logging</addModule>
 							</addModules>
+							<javaOptions>
+								<!-- -XX:+UnlockExperimentalVMOptions will not be needed in Java 25+ (see JEP 519) -->
+								<javaOption>-XX:+UnlockExperimentalVMOptions</javaOption>
+								<javaOption>-XX:+UseCompactObjectHeaders</javaOption>
+							</javaOptions>
 							<mainJar>${project.build.finalName}.${project.packaging}</mainJar>
 							<mainClass>com.github.rcd47.x2data.explorer.jfx.MainLauncher</mainClass>
 							<name>X2 Data Explorer</name>

x2-data-explorer/src/main/java/com/github/rcd47/x2data/explorer/file/GameStateContext.java

@@ -1,73 +1,88 @@
 package com.github.rcd47.x2data.explorer.file;
 
-import java.util.HashMap;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
+import com.github.rcd47.x2data.explorer.file.data.PrimitiveInterner;
+import com.github.rcd47.x2data.explorer.file.data.X2VersionedDatumTreeItem;
+import com.github.rcd47.x2data.explorer.file.data.X2VersionedMap;
 import com.github.rcd47.x2data.lib.unreal.mappings.UnrealName;
 import com.github.rcd47.x2data.lib.unreal.mappings.base.EInterruptionStatus;
 import com.google.common.base.Throwables;
 
 import groovy.lang.Script;
+import javafx.scene.control.TreeItem;
 
 public class GameStateContext implements ISizedObject {
 
 	private static final UnrealName INTERRUPTION_HISTORY_INDEX = new UnrealName("InterruptionHistoryIndex");
 	private static final UnrealName HISTORY_INDEX_INTERRUPTED_BY_SELF = new UnrealName("HistoryIndexInterruptedBySelf");
 	private static final UnrealName INTERRUPTION_STATUS = new UnrealName("InterruptionStatus");
+	private static final Map<UnrealName, EInterruptionStatus> STATUS_MAP =
+			Arrays.stream(EInterruptionStatus.values()).collect(Collectors.toMap(s -> new UnrealName(s.name()), Function.identity()));
 
 	private final int sizeInFile;
 	private final UnrealName type;
-	private final Map<UnrealName, NonVersionedField> fields;
+	private final X2VersionedMap properties;
 	private final HistoryFrame frame;
 	private final String summary;
 	private final EInterruptionStatus interruptionStatus;
 	private final HistoryFrame resumedFrom;
 	private final HistoryFrame interruptedByThis;
 	private HistoryFrame resumedBy;
 
-	public GameStateContext(int sizeInFile, GenericObject object, HistoryFrame frame, Map<Integer, HistoryFrame> frames, Script summarizer,
+	public GameStateContext(int sizeInFile, UnrealName type, X2VersionedMap properties, HistoryFrame frame,
+			int frameOffset, HistoryFrame[] frames, PrimitiveInterner interner, Script summarizer,
 			List<HistoryFileProblem> problemsDetected) {
 		this.sizeInFile = sizeInFile;
+		this.type = type;
+		this.properties = properties;
 		this.frame = frame;
 
-		type = object.type;
-
-		fields = new HashMap<>();
-		object.properties.forEach((k, v) -> fields.put(k, new NonVersionedField(v)));
+		var fields = properties.getValueAt(0);
 
 		interruptionStatus = Optional
 				.ofNullable(fields.get(INTERRUPTION_STATUS))
-				.map(s -> EInterruptionStatus.valueOf(((UnrealName) s.getValue()).getOriginal()))
+				.map(STATUS_MAP::get)
 				.orElse(EInterruptionStatus.eInterruptionStatus_None);
 
 		String summaryTemp;
 		try {
-			summarizer.getBinding().setProperty("ctx", this);
-			summaryTemp = (String) summarizer.run();
+			synchronized (summarizer) {
+				summarizer.getBinding().setProperty("ctx", this);
+				summaryTemp = (String) summarizer.run();
+			}
 		} catch (Exception e) {
-			summaryTemp = "";
+			summaryTemp = null;
 			problemsDetected.add(new HistoryFileProblem(
 					frame, this, null, "Summary script failed. Stack trace:\n" + Throwables.getStackTraceAsString(e)));
 		}
-		summary = summaryTemp;
+		summary = summaryTemp == null ? null : interner.internString(summaryTemp);
 
 		var resumedFromIndex = fields.get(INTERRUPTION_HISTORY_INDEX);
-		if (resumedFromIndex == null) {
-			resumedFrom = null;
-		} else {
-			resumedFrom = frames.get(resumedFromIndex.getValue());
-			resumedFrom.getContext().resumedBy = frame;
-		}
+		resumedFrom = resumedFromIndex == null ? null : frames[(int) resumedFromIndex - frameOffset];
 
 		var interruptingIndex = fields.get(HISTORY_INDEX_INTERRUPTED_BY_SELF);
-		interruptedByThis = interruptingIndex == null ? null : frames.get(interruptingIndex.getValue());
+		interruptedByThis = interruptingIndex == null ? null : frames[(int) interruptingIndex - frameOffset];
+	}
+
+	void finishBuilding() {
+		if (resumedFrom != null) {
+			resumedFrom.getContext().resumedBy = frame;
+		}
 	}
 
 	// Groovy support
 	public Object propertyMissing(String name) {
-		return fields.get(new UnrealName(name));
+		return properties.getValueAt(0).get(name);
+	}
+
+	public TreeItem<X2VersionedDatumTreeItem> getTree(PrimitiveInterner interner) {
+		return properties.getTreeNodeAt(interner, null, 0, false);
 	}
 
 	@Override
@@ -79,10 +94,6 @@ public UnrealName getType() {
 		return type;
 	}
 
-	public Map<UnrealName, NonVersionedField> getFields() {
-		return fields;
-	}
-
 	public HistoryFrame getFrame() {
 		return frame;
 	}

x2-data-explorer/src/main/java/com/github/rcd47/x2data/explorer/file/GameStateObject.java

@@ -1,17 +1,13 @@
 package com.github.rcd47.x2data.explorer.file;
 
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
 
+import com.github.rcd47.x2data.explorer.file.data.PrimitiveInterner;
+import com.github.rcd47.x2data.explorer.file.data.X2VersionedMap;
 import com.github.rcd47.x2data.lib.unreal.mappings.UnrealName;
 import com.google.common.base.Throwables;
 
 import groovy.lang.Script;
-import javafx.scene.control.TreeItem;
 
 public class GameStateObject implements ISizedObject {
 
@@ -23,54 +19,45 @@ public class GameStateObject implements ISizedObject {
 	private final boolean removed; // note that it is possible for an object to be added and removed in the same state
 	private final UnrealName type;
 	private final String summary;
-	private final Map<UnrealName, GameStateObjectField> fields;
+	private final X2VersionedMap fields;
 	private final HistoryFrame frame;
 	private final GameStateObject previousVersion;
 	private GameStateObject nextVersion;
 
-	public GameStateObject(int sizeInFile, Map<Integer, GameStateObject> stateObjects, GenericObject currentVersion,
-			HistoryFrame frame, Script summarizer, List<HistoryFileProblem> problemsDetected) {
+	public GameStateObject(int sizeInFile, GameStateObject previousVersion, X2VersionedMap fields,
+			UnrealName type, HistoryFrame frame, PrimitiveInterner interner, Script summarizer,
+			List<HistoryFileProblem> problemsDetected) {
 		this.sizeInFile = sizeInFile;
+		this.type = type;
+		this.fields = fields;
 		this.frame = frame;
+		this.previousVersion = previousVersion;
 
-		objectId = (int) currentVersion.properties.get(OBJECT_ID);
-		removed = Boolean.TRUE.equals(currentVersion.properties.get(REMOVED));
-		type = currentVersion.type;
-		previousVersion = stateObjects.put(objectId, this);
-		if (previousVersion == null) {
-			fields = diffFields(this, null, currentVersion.properties);
-		} else {
-			fields = diffFields(this, previousVersion.fields, currentVersion.properties);
+		if (previousVersion != null) {
 			previousVersion.nextVersion = this;
 		}
 
+		var currentVersion = fields.getValueAt(frame.getNumber());
+		objectId = (int) currentVersion.get(OBJECT_ID);
+		removed = Boolean.TRUE.equals(currentVersion.get(REMOVED));
+
 		String summaryTemp;
 		try {
-			summarizer.getBinding().setProperty("gso", this);
-			summaryTemp = (String) summarizer.run();
+			synchronized (summarizer) {
+				summarizer.getBinding().setProperty("gso", this);
+				summaryTemp = (String) summarizer.run();
+			}
 		} catch (Exception e) {
-			summaryTemp = "";
+			summaryTemp = null;
 			problemsDetected.add(new HistoryFileProblem(
 					frame, null, this, "Summary script failed. Stack trace:\n" + Throwables.getStackTraceAsString(e)));
 		}
-		summary = summaryTemp;
+		summary = summaryTemp == null ? null : interner.internString(summaryTemp);
 	}
 
 	// Groovy support
 	public Object propertyMissing(String name) {
-		return fields == null ? null : fields.get(new UnrealName(name));
-	}
-
-	public TreeItem<GameStateObjectFieldTreeNode> getFieldsAsTreeNode(boolean onlyModified) {
-		var root = new TreeItem<GameStateObjectFieldTreeNode>();
-		var rootChildren = root.getChildren();
-		fields.values()
-				.stream()
-				.sorted((a, b) -> a.getName().compareTo(b.getName()))
-				.map(f -> f.asTreeNode(this, onlyModified))
-				.filter(f -> f != null)
-				.forEachOrdered(rootChildren::add);
-		return root;
+		return fields.getValueAt(frame.getNumber()).get(name);
 	}
 
 	@Override
@@ -94,7 +81,7 @@ public String getSummary() {
 		return summary;
 	}
 
-	public Map<UnrealName, GameStateObjectField> getFields() {
+	public X2VersionedMap getFields() {
 		return fields;
 	}
 
@@ -109,70 +96,5 @@ public GameStateObject getPreviousVersion() {
 	public GameStateObject getNextVersion() {
 		return nextVersion;
 	}
-
-	private static Map<UnrealName, GameStateObjectField> diffFields(
-			GameStateObject newStateObject, Map<UnrealName, GameStateObjectField> oldFields, Map<UnrealName, Object> newGenericFields) {
-		if (oldFields == null) {
-			oldFields = Map.of();
-		}
-		if (newGenericFields == null) {
-			newGenericFields = Map.of();
-		}
-		Set<UnrealName> fieldNames = new HashSet<>(newGenericFields.keySet());
-		for (var oldField : oldFields.values()) {
-			if (!oldField.isTombstone()) {
-				fieldNames.add(oldField.getName());
-			}
-		}
-
-		boolean changed = false;
-		Map<UnrealName, GameStateObjectField> newFields = new HashMap<>();
-		for (var fieldName : fieldNames) {
-			var oldField = oldFields.get(fieldName);
-			var newField = diffField(newStateObject, fieldName, oldField, newGenericFields.get(fieldName));
-			newFields.put(fieldName, newField);
-			if (oldField != newField) { // deliberate identity comparison
-				changed = true;
-			}
-		}
-
-		return changed ? newFields : oldFields;
-	}
-
-	@SuppressWarnings("unchecked")
-	private static GameStateObjectField diffField(
-			GameStateObject newStateObject, UnrealName fieldName, GameStateObjectField oldField, Object newValue) {
-		if (newValue != null) {
-			Map<UnrealName, Object> newValueMap;
-			if (newValue instanceof List<?> list) {
-				newValueMap = new HashMap<>();
-				for (int i = 0; i < list.size(); i++) {
-					var child = list.get(i);
-					if (child != null) { // can be null for static arrays, if child is the default value and thus not serialized
-						newValueMap.put(new UnrealName(Integer.toString(i)), child);
-					}
-				}
-			} else if (newValue instanceof GenericObject obj) {
-				newValueMap = obj.properties;
-			} else if (newValue instanceof Map) {
-				newValueMap = (Map<UnrealName, Object>) newValue;
-			} else {
-				// must be a scalar
-				return oldField != null && Objects.equals(oldField.getValue(), newValue) ?
-						oldField : new GameStateObjectField(fieldName, newValue, null, newStateObject, oldField);
-			}
-
-			var oldFieldChildren = oldField == null ? null : oldField.getChildren();
-			var newFieldChildren = diffFields(newStateObject, oldFieldChildren, newValueMap);
-			return newFieldChildren.equals(oldFieldChildren) ?
-					oldField : new GameStateObjectField(fieldName, null, newFieldChildren, newStateObject, oldField);
-		}
-
-		var oldFieldChildren = oldField.getChildren();
-		return new GameStateObjectField(
-				fieldName, null,
-				oldFieldChildren == null ? null : diffFields(newStateObject, oldFieldChildren, null),
-				newStateObject, oldField);
-	}
 
 }