fix: remove fields that are duplicates from parent

README.md

@@ -5,4 +5,47 @@
 This library provides definitions of structures used in the output of the THOR APT Forensic Scanner. These structures can be used for different use cases:
 - generate a schema for THOR JSON logs
 - convert JSON logs into text logs
-- parse JSON logs
+- parse JSON logs
+
+## Usage
+
+Each object in the THOR log contains a `type` field that indicates the object type.
+This type determines how the object should be interpreted and what fields it contains.
+
+### Event Types
+
+The object types contained in a THOR log are `THOR finding` and `THOR message`:
+ - Findings are the results of THOR's analysis, such as detected threats or anomalies.
+ - Messages are informational or status updates from THOR, such as progress updates.
+
+Both findings and messages are together called _events_.
+
+### Parsing Events
+
+There is a parser in the `thorlog/parser` package which can be used to parse an event independently of its type.
+This parser is also version aware, meaning it can handle different versions of the THOR log format.
+
+### Reportable Objects
+
+Findings may contain more objects, e.g. as a subject that they report. 
+Object types that can appear as subjects are called _reportable objects_.
+The most common reportable objects are:
+- `file`
+- `process`
+
+Reportable objects should contain only fields that relate directly to the object itself.
+E.g. when extracting a file from an archive, the file object should contain only fields 
+that relate to the file itself, not to the archive.
+The archive data will instead appear in the _context_ of the finding.
+
+### Textlog conversion
+
+The `jsonlog.TextlogFormatter` type provides a way to convert an object to a text log format.
+
+This formatter can be used to convert findings and messages to a human-readable format.
+However, the text log format is not as rich as the JSON format and may not contain all fields.
+When in doubt, use the JSON format for analysis.
+
+## Schema
+
+A schema for the THOR log format can be generated using the `thorlog/jsonschema` package.

thorlog/v3/atjob.go

@@ -7,7 +7,6 @@ import (
 type AtJob struct {
 	jsonlog.ObjectHeader
 
-	File      string `json:"file" textlog:"job"`
 	Command   string `json:"command" textlog:"command"`
 	Start     string `json:"start" textlog:"start"`
 	User      string `json:"user" textlog:"user"`

thorlog/v3/authorizedkeys.go

@@ -7,7 +7,6 @@ import (
 type AuthorizedKeysEntry struct {
 	jsonlog.ObjectHeader
 
-	Path    string `json:"path" textlog:"path"`
 	Type    string `json:"key_type" textlog:"type"`
 	Key     string `json:"key" textlog:"key"`
 	Comment string `json:"comment" textlog:"comment"`

thorlog/v3/crontab.go

@@ -3,7 +3,6 @@ package thorlog
 type CronJob struct {
 	LogObjectHeader
 
-	File     string `json:"file" textlog:"file"`
 	User     string `json:"user" textlog:"user"`
 	Schedule string `json:"schedule" textlog:"schedule"`
 	Command  string `json:"command" textlog:"command"`

thorlog/v3/deepdive.go

@@ -9,8 +9,6 @@ import (
 type DeepDiveChunk struct {
 	jsonlog.ObjectHeader
 
-	Target *File `json:"file" textlog:"file"`
-
 	ChunkOffset HexNumber   `json:"chunk_offset" textlog:"chunk_offset"`
 	ChunkEnd    HexNumber   `json:"chunk_end" textlog:"chunk_end"`
 	Content     *SparseData `json:"content" textlog:"content,expand"`

thorlog/v3/envvar.go

@@ -3,9 +3,6 @@ package thorlog
 type EnvironmentVariable struct {
 	LogObjectHeader
 
-	File    string   `json:"file,omitempty" textlog:"file,omitempty"`
-	Process *Process `json:"process,omitempty" textlog:"process,expand,omitempty"`
-
 	Variable string `json:"variable" textlog:"var"`
 	Value    string `json:"value" textlog:"value"`
 }

thorlog/v3/eventlog.go

@@ -9,8 +9,6 @@ import (
 type WindowsEventlogEntry struct {
 	jsonlog.ObjectHeader
 
-	File string `json:"file,omitempty" textlog:"file,omitempty"`
-
 	EventId       uint16    `json:"-" textlog:"event_id"`
 	EventLevel    int       `json:"-" textlog:"event_level"`
 	EventTime     time.Time `json:"-" textlog:"event_time"`

thorlog/v3/groupsxml.go

@@ -6,7 +6,6 @@ import (
 
 type GroupsXmlUser struct {
 	jsonlog.ObjectHeader
-	File     string `json:"file" textlog:"file"`
 	User     string `json:"user" textlog:"user"`
 	Password string `json:"password" textlog:"password"`
 }
@@ -17,12 +16,11 @@ const typeGroupsXmlPassword = "groups.xml user"
 
 func init() { AddLogObjectType(typeGroupsXmlPassword, &GroupsXmlUser{}) }
 
-func NewGroupsXmlPassword(file, user, password string) *GroupsXmlUser {
+func NewGroupsXmlPassword(user, password string) *GroupsXmlUser {
 	return &GroupsXmlUser{
 		ObjectHeader: jsonlog.ObjectHeader{
 			Type: typeGroupsXmlPassword,
 		},
-		File:     file,
 		User:     user,
 		Password: password,
 	}

thorlog/v3/processconnection.go

@@ -6,8 +6,7 @@ import (
 
 type ProcessConnectionObject struct {
 	jsonlog.ObjectHeader
-	Process           *Process          `json:"process" textlog:"process,expand"`
-	ConnectionDetails ProcessConnection `json:"connection" textlog:",expand"`
+	ProcessConnection
 }
 
 func (ProcessConnectionObject) reportable() {}

thorlog/v3/processhandle.go

@@ -7,10 +7,9 @@ import (
 type ProcessHandle struct {
 	jsonlog.ObjectHeader
 
-	Process *Process `json:"process" textlog:"process,expand"`
-	Name    string   `json:"name" textlog:"name"`
-	Handle  uint64   `json:"handle" textlog:"handle,omitempty"`
-	Type    string   `json:"type,omitempty" textlog:"type,omitempty"`
+	Name   string `json:"name" textlog:"name"`
+	Handle uint64 `json:"handle" textlog:"handle,omitempty"`
+	Type   string `json:"type,omitempty" textlog:"type,omitempty"`
 }
 
 func (ProcessHandle) reportable() {}

thorlog/v3/registry.go

@@ -9,7 +9,6 @@ import (
 type RegistryValue struct {
 	jsonlog.ObjectHeader
 
-	File        string    `json:"file,omitempty" textlog:"file"`
 	Key         string    `json:"key" textlog:"key"`
 	Modified    time.Time `json:"modified" textlog:"modified"`
 	ParsedValue string    `json:"value" textlog:"value"`
@@ -32,7 +31,6 @@ func NewRegistryValue() *RegistryValue {
 
 type RegistryKey struct {
 	jsonlog.ObjectHeader
-	File            string    `json:"path,omitempty" textlog:"path,omitempty"`
 	Key             string    `json:"key" textlog:"key"`
 	Modified        time.Time `json:"modified" textlog:"modified"`
 	FormattedValues string    `json:"values" textlog:"values"`
@@ -64,10 +62,9 @@ func init() {
 
 type MsOfficeConnectionCacheEntry struct {
 	jsonlog.ObjectHeader
-	RegistryHive string    `json:"registry_hive" textlog:"path"`
-	Entry        string    `json:"entry" textlog:"entry"`
-	Modified     time.Time `json:"modified" textlog:"modified"`
-	Key          string    `json:"key" textlog:"key"`
+	Entry    string    `json:"entry" textlog:"entry"`
+	Modified time.Time `json:"modified" textlog:"modified"`
+	Key      string    `json:"key" textlog:"key"`
 }
 
 func (MsOfficeConnectionCacheEntry) reportable() {}

thorlog/v3/scheduledtask.go

@@ -33,16 +33,14 @@ func NewScheduledTask() *ScheduledTask {
 
 type RegistryScheduledTask struct {
 	jsonlog.ObjectHeader
-	RegistryHive string    `json:"registry_hive" textlog:"hive"`
-	Key          string    `json:"key" textlog:"registry_path"`
-	Guid         string    `json:"guid" textlog:"guid"`
-	Path         string    `json:"path" textlog:"path"`
-	Version      int       `json:"version" textlog:"version"`
-	Created      time.Time `json:"created" textlog:"created"`
-	LastRun      time.Time `json:"last_run" textlog:"last_run"`
-	LastStopped  time.Time `json:"last_stopped" textlog:"last_stopped"`
-	Status       string    `json:"status" textlog:"status"`
-	LastResult   string    `json:"last_result" textlog:"last_result"`
+	Guid        string    `json:"guid" textlog:"guid"`
+	Path        string    `json:"path" textlog:"path"`
+	Version     int       `json:"version" textlog:"version"`
+	Created     time.Time `json:"created" textlog:"created"`
+	LastRun     time.Time `json:"last_run" textlog:"last_run"`
+	LastStopped time.Time `json:"last_stopped" textlog:"last_stopped"`
+	Status      string    `json:"status" textlog:"status"`
+	LastResult  string    `json:"last_result" textlog:"last_result"`
 }
 
 func (RegistryScheduledTask) reportable() {}

thorlog/v3/sdb.go

@@ -7,8 +7,6 @@ import (
 type SdbEntry struct {
 	jsonlog.ObjectHeader
 
-	File string `json:"file,omitempty" textlog:"file,omitempty"`
-
 	Entry KeyValueList `json:"entry" textlog:"entry"`
 }
 

thorlog/v3/shellbag.go

@@ -14,7 +14,6 @@ type ShellbagEntry struct {
 	Path       string    `json:"path" textlog:"path"`
 	Name       string    `json:"name" textlog:"name"`
 	DateAccess time.Time `json:"date_access" textlog:"date_access"`
-	Hive       string    `json:"hive" textlog:"hive"`
 }
 
 func (ShellbagEntry) reportable() {}

thorlog/v3/shimcache.go

@@ -12,7 +12,6 @@ type ShimCacheEntry struct {
 	Timestamp time.Time `json:"timestamp" textlog:"timestamp"`
 	ExecFlag  *bool     `json:"exec_flag" textlog:"exec_flag,omitempty"`
 	Path      string    `json:"path" textlog:"path"`
-	Hive      string    `json:"hive" textlog:"hive"`
 }
 
 func (ShimCacheEntry) reportable() {}

thorlog/v3/teamviewer.go

@@ -6,10 +6,8 @@ import (
 
 type TeamViewerPassword struct {
 	jsonlog.ObjectHeader
-	Password     string `json:"password" textlog:"password"`
-	Path         string `json:"path" textlog:"path"`
-	RegistryPath string `json:"registry_path" textlog:"registry_path"`
-	Name         string `json:"name" textlog:"name"`
+	Password string `json:"password" textlog:"password"`
+	Name     string `json:"name" textlog:"name"`
 }
 
 func (TeamViewerPassword) reportable() {}

thorlog/v3/tomcatusers.go

@@ -7,7 +7,6 @@ import (
 type TomcatUser struct {
 	jsonlog.ObjectHeader
 	User string `json:"user" textlog:"user"`
-	File string `json:"file" textlog:"file"`
 }
 
 func (TomcatUser) reportable() {}
@@ -16,12 +15,11 @@ const typeTomcatUser = "Tomcat user"
 
 func init() { AddLogObjectType(typeTomcatUser, &TomcatUser{}) }
 
-func NewTomcatUser(user, file string) *TomcatUser {
+func NewTomcatUser(user string) *TomcatUser {
 	return &TomcatUser{
 		ObjectHeader: jsonlog.ObjectHeader{
 			Type: typeTomcatUser,
 		},
 		User: user,
-		File: file,
 	}
 }