From a066834aac969056197345c6de06ba1ab059119a Mon Sep 17 00:00:00 2001 From: Max Altgelt Date: Tue, 8 Jul 2025 13:26:34 +0200 Subject: [PATCH 1/2] fix: remove fields that are duplicates from parent --- thorlog/v3/atjob.go | 1 - thorlog/v3/authorizedkeys.go | 1 - thorlog/v3/crontab.go | 1 - thorlog/v3/deepdive.go | 2 -- thorlog/v3/envvar.go | 3 --- thorlog/v3/eventlog.go | 2 -- thorlog/v3/groupsxml.go | 4 +--- thorlog/v3/processconnection.go | 3 +-- thorlog/v3/processhandle.go | 7 +++---- thorlog/v3/registry.go | 9 +++------ thorlog/v3/scheduledtask.go | 18 ++++++++---------- thorlog/v3/sdb.go | 2 -- thorlog/v3/shellbag.go | 1 - thorlog/v3/shimcache.go | 1 - thorlog/v3/teamviewer.go | 6 ++---- thorlog/v3/tomcatusers.go | 4 +--- 16 files changed, 19 insertions(+), 46 deletions(-) diff --git a/thorlog/v3/atjob.go b/thorlog/v3/atjob.go index 0e0c1cb..67391ff 100644 --- a/thorlog/v3/atjob.go +++ b/thorlog/v3/atjob.go @@ -7,7 +7,6 @@ import ( type AtJob struct { jsonlog.ObjectHeader - File string `json:"file" textlog:"job"` Command string `json:"command" textlog:"command"` Start string `json:"start" textlog:"start"` User string `json:"user" textlog:"user"` diff --git a/thorlog/v3/authorizedkeys.go b/thorlog/v3/authorizedkeys.go index 6d4fcb1..8f50e9b 100644 --- a/thorlog/v3/authorizedkeys.go +++ b/thorlog/v3/authorizedkeys.go @@ -7,7 +7,6 @@ import ( type AuthorizedKeysEntry struct { jsonlog.ObjectHeader - Path string `json:"path" textlog:"path"` Type string `json:"key_type" textlog:"type"` Key string `json:"key" textlog:"key"` Comment string `json:"comment" textlog:"comment"` diff --git a/thorlog/v3/crontab.go b/thorlog/v3/crontab.go index 022e3db..b90cfaa 100644 --- a/thorlog/v3/crontab.go +++ b/thorlog/v3/crontab.go @@ -3,7 +3,6 @@ package thorlog type CronJob struct { LogObjectHeader - File string `json:"file" textlog:"file"` User string `json:"user" textlog:"user"` Schedule string `json:"schedule" textlog:"schedule"` Command string `json:"command" textlog:"command"` diff --git a/thorlog/v3/deepdive.go b/thorlog/v3/deepdive.go index 1982644..9803e9b 100644 --- a/thorlog/v3/deepdive.go +++ b/thorlog/v3/deepdive.go @@ -9,8 +9,6 @@ import ( type DeepDiveChunk struct { jsonlog.ObjectHeader - Target *File `json:"file" textlog:"file"` - ChunkOffset HexNumber `json:"chunk_offset" textlog:"chunk_offset"` ChunkEnd HexNumber `json:"chunk_end" textlog:"chunk_end"` Content *SparseData `json:"content" textlog:"content,expand"` diff --git a/thorlog/v3/envvar.go b/thorlog/v3/envvar.go index 7a6ecc9..747d5ab 100644 --- a/thorlog/v3/envvar.go +++ b/thorlog/v3/envvar.go @@ -3,9 +3,6 @@ package thorlog type EnvironmentVariable struct { LogObjectHeader - File string `json:"file,omitempty" textlog:"file,omitempty"` - Process *Process `json:"process,omitempty" textlog:"process,expand,omitempty"` - Variable string `json:"variable" textlog:"var"` Value string `json:"value" textlog:"value"` } diff --git a/thorlog/v3/eventlog.go b/thorlog/v3/eventlog.go index 3ed4835..f686c83 100644 --- a/thorlog/v3/eventlog.go +++ b/thorlog/v3/eventlog.go @@ -9,8 +9,6 @@ import ( type WindowsEventlogEntry struct { jsonlog.ObjectHeader - File string `json:"file,omitempty" textlog:"file,omitempty"` - EventId uint16 `json:"-" textlog:"event_id"` EventLevel int `json:"-" textlog:"event_level"` EventTime time.Time `json:"-" textlog:"event_time"` diff --git a/thorlog/v3/groupsxml.go b/thorlog/v3/groupsxml.go index f88eee0..ac86e87 100644 --- a/thorlog/v3/groupsxml.go +++ b/thorlog/v3/groupsxml.go @@ -6,7 +6,6 @@ import ( type GroupsXmlUser struct { jsonlog.ObjectHeader - File string `json:"file" textlog:"file"` User string `json:"user" textlog:"user"` Password string `json:"password" textlog:"password"` } @@ -17,12 +16,11 @@ const typeGroupsXmlPassword = "groups.xml user" func init() { AddLogObjectType(typeGroupsXmlPassword, &GroupsXmlUser{}) } -func NewGroupsXmlPassword(file, user, password string) *GroupsXmlUser { +func NewGroupsXmlPassword(user, password string) *GroupsXmlUser { return &GroupsXmlUser{ ObjectHeader: jsonlog.ObjectHeader{ Type: typeGroupsXmlPassword, }, - File: file, User: user, Password: password, } diff --git a/thorlog/v3/processconnection.go b/thorlog/v3/processconnection.go index f319aca..75240a9 100644 --- a/thorlog/v3/processconnection.go +++ b/thorlog/v3/processconnection.go @@ -6,8 +6,7 @@ import ( type ProcessConnectionObject struct { jsonlog.ObjectHeader - Process *Process `json:"process" textlog:"process,expand"` - ConnectionDetails ProcessConnection `json:"connection" textlog:",expand"` + ProcessConnection } func (ProcessConnectionObject) reportable() {} diff --git a/thorlog/v3/processhandle.go b/thorlog/v3/processhandle.go index 777d2fd..99694fe 100644 --- a/thorlog/v3/processhandle.go +++ b/thorlog/v3/processhandle.go @@ -7,10 +7,9 @@ import ( type ProcessHandle struct { jsonlog.ObjectHeader - Process *Process `json:"process" textlog:"process,expand"` - Name string `json:"name" textlog:"name"` - Handle uint64 `json:"handle" textlog:"handle,omitempty"` - Type string `json:"type,omitempty" textlog:"type,omitempty"` + Name string `json:"name" textlog:"name"` + Handle uint64 `json:"handle" textlog:"handle,omitempty"` + Type string `json:"type,omitempty" textlog:"type,omitempty"` } func (ProcessHandle) reportable() {} diff --git a/thorlog/v3/registry.go b/thorlog/v3/registry.go index 2bde580..8eefc1f 100644 --- a/thorlog/v3/registry.go +++ b/thorlog/v3/registry.go @@ -9,7 +9,6 @@ import ( type RegistryValue struct { jsonlog.ObjectHeader - File string `json:"file,omitempty" textlog:"file"` Key string `json:"key" textlog:"key"` Modified time.Time `json:"modified" textlog:"modified"` ParsedValue string `json:"value" textlog:"value"` @@ -32,7 +31,6 @@ func NewRegistryValue() *RegistryValue { type RegistryKey struct { jsonlog.ObjectHeader - File string `json:"path,omitempty" textlog:"path,omitempty"` Key string `json:"key" textlog:"key"` Modified time.Time `json:"modified" textlog:"modified"` FormattedValues string `json:"values" textlog:"values"` @@ -64,10 +62,9 @@ func init() { type MsOfficeConnectionCacheEntry struct { jsonlog.ObjectHeader - RegistryHive string `json:"registry_hive" textlog:"path"` - Entry string `json:"entry" textlog:"entry"` - Modified time.Time `json:"modified" textlog:"modified"` - Key string `json:"key" textlog:"key"` + Entry string `json:"entry" textlog:"entry"` + Modified time.Time `json:"modified" textlog:"modified"` + Key string `json:"key" textlog:"key"` } func (MsOfficeConnectionCacheEntry) reportable() {} diff --git a/thorlog/v3/scheduledtask.go b/thorlog/v3/scheduledtask.go index 0a6fe81..7d3edc0 100644 --- a/thorlog/v3/scheduledtask.go +++ b/thorlog/v3/scheduledtask.go @@ -33,16 +33,14 @@ func NewScheduledTask() *ScheduledTask { type RegistryScheduledTask struct { jsonlog.ObjectHeader - RegistryHive string `json:"registry_hive" textlog:"hive"` - Key string `json:"key" textlog:"registry_path"` - Guid string `json:"guid" textlog:"guid"` - Path string `json:"path" textlog:"path"` - Version int `json:"version" textlog:"version"` - Created time.Time `json:"created" textlog:"created"` - LastRun time.Time `json:"last_run" textlog:"last_run"` - LastStopped time.Time `json:"last_stopped" textlog:"last_stopped"` - Status string `json:"status" textlog:"status"` - LastResult string `json:"last_result" textlog:"last_result"` + Guid string `json:"guid" textlog:"guid"` + Path string `json:"path" textlog:"path"` + Version int `json:"version" textlog:"version"` + Created time.Time `json:"created" textlog:"created"` + LastRun time.Time `json:"last_run" textlog:"last_run"` + LastStopped time.Time `json:"last_stopped" textlog:"last_stopped"` + Status string `json:"status" textlog:"status"` + LastResult string `json:"last_result" textlog:"last_result"` } func (RegistryScheduledTask) reportable() {} diff --git a/thorlog/v3/sdb.go b/thorlog/v3/sdb.go index 18acd6c..adb4f85 100644 --- a/thorlog/v3/sdb.go +++ b/thorlog/v3/sdb.go @@ -7,8 +7,6 @@ import ( type SdbEntry struct { jsonlog.ObjectHeader - File string `json:"file,omitempty" textlog:"file,omitempty"` - Entry KeyValueList `json:"entry" textlog:"entry"` } diff --git a/thorlog/v3/shellbag.go b/thorlog/v3/shellbag.go index 38efa3d..15a940a 100644 --- a/thorlog/v3/shellbag.go +++ b/thorlog/v3/shellbag.go @@ -14,7 +14,6 @@ type ShellbagEntry struct { Path string `json:"path" textlog:"path"` Name string `json:"name" textlog:"name"` DateAccess time.Time `json:"date_access" textlog:"date_access"` - Hive string `json:"hive" textlog:"hive"` } func (ShellbagEntry) reportable() {} diff --git a/thorlog/v3/shimcache.go b/thorlog/v3/shimcache.go index 8d23b13..56373eb 100644 --- a/thorlog/v3/shimcache.go +++ b/thorlog/v3/shimcache.go @@ -12,7 +12,6 @@ type ShimCacheEntry struct { Timestamp time.Time `json:"timestamp" textlog:"timestamp"` ExecFlag *bool `json:"exec_flag" textlog:"exec_flag,omitempty"` Path string `json:"path" textlog:"path"` - Hive string `json:"hive" textlog:"hive"` } func (ShimCacheEntry) reportable() {} diff --git a/thorlog/v3/teamviewer.go b/thorlog/v3/teamviewer.go index afc790a..780147c 100644 --- a/thorlog/v3/teamviewer.go +++ b/thorlog/v3/teamviewer.go @@ -6,10 +6,8 @@ import ( type TeamViewerPassword struct { jsonlog.ObjectHeader - Password string `json:"password" textlog:"password"` - Path string `json:"path" textlog:"path"` - RegistryPath string `json:"registry_path" textlog:"registry_path"` - Name string `json:"name" textlog:"name"` + Password string `json:"password" textlog:"password"` + Name string `json:"name" textlog:"name"` } func (TeamViewerPassword) reportable() {} diff --git a/thorlog/v3/tomcatusers.go b/thorlog/v3/tomcatusers.go index c22ef2f..e9da2f3 100644 --- a/thorlog/v3/tomcatusers.go +++ b/thorlog/v3/tomcatusers.go @@ -7,7 +7,6 @@ import ( type TomcatUser struct { jsonlog.ObjectHeader User string `json:"user" textlog:"user"` - File string `json:"file" textlog:"file"` } func (TomcatUser) reportable() {} @@ -16,12 +15,11 @@ const typeTomcatUser = "Tomcat user" func init() { AddLogObjectType(typeTomcatUser, &TomcatUser{}) } -func NewTomcatUser(user, file string) *TomcatUser { +func NewTomcatUser(user string) *TomcatUser { return &TomcatUser{ ObjectHeader: jsonlog.ObjectHeader{ Type: typeTomcatUser, }, User: user, - File: file, } } From 2427ddf52e41bf0207f377a28868e5744142aea4 Mon Sep 17 00:00:00 2001 From: Max Altgelt Date: Tue, 8 Jul 2025 16:51:53 +0200 Subject: [PATCH 2/2] feat: improve readme --- README.md | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ebd618f..148d509 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,47 @@ This library provides definitions of structures used in the output of the THOR APT Forensic Scanner. These structures can be used for different use cases: - generate a schema for THOR JSON logs - convert JSON logs into text logs -- parse JSON logs \ No newline at end of file +- parse JSON logs + +## Usage + +Each object in the THOR log contains a `type` field that indicates the object type. +This type determines how the object should be interpreted and what fields it contains. + +### Event Types + +The object types contained in a THOR log are `THOR finding` and `THOR message`: + - Findings are the results of THOR's analysis, such as detected threats or anomalies. + - Messages are informational or status updates from THOR, such as progress updates. + +Both findings and messages are together called _events_. + +### Parsing Events + +There is a parser in the `thorlog/parser` package which can be used to parse an event independently of its type. +This parser is also version aware, meaning it can handle different versions of the THOR log format. + +### Reportable Objects + +Findings may contain more objects, e.g. as a subject that they report. +Object types that can appear as subjects are called _reportable objects_. +The most common reportable objects are: +- `file` +- `process` + +Reportable objects should contain only fields that relate directly to the object itself. +E.g. when extracting a file from an archive, the file object should contain only fields +that relate to the file itself, not to the archive. +The archive data will instead appear in the _context_ of the finding. + +### Textlog conversion + +The `jsonlog.TextlogFormatter` type provides a way to convert an object to a text log format. + +This formatter can be used to convert findings and messages to a human-readable format. +However, the text log format is not as rich as the JSON format and may not contain all fields. +When in doubt, use the JSON format for analysis. + +## Schema + +A schema for the THOR log format can be generated using the `thorlog/jsonschema` package.