Bugfix: Support 32kb page size

.github/workflows/test.yaml

@@ -3,15 +3,15 @@ on: [pull_request]
 jobs:
   build:
     name: CI Test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - name: Set up Go 1.19
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v4
       with:
-        go-version: '^1.20'
+        go-version: '^1.26'
       id: go
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Build
       run: |

bin/catalog.go

@@ -20,7 +20,11 @@ var (
 )
 
 func doCatalog() {
-	ese_ctx, err := parser.NewESEContext(*catalog_command_file_arg)
+	stat, err := (*catalog_command_file_arg).Stat()
+	kingpin.FatalIfError(err, "Unable to open ese file")
+
+	ese_ctx, err := parser.NewESEContext(
+		*catalog_command_file_arg, stat.Size())
 	kingpin.FatalIfError(err, "Unable to open ese file")
 
 	catalog, err := parser.ReadCatalog(ese_ctx)

bin/dump_table.go

@@ -26,11 +26,18 @@ var (
 	dump_command_table_name_limit = dump_command.Flag(
 		"limit", "Only dump this many rows").Int()
 
+	dump_command_pages = dump_command.Flag(
+		"pages", "Only show the pages").Bool()
+
 	STOP_ERROR = errors.New("Stop")
 )
 
 func doDump() {
-	ese_ctx, err := parser.NewESEContext(*dump_command_file_arg)
+	stat, err := (*dump_command_file_arg).Stat()
+	kingpin.FatalIfError(err, "Unable to open ese file")
+
+	ese_ctx, err := parser.NewESEContext(
+		*dump_command_file_arg, stat.Size())
 	kingpin.FatalIfError(err, "Unable to open ese file")
 
 	catalog, err := parser.ReadCatalog(ese_ctx)
@@ -44,6 +51,22 @@ func doDump() {
 	for _, t := range tables {
 		count := 0
 
+		if *dump_command_pages {
+			table_any, pres := catalog.Tables.Get(t)
+			if !pres {
+				continue
+			}
+
+			table := table_any.(*parser.Table)
+			err := parser.WalkPages(ese_ctx, int64(table.FatherDataPageNumber),
+				func(header *parser.PageHeader, id int64, value *parser.Value) error {
+					fmt.Printf("Page %v: %v\n", id, header.DebugString())
+					return nil
+				})
+			kingpin.FatalIfError(err, "Unable to walk table pages")
+			continue
+		}
+
 		err = catalog.DumpTable(t, func(row *ordereddict.Dict) error {
 			serialized, err := json.Marshal(row)
 			if err != nil {

bin/page.go

@@ -21,7 +21,11 @@ var (
 )
 
 func doPage() {
-	ese_ctx, err := parser.NewESEContext(*page_command_file_arg)
+	stat, err := (*page_command_file_arg).Stat()
+	kingpin.FatalIfError(err, "Unable to open ese file")
+
+	ese_ctx, err := parser.NewESEContext(
+		*page_command_file_arg, stat.Size())
 	kingpin.FatalIfError(err, "Unable to open ese file")
 
 	parser.DumpPage(ese_ctx, *page_command_page_number)

parser/catalog.go

@@ -516,6 +516,11 @@ func (self *Table) ParseMultiValue(buffer []byte, parseFn func([]byte) any, fLon
 		} else {
 			ib2 = int(binary.LittleEndian.Uint16(buffer[2*(imv+1):2*(imv+1)+2]) & maskIb)
 		}
+
+		if ib1 > len(buffer) || ib2 > len(buffer) || ib1 > ib2 {
+			return nil
+		}
+
 		data := buffer[ib1:ib2]
 		fSeparatedInstance := (mv1 & 0x8000) != 0
 		if Debug && fCompressed && imv == 0 && fSeparatedInstance {
@@ -548,6 +553,10 @@ func (self *Table) ParseMultiValue(buffer []byte, parseFn func([]byte) any, fLon
 func ParseTwoValue(buffer []byte, parseFn func([]byte) any) []any {
 	var twoValues []any
 	lenFstValue := int(buffer[0])
+	if lenFstValue > len(buffer) {
+		return nil
+	}
+
 	twoValues = append(twoValues, parseFn(buffer[1:1+lenFstValue]))
 	twoValues = append(twoValues, parseFn(buffer[1+lenFstValue:]))
 	return twoValues

parser/compression.go

@@ -39,6 +39,10 @@ func Decompress7BitCompression(buf []byte) []byte {
 }
 
 func DecompressLongValue(buf []byte) []byte {
+	if len(buf) == 0 {
+		return nil
+	}
+
 	compression_flag := buf[0] >> 3
 	switch {
 	case compression_flag == 0x1:

parser/context.go

@@ -10,12 +10,13 @@ type ESEContext struct {
 	Reader   io.ReaderAt
 	Profile  *ESEProfile
 	PageSize int64
+	MaxPage  int64
 	Header   *FileHeader
 	Version  uint32
 	Revision uint32
 }
 
-func NewESEContext(reader io.ReaderAt) (*ESEContext, error) {
+func NewESEContext(reader io.ReaderAt, size int64) (*ESEContext, error) {
 	result := &ESEContext{
 		Profile: NewESEProfile(),
 		Reader:  reader,
@@ -38,16 +39,22 @@ func NewESEContext(reader io.ReaderAt) (*ESEContext, error) {
 
 	result.Version = result.Header.FormatVersion()
 	result.Revision = result.Header.FormatRevision()
+	result.MaxPage = size/result.PageSize + 1
 	return result, nil
 }
 
-func (self *ESEContext) GetPage(id int64) *PageHeader {
+func (self *ESEContext) GetPage(id int64) (*PageHeader, error) {
+	if self.MaxPage > 0 && id > self.MaxPage {
+		return nil, fmt.Errorf("Page %v exceeds max page %v",
+			id, self.MaxPage)
+	}
+
 	// First file page is file header, second page is backup of file
 	// header.
 	return &PageHeader{
 		PageHeader_: self.Profile.PageHeader_(
 			self.Reader, (id+1)*self.PageSize),
-	}
+	}, nil
 }
 
 func (self *ESEContext) IsSmallPage() bool {

parser/debug.go

@@ -10,7 +10,7 @@ var (
 	DebugWalk = false
 )
 
-func DlvDebug() {
+func DlvBreak() {
 
 }
 

parser/pages.go

@@ -89,6 +89,16 @@ func GetPageValues(ctx *ESEContext, header *PageHeader, id int64) []*Value {
 	available_tags := header.AvailablePageTag()
 	var largest_value_offset int64
 
+	/*  https://github.com/libyal/libesedb/issues/78
+
+	In the 32KiB page format the upper 4 bits of the available page tag
+	are reserved (ctagReserved) and the lower 12 bits contain the
+	actual number of page tags
+	*/
+	if ctx.PageSize >= 32768 {
+		available_tags &= 0x0fff
+	}
+
 	for tag_count := available_tags - 1; tag_count > 0; tag_count-- {
 		// Handle the case where available_tags lies and is way too
 		// large. The tags are stored in the end of the page and go
@@ -179,7 +189,11 @@ func (self *PageHeader) EndOffset(ctx *ESEContext) int64 {
 }
 
 func DumpPage(ctx *ESEContext, id int64) {
-	header := ctx.GetPage(id)
+	header, err := ctx.GetPage(id)
+	if err != nil {
+		fmt.Printf("Page %v: %v\n", id, err)
+		return
+	}
 	fmt.Printf("Page %v: %v\n", id, header.DebugString())
 
 	// Show the tags
@@ -319,7 +333,10 @@ func _walkPages(ctx *ESEContext,
 	}
 	seen[id] = true
 
-	header := ctx.GetPage(id)
+	header, err := ctx.GetPage(id)
+	if err != nil {
+		return err
+	}
 	values := GetPageValues(ctx, header, id)
 	if DebugWalk {
 		fmt.Printf("Walking page %v %v\n", id, header.DebugString())