From 910f8ea436d02688c0625356bff6b77602216f08 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Mon, 25 Aug 2025 09:38:51 +0000 Subject: [PATCH] feat: Enable tgz takeouts import This change adds support for importing `.tgz` takeout files to the Immich server. It introduces a new `TgzReadCloser` to handle `.tgz` archives, updates the documentation, and adds tests for the new functionality. --- docs/google-takeout.md | 2 + go.mod | 1 + go.sum | 2 + internal/fshelper/parseArgs.go | 13 ++- internal/fshelper/tgzname/tgz.go | 148 ++++++++++++++++++++++++++ internal/fshelper/tgzname/tgz_test.go | 76 +++++++++++++ 6 files changed, 237 insertions(+), 5 deletions(-) create mode 100644 internal/fshelper/tgzname/tgz.go create mode 100644 internal/fshelper/tgzname/tgz_test.go diff --git a/docs/google-takeout.md b/docs/google-takeout.md index b31e94928..5021925f1 100644 --- a/docs/google-takeout.md +++ b/docs/google-takeout.md @@ -2,6 +2,8 @@ This project aims to make the process of importing Google Photos takeouts as easy and accurate as possible. But keep in mind that Google takeout structure is complex and not documented. Some information may be missed or may even be wrong. +When downloading your photos from Google Photos, you have the choice between .zip and .tgz format. `immich-go` supports both formats. + ## Folders in takeout - The Year folder contains all images taken that year - Albums are in separate folders named as the album diff --git a/go.mod b/go.mod index a638bd068..ae5f4d9c0 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.12.0 // indirect github.com/spf13/cast v1.7.1 // indirect + github.com/stretchr/testify v1.11.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 943eafd45..f05a0bd2c 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= +github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4= diff --git a/internal/fshelper/parseArgs.go b/internal/fshelper/parseArgs.go index f12ef8ca5..528eec62b 100644 --- a/internal/fshelper/parseArgs.go +++ b/internal/fshelper/parseArgs.go @@ -7,16 +7,14 @@ import ( "path/filepath" "strings" + "github.com/simulot/immich-go/internal/fshelper/tgzname" zipname "github.com/simulot/immich-go/internal/fshelper/zipName" ) // ParsePath return a list of FS bases on args // -// Zip files are opened and returned as FS +// Zip and tgz files are opened and returned as FS // Manage wildcards in path -// -// TODO: Implement a tgz reader for non google-photos archives - func ParsePath(args []string) ([]fs.FS, error) { var errs error fsyss := []fs.FS{} @@ -32,7 +30,12 @@ func ParsePath(args []string) ([]fs.FS, error) { lowF := strings.ToLower(f) switch { case strings.HasSuffix(lowF, ".tgz") || strings.HasSuffix(lowF, ".tar.gz"): - errs = errors.Join(fmt.Errorf("immich-go can't use tgz archives: %s", filepath.Base(a))) + fsys, err := tgzname.OpenReader(f) + if err != nil { + errs = errors.Join(errs, fmt.Errorf("%s: %w", a, err)) + continue + } + fsyss = append(fsyss, fsys) case strings.HasSuffix(lowF, ".zip"): fsys, err := zipname.OpenReader(f) // zip.OpenReader(f) if err != nil { diff --git a/internal/fshelper/tgzname/tgz.go b/internal/fshelper/tgzname/tgz.go new file mode 100644 index 000000000..a658060ff --- /dev/null +++ b/internal/fshelper/tgzname/tgz.go @@ -0,0 +1,148 @@ +package tgzname + +import ( + "archive/tar" + "compress/gzip" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "github.com/simulot/immich-go/internal/fshelper/debugfiles" +) + +type TgzReadCloser struct { + f *os.File + name string + zr *gzip.Reader + tr *tar.Reader +} + +type fileInfo struct { + name string + size int64 + mode fs.FileMode + modTime time.Time +} + +func (fi fileInfo) Name() string { + return fi.name +} +func (fi fileInfo) Size() int64 { + return fi.size +} +func (fi fileInfo) Mode() fs.FileMode { + return fi.mode +} +func (fi fileInfo) ModTime() time.Time { + return fi.modTime +} +func (fi fileInfo) IsDir() bool { + return fi.mode.IsDir() +} +func (fi fileInfo) Sys() any { + return nil +} + +var ( + _ fs.File = (*file)(nil) + _ fs.FileInfo = (*fileInfo)(nil) +) + +type file struct { + io.Reader + io.Closer + fi os.FileInfo +} + +func (f file) Stat() (fs.FileInfo, error) { + return f.fi, nil +} + +func OpenReader(name string) (*TgzReadCloser, error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + + s, err := f.Stat() + if err != nil { + return nil, err + } + if s.IsDir() { + return nil, fs.ErrInvalid + } + zr, err := gzip.NewReader(f) + if err != nil { + return nil, err + } + + tr := tar.NewReader(zr) + + debugfiles.TrackOpenFile(f, name) + baseName := filepath.Base(name) + baseName = strings.TrimSuffix(baseName, filepath.Ext(baseName)) + if strings.HasSuffix(strings.ToLower(baseName), ".tar") { + baseName = strings.TrimSuffix(baseName, ".tar") + } + + return &TgzReadCloser{ + f: f, + name: baseName, + zr: zr, + tr: tr, + }, nil +} + +func (z TgzReadCloser) Open(name string) (fs.File, error) { + err := z.rewind() + if err != nil { + return nil, err + } + for { + h, err := z.tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if h.Name == name { + return file{ + Reader: z.tr, + Closer: io.NopCloser(nil), + fi: h.FileInfo(), + }, nil + } + } + return nil, fs.ErrNotExist +} + +func (z *TgzReadCloser) rewind() error { + _, err := z.f.Seek(0, 0) + if err != nil { + return err + } + err = z.zr.Reset(z.f) + if err != nil { + return err + } + z.tr = tar.NewReader(z.zr) + return nil +} + +func (z TgzReadCloser) Close() error { + debugfiles.TrackCloseFile(z.f) + err := z.zr.Close() + err2 := z.f.Close() + if err != nil { + return err + } + return err2 +} + +func (z TgzReadCloser) Name() string { + return z.name +} diff --git a/internal/fshelper/tgzname/tgz_test.go b/internal/fshelper/tgzname/tgz_test.go new file mode 100644 index 000000000..aeafab963 --- /dev/null +++ b/internal/fshelper/tgzname/tgz_test.go @@ -0,0 +1,76 @@ +package tgzname + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTgzReader(t *testing.T) { + // Create a temporary tgz file for testing + tmpfile, err := os.CreateTemp("", "test.tgz") + assert.NoError(t, err) + defer os.Remove(tmpfile.Name()) + + err = createTestTgz(tmpfile, []struct{ Name, Body string }{ + {"test.txt", "This is a test file."}, + {"folder/test2.txt", "This is another test file."}, + }) + assert.NoError(t, err) + tmpfile.Close() + + // Open the tgz file with the reader + r, err := OpenReader(tmpfile.Name()) + assert.NoError(t, err) + defer r.Close() + + // Test reading a file from the archive + file, err := r.Open("test.txt") + assert.NoError(t, err) + defer file.Close() + + content, err := io.ReadAll(file) + assert.NoError(t, err) + assert.Equal(t, "This is a test file.", string(content)) + + // Test reading a file from a folder in the archive + file, err = r.Open("folder/test2.txt") + assert.NoError(t, err) + defer file.Close() + + content, err = io.ReadAll(file) + assert.NoError(t, err) + assert.Equal(t, "This is another test file.", string(content)) + + // Test reading a non-existent file + _, err = r.Open("nonexistent.txt") + assert.Error(t, err) +} + +func createTestTgz(file *os.File, files []struct{ Name, Body string }) error { + gw := gzip.NewWriter(file) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + for _, f := range files { + hdr := &tar.Header{ + Name: f.Name, + Mode: 0600, + Size: int64(len(f.Body)), + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if _, err := tw.Write([]byte(f.Body)); err != nil { + return err + } + } + + return nil +}