Skip to content

Fix/fix failing tests #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jzunigax2 wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

Fix/fix failing tests #13

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4149392
internxt: Integrate pacer for retry logic in various folder and file …
jzunigax2 Dec 23, 2025
43a6346
internxt: fix pre-upload file existence check and enhance backup file…
jzunigax2 Dec 24, 2025
6e97326
internxt: fix folder creation conflict resolution, fix pacer no retry…
jzunigax2 Dec 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 102 additions & 39 deletions backend/internxt/internxt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ func init() {
encoder.EncodeSlash |
encoder.EncodeBackSlash |
encoder.EncodeRightPeriod |
encoder.EncodeDot,
encoder.EncodeDot |
encoder.EncodeCrLf,
},
}},
)
Expand Down Expand Up @@ -388,7 +389,12 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
// FindLeaf looks for a sub‑folder named `leaf` under the Internxt folder `pathID`.
// If found, it returns its UUID and true. If not found, returns "", false.
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (string, bool, error) {
entries, err := folders.ListAllFolders(ctx, f.cfg, pathID)
var entries []folders.Folder
err := f.pacer.Call(func() (bool, error) {
var err error
entries, err = folders.ListAllFolders(ctx, f.cfg, pathID)
return shouldRetry(ctx, err)
})
if err != nil {
return "", false, err
}
Expand All @@ -409,12 +415,20 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (string, error)
}

var resp *folders.Folder
err := f.pacer.Call(func() (bool, error) {
err := f.pacer.CallNoRetry(func() (bool, error) {
var err error
resp, err = folders.CreateFolder(ctx, f.cfg, request)
return shouldRetry(ctx, err)
})
if err != nil {
// If folder already exists (409 conflict), try to find it
if strings.Contains(err.Error(), "409") || strings.Contains(err.Error(), "Conflict") {
existingID, found, findErr := f.FindLeaf(ctx, pathID, leaf)
if findErr == nil && found {
fs.Debugf(f, "Folder %q already exists in %q, using existing UUID: %s", leaf, pathID, existingID)
return existingID, nil
}
}
return "", fmt.Errorf("can't create folder, %w", err)
}

Expand Down Expand Up @@ -442,7 +456,7 @@ func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string) (*fol
return nil, nil
}

if len(checkResult.Files) > 0 && checkResult.Files[0].Exists {
if len(checkResult.Files) > 0 && checkResult.Files[0].FileExists() {
existingUUID := checkResult.Files[0].UUID
if existingUUID != "" {
fileMeta, err := files.GetFileMeta(ctx, f.cfg, existingUUID)
Expand All @@ -455,7 +469,6 @@ func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string) (*fol
}
}
}

return nil, nil
}

Expand Down Expand Up @@ -484,15 +497,25 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
}
var out fs.DirEntries

foldersList, err := folders.ListAllFolders(ctx, f.cfg, dirID)
var foldersList []folders.Folder
err = f.pacer.Call(func() (bool, error) {
var err error
foldersList, err = folders.ListAllFolders(ctx, f.cfg, dirID)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
for _, e := range foldersList {
remote := filepath.Join(dir, f.opt.Encoding.ToStandardName(e.PlainName))
out = append(out, fs.NewDir(remote, e.ModificationTime))
}
filesList, err := folders.ListAllFiles(ctx, f.cfg, dirID)
var filesList []folders.File
err = f.pacer.Call(func() (bool, error) {
var err error
filesList, err = folders.ListAllFiles(ctx, f.cfg, dirID)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -571,7 +594,11 @@ func (f *Fs) Remove(ctx context.Context, remote string) error {
if err != nil {
return err
}
if err := folders.DeleteFolder(ctx, f.cfg, dirID); err != nil {
err = f.pacer.Call(func() (bool, error) {
err := folders.DeleteFolder(ctx, f.cfg, dirID)
return shouldRetry(ctx, err)
})
if err != nil {
return err
}
f.dirCache.FlushDir(remote)
Expand All @@ -591,7 +618,12 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return nil, fs.ErrorObjectNotFound
}

files, err := folders.ListAllFiles(ctx, f.cfg, dirID)
var files []folders.File
err = f.pacer.Call(func() (bool, error) {
var err error
files, err = folders.ListAllFiles(ctx, f.cfg, dirID)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -669,12 +701,22 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) error {

// About gets quota information
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
internxtLimit, err := users.GetLimit(ctx, f.cfg)
var internxtLimit *users.LimitResponse
err := f.pacer.Call(func() (bool, error) {
var err error
internxtLimit, err = users.GetLimit(ctx, f.cfg)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}

internxtUsage, err := users.GetUsage(ctx, f.cfg)
var internxtUsage *users.UsageResponse
err = f.pacer.Call(func() (bool, error) {
var err error
internxtUsage, err = users.GetUsage(ctx, f.cfg)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -711,7 +753,16 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
if o.f.opt.SimulateEmptyFiles && o.size == 0 {
return io.NopCloser(bytes.NewReader(nil)), nil
}
return buckets.DownloadFileStream(ctx, o.f.cfg, o.id, rangeValue)
var stream io.ReadCloser
err := o.f.pacer.Call(func() (bool, error) {
var err error
stream, err = buckets.DownloadFileStream(ctx, o.f.cfg, o.id, rangeValue)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
return stream, nil
}

// Update updates an existing file or creates a new one
Expand Down Expand Up @@ -768,7 +819,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
backupType = ext

// Rename existing file to backup name
err = files.RenameFile(ctx, o.f.cfg, oldUUID, backupName, backupType)
err = o.f.pacer.Call(func() (bool, error) {
err := files.RenameFile(ctx, o.f.cfg, oldUUID, backupName, backupType)
return shouldRetry(ctx, err)
})
if err != nil {
return fmt.Errorf("failed to rename existing file to backup: %w", err)
}
Expand All @@ -777,45 +831,38 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
fs.Debugf(o.f, "Renamed existing file %s to backup %s.%s (UUID: %s)", remote, backupName, backupType, backupUUID)
}

// Step 2: Upload new file to original location
meta, err := buckets.UploadFileStreamAuto(ctx,
o.f.cfg,
dirID,
o.f.opt.Encoding.FromStandardName(filepath.Base(remote)),
in,
src.Size(),
src.ModTime(ctx),
)
var meta *buckets.CreateMetaResponse
err = o.f.pacer.CallNoRetry(func() (bool, error) {
var err error
meta, err = buckets.UploadFileStreamAuto(ctx,
o.f.cfg,
dirID,
o.f.opt.Encoding.FromStandardName(filepath.Base(remote)),
in,
src.Size(),
src.ModTime(ctx),
)
return shouldRetry(ctx, err)
})

if err != nil {
// Upload failed - restore backup if it exists
if backupUUID != "" {
fs.Debugf(o.f, "Upload failed, attempting to restore backup %s.%s to %s", backupName, backupType, remote)

restoreErr := files.RenameFile(ctx, o.f.cfg, backupUUID,
o.f.opt.Encoding.FromStandardName(origName), origType)
restoreErr := o.f.pacer.Call(func() (bool, error) {
err := files.RenameFile(ctx, o.f.cfg, backupUUID,
o.f.opt.Encoding.FromStandardName(origName), origType)
return shouldRetry(ctx, err)
})
if restoreErr != nil {
fs.Errorf(o.f, "CRITICAL: Upload failed AND backup restore failed: %v. Backup file remains as %s.%s (UUID: %s)",
restoreErr, backupName, backupType, backupUUID)
return fmt.Errorf("upload failed: %w (backup restore also failed: %v)", err, restoreErr)
}
fs.Debugf(o.f, "Upload failed, successfully restored backup file to original name")
}
return fmt.Errorf("upload failed: %w", err)
}

// Step 3: Upload succeeded - delete backup file
if backupUUID != "" {
fs.Debugf(o.f, "Upload succeeded, deleting backup %s.%s (UUID: %s)", backupName, backupType, backupUUID)

if err := files.DeleteFile(ctx, o.f.cfg, backupUUID); err != nil {
if !strings.Contains(err.Error(), "404") {
fs.Logf(o.f, "Warning: uploaded new version but failed to delete backup %s.%s (UUID: %s): %v. You may need to manually delete this orphaned file.",
backupName, backupType, backupUUID, err)
}
} else {
fs.Debugf(o.f, "Successfully deleted backup file after upload")
}
return err
}

// Update object metadata
Expand All @@ -826,6 +873,22 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
o.size = 0
}

// Step 3: Upload succeeded - delete the backup file
if backupUUID != "" {
fs.Debugf(o.f, "Upload succeeded, deleting backup file %s.%s (UUID: %s)", backupName, backupType, backupUUID)
err := o.f.pacer.Call(func() (bool, error) {
err := files.DeleteFile(ctx, o.f.cfg, backupUUID)
return shouldRetry(ctx, err)
})
if err != nil {
fs.Errorf(o.f, "Failed to delete backup file %s.%s (UUID: %s): %v. This may leave an orphaned backup file.",
backupName, backupType, backupUUID, err)
// Don't fail the upload just because backup deletion failed
} else {
fs.Debugf(o.f, "Successfully deleted backup file")
}
}

return nil
}

Expand Down