Skip to content
Open
106 changes: 50 additions & 56 deletions src/commands/upload-file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,11 @@ import { CLIUtils } from '../utils/cli.utils';
import { ConfigService } from '../services/config.service';
import path from 'node:path';
import { DriveFileService } from '../services/drive/drive-file.service';
import { ErrorUtils } from '../utils/errors.utils';
import { NotValidDirectoryError } from '../types/command.types';
import { ValidationService } from '../services/validation.service';
import { EncryptionVersion } from '@internxt/sdk/dist/drive/storage/types';
import { ThumbnailService } from '../services/thumbnail.service';
import { BufferStream } from '../utils/stream.utils';
import { isFileThumbnailable } from '../utils/thumbnail.utils';
import { isFileThumbnailable, tryUploadThumbnail } from '../utils/thumbnail.utils';
import { Readable } from 'node:stream';

export default class UploadFile extends Command {
Expand Down Expand Up @@ -46,9 +44,6 @@ export default class UploadFile extends Command {
const filePath = await this.getFilePath(flags['file'], nonInteractive);

const stats = await stat(filePath);
if (!stats.size) {
throw new Error('The file is empty. Uploading empty files is not allowed.');
}

const fileInfo = path.parse(filePath);
const fileType = fileInfo.ext.replaceAll('.', '');
Expand All @@ -62,11 +57,9 @@ export default class UploadFile extends Command {
reporter: this.log.bind(this),
})) ?? user.rootFolderId;

// 1. Prepare the network
const networkFacade = await CLIUtils.prepareNetwork({ loginUserDetails: user, jsonFlag: flags['json'] });
// Prepare the network
const networkFacade = CLIUtils.prepareNetwork({ loginUserDetails: user, jsonFlag: flags['json'] });

// 2. Upload file to the Network
const readStream = createReadStream(filePath);
const timer = CLIUtils.timer();
const progressBar = CLIUtils.progress(
{
Expand All @@ -77,42 +70,50 @@ export default class UploadFile extends Command {
);
progressBar?.start(100, 0);

let fileId: string | undefined;
let bufferStream: BufferStream | undefined;
let fileStream: Readable = readStream;
const isThumbnailable = isFileThumbnailable(fileType);
if (isThumbnailable) {
bufferStream = new BufferStream();
fileStream = readStream.pipe(bufferStream);
}
const fileSize = stats.size ?? 0;

const progressCallback = (progress: number) => {
progressBar?.update(progress * 100 * 0.99);
};
if (fileSize > 0) {
// Upload file to the Network
const readStream = createReadStream(filePath);
let fileStream: Readable = readStream;

const fileId = await new Promise((resolve: (fileId: string) => void, reject) => {
const state = networkFacade.uploadFile(
fileStream,
stats.size,
user.bucket,
(err: Error | null, res: string | null) => {
if (err) {
return reject(err);
}
resolve(res as string);
},
progressCallback,
);
process.on('SIGINT', () => {
state.stop();
process.exit(1);
if (isThumbnailable) {
bufferStream = new BufferStream();
fileStream = readStream.pipe(bufferStream);
}

const progressCallback = (progress: number) => {
progressBar?.update(progress * 100 * 0.99);
};

fileId = await new Promise((resolve: (fileId: string) => void, reject) => {
const state = networkFacade.uploadFile(
fileStream,
fileSize,
user.bucket,
(err: Error | null, res: string | null) => {
if (err) {
return reject(err);
}
resolve(res as string);
},
progressCallback,
);
process.on('SIGINT', () => {
state.stop();
process.exit(1);
});
});
});
}

// 3. Create the file in Drive
// Create the file in Drive
const createdDriveFile = await DriveFileService.instance.createFile({
plainName: fileInfo.name,
type: fileType,
size: stats.size,
size: fileSize,
folderUuid: destinationFolderUuid,
fileId: fileId,
bucket: user.bucket,
Expand All @@ -121,38 +122,31 @@ export default class UploadFile extends Command {
modificationTime: stats.mtime?.toISOString(),
});

try {
if (isThumbnailable && bufferStream) {
const thumbnailBuffer = bufferStream.getBuffer();

if (thumbnailBuffer) {
await ThumbnailService.instance.uploadThumbnail(
thumbnailBuffer,
fileType,
user.bucket,
createdDriveFile.uuid,
networkFacade,
);
}
}
} catch (error) {
ErrorUtils.report(error, { command: this.id });
if (fileSize > 0 && isThumbnailable && bufferStream) {
void tryUploadThumbnail({
bufferStream,
fileType,
userBucket: user.bucket,
fileUuid: createdDriveFile.uuid,
networkFacade,
});
}

progressBar?.update(100);
progressBar?.stop();

const uploadTime = timer.stop();
this.log('\n');
// eslint-disable-next-line max-len
const message = `File uploaded in ${uploadTime}ms, view it at ${ConfigService.instance.get('DRIVE_WEB_URL')}/file/${createdDriveFile.uuid}`;
const message =
`File uploaded in ${uploadTime}ms, view it at ` +
`${ConfigService.instance.get('DRIVE_WEB_URL')}/file/${createdDriveFile.uuid}`;
CLIUtils.success(this.log.bind(this), message);
return {
success: true,
message,
file: {
...createdDriveFile,
plainName: fileInfo.name,
plainName: createdDriveFile.name,
},
};
};
Expand Down
62 changes: 35 additions & 27 deletions src/services/network/upload/upload-file.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import { isAlreadyExistsError } from '../../../utils/errors.utils';
import { stat } from 'node:fs/promises';
import { EncryptionVersion } from '@internxt/sdk/dist/drive/storage/types';
import { createFileStreamWithBuffer, tryUploadThumbnail } from '../../../utils/thumbnail.utils';
import { BufferStream } from '../../../utils/stream.utils';
import { DriveFileItem } from '../../../types/drive.types';

export class UploadFileService {
static readonly instance = new UploadFileService();
Expand Down Expand Up @@ -63,40 +65,45 @@ export class UploadFileService {
network,
bucket,
parentFolderUuid,
}: UploadFileWithRetryParams): Promise<string | null> {
}: UploadFileWithRetryParams): Promise<DriveFileItem | null> {
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
try {
const stats = await stat(file.absolutePath);
if (!stats.size) {
logger.warn(`Skipping empty file: ${file.relativePath}`);
return null;
}
const fileSize = stats.size ?? 0;

const fileType = extname(file.absolutePath).replaceAll('.', '');
const { fileStream, bufferStream } = createFileStreamWithBuffer({
path: file.absolutePath,
fileType,
});

const fileId = await new Promise<string>((resolve, reject) => {
network.uploadFile(
fileStream,
stats.size,
bucket,
(err: Error | null, res: string | null) => {
if (err) {
return reject(err);
}
resolve(res as string);
},
() => {},
);
});
let fileId: string | undefined;
let thumbnailStream: BufferStream | undefined;

if (fileSize > 0) {
const { fileStream, bufferStream } = createFileStreamWithBuffer({
path: file.absolutePath,
fileType,
});

thumbnailStream = bufferStream;

fileId = await new Promise<string>((resolve, reject) => {
network.uploadFile(
fileStream,
fileSize,
bucket,
(err: Error | null, res: string | null) => {
if (err) {
return reject(err);
}
resolve(res as string);
},
() => {},
);
});
}

const createdDriveFile = await DriveFileService.instance.createFile({
plainName: file.name,
type: fileType,
size: stats.size,
size: fileSize,
folderUuid: parentFolderUuid,
fileId,
bucket,
Expand All @@ -105,17 +112,17 @@ export class UploadFileService {
modificationTime: stats.mtime?.toISOString(),
});

if (bufferStream) {
if (thumbnailStream && fileSize > 0) {
void tryUploadThumbnail({
bufferStream,
bufferStream: thumbnailStream,
fileType,
userBucket: bucket,
fileUuid: createdDriveFile.uuid,
networkFacade: network,
});
}

return createdDriveFile.fileId;
return createdDriveFile;
} catch (error: unknown) {
if (isAlreadyExistsError(error)) {
const msg = `File ${file.name} already exists, skipping...`;
Expand All @@ -136,6 +143,7 @@ export class UploadFileService {
}
return null;
}

private chunkArray<T>(array: T[], chunkSize: number): T[][] {
const chunks: T[][] = [];
for (let i = 0; i < array.length; i += chunkSize) {
Expand Down
4 changes: 2 additions & 2 deletions src/utils/thumbnail.utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,14 @@ export const tryUploadThumbnail = async ({
fileUuid,
networkFacade,
}: {
bufferStream: BufferStream;
bufferStream?: BufferStream;
fileType: string;
userBucket: string;
fileUuid: string;
networkFacade: NetworkFacade;
}) => {
try {
const thumbnailBuffer = bufferStream.getBuffer();
const thumbnailBuffer = bufferStream?.getBuffer();
if (thumbnailBuffer) {
await ThumbnailService.instance.uploadThumbnail(thumbnailBuffer, fileType, userBucket, fileUuid, networkFacade);
}
Expand Down
2 changes: 0 additions & 2 deletions src/webdav/handlers/GET.handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ export class GETRequestHandler implements WebDavMethodHandler {
const { driveFileService, authService, networkFacade } = this.dependencies;
const resource = await WebDavUtils.getRequestedResource(req.url);

if (resource.name.startsWith('._')) throw new NotFoundError('File not found');

webdavLogger.info(`[GET] Request received item at ${resource.url}`);
const driveFile = await WebDavUtils.getDriveFileFromResource({
url: resource.url,
Expand Down
Loading