From 82f8898a0325ccc572c1ec98c9cdf6a6d442ac55 Mon Sep 17 00:00:00 2001 From: Joy Majumdar <49155965+Joy-Majumder@users.noreply.github.com> Date: Sun, 7 Dec 2025 06:31:26 +0000 Subject: [PATCH] Add configurable input buffer size for decompression (--ibuf-size) This feature addresses performance issues when decompressing large files on mechanical hard drives with excessive disk seek operations. Changes: - Add 'inputBufferSize' field to FIO_prefs_t in fileio_types.h - Implement FIO_setInputBufferSize() setter function - Use configured buffer size in FIO_createDResources() - Add --ibuf-size command-line option with help text The option allows users to specify input buffer size (default: 0 for automatic). For large files on slow drives, using --ibuf-size=1024M to --ibuf-size=5120M can significantly improve throughput by reducing disk seek operations. Backward compatible: defaults to ZSTD_DStreamInSize when not specified. --- programs/fileio.c | 8 +++++++- programs/fileio.h | 1 + programs/fileio_types.h | 1 + programs/zstdcli.c | 4 ++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 4000c5b62f4..c0df3f25469 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -573,6 +573,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->allowBlockDevices = 0; ret->asyncIO = AIO_supported(); ret->passThrough = -1; + ret->inputBufferSize = 0; /* 0 = use default */ return ret; } @@ -682,6 +683,10 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) { prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint); } +void FIO_setInputBufferSize(FIO_prefs_t* const prefs, size_t inputBufferSize) { + prefs->inputBufferSize = inputBufferSize; +} + void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) { prefs->testMode = (testMode!=0); } @@ -2611,7 +2616,8 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi } ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_DStreamOutSize()); - ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_DStreamInSize()); + ress.readCtx = AIO_ReadPool_create(prefs, + prefs->inputBufferSize > 0 ? prefs->inputBufferSize : ZSTD_DStreamInSize()); return ress; } diff --git a/programs/fileio.h b/programs/fileio.h index 5d7334ef5f0..154042997b3 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -87,6 +87,7 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable); void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize); void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize); void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint); +void FIO_setInputBufferSize(FIO_prefs_t* const prefs, size_t inputBufferSize); void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode); void FIO_setLiteralCompressionMode( FIO_prefs_t* const prefs, diff --git a/programs/fileio_types.h b/programs/fileio_types.h index 9bbb5154979..3f827c3b990 100644 --- a/programs/fileio_types.h +++ b/programs/fileio_types.h @@ -59,6 +59,7 @@ typedef struct FIO_prefs_s { int removeSrcFile; int overwrite; int asyncIO; + size_t inputBufferSize; /* size of the input buffer for decompression (0 = default) */ /* Computation resources preferences */ unsigned memLimit; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 01760ff8c45..48aecbb8930 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -245,6 +245,7 @@ static void usageAdvanced(const char* programName) DISPLAYOUT(" --exclude-compressed Only compress files that are not already compressed.\n\n"); DISPLAYOUT(" --stream-size=# Specify size of streaming input from STDIN.\n"); + DISPLAYOUT(" --ibuf-size=# Specify input buffer size for decompression. Helps with large files on slow disks.\n"); DISPLAYOUT(" --size-hint=# Optimize compression parameters for streaming input of approximately size #.\n"); DISPLAYOUT(" --target-compressed-block-size=#\n"); DISPLAYOUT(" Generate compressed blocks of approximately # size.\n\n"); @@ -912,6 +913,7 @@ int main(int argCount, const char* argv[]) size_t streamSrcSize = 0; size_t targetCBlockSize = 0; size_t srcSizeHint = 0; + size_t inputBufferSize = 0; size_t nbInputFileNames = 0; int dictCLevel = g_defaultDictCLevel; unsigned dictSelect = g_defaultSelectivityLevel; @@ -1096,6 +1098,7 @@ int main(int argCount, const char* argv[]) if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; } if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badUsage(programName, originalArgument); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; } if (longCommandWArg(&argument, "--stream-size")) { NEXT_TSIZE(streamSrcSize); continue; } + if (longCommandWArg(&argument, "--ibuf-size")) { NEXT_TSIZE(inputBufferSize); continue; } if (longCommandWArg(&argument, "--target-compressed-block-size")) { NEXT_TSIZE(targetCBlockSize); continue; } if (longCommandWArg(&argument, "--size-hint")) { NEXT_TSIZE(srcSizeHint); continue; } if (longCommandWArg(&argument, "--output-dir-flat")) { @@ -1615,6 +1618,7 @@ int main(int argCount, const char* argv[]) FIO_setStreamSrcSize(prefs, streamSrcSize); FIO_setTargetCBlockSize(prefs, targetCBlockSize); FIO_setSrcSizeHint(prefs, srcSizeHint); + FIO_setInputBufferSize(prefs, inputBufferSize); FIO_setLiteralCompressionMode(prefs, literalCompressionMode); FIO_setSparseWrite(prefs, 0); if (adaptMin > cLevel) cLevel = adaptMin;