summaryrefslogtreecommitdiff
path: root/programs/bench.c
diff options
context:
space:
mode:
Diffstat (limited to 'programs/bench.c')
-rw-r--r--programs/bench.c228
1 files changed, 124 insertions, 104 deletions
diff --git a/programs/bench.c b/programs/bench.c
index ec99c61cebf68..5d2568f104f50 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -34,14 +34,12 @@
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen */
-#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"
-#include "zstdmt_compress.h"
/* *************************************
@@ -73,12 +71,13 @@ static U32 g_compressibilityDefault = 50;
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
- if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
- { g_time = clock(); DISPLAY(__VA_ARGS__); \
- if (g_displayLevel>=4) fflush(stderr); } }
-static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
-static clock_t g_time = 0;
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
+
+#define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \
+ if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+ { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+ if (g_displayLevel>=4) fflush(stderr); } } }
/* *************************************
@@ -130,6 +129,17 @@ void BMK_setNbThreads(unsigned nbThreads) {
#endif
g_nbThreads = nbThreads;
}
+
+static U32 g_realTime = 0;
+void BMK_setRealTime(unsigned priority) {
+ g_realTime = (priority>0);
+}
+
+static U32 g_separateFiles = 0;
+void BMK_setSeparateFiles(unsigned separate) {
+ g_separateFiles = (separate>0);
+}
+
static U32 g_ldmFlag = 0;
void BMK_setLdmFlag(unsigned ldmFlag) {
g_ldmFlag = ldmFlag;
@@ -181,7 +191,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel,
const size_t* fileSizes, U32 nbFiles,
const void* dictBuffer, size_t dictBufferSize,
- const ZSTD_compressionParameters* comprParams)
+ const ZSTD_compressionParameters* const comprParams)
{
size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
@@ -189,7 +199,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
void* resultBuffer = malloc(srcSize);
- ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(g_nbThreads);
ZSTD_CCtx* const ctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t const loadedCompressedSize = srcSize;
@@ -286,8 +295,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (!cCompleted) { /* still some time to do compression tests */
U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
U32 nbLoops = 0;
- ZSTD_CDict* cdict = NULL;
-#ifdef ZSTD_NEWAPI
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
@@ -306,84 +313,64 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
-#else
- size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
- ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
- ZSTD_customMem const cmem = { NULL, NULL, NULL };
- if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
- if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
- if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
- if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
- if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
- if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
- if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
- cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dm_auto, zparams.cParams, cmem);
- if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
-#endif
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
- size_t rSize;
-#ifdef ZSTD_NEWAPI
- ZSTD_outBuffer out = { blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, 0 };
- ZSTD_inBuffer in = { blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize, 0 };
- size_t cError = 1;
- while (cError) {
- cError = ZSTD_compress_generic(ctx,
+#if 0 /* direct compression function, for occasional comparison */
+ ZSTD_parameters const params = ZSTD_getParams(cLevel, blockTable[blockNb].srcSize, dictBufferSize);
+ blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
+ blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
+ blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
+ dictBuffer, dictBufferSize,
+ params);
+#else
+ size_t moreToFlush = 1;
+ ZSTD_outBuffer out;
+ ZSTD_inBuffer in;
+ in.src = blockTable[blockNb].srcPtr;
+ in.size = blockTable[blockNb].srcSize;
+ in.pos = 0;
+ out.dst = blockTable[blockNb].cPtr;
+ out.size = blockTable[blockNb].cRoom;
+ out.pos = 0;
+ while (moreToFlush) {
+ moreToFlush = ZSTD_compress_generic(ctx,
&out, &in, ZSTD_e_end);
- if (ZSTD_isError(cError))
+ if (ZSTD_isError(moreToFlush))
EXM_THROW(1, "ZSTD_compress_generic() error : %s",
- ZSTD_getErrorName(cError));
- }
- rSize = out.pos;
-#else /* ! ZSTD_NEWAPI */
- if (dictBufferSize) {
- rSize = ZSTD_compress_usingCDict(ctx,
- blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
- blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
- cdict);
- } else {
-# ifdef ZSTD_MULTITHREAD /* note : limitation : MT single-pass does not support compression with dictionary */
- rSize = ZSTDMT_compressCCtx(mtctx,
- blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
- blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
- cLevel);
-# else
- rSize = ZSTD_compress_advanced (ctx,
- blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
- blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
- NULL, 0, zparams);
-# endif
+ ZSTD_getErrorName(moreToFlush));
}
- if (ZSTD_isError(rSize))
- EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s",
- ZSTD_getErrorName(rSize));
-#endif /* ZSTD_NEWAPI */
- blockTable[blockNb].cSize = rSize;
+ blockTable[blockNb].cSize = out.pos;
+#endif
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart) < clockLoop);
- ZSTD_freeCDict(cdict);
- { U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart);
- if (clockSpanMicro < fastestC*nbLoops) fastestC = clockSpanMicro / nbLoops;
- totalCTime += clockSpanMicro;
- cCompleted = (totalCTime >= maxTime);
+ { U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
+ if (loopDuration < fastestC*nbLoops)
+ fastestC = loopDuration / nbLoops;
+ totalCTime += loopDuration;
+ cCompleted = (totalCTime >= maxTime); /* end compression tests */
} }
cSize = 0;
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
ratio = (double)srcSize / (double)cSize;
markNb = (markNb+1) % NB_MARKS;
- DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
- marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
- (double)srcSize / fastestC );
+ { int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+ double const compressionSpeed = (double)srcSize / fastestC;
+ int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
+ marks[markNb], displayName, (U32)srcSize, (U32)cSize,
+ ratioAccuracy, ratio,
+ cSpeedAccuracy, compressionSpeed );
+ }
} else { /* g_decodeOnly */
memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
}
#if 0 /* disable decompression test */
dCompleted=1;
- (void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */
+ (void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */
#else
/* Decompression */
if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
@@ -413,17 +400,24 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart) < clockLoop);
ZSTD_freeDDict(ddict);
- { U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart);
- if (clockSpanMicro < fastestD*nbLoops) fastestD = clockSpanMicro / nbLoops;
- totalDTime += clockSpanMicro;
+ { U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
+ if (loopDuration < fastestD*nbLoops)
+ fastestD = loopDuration / nbLoops;
+ totalDTime += loopDuration;
dCompleted = (totalDTime >= maxTime);
} }
markNb = (markNb+1) % NB_MARKS;
- DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
- marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
- (double)srcSize / fastestC,
- (double)srcSize / fastestD );
+ { int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
+ double const compressionSpeed = (double)srcSize / fastestC;
+ int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
+ double const decompressionSpeed = (double)srcSize / fastestD;
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
+ marks[markNb], displayName, (U32)srcSize, (U32)cSize,
+ ratioAccuracy, ratio,
+ cSpeedAccuracy, compressionSpeed,
+ decompressionSpeed);
+ }
/* CRC Checking */
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
@@ -444,10 +438,12 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos);
if (u>5) {
int n;
+ DISPLAY("origin: ");
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
DISPLAY(" :%02X: ", ((const BYTE*)srcBuffer)[u]);
for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
DISPLAY(" \n");
+ DISPLAY("decode: ");
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
DISPLAY(" :%02X: ", ((const BYTE*)resultBuffer)[u]);
for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
@@ -463,7 +459,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
#endif
} /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
- if (g_displayLevel == 1) {
+ if (g_displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
double cSpeed = (double)srcSize / fastestC;
double dSpeed = (double)srcSize / fastestD;
if (g_additionalParam)
@@ -478,7 +474,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
free(blockTable);
free(compressedBuffer);
free(resultBuffer);
- ZSTDMT_freeCCtx(mtctx);
ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx);
return 0;
@@ -503,11 +498,11 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t)(requiredMem);
}
-static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
+static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast,
const size_t* fileSizes, unsigned nbFiles,
const void* dictBuffer, size_t dictBufferSize,
- ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
+ const ZSTD_compressionParameters* const compressionParams)
{
int l;
@@ -515,8 +510,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1;
- if (setRealTimePrio) {
- DISPLAYLEVEL(2, "Note : switching to a real-time priority \n");
+ if (g_realTime) {
+ DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
SET_REALTIME_PRIORITY;
}
@@ -539,7 +534,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
At most, fills `buffer` entirely */
static void BMK_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes,
- const char** fileNamesTable, unsigned nbFiles)
+ const char* const * const fileNamesTable, unsigned nbFiles)
{
size_t pos = 0, totalSize = 0;
unsigned n;
@@ -551,6 +546,11 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
fileSizes[n] = 0;
continue;
}
+ if (fileSize == UTIL_FILESIZE_UNKNOWN) {
+ DISPLAYLEVEL(2, "Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
+ fileSizes[n] = 0;
+ continue;
+ }
f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
@@ -566,26 +566,30 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
if (totalSize == 0) EXM_THROW(12, "no data to bench");
}
-static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel,
- int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
+static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles,
+ const char* const dictFileName,
+ int const cLevel, int const cLevelLast,
+ const ZSTD_compressionParameters* const compressionParams)
{
void* srcBuffer;
size_t benchedSize;
void* dictBuffer = NULL;
size_t dictBufferSize = 0;
- size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
+ size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
- char mfName[20] = {0};
if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
/* Load dictionary */
if (dictFileName != NULL) {
- U64 dictFileSize = UTIL_getFileSize(dictFileName);
- if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
+ U64 const dictFileSize = UTIL_getFileSize(dictFileName);
+ if (dictFileSize > 64 MB)
+ EXM_THROW(10, "dictionary file %s too large", dictFileName);
dictBufferSize = (size_t)dictFileSize;
dictBuffer = malloc(dictBufferSize);
- if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
+ if (dictBuffer==NULL)
+ EXM_THROW(11, "not enough memory for dictionary (%u bytes)",
+ (U32)dictBufferSize);
BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
}
@@ -601,13 +605,26 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
/* Bench */
- snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
- { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
- BMK_benchCLevel(srcBuffer, benchedSize,
- displayName, cLevel, cLevelLast,
- fileSizes, nbFiles,
- dictBuffer, dictBufferSize, compressionParams, setRealTimePrio);
- }
+ if (g_separateFiles) {
+ const BYTE* srcPtr = (const BYTE*)srcBuffer;
+ U32 fileNb;
+ for (fileNb=0; fileNb<nbFiles; fileNb++) {
+ size_t const fileSize = fileSizes[fileNb];
+ BMK_benchCLevel(srcPtr, fileSize,
+ fileNamesTable[fileNb], cLevel, cLevelLast,
+ fileSizes+fileNb, 1,
+ dictBuffer, dictBufferSize, compressionParams);
+ srcPtr += fileSize;
+ }
+ } else {
+ char mfName[20] = {0};
+ snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
+ { const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
+ BMK_benchCLevel(srcBuffer, benchedSize,
+ displayName, cLevel, cLevelLast,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize, compressionParams);
+ } }
/* clean up */
free(srcBuffer);
@@ -616,7 +633,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
}
-static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
+static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, const ZSTD_compressionParameters* compressionParams)
{
char name[20] = {0};
size_t benchedSize = 10000000;
@@ -630,15 +647,17 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
/* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
- BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio);
+ BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams);
/* clean up */
free(srcBuffer);
}
-int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
- int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
+ const char* dictFileName,
+ int cLevel, int cLevelLast,
+ const ZSTD_compressionParameters* compressionParams)
{
double const compressibility = (double)g_compressibilityDefault / 100;
@@ -646,11 +665,12 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
if (cLevelLast < cLevel) cLevelLast = cLevel;
- if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
+ if (cLevelLast > cLevel)
+ DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (nbFiles == 0)
- BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio);
+ BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams);
else
- BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio);
+ BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams);
return 0;
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-13 11:49:14 +0000