diff options
Diffstat (limited to 'tests/playTests.sh')
| -rwxr-xr-x | tests/playTests.sh | 229 |
1 files changed, 195 insertions, 34 deletions
diff --git a/tests/playTests.sh b/tests/playTests.sh index 41d8263b6a674..b86a0dc40cb20 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -48,6 +48,12 @@ fileRoundTripTest() { $DIFF -q tmp.md5.1 tmp.md5.2 } +truncateLastByte() { + dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" status=none +} + +UNAME=$(uname) + isTerminal=false if [ -t 0 ] && [ -t 1 ] then @@ -56,7 +62,10 @@ fi isWindows=false INTOVOID="/dev/null" -DEVDEVICE="/dev/zero" +case "$UNAME" in + GNU) DEVDEVICE="/dev/random" ;; + *) DEVDEVICE="/dev/zero" ;; +esac case "$OS" in Windows*) isWindows=true @@ -65,10 +74,10 @@ case "$OS" in ;; esac -UNAME=$(uname) case "$UNAME" in Darwin) MD5SUM="md5 -r" ;; FreeBSD) MD5SUM="gmd5sum" ;; + OpenBSD) MD5SUM="md5" ;; *) MD5SUM="md5sum" ;; esac @@ -94,6 +103,7 @@ else fi + $ECHO "\n===> simple tests " ./datagen > tmp @@ -103,10 +113,15 @@ $ECHO "test : basic decompression" $ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) $ECHO "test : too large compression level => auto-fix" $ZSTD -99 -f tmp # too large compression level, automatic sized down +$ZSTD -5000000000 -f tmp && die "too large numeric value : must fail" $ECHO "test : --fast aka negative compression levels" $ZSTD --fast -f tmp # == -1 $ZSTD --fast=3 -f tmp # == -3 -$ZSTD --fast=200000 -f tmp # == no compression +$ZSTD --fast=200000 -f tmp # too low compression level, automatic fixed +$ZSTD --fast=5000000000 -f tmp && die "too large numeric value : must fail" +$ZSTD -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0" +$ECHO "test : too large numeric argument" +$ZSTD --fast=9999999999 -f tmp && die "should have refused numeric value" $ECHO "test : compress to stdout" $ZSTD tmp -c > tmpCompressed $ZSTD tmp --stdout > tmpCompressed # long command format @@ -165,6 +180,8 @@ chmod 400 tmpro.zst $ZSTD -q tmpro && die "should have refused to overwrite read-only file" $ZSTD -q -f tmpro rm -f tmpro tmpro.zst + + $ECHO "test : file removal" $ZSTD -f --rm tmp test ! -f tmp # tmp should no longer be present @@ -175,12 +192,23 @@ $ECHO hello > tmp $ZSTD tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID" grep -v "Refusing to remove non-regular file" tmplog rm -f tmplog -$ZSTD tmp -f -o "$INTONULL" 2>&1 | grep -v "Refusing to remove non-regular file" +$ZSTD tmp -f -o "$INTOVOID" 2>&1 | grep -v "Refusing to remove non-regular file" $ECHO "test : --rm on stdin" $ECHO a | $ZSTD --rm > $INTOVOID # --rm should remain silent rm tmp $ZSTD -f tmp && die "tmp not present : should have failed" test ! -f tmp.zst # tmp.zst should not be created +$ECHO "test : -d -f do not delete destination when source is not present" +touch tmp # create destination file +$ZSTD -d -f tmp.zst && die "attempt to decompress a non existing file" +test -f tmp # destination file should still be present +$ECHO "test : -f do not delete destination when source is not present" +rm tmp # erase source file +touch tmp.zst # create destination file +$ZSTD -f tmp && die "attempt to compress a non existing file" +test -f tmp.zst # destination file should still be present +rm tmp* + $ECHO "test : compress multiple files" $ECHO hello > tmp1 @@ -258,7 +286,7 @@ rm ./*.tmp ./*.zstd $ECHO "frame concatenation tests completed" -if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then +if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] ; then $ECHO "\n**** flush write error test **** " $ECHO "$ECHO foo | $ZSTD > /dev/full" @@ -395,28 +423,54 @@ $ECHO "Hello World" > tmp $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" ./datagen -P0 -g10M > tmp $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" -rm tmp* +$ECHO "- Test -o before --train" +rm -f tmpDict dictionary +$ZSTD -o tmpDict --train *.c ../programs/*.c +test -f tmpDict +$ZSTD --train *.c ../programs/*.c +test -f dictionary +rm tmp* dictionary -$ECHO "\n===> cover dictionary builder : advanced options " +$ECHO "\n===> fastCover dictionary builder : advanced options " TESTFILE=../programs/zstdcli.c ./datagen > tmpDict $ECHO "- Create first dictionary" -$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c -o tmpDict +$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 *.c ../programs/*.c -o tmpDict cp $TESTFILE tmp $ZSTD -f tmp -D tmpDict $ZSTD -d tmp.zst -D tmpDict -fo result $DIFF $TESTFILE result $ECHO "- Create second (different) dictionary" -$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC +$ZSTD --train-fastcover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" $ECHO "- Create dictionary with short dictID" -$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c --dictID=1 -o tmpDict1 +$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "- Create dictionary with size limit" -$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K -rm tmp* +$ZSTD --train-fastcover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K +$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples" +$ZSTD --train-fastcover=split=90 -r *.c ../programs/*.c +$ZSTD --train-fastcover=split=80 -r *.c ../programs/*.c +$ECHO "- Create dictionary using all samples for both training and testing" +$ZSTD --train-fastcover=split=100 -r *.c ../programs/*.c +$ECHO "- Create dictionary using f=16" +$ZSTD --train-fastcover=f=16 -r *.c ../programs/*.c +$ECHO "- Create dictionary using accel=2" +$ZSTD --train-fastcover=accel=2 -r *.c ../programs/*.c +$ECHO "- Create dictionary using accel=10" +$ZSTD --train-fastcover=accel=10 -r *.c ../programs/*.c +$ECHO "- Create dictionary with multithreading" +$ZSTD --train-fastcover -T4 -r *.c ../programs/*.c +$ECHO "- Test -o before --train-fastcover" +rm -f tmpDict dictionary +$ZSTD -o tmpDict --train-fastcover *.c ../programs/*.c +test -f tmpDict +$ZSTD --train-fastcover *.c ../programs/*.c +test -f dictionary +rm tmp* dictionary + $ECHO "\n===> legacy dictionary builder " @@ -436,7 +490,13 @@ $ZSTD --train-legacy -s5 *.c ../programs/*.c --dictID=1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "- Create dictionary with size limit" $ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K -rm tmp* +$ECHO "- Test -o before --train-legacy" +rm -f tmpDict dictionary +$ZSTD -o tmpDict --train-legacy *.c ../programs/*.c +test -f tmpDict +$ZSTD --train-legacy *.c ../programs/*.c +test -f dictionary +rm tmp* dictionary $ECHO "\n===> integrity tests " @@ -482,6 +542,12 @@ $ZSTD -bi0 --fast tmp1 $ECHO "with recursive and quiet modes" $ZSTD -rqi1b1e2 tmp1 +$ECHO "\n===> zstd compatibility tests " + +./datagen > tmp +rm -f tmp.zst +$ZSTD --format=zstd -f tmp +test -f tmp.zst $ECHO "\n===> gzip compatibility tests " @@ -513,12 +579,18 @@ if [ $GZIPMODE -eq 1 ]; then $ZSTD -f --format=gzip tmp $ZSTD -f tmp cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp - head -c -1 tmp.gz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" + truncateLastByte tmp.gz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" rm tmp* else $ECHO "gzip mode not supported" fi +if [ $GZIPMODE -eq 1 ]; then + ./datagen > tmp + rm -f tmp.zst + $ZSTD --format=gzip --format=zstd -f tmp + test -f tmp.zst +fi $ECHO "\n===> xz compatibility tests " @@ -527,16 +599,16 @@ $ZSTD --format=xz -V || LZMAMODE=0 if [ $LZMAMODE -eq 1 ]; then $ECHO "xz support detected" XZEXE=1 - xz -V && lzma -V || XZEXE=0 + xz -Q -V && lzma -Q -V || XZEXE=0 if [ $XZEXE -eq 1 ]; then $ECHO "Testing zstd xz and lzma support" ./datagen > tmp $ZSTD --format=lzma -f tmp $ZSTD --format=xz -f tmp - xz -t -v tmp.xz - xz -t -v tmp.lzma - xz -f -k tmp - lzma -f -k --lzma1 tmp + xz -Q -t -v tmp.xz + xz -Q -t -v tmp.lzma + xz -Q -f -k tmp + lzma -Q -f -k --lzma1 tmp $ZSTD -d -f -v tmp.xz $ZSTD -d -f -v tmp.lzma rm tmp* @@ -548,13 +620,13 @@ if [ $LZMAMODE -eq 1 ]; then $ECHO "Testing xz and lzma symlinks" ./datagen > tmp ./xz tmp - xz -d tmp.xz + xz -Q -d tmp.xz ./lzma tmp - lzma -d tmp.lzma + lzma -Q -d tmp.lzma $ECHO "Testing unxz and unlzma symlinks" - xz tmp + xz -Q tmp ./xz -d tmp.xz - lzma tmp + lzma -Q tmp ./lzma -d tmp.lzma rm xz unxz lzma unlzma rm tmp* @@ -574,8 +646,8 @@ if [ $LZMAMODE -eq 1 ]; then $ZSTD -f --format=lzma tmp $ZSTD -f tmp cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | $ZSTD -d -f -o tmp - head -c -1 tmp.xz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" - head -c -1 tmp.lzma | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" + truncateLastByte tmp.xz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" + truncateLastByte tmp.lzma | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" rm tmp* else $ECHO "xz mode not supported" @@ -611,12 +683,29 @@ if [ $LZ4MODE -eq 1 ]; then $ZSTD -f --format=lz4 tmp $ZSTD -f tmp cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | $ZSTD -d -f -o tmp - head -c -1 tmp.lz4 | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" + truncateLastByte tmp.lz4 | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" rm tmp* else $ECHO "lz4 mode not supported" fi +$ECHO "\n===> suffix list test" + +! $ZSTD -d tmp.abc 2> tmplg + +if [ $GZIPMODE -ne 1 ]; then + grep ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed" +fi + +if [ $LZMAMODE -ne 1 ]; then + grep ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed" + grep ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed" +fi + +if [ $LZ4MODE -ne 1 ]; then + grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" +fi + $ECHO "\n===> zstd round-trip tests " roundTripTest @@ -650,6 +739,25 @@ then $ECHO "\n===> zstdmt long distance matching round-trip tests " roundTripTest -g8M "3 --long=24 -T2" + + $ECHO "\n===> ovLog tests " + ./datagen -g2MB > tmp + refSize=$($ZSTD tmp -6 -c --zstd=wlog=18 | wc -c) + ov9Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c) + ov0Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=0 | wc -c) + if [ $refSize -eq $ov9Size ]; then + echo ov9Size should be different from refSize + exit 1 + fi + if [ $refSize -eq $ov0Size ]; then + echo ov0Size should be different from refSize + exit 1 + fi + if [ $ov9Size -ge $ov0Size ]; then + echo ov9Size=$ov9Size should be smaller than ov0Size=$ov0Size + exit 1 + fi + else $ECHO "\n===> no multithreading, skipping zstdmt tests " fi @@ -673,23 +781,32 @@ $ZSTD -l *.zst $ZSTD -lv *.zst $ECHO "\n===> zstd --list/-l error detection tests " -! $ZSTD -l tmp1 tmp1.zst -! $ZSTD --list tmp* -! $ZSTD -lv tmp1* -! $ZSTD --list -v tmp2 tmp12.zst +$ZSTD -l tmp1 tmp1.zst && die "-l must fail on non-zstd file" +$ZSTD --list tmp* && die "-l must fail on non-zstd file" +$ZSTD -lv tmp1* && die "-l must fail on non-zstd file" +$ZSTD --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file" + +$ECHO "\n===> zstd --list/-l errors when presented with stdin / no files" +$ZSTD -l && die "-l must fail on empty list of files" +$ZSTD -l - && die "-l does not work on stdin" +$ZSTD -l < tmp1.zst && die "-l does not work on stdin" +$ZSTD -l - < tmp1.zst && die "-l does not work on stdin" +$ZSTD -l - tmp1.zst && die "-l does not work on stdin" +$ZSTD -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin" +$ZSTD -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty $ECHO "\n===> zstd --list/-l test with null files " ./datagen -g0 > tmp5 $ZSTD tmp5 $ZSTD -l tmp5.zst -! $ZSTD -l tmp5* +$ZSTD -l tmp5* && die "-l must fail on non-zstd file" $ZSTD -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 size is present in header -! $ZSTD -lv tmp5* +$ZSTD -lv tmp5* && die "-l must fail on non-zstd file" $ECHO "\n===> zstd --list/-l test with no content size field " ./datagen -g513K | $ZSTD > tmp6.zst $ZSTD -l tmp6.zst -! $ZSTD -lv tmp6.zst | grep "Decompressed Size:" # must NOT be present in header +$ZSTD -lv tmp6.zst | grep "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" $ECHO "\n===> zstd --list/-l test with no checksum " $ZSTD -f --no-check tmp1 @@ -709,11 +826,22 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" +$ECHO "\n===> adaptive mode " +roundTripTest -g270000000 " --adapt" +roundTripTest -g27000000 " --adapt=min=1,max=4" +$ECHO "===> test: --adapt must fail on incoherent bounds " +./datagen > tmp +$ZSTD -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" + + if [ "$1" != "--test-large-data" ]; then $ECHO "Skipping large data tests" exit 0 fi + +############################################################################# + $ECHO "\n===> large files tests " roundTripTest -g270000000 1 @@ -768,4 +896,37 @@ else $ECHO "\n**** no multithreading, skipping zstdmt tests **** " fi -rm tmp* + +$ECHO "\n===> cover dictionary builder : advanced options " + +TESTFILE=../programs/zstdcli.c +./datagen > tmpDict +$ECHO "- Create first dictionary" +$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c -o tmpDict +cp $TESTFILE tmp +$ZSTD -f tmp -D tmpDict +$ZSTD -d tmp.zst -D tmpDict -fo result +$DIFF $TESTFILE result +$ECHO "- Create second (different) dictionary" +$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC +$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" +$ECHO "- Create dictionary with short dictID" +$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1 +cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" +$ECHO "- Create dictionary with size limit" +$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K +$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples" +$ZSTD --train-cover=split=90 -r *.c ../programs/*.c +$ZSTD --train-cover=split=80 -r *.c ../programs/*.c +$ECHO "- Create dictionary using all samples for both training and testing" +$ZSTD --train-cover=split=100 -r *.c ../programs/*.c +$ECHO "- Test -o before --train-cover" +rm -f tmpDict dictionary +$ZSTD -o tmpDict --train-cover *.c ../programs/*.c +test -f tmpDict +$ZSTD --train-cover *.c ../programs/*.c +test -f dictionary +rm -f tmp* dictionary + + +rm -f tmp* |