fuzzer.c

/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
 * in the COPYING file in the root directory of this source tree).
 * You may select, at your option, one of the above-listed licenses.
 */


/*-************************************
*  Compiler specific
**************************************/
#ifdef _MSC_VER    /* Visual Studio */
#  define _CRT_SECURE_NO_WARNINGS   /* fgets */
#  pragma warning(disable : 4127)   /* disable: C4127: conditional expression is constant */
#  pragma warning(disable : 4204)   /* disable: C4204: non-constant aggregate initializer */
#endif


/*-************************************
*  Includes
**************************************/
#include <stdlib.h>       /* free */
#include <stdio.h>        /* fgets, sscanf */
#include <string.h>       /* strcmp */
#include <time.h>         /* time(), time_t */
#undef NDEBUG             /* always enable assert() */
#include <assert.h>
#define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "debug.h"        /* DEBUG_STATIC_ASSERT */
#include "fse.h"
#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
#include "zstd.h"         /* ZSTD_VERSION_STRING */
#include "zstd_errors.h"  /* ZSTD_getErrorCode */
#define ZDICT_STATIC_LINKING_ONLY
#include "zdict.h"        /* ZDICT_trainFromBuffer */
#include "mem.h"
#include "datagen.h"      /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
#include "xxhash.h"       /* XXH64 */
#include "timefn.h"       /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
/* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
#include "threading.h"    /* ZSTD_pthread_create, ZSTD_pthread_join */


/*-************************************
*  Constants
**************************************/
#define GB *(1U<<30)

static const int FUZ_compressibility_default = 50;
static const int nbTestsDefault = 30000;


/*-************************************
*  Display Macros
**************************************/
#define DISPLAY(...)          fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...)  if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2;

static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;

#define DISPLAYUPDATE(l, ...) \
    if (g_displayLevel>=l) { \
        if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
        { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
        if (g_displayLevel>=4) fflush(stderr); } \
    }


/*-*******************************************************
*  Compile time test
*********************************************************/
#undef MIN
#undef MAX
/* Declaring the function, to avoid -Wmissing-prototype */
void FUZ_bug976(void);
void FUZ_bug976(void)
{   /* these constants shall not depend on MIN() macro */
    DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
    DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
}


/*-*******************************************************
*  Internal functions
*********************************************************/
#define MIN(a,b) ((a)<(b)?(a):(b))
#define MAX(a,b) ((a)>(b)?(a):(b))

#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static U32 FUZ_rand(U32* src)
{
    static const U32 prime1 = 2654435761U;
    static const U32 prime2 = 2246822519U;
    U32 rand32 = *src;
    rand32 *= prime1;
    rand32 += prime2;
    rand32  = FUZ_rotl32(rand32, 13);
    *src = rand32;
    return rand32 >> 5;
}

static U32 FUZ_highbit32(U32 v32)
{
    unsigned nbBits = 0;
    if (v32==0) return 0;
    while (v32) v32 >>= 1, nbBits++;
    return nbBits;
}


/*=============================================
*   Test macros
=============================================*/
#define CHECK(fn)  { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }

#define CHECK_Z(f) {                          \
    size_t const err = f;                     \
    if (ZSTD_isError(err)) {                  \
        DISPLAY("Error => %s : %s ",          \
                #f, ZSTD_getErrorName(err));  \
        exit(1);                              \
}   }

#define CHECK_VAR(var, fn)  var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
#define CHECK_NEWV(var, fn)  size_t const CHECK_VAR(var, fn)
#define CHECKPLUS(var, fn, more)  { CHECK_NEWV(var, fn); more; }

#define CHECK_OP(op, lhs, rhs) {                                  \
    if (!((lhs) op (rhs))) {                                      \
        DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs);  \
         exit(1);                                                 \
    }                                                             \
}
#define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
#define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs)


/*=============================================
*   Memory Tests
=============================================*/
#if defined(__APPLE__) && defined(__MACH__)

#include <malloc/malloc.h>    /* malloc_size */

typedef struct {
    unsigned long long totalMalloc;
    size_t currentMalloc;
    size_t peakMalloc;
    unsigned nbMalloc;
    unsigned nbFree;
} mallocCounter_t;

static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 };

static void* FUZ_mallocDebug(void* counter, size_t size)
{
    mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
    void* const ptr = malloc(size);
    if (ptr==NULL) return NULL;
    DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n",
        (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10));  /* OS-X specific */
    mcPtr->totalMalloc += size;
    mcPtr->currentMalloc += size;
    if (mcPtr->currentMalloc > mcPtr->peakMalloc)
        mcPtr->peakMalloc = mcPtr->currentMalloc;
    mcPtr->nbMalloc += 1;
    return ptr;
}

static void FUZ_freeDebug(void* counter, void* address)
{
    mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
    DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10));
    mcPtr->nbFree += 1;
    mcPtr->currentMalloc -= malloc_size(address);  /* OS-X specific */
    free(address);
}

static void FUZ_displayMallocStats(mallocCounter_t count)
{
    DISPLAYLEVEL(3, "peak:%6u KB,  nbMallocs:%2u, total:%6u KB \n",
        (unsigned)(count.peakMalloc >> 10),
        count.nbMalloc,
        (unsigned)(count.totalMalloc >> 10));
}

static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part,
                void* inBuffer, size_t inSize, void* outBuffer, size_t outSize)
{
    /* test only played in verbose mode, as they are long */
    if (g_displayLevel<3) return 0;

    /* Create compressible noise */
    if (!inBuffer || !outBuffer) {
        DISPLAY("Not enough memory, aborting\n");
        exit(1);
    }
    RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed);

    /* simple compression tests */
    if (part <= 1)
    {   int compressionLevel;
        for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
            mallocCounter_t malcount = INIT_MALLOC_COUNTER;
            ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
            ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
            CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) );
            ZSTD_freeCCtx(cctx);
            DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
            FUZ_displayMallocStats(malcount);
    }   }

    /* streaming compression tests */
    if (part <= 2)
    {   int compressionLevel;
        for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
            mallocCounter_t malcount = INIT_MALLOC_COUNTER;
            ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
            ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
            ZSTD_outBuffer out = { outBuffer, outSize, 0 };
            ZSTD_inBuffer in = { inBuffer, inSize, 0 };
            CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) );
            CHECK_Z( ZSTD_compressStream(cstream, &out, &in) );
            CHECK_Z( ZSTD_endStream(cstream, &out) );
            ZSTD_freeCStream(cstream);
            DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
            FUZ_displayMallocStats(malcount);
    }   }

    /* advanced MT API test */
    if (part <= 3)
    {   int nbThreads;
        for (nbThreads=1; nbThreads<=4; nbThreads++) {
            int compressionLevel;
            for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
                mallocCounter_t malcount = INIT_MALLOC_COUNTER;
                ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
                ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
                CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
                CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
                CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
                ZSTD_freeCCtx(cctx);
                DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ",
                                nbThreads, compressionLevel);
                FUZ_displayMallocStats(malcount);
    }   }   }

    /* advanced MT streaming API test */
    if (part <= 4)
    {   int nbThreads;
        for (nbThreads=1; nbThreads<=4; nbThreads++) {
            int compressionLevel;
            for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
                mallocCounter_t malcount = INIT_MALLOC_COUNTER;
                ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
                ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
                ZSTD_outBuffer out = { outBuffer, outSize, 0 };
                ZSTD_inBuffer in = { inBuffer, inSize, 0 };
                CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
                CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
                CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
                while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
                ZSTD_freeCCtx(cctx);
                DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ",
                                nbThreads, compressionLevel);
                FUZ_displayMallocStats(malcount);
    }   }   }

    return 0;
}

static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
{
    size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
    size_t const outSize = ZSTD_compressBound(inSize);
    void* const inBuffer = malloc(inSize);
    void* const outBuffer = malloc(outSize);
    int result;

    /* Create compressible noise */
    if (!inBuffer || !outBuffer) {
        DISPLAY("Not enough memory, aborting \n");
        exit(1);
    }

    result = FUZ_mallocTests_internal(seed, compressibility, part,
                    inBuffer, inSize, outBuffer, outSize);

    free(inBuffer);
    free(outBuffer);
    return result;
}

#else

static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
{
    (void)seed; (void)compressibility; (void)part;
    return 0;
}

#endif

static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
                                BYTE* src, size_t size, ZSTD_SequenceFormat_e format)
{
    size_t i;
    size_t j;
    for(i = 0; i < seqsSize; ++i) {
        assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size);
        assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size);
        if (format == ZSTD_sf_noBlockDelimiters) {
            assert(seqs[i].matchLength != 0 || seqs[i].offset != 0);
        }

        memcpy(dst, src, seqs[i].litLength);
        dst += seqs[i].litLength;
        src += seqs[i].litLength;
        size -= seqs[i].litLength;

        if (seqs[i].offset != 0) {
            for (j = 0; j < seqs[i].matchLength; ++j)
                dst[j] = dst[(ptrdiff_t)(j - seqs[i].offset)];
            dst += seqs[i].matchLength;
            src += seqs[i].matchLength;
            size -= seqs[i].matchLength;
        }
    }
    if (format == ZSTD_sf_noBlockDelimiters) {
        memcpy(dst, src, size);
    }
}

static size_t FUZ_getLitSize(const ZSTD_Sequence* seqs, size_t nbSeqs)
{
    size_t n, litSize = 0;
    assert(seqs != NULL);
    for (n=0; n<nbSeqs; n++) {
        litSize += seqs[n].litLength;
    }
    return litSize;
}

static void
FUZ_transferLiterals(void* dst, size_t dstCapacity,
                const void* src, size_t srcSize,
                const ZSTD_Sequence* seqs, size_t nbSeqs)
{
    size_t n;
    const char* ip = (const char*)src;
    char* op = (char*)dst;
    size_t const litSize = FUZ_getLitSize(seqs, nbSeqs);
    assert(litSize <= dstCapacity);
    for (n=0; n<nbSeqs; n++) {
        size_t const ll = seqs[n].litLength;
        memcpy(op, ip, ll);
        op += ll;
        ip += ll + seqs[n].matchLength;
    }
    assert((size_t)(ip - (const char*)src) == srcSize);
}

#ifdef ZSTD_MULTITHREAD

typedef struct {
    ZSTD_CCtx* cctx;
    ZSTD_threadPool* pool;
    void* CNBuffer;
    size_t CNBuffSize;
    void* compressedBuffer;
    size_t compressedBufferSize;
    void* decodedBuffer;
    int err;
} threadPoolTests_compressionJob_payload;

static void* threadPoolTests_compressionJob(void* payload) {
    threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload;
    size_t cSize;
    if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1;
    cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize);
    if (ZSTD_isError(cSize)) args->err = 1;
    if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1;
    return payload;
}

static int threadPoolTests(void) {
    int testResult = 0;
    size_t err;

    size_t const CNBuffSize = 5 MB;
    void* const CNBuffer = malloc(CNBuffSize);
    size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
    void* const compressedBuffer = malloc(compressedBufferSize);
    void* const decodedBuffer = malloc(CNBuffSize);

    size_t const kPoolNumThreads = 8;

    RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0);

    DISPLAYLEVEL(3, "thread pool test : threadPool reuse roundtrips: ");
    {
        ZSTD_CCtx* cctx = ZSTD_createCCtx();
        ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);

        size_t nbThreads = 1;
        for (; nbThreads <= kPoolNumThreads; ++nbThreads) {
            ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
            ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads);
            err = ZSTD_CCtx_refThreadPool(cctx, pool);
            if (ZSTD_isError(err)) {
                DISPLAYLEVEL(3, "refThreadPool error!\n");
                ZSTD_freeCCtx(cctx);
                goto _output_error;
            }
            err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
            if (ZSTD_isError(err)) {
                DISPLAYLEVEL(3, "Compression error!\n");
                ZSTD_freeCCtx(cctx);
                goto _output_error;
            }
            err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err);
            if (ZSTD_isError(err)) {
                DISPLAYLEVEL(3, "Decompression error!\n");
                ZSTD_freeCCtx(cctx);
                goto _output_error;
            }
        }

        ZSTD_freeCCtx(cctx);
        ZSTD_freeThreadPool(pool);
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: ");
    {
        void* const decodedBuffer2 = malloc(CNBuffSize);
        void* const compressedBuffer2 = malloc(compressedBufferSize);
        ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
        ZSTD_CCtx* cctx1 = ZSTD_createCCtx();
        ZSTD_CCtx* cctx2 = ZSTD_createCCtx();

        ZSTD_pthread_t t1;
        ZSTD_pthread_t t2;
        threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize,
                                                     compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */};
        threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize,
                                                     compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */};

        ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2);
        ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2);
        ZSTD_CCtx_refThreadPool(cctx1, pool);
        ZSTD_CCtx_refThreadPool(cctx2, pool);

        ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
        ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
        ZSTD_pthread_join(t1);
        ZSTD_pthread_join(t2);

        assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
        free(decodedBuffer2);
        free(compressedBuffer2);

        ZSTD_freeThreadPool(pool);
        ZSTD_freeCCtx(cctx1);
        ZSTD_freeCCtx(cctx2);

        if (p1.err || p2.err) goto _output_error;
    }
    DISPLAYLEVEL(3, "OK \n");

_end:
    free(CNBuffer);
    free(compressedBuffer);
    free(decodedBuffer);
    return testResult;

_output_error:
    testResult = 1;
    DISPLAY("Error detected in Unit tests ! \n");
    goto _end;
}
#endif /* ZSTD_MULTITHREAD */

/*=============================================
*   Unit tests
=============================================*/

static void test_compressBound(unsigned tnb)
{
    DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb);

    /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
     * for a large range of known valid values */
    DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
    {   int s;
        for (s=0; s<30; s++) {
            size_t const w = (size_t)1 << s;
            CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
    }   }

    /* Ensure error if srcSize too big */
    {   size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
        CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
        CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
    }

    DISPLAYLEVEL(3, "OK \n");
}

static void test_decompressBound(unsigned tnb)
{
    DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);

    /* Simple compression, with size : should provide size; */
    {   const char example[] = "abcd";
        char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
        size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
        CHECK_Z(cSize);
        CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
    }

    /* Simple small compression without size : should provide 1 block size */
    {   char cBuffer[ZSTD_COMPRESSBOUND(0)];
        ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
        ZSTD_inBuffer in = { NULL, 0, 0 };
        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
        assert(cctx);
        CHECK_Z( ZSTD_initCStream(cctx, 0) );
        CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
        CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
        CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX );
        ZSTD_freeCCtx(cctx);
    }

    /* Attempt to overflow 32-bit intermediate multiplication result
     * This requires dBound >= 4 GB, aka 2^32.
     * This requires 2^32 / 2^17 = 2^15 blocks
     * => create 2^15 blocks (can be empty, or just 1 byte). */
    {   const char input[] = "a";
        size_t const nbBlocks = (1 << 15) + 1;
        size_t blockNb;
        size_t const outCapacity = 1 << 18; /* large margin */
        char* const outBuffer = malloc (outCapacity);
        ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
        assert(cctx);
        assert(outBuffer);
        CHECK_Z( ZSTD_initCStream(cctx, 0) );
        for (blockNb=0; blockNb<nbBlocks; blockNb++) {
            ZSTD_inBuffer in = { input, sizeof(input), 0 };
            CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
            CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 );
        }
        CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
        CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000ULL /* 4 GB */ );
        ZSTD_freeCCtx(cctx);
        free(outBuffer);
    }

    DISPLAYLEVEL(3, "OK \n");
}

static void test_setCParams(unsigned tnb)
{
    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
    ZSTD_compressionParameters cparams;
    assert(cctx);

    DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb);

    /* valid cparams */
    cparams = ZSTD_getCParams(1, 0, 0);
    CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));

    /* invalid cparams (must fail) */
    cparams.windowLog = 99;
    CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));

    free(cctx);
    DISPLAYLEVEL(3, "OK \n");
}

static void test_blockSplitter_incompressibleExpansionProtection(unsigned testNb, unsigned seed)
{
    DISPLAYLEVEL(3, "test%3i : Check block splitter doesn't oversplit incompressible data (seed %u): ", testNb, seed);
    {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
        size_t const srcSize = 256 * 1024; /* needs to be at least 2 blocks */
        void* incompressible = malloc(srcSize);
        size_t const dstCapacity = ZSTD_compressBound(srcSize);
        void* cBuffer = malloc(dstCapacity);
        size_t const chunkSize = 8 KB;
        size_t const nbChunks = srcSize / chunkSize;
        size_t chunkNb, cSizeNoSplit, cSizeWithSplit;
        assert(cctx != NULL);
        assert(incompressible != NULL);
        assert(cBuffer != NULL);

        /* let's fill input with random noise (incompressible) */
        RDG_genBuffer(incompressible, srcSize, 0.0, 0.0, seed);

        /* this pattern targets the fastest _byChunk variant's sampling (level 3).
         * manually checked that, without the @savings protection, it would over-split.
         */
        for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
            BYTE* const p = (BYTE*)incompressible + chunkNb * chunkSize;
            size_t const samplingRate = 43;
            int addOrRemove = chunkNb % 2;
            size_t n;
            for (n=0; n<chunkSize; n+=samplingRate) {
                if (addOrRemove) {
                    p[n] &= 0x80;
                } else {
                    p[n] |= 0x80;
                }
            }
        }

        /* run first without splitting */
        ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockSplitterLevel, 1 /* no split */);
        cSizeNoSplit = ZSTD_compress2(cctx, cBuffer, dstCapacity, incompressible, srcSize);

        /* run with sample43 splitter, check it's still the same */
        ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockSplitterLevel, 3 /* sample43, fastest _byChunk variant */);
        cSizeWithSplit = ZSTD_compress2(cctx, cBuffer, dstCapacity, incompressible, srcSize);

        if (cSizeWithSplit != cSizeNoSplit) {
            DISPLAYLEVEL(1, "invalid compressed size: cSizeWithSplit %u != %u cSizeNoSplit \n",
                    (unsigned)cSizeWithSplit, (unsigned)cSizeNoSplit);
            abort();
        }
        DISPLAYLEVEL(4, "compressed size: cSizeWithSplit %u == %u cSizeNoSplit : ",
                (unsigned)cSizeWithSplit, (unsigned)cSizeNoSplit);

        free(incompressible);
        free(cBuffer);
        ZSTD_freeCCtx(cctx);
    }
    DISPLAYLEVEL(3, "OK \n");
}

/* ============================================================= */

static int basicUnitTests(U32 const seed, double compressibility)
{
    size_t const CNBuffSize = 5 MB;
    void* const CNBuffer = malloc(CNBuffSize);
    size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
    void* const compressedBuffer = malloc(compressedBufferSize);
    void* const decodedBuffer = malloc(CNBuffSize);
    int testResult = 0;
    unsigned testNb=0;
    size_t cSize;

    /* Create compressible noise */
    if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
        DISPLAY("Not enough memory, aborting\n");
        testResult = 1;
        goto _end;
    }
    RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);

    /* Basic tests */
    DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++);
    {   const char* errorString = ZSTD_getErrorName(0);
        DISPLAYLEVEL(3, "OK : %s \n", errorString);
    }

    DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++);
    {   const char* errorString = ZSTD_getErrorName(499);
        DISPLAYLEVEL(3, "OK : %s \n", errorString);
    }

    DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++);
    {   int const mcl = ZSTD_minCLevel();
        DISPLAYLEVEL(3, "%i (OK) \n", mcl);
    }

    DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++);
    {   int const defaultCLevel = ZSTD_defaultCLevel();
        if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error;
        DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel);
    }

    DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++);
    {   unsigned const vn = ZSTD_versionNumber();
        DISPLAYLEVEL(3, "%u (OK) \n", vn);
    }

    test_compressBound(testNb++);

    test_decompressBound(testNb++);

    test_setCParams(testNb++);

    DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
    {
        ZSTD_compressionParameters params;
        memset(&params, 0, sizeof(params));
        params.windowLog = 10;
        params.hashLog = 19;
        params.chainLog = 19;
        params = ZSTD_adjustCParams(params, 1000, 100000);
        if (params.hashLog != 18) goto _output_error;
        if (params.chainLog != 17) goto _output_error;
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
    {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
        if (cctx==NULL) goto _output_error;
        CHECK_VAR(cSize, ZSTD_compressCCtx(cctx,
                            compressedBuffer, compressedBufferSize,
                            CNBuffer, CNBuffSize, 1) );
        DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);

        DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++);
        {   size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
            DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize);
        }
        ZSTD_freeCCtx(cctx);
    }

    DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++);
    {
       char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff";
       size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8);
       if (!ZSTD_isError(size)) goto _output_error;
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
    {   unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
        if (rSize != CNBuffSize) goto _output_error;
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++);
    {   unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
        if (rSize != CNBuffSize) goto _output_error;
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
    {   unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
        if (rSize != CNBuffSize) goto _output_error;
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
    {
        unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
        if (bound != CNBuffSize) goto _output_error;
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++);
    {
        unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1);
        if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error;
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
    { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
      if (r != CNBuffSize) goto _output_error; }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize);
    {
        ZSTD_DCtx* dctx = ZSTD_createDCtx();
        size_t r;
        CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1));
        r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
        if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
        ZSTD_freeDCtx(dctx);
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
    {   size_t u;
        for (u=0; u<CNBuffSize; u++) {
            if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
    }   }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++);
    {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
        ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 };
        ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 };
        if (cctx==NULL) goto _output_error;
        CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) );  /* must fail */
        CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) );  /* must fail */
        ZSTD_freeCCtx(cctx);
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++);
    {
        ZSTD_parameters params = ZSTD_getParams(3, 0, 0);
        assert(!ZSTD_checkCParams(params.cParams));
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++);
    {
        ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
        assert(dctx != NULL);
        assert(ZSTD_sizeof_DCtx(dctx) != 0);
        ZSTD_freeDCtx(dctx);
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++);
    {
        /* %p takes a void*. In ISO C, it's illegal to cast a function pointer
         * to a data pointer. (Although in POSIX you're required to be allowed
         * to do it...) So we have to fall back to our trusty friend memcpy. */
        unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) =
            ZSTD_getDictID_fromDDict;
        ZSTD_DStream* (* const funcptr_createDStream)(
            ZSTD_customMem customMem) = ZSTD_createDStream_advanced;
        void (* const funcptr_copyDCtx)(
            ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx;
        ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) =
            ZSTD_nextInputType;
        const void *voidptr_getDictID;
        const void *voidptr_createDStream;
        const void *voidptr_copyDCtx;
        const void *voidptr_nextInputType;
        DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID));
        memcpy(
            (void*)&voidptr_getDictID,
            (const void*)&funcptr_getDictID,
            sizeof(void*));
        memcpy(
            (void*)&voidptr_createDStream,
            (const void*)&funcptr_createDStream,
            sizeof(void*));
        memcpy(
            (void*)&voidptr_copyDCtx,
            (const void*)&funcptr_copyDCtx,
            sizeof(void*));
        memcpy(
            (void*)&voidptr_nextInputType,
            (const void*)&funcptr_nextInputType,
            sizeof(void*));
        DISPLAYLEVEL(3, "%p ", voidptr_getDictID);
        DISPLAYLEVEL(3, "%p ", voidptr_createDStream);
        DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx);
        DISPLAYLEVEL(3, "%p ", voidptr_nextInputType);
    }
    DISPLAYLEVEL(3, ": OK \n");

    DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
    {   ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
        {   size_t const r = ZSTD_decompress_usingDict(dctx,
                                                    decodedBuffer, CNBuffSize,
                                                    compressedBuffer, cSize,
                                                    NULL, 0);
            if (r != CNBuffSize) goto _output_error;
        }
        ZSTD_freeDCtx(dctx);
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
    {   ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
        {   size_t const r = ZSTD_decompress_usingDDict(dctx,
                                                    decodedBuffer, CNBuffSize,
                                                    compressedBuffer, cSize,
                                                    NULL);
            if (r != CNBuffSize) goto _output_error;
        }
        ZSTD_freeDCtx(dctx);
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
    { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
      if (!ZSTD_isError(r)) goto _output_error;
      if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++);
    { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
      if (!ZSTD_isError(r)) goto _output_error;
      if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++);
    { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize);
      if (!ZSTD_isError(r)) goto _output_error;
      if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++);
    { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize);
      if (!ZSTD_isError(r)) goto _output_error;
      if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++);
    {   /* create compressed buffer with checksumming enabled */
        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
        if (!cctx) {
            DISPLAY("Not enough memory, aborting\n");
            testResult = 1;
            goto _end;
        }
        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
        CHECK_VAR(cSize, ZSTD_compress2(cctx,
                            compressedBuffer, compressedBufferSize,
                            CNBuffer, CNBuffSize) );
        ZSTD_freeCCtx(cctx);
    }
    {   /* copy the compressed buffer and corrupt the checksum */
        size_t r;
        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
        if (!dctx) {
            DISPLAY("Not enough memory, aborting\n");
            testResult = 1;
            goto _end;
        }

        ((char*)compressedBuffer)[cSize-1] += 1;
        r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
        if (!ZSTD_isError(r)) goto _output_error;
        if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error;

        CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum));
        r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
        if (!ZSTD_isError(r)) goto _output_error;   /* wrong checksum size should still throw error */
        r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
        if (ZSTD_isError(r)) goto _output_error;

        ZSTD_freeDCtx(dctx);
    }
    DISPLAYLEVEL(3, "OK \n");


    DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++);
    {   /* create compressed buffer with content size missing */
        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
        CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
        CHECK_VAR(cSize, ZSTD_compress2(cctx,
                            compressedBuffer, compressedBufferSize,
                            CNBuffer, CNBuffSize) );
        ZSTD_freeCCtx(cctx);
    }
    {   /* ensure frame content size is missing */
        ZSTD_FrameHeader zfh;
        size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
        if (ret != 0 || zfh.frameContentSize !=  ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
    }
    {   /* ensure CNBuffSize <= decompressBound */
        unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
        if (CNBuffSize > bound) goto _output_error;
    }
    DISPLAYLEVEL(3, "OK \n");

    DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++);
    {
        size_t const largeFrameSrcSize = 200;
        size_t const smallFrameSrcSize = 10;
        size_t const nbFrames = 256;

        size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0;
        int sizeReduced = 0;

        BYTE* const dst = (BYTE*)compressedBuffer;
        ZSTD_DCtx* dctx = ZSTD_createDCtx();

        /* create a large frame and then a bunch of small frames */
        size_t srcSize = ZSTD_compress((void*)dst,
            compressedBufferSize, CNBuffer, largeFrameSrcSize, 3);
        for (i = 0; i < nbFrames; i++)
            srcSize += ZSTD_compress((void*)(dst + srcSize),
                compressedBufferSize - srcSize, CNBuffer,
                smallFrameSrcSize, 3);

        /* decompressStream and make sure that dctx size was reduced at least once */
        while (consumed < srcSize) {
            ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0};
            ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0};
            ZSTD_decompressStream(dctx, &out, &in);
            consumed += in.pos;
            produced += out.pos;

            /* success! size was reduced from the previous frame */
            if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx))
                sizeReduced = 1;

            prevDCtxSize = ZSTD_sizeof_DCtx(dctx);