fileio.c

/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
 * in the COPYING file in the root directory of this source tree).
 * You may select, at your option, one of the above-listed licenses.
 */


/* *************************************
*  Compiler Options
***************************************/
#ifdef _MSC_VER   /* Visual */
#  pragma warning(disable : 4127)  /* disable: C4127: conditional expression is constant */
#  pragma warning(disable : 4204)  /* non-constant aggregate initializer */
#endif
#if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
#  define _POSIX_SOURCE 1          /* disable %llu warnings with MinGW on Windows */
#endif

/*-*************************************
*  Includes
***************************************/
#include "platform.h"   /* Large Files support, SET_BINARY_MODE */
#include "util.h"       /* UTIL_getFileSize, UTIL_isRegularFile, UTIL_isSameFile */
#include <stdio.h>      /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */
#include <stdlib.h>     /* malloc, free */
#include <string.h>     /* strcmp, strlen */
#include <time.h>       /* clock_t, to measure process time */
#include <fcntl.h>      /* O_WRONLY */
#include <assert.h>
#include <errno.h>      /* errno */
#include <limits.h>     /* INT_MAX */
#include <signal.h>
#include "timefn.h"     /* UTIL_getTime, UTIL_clockSpanMicro */

#if defined (_MSC_VER)
#  include <sys/stat.h>
#  include <io.h>
#endif

#include "fileio.h"
#include "fileio_asyncio.h"
#include "fileio_common.h"

FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto};
UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;

#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "../lib/zstd.h"
#include "../lib/zstd_errors.h"  /* ZSTD_error_frameParameter_windowTooLarge */

#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
#  include <zlib.h>
#  if !defined(z_const)
#    define z_const
#  endif
#endif

#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS)
#  include <lzma.h>
#endif

#define LZ4_MAGICNUMBER 0x184D2204
#if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS)
#  define LZ4F_ENABLE_OBSOLETE_ENUMS
#  include <lz4frame.h>
#  include <lz4.h>
#endif

char const* FIO_zlibVersion(void)
{
#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
    return zlibVersion();
#else
    return "Unsupported";
#endif
}

char const* FIO_lz4Version(void)
{
#if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS)
    /* LZ4_versionString() added in v1.7.3 */
#   if LZ4_VERSION_NUMBER >= 10703
        return LZ4_versionString();
#   else
#       define ZSTD_LZ4_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE
#       define ZSTD_LZ4_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LZ4_VERSION)
        return ZSTD_LZ4_VERSION_STRING;
#   endif
#else
    return "Unsupported";
#endif
}

char const* FIO_lzmaVersion(void)
{
#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS)
    return lzma_version_string();
#else
    return "Unsupported";
#endif
}


/*-*************************************
*  Constants
***************************************/
#define ADAPT_WINDOWLOG_DEFAULT 23   /* 8 MB */
#define DICTSIZE_MAX (32 MB)   /* protection against large input (attack scenario) */

#define FNSPACE 30

/* Default file permissions 0666 (modulated by umask) */
/* Temporary restricted file permissions are used when we're going to
 * chmod/chown at the end of the operation. */
#if !defined(_WIN32)
/* These macros aren't defined on windows. */
#define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)
#define TEMPORARY_FILE_PERMISSIONS (S_IRUSR|S_IWUSR)
#else
#define DEFAULT_FILE_PERMISSIONS (0666)
#define TEMPORARY_FILE_PERMISSIONS (0600)
#endif

/*-************************************
*  Signal (Ctrl-C trapping)
**************************************/
static const char* g_artefact = NULL;
static void INThandler(int sig)
{
    assert(sig==SIGINT); (void)sig;
#if !defined(_MSC_VER)
    signal(sig, SIG_IGN);  /* this invocation generates a buggy warning in Visual Studio */
#endif
    if (g_artefact) {
        assert(UTIL_isRegularFile(g_artefact));
        remove(g_artefact);
    }
    DISPLAY("\n");
    exit(2);
}
static void addHandler(char const* dstFileName)
{
    if (UTIL_isRegularFile(dstFileName)) {
        g_artefact = dstFileName;
        signal(SIGINT, INThandler);
    } else {
        g_artefact = NULL;
    }
}
/* Idempotent */
static void clearHandler(void)
{
    if (g_artefact) signal(SIGINT, SIG_DFL);
    g_artefact = NULL;
}


/*-*********************************************************
*  Termination signal trapping (Print debug stack trace)
***********************************************************/
#if defined(__has_feature) && !defined(BACKTRACE_ENABLE) /* Clang compiler */
#  if (__has_feature(address_sanitizer))
#    define BACKTRACE_ENABLE 0
#  endif /* __has_feature(address_sanitizer) */
#elif defined(__SANITIZE_ADDRESS__) && !defined(BACKTRACE_ENABLE) /* GCC compiler */
#  define BACKTRACE_ENABLE 0
#endif

#if !defined(BACKTRACE_ENABLE)
/* automatic detector : backtrace enabled by default on linux+glibc and osx */
#  if (defined(__linux__) && (defined(__GLIBC__) && !defined(__UCLIBC__))) \
     || (defined(__APPLE__) && defined(__MACH__))
#    define BACKTRACE_ENABLE 1
#  else
#    define BACKTRACE_ENABLE 0
#  endif
#endif

/* note : after this point, BACKTRACE_ENABLE is necessarily defined */


#if BACKTRACE_ENABLE

#include <execinfo.h>   /* backtrace, backtrace_symbols */

#define MAX_STACK_FRAMES    50

static void ABRThandler(int sig) {
    const char* name;
    void* addrlist[MAX_STACK_FRAMES];
    char** symbollist;
    int addrlen, i;

    switch (sig) {
        case SIGABRT: name = "SIGABRT"; break;
        case SIGFPE: name = "SIGFPE"; break;
        case SIGILL: name = "SIGILL"; break;
        case SIGINT: name = "SIGINT"; break;
        case SIGSEGV: name = "SIGSEGV"; break;
        default: name = "UNKNOWN";
    }

    DISPLAY("Caught %s signal, printing stack:\n", name);
    /* Retrieve current stack addresses. */
    addrlen = backtrace(addrlist, MAX_STACK_FRAMES);
    if (addrlen == 0) {
        DISPLAY("\n");
        return;
    }
    /* Create readable strings to each frame. */
    symbollist = backtrace_symbols(addrlist, addrlen);
    /* Print the stack trace, excluding calls handling the signal. */
    for (i = ZSTD_START_SYMBOLLIST_FRAME; i < addrlen; i++) {
        DISPLAY("%s\n", symbollist[i]);
    }
    free(symbollist);
    /* Reset and raise the signal so default handler runs. */
    signal(sig, SIG_DFL);
    raise(sig);
}
#endif

void FIO_addAbortHandler(void)
{
#if BACKTRACE_ENABLE
    signal(SIGABRT, ABRThandler);
    signal(SIGFPE, ABRThandler);
    signal(SIGILL, ABRThandler);
    signal(SIGSEGV, ABRThandler);
    signal(SIGBUS, ABRThandler);
#endif
}

/*-*************************************
*  Parameters: FIO_ctx_t
***************************************/

/* typedef'd to FIO_ctx_t within fileio.h */
struct FIO_ctx_s {

    /* file i/o info */
    int nbFilesTotal;
    int hasStdinInput;
    int hasStdoutOutput;

    /* file i/o state */
    int currFileIdx;
    int nbFilesProcessed;
    size_t totalBytesInput;
    size_t totalBytesOutput;
};

static int FIO_shouldDisplayFileSummary(FIO_ctx_t const* fCtx)
{
    return fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3;
}

static int FIO_shouldDisplayMultipleFileSummary(FIO_ctx_t const* fCtx)
{
    int const shouldDisplay = (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1);
    assert(shouldDisplay || FIO_shouldDisplayFileSummary(fCtx) || fCtx->nbFilesProcessed == 0);
    return shouldDisplay;
}


/*-*************************************
*  Parameters: Initialization
***************************************/

#define FIO_OVERLAP_LOG_NOTSET 9999
#define FIO_LDM_PARAM_NOTSET 9999


FIO_prefs_t* FIO_createPreferences(void)
{
    FIO_prefs_t* const ret = (FIO_prefs_t*)malloc(sizeof(FIO_prefs_t));
    if (!ret) EXM_THROW(21, "Allocation error : not enough memory");

    ret->compressionType = FIO_zstdCompression;
    ret->overwrite = 0;
    ret->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
    ret->dictIDFlag = 1;
    ret->checksumFlag = 1;
    ret->removeSrcFile = 0;
    ret->memLimit = 0;
    ret->nbWorkers = 1;
    ret->blockSize = 0;
    ret->overlapLog = FIO_OVERLAP_LOG_NOTSET;
    ret->adaptiveMode = 0;
    ret->rsyncable = 0;
    ret->minAdaptLevel = -50;   /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */
    ret->maxAdaptLevel = 22;   /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */
    ret->ldmFlag = 0;
    ret->ldmHashLog = 0;
    ret->ldmMinMatch = 0;
    ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET;
    ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET;
    ret->streamSrcSize = 0;
    ret->targetCBlockSize = 0;
    ret->srcSizeHint = 0;
    ret->testMode = 0;
    ret->literalCompressionMode = ZSTD_ps_auto;
    ret->excludeCompressedFiles = 0;
    ret->allowBlockDevices = 0;
    ret->asyncIO = AIO_supported();
    ret->passThrough = -1;
    return ret;
}

FIO_ctx_t* FIO_createContext(void)
{
    FIO_ctx_t* const ret = (FIO_ctx_t*)malloc(sizeof(FIO_ctx_t));
    if (!ret) EXM_THROW(21, "Allocation error : not enough memory");

    ret->currFileIdx = 0;
    ret->hasStdinInput = 0;
    ret->hasStdoutOutput = 0;
    ret->nbFilesTotal = 1;
    ret->nbFilesProcessed = 0;
    ret->totalBytesInput = 0;
    ret->totalBytesOutput = 0;
    return ret;
}

void FIO_freePreferences(FIO_prefs_t* const prefs)
{
    free(prefs);
}

void FIO_freeContext(FIO_ctx_t* const fCtx)
{
    free(fCtx);
}


/*-*************************************
*  Parameters: Display Options
***************************************/

void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; }

void FIO_setProgressSetting(FIO_progressSetting_e setting) { g_display_prefs.progressSetting = setting; }


/*-*************************************
*  Parameters: Setters
***************************************/

/* FIO_prefs_t functions */

void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; }

void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; }

void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse) { prefs->sparseFileSupport = sparse; }

void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; }

void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; }

void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, int flag) { prefs->removeSrcFile = (flag!=0); }

void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; }

void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) {
#ifndef ZSTD_MULTITHREAD
    if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
#endif
    prefs->nbWorkers = nbWorkers;
}

void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; }

void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; }

void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
    if (blockSize && prefs->nbWorkers==0)
        DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
    prefs->blockSize = blockSize;
}

void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
    if (overlapLog && prefs->nbWorkers==0)
        DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
    prefs->overlapLog = overlapLog;
}

void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt) {
    if ((adapt>0) && (prefs->nbWorkers==0))
        EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
    prefs->adaptiveMode = adapt;
}

void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder) {
    prefs->useRowMatchFinder = useRowMatchFinder;
}

void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) {
    if ((rsyncable>0) && (prefs->nbWorkers==0))
        EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n");
    prefs->rsyncable = rsyncable;
}

void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize) {
    prefs->streamSrcSize = streamSrcSize;
}

void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize) {
    prefs->targetCBlockSize = targetCBlockSize;
}

void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) {
    prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint);
}

void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) {
    prefs->testMode = (testMode!=0);
}

void FIO_setLiteralCompressionMode(
        FIO_prefs_t* const prefs,
        ZSTD_ParamSwitch_e mode) {
    prefs->literalCompressionMode = mode;
}

void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel)
{
#ifndef ZSTD_NOCOMPRESS
    assert(minCLevel >= ZSTD_minCLevel());
#endif
    prefs->minAdaptLevel = minCLevel;
}

void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel)
{
    prefs->maxAdaptLevel = maxCLevel;
}

void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) {
    prefs->ldmFlag = (ldmFlag>0);
}

void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog) {
    prefs->ldmHashLog = ldmHashLog;
}

void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch) {
    prefs->ldmMinMatch = ldmMinMatch;
}

void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog) {
    prefs->ldmBucketSizeLog = ldmBucketSizeLog;
}


void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog) {
    prefs->ldmHashRateLog = ldmHashRateLog;
}

void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value)
{
    prefs->patchFromMode = value != 0;
}

void FIO_setContentSize(FIO_prefs_t* const prefs, int value)
{
    prefs->contentSize = value != 0;
}

void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) {
#ifdef ZSTD_MULTITHREAD
    prefs->asyncIO = value;
#else
    (void) prefs;
    (void) value;
    DISPLAYLEVEL(2, "Note : asyncio is disabled (lack of multithreading support) \n");
#endif
}

void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
    prefs->passThrough = (value != 0);
}

void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_ParamSwitch_e value)
{
    prefs->mmapDict = value;
}

/* FIO_ctx_t functions */

void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) {
    fCtx->hasStdoutOutput = value;
}

void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value)
{
    fCtx->nbFilesTotal = value;
}

void FIO_determineHasStdinInput(FIO_ctx_t* const fCtx, const FileNamesTable* const filenames) {
    size_t i = 0;
    for ( ; i < filenames->tableSize; ++i) {
        if (!strcmp(stdinmark, filenames->fileNames[i])) {
            fCtx->hasStdinInput = 1;
            return;
        }
    }
}

/*-*************************************
*  Functions
***************************************/
/** FIO_removeFile() :
 * @result : Unlink `fileName`, even if it's read-only */
static int FIO_removeFile(const char* path)
{
    stat_t statbuf;
    if (!UTIL_stat(path, &statbuf)) {
        DISPLAYLEVEL(2, "zstd: Failed to stat %s while trying to remove it\n", path);
        return 0;
    }
    if (!UTIL_isRegularFileStat(&statbuf)) {
        DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path);
        return 0;
    }
#if defined(_WIN32)
    /* windows doesn't allow remove read-only files,
     * so try to make it writable first */
    if (!(statbuf.st_mode & _S_IWRITE)) {
        UTIL_chmod(path, &statbuf, _S_IWRITE);
    }
#endif
    return remove(path);
}

/** FIO_openSrcFile() :
 *  condition : `srcFileName` must be non-NULL. `prefs` may be NULL.
 * @result : FILE* to `srcFileName`, or NULL if it fails */
static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName, stat_t* statbuf)
{
    int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0;
    assert(srcFileName != NULL);
    assert(statbuf != NULL);
    if (!strcmp (srcFileName, stdinmark)) {
        DISPLAYLEVEL(4,"Using stdin for input \n");
        SET_BINARY_MODE(stdin);
        return stdin;
    }

    if (!UTIL_stat(srcFileName, statbuf)) {
        DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n",
                        srcFileName, strerror(errno));
        return NULL;
    }

    if (!UTIL_isRegularFileStat(statbuf)
     && !UTIL_isFIFOStat(statbuf)
     && !(allowBlockDevices && UTIL_isBlockDevStat(statbuf))
    ) {
        DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
                        srcFileName);
        return NULL;
    }

    {   FILE* const f = fopen(srcFileName, "rb");
        if (f == NULL)
            DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
        return f;
    }
}

/** FIO_openDstFile() :
 *  condition : `dstFileName` must be non-NULL.
 * @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE*
FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
                const char* srcFileName, const char* dstFileName,
                const int mode)
{
    int isDstRegFile;

    if (prefs->testMode) return NULL;  /* do not open file in test mode */

    assert(dstFileName != NULL);
    if (!strcmp (dstFileName, stdoutmark)) {
        DISPLAYLEVEL(4,"Using stdout for output \n");
        SET_BINARY_MODE(stdout);
        if (prefs->sparseFileSupport == 1) {
            prefs->sparseFileSupport = 0;
            DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
        }
        return stdout;
    }

    /* ensure dst is not the same as src */
    if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) {
        DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n");
        return NULL;
    }

    isDstRegFile = UTIL_isRegularFile(dstFileName);  /* invoke once */
    if (prefs->sparseFileSupport == 1) {
        prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
        if (!isDstRegFile) {
            prefs->sparseFileSupport = 0;
            DISPLAYLEVEL(4, "Sparse File Support is disabled when output is not a file \n");
        }
    }

    if (isDstRegFile) {
        /* Check if destination file already exists */
#if !defined(_WIN32)
        /* this test does not work on Windows :
         * `NUL` and `nul` are detected as regular files */
        if (!strcmp(dstFileName, nulmark)) {
            EXM_THROW(40, "%s is unexpectedly categorized as a regular file",
                        dstFileName);
        }
#endif
        if (!prefs->overwrite) {
            if (g_display_prefs.displayLevel <= 1) {
                /* No interaction possible */
                DISPLAYLEVEL(1, "zstd: %s already exists; not overwritten  \n",
                        dstFileName);
                return NULL;
            }
            DISPLAY("zstd: %s already exists; ", dstFileName);
            if (UTIL_requireUserConfirmation("overwrite (y/n) ? ", "Not overwritten  \n", "yY", fCtx->hasStdinInput))
                return NULL;
        }
        /* need to unlink */
        FIO_removeFile(dstFileName);
    }

    {
#if defined(_WIN32)
        /* Windows requires opening the file as a "binary" file to avoid
         * mangling. This macro doesn't exist on unix. */
        const int openflags = O_WRONLY|O_CREAT|O_TRUNC|O_BINARY;
        const int fd = _open(dstFileName, openflags, mode);
        FILE* f = NULL;
        if (fd != -1) {
            f = _fdopen(fd, "wb");
        }
#else
        const int openflags = O_WRONLY|O_CREAT|O_TRUNC;
        const int fd = open(dstFileName, openflags, mode);
        FILE* f = NULL;
        if (fd != -1) {
            f = fdopen(fd, "wb");
        }
#endif
        if (f == NULL) {
            DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
        } else {
            /* An increased buffer size can provide a significant performance
             * boost on some platforms. Note that providing a NULL buf with a
             * size that's not 0 is not defined in ANSI C, but is defined in an
             * extension. There are three possibilities here:
             * 1. Libc supports the extended version and everything is good.
             * 2. Libc ignores the size when buf is NULL, in which case
             *    everything will continue as if we didn't call `setvbuf()`.
             * 3. We fail the call and execution continues but a warning
             *    message might be shown.
             * In all cases due execution continues. For now, I believe that
             * this is a more cost-effective solution than managing the buffers
             * allocations ourselves (will require an API change).
             */
            if (setvbuf(f, NULL, _IOFBF, 1 MB)) {
                DISPLAYLEVEL(2, "Warning: setvbuf failed for %s\n", dstFileName);
            }
        }
        return f;
    }
}


/* FIO_getDictFileStat() :
 */
static void FIO_getDictFileStat(const char* fileName, stat_t* dictFileStat) {
    assert(dictFileStat != NULL);
    if (fileName == NULL) return;

    if (!UTIL_stat(fileName, dictFileStat)) {
        EXM_THROW(31, "Stat failed on dictionary file %s: %s", fileName, strerror(errno));
    }

    if (!UTIL_isRegularFileStat(dictFileStat)) {
        EXM_THROW(32, "Dictionary %s must be a regular file.", fileName);
    }
}

/*  FIO_setDictBufferMalloc() :
 *  allocates a buffer, pointed by `dict->dictBuffer`,
 *  loads `filename` content into it, up to DICTSIZE_MAX bytes.
 * @return : loaded size
 *  if fileName==NULL, returns 0 and a NULL pointer
 */
static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
    FILE* fileHandle;
    U64 fileSize;
    void** bufferPtr = &dict->dictBuffer;

    assert(bufferPtr != NULL);
    assert(dictFileStat != NULL);
    *bufferPtr = NULL;
    if (fileName == NULL) return 0;

    DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);

    fileHandle = fopen(fileName, "rb");

    if (fileHandle == NULL) {
        EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
    }

    fileSize = UTIL_getFileSizeStat(dictFileStat);
    {
        size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
        if (fileSize >  dictSizeMax) {
            EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
                            fileName,  (unsigned)dictSizeMax);   /* avoid extreme cases */
        }
    }
    *bufferPtr = malloc((size_t)fileSize);
    if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
    {   size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
        if (readSize != fileSize) {
            EXM_THROW(35, "Error reading dictionary file %s : %s",
                    fileName, strerror(errno));
        }
    }
    fclose(fileHandle);
    return (size_t)fileSize;
}

#if (PLATFORM_POSIX_VERSION > 0)
#include <sys/mman.h>
static void FIO_munmap(FIO_Dict_t* dict)
{
    munmap(dict->dictBuffer, dict->dictBufferSize);
    dict->dictBuffer = NULL;
    dict->dictBufferSize = 0;
}
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
    int fileHandle;
    U64 fileSize;
    void** bufferPtr = &dict->dictBuffer;

    assert(bufferPtr != NULL);
    assert(dictFileStat != NULL);
    *bufferPtr = NULL;
    if (fileName == NULL) return 0;

    DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);

    fileHandle = open(fileName, O_RDONLY);

    if (fileHandle == -1) {
        EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
    }

    fileSize = UTIL_getFileSizeStat(dictFileStat);
    {
        size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
        if (fileSize >  dictSizeMax) {
            EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
                            fileName,  (unsigned)dictSizeMax);   /* avoid extreme cases */
        }
    }

    *bufferPtr = mmap(NULL, (size_t)fileSize, PROT_READ, MAP_PRIVATE, fileHandle, 0);
    if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));

    close(fileHandle);
    return (size_t)fileSize;
}
#elif defined(_MSC_VER) || defined(_WIN32)
#include <windows.h>
static void FIO_munmap(FIO_Dict_t* dict)
{
    UnmapViewOfFile(dict->dictBuffer);
    CloseHandle(dict->dictHandle);
    dict->dictBuffer = NULL;
    dict->dictBufferSize = 0;
}
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
    HANDLE fileHandle, mapping;
    U64 fileSize;
    void** bufferPtr = &dict->dictBuffer;

    assert(bufferPtr != NULL);
    assert(dictFileStat != NULL);
    *bufferPtr = NULL;
    if (fileName == NULL) return 0;

    DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);

    fileHandle = CreateFileA(fileName, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_READONLY, NULL);

    if (fileHandle == INVALID_HANDLE_VALUE) {
        EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
    }

    fileSize = UTIL_getFileSizeStat(dictFileStat);
    {
        size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
        if (fileSize >  dictSizeMax) {
            EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
                            fileName,  (unsigned)dictSizeMax);   /* avoid extreme cases */
        }
    }

    mapping = CreateFileMapping(fileHandle, NULL, PAGE_READONLY, 0, 0, NULL);
	if (mapping == NULL) {
        EXM_THROW(35, "Couldn't map dictionary %s: %s", fileName, strerror(errno));
    }

	*bufferPtr = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, (DWORD)fileSize); /* we can only cast to DWORD here because dictSize <= 2GB */
	if (*bufferPtr==NULL) EXM_THROW(36, "%s", strerror(errno));

    dict->dictHandle = fileHandle;
    return (size_t)fileSize;
}
#else
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
   return FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat);
}
static void FIO_munmap(FIO_Dict_t* dict) {
   free(dict->dictBuffer);
   dict->dictBuffer = NULL;
   dict->dictBufferSize = 0;
}
#endif

static void FIO_freeDict(FIO_Dict_t* dict) {
    if (dict->dictBufferType == FIO_mallocDict) {
        free(dict->dictBuffer);
        dict->dictBuffer = NULL;
        dict->dictBufferSize = 0;
    } else if (dict->dictBufferType == FIO_mmapDict)  {
        FIO_munmap(dict);
    } else {
        assert(0); /* Should not reach this case */
    }
}

static void FIO_initDict(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat, FIO_dictBufferType_t dictBufferType) {
    dict->dictBufferType = dictBufferType;
    if (dict->dictBufferType == FIO_mallocDict) {
        dict->dictBufferSize = FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat);
    } else if (dict->dictBufferType == FIO_mmapDict)  {
        dict->dictBufferSize = FIO_setDictBufferMMap(dict, fileName, prefs, dictFileStat);
    } else {
        assert(0); /* Should not reach this case */
    }
}


/* FIO_checkFilenameCollisions() :
 * Checks for and warns if there are any files that would have the same output path
 */
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
    const char **filenameTableSorted, *prevElem, *filename;
    unsigned u;

    filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles);
    if (!filenameTableSorted) {
        DISPLAYLEVEL(1, "Allocation error during filename collision checking \n");
        return 1;
    }

    for (u = 0; u < nbFiles; ++u) {
        filename = strrchr(filenameTable[u], PATH_SEP);
        if (filename == NULL) {
            filenameTableSorted[u] = filenameTable[u];
        } else {
            filenameTableSorted[u] = filename+1;
        }
    }

    qsort((void*)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
    prevElem = filenameTableSorted[0];
    for (u = 1; u < nbFiles; ++u) {
        if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
            DISPLAYLEVEL(2, "WARNING: Two files have same filename: %s\n", prevElem);
        }
        prevElem = filenameTableSorted[u];
    }

    free((void*)filenameTableSorted);
    return 0;
}

static const char*
extractFilename(const char* path, char separator)
{
    const char* search = strrchr(path, separator);
    if (search == NULL) return path;
    return search+1;
}

/* FIO_createFilename_fromOutDir() :
 * Takes a source file name and specified output directory, and
 * allocates memory for and returns a pointer to final path.
 * This function never returns an error (it may abort() in case of pb)
 */
static char*
FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen)
{
    const char* filenameStart;
    char separator;
    char* result;

#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
    separator = '\\';
#else
    separator = '/';
#endif

    filenameStart = extractFilename(path, separator);
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
    filenameStart = extractFilename(filenameStart, '/');  /* sometimes, '/' separator is also used on Windows (mingw+msys2) */
#endif

    result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1);
    if (!result) {
        EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno));
    }

    memcpy(result, outDirName, strlen(outDirName));
    if (outDirName[strlen(outDirName)-1] == separator) {
        memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart));
    } else {
        memcpy(result + strlen(outDirName), &separator, 1);
        memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart));
    }

    return result;
}

/* FIO_highbit64() :
 * gives position of highest bit.
 * note : only works for v > 0 !
 */
static unsigned FIO_highbit64(unsigned long long v)
{
    unsigned count = 0;
    assert(v != 0);
    v >>= 1;
    while (v) { v >>= 1; count++; }
    return count;
}

static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
                                    unsigned long long const dictSize,
                                    unsigned long long const maxSrcFileSize)
{
    unsigned long long maxSize = MAX(prefs->memLimit, MAX(dictSize, maxSrcFileSize));
    unsigned const maxWindowSize = (1U << ZSTD_WINDOWLOG_MAX);
    if (maxSize == UTIL_FILESIZE_UNKNOWN)
        EXM_THROW(42, "Using --patch-from with stdin requires --stream-size");
    assert(maxSize != UTIL_FILESIZE_UNKNOWN);
    if (maxSize > maxWindowSize)
        EXM_THROW(42, "Can't handle files larger than %u GB\n", maxWindowSize/(1 GB));
    FIO_setMemLimit(prefs, (unsigned)maxSize);
}

/* FIO_multiFilesConcatWarning() :
 * This function handles logic when processing multiple files with -o or -c, displaying the appropriate warnings/prompts.
 * Returns 1 if the console should abort, 0 if console should proceed.
 *
 * If output is stdout or test mode is active, check that `--rm` disabled.
 *
 * If there is just 1 file to process, zstd will proceed as usual.
 * If each file get processed into its own separate destination file, proceed as usual.
 *
 * When multiple files are processed into a single output,
 * display a warning message, then disable --rm if it's set.
 *
 * If -f is specified or if output is stdout, just proceed.
 * If output is set with -o, prompt for confirmation.
 */
static int FIO_multiFilesConcatWarning(const FIO_ctx_t* fCtx, FIO_prefs_t* prefs, const char* outFileName, int displayLevelCutoff)
{
    if (fCtx->hasStdoutOutput) {
        if (prefs->removeSrcFile)
            /* this should not happen ; hard fail, to protect user's data
             * note: this should rather be an assert(), but we want to be certain that user's data will not be wiped out in case it nonetheless happen */
            EXM_THROW(43, "It's not allowed to remove input files when processed output is piped to stdout. "
                "This scenario is not supposed to be possible. "
                "This is a programming error. File an issue for it to be fixed.");