util.c

/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
 * in the COPYING file in the root directory of this source tree).
 * You may select, at your option, one of the above-listed licenses.
 */

#if defined (__cplusplus)
extern "C" {
#endif


/*-****************************************
*  Dependencies
******************************************/
#include "util.h"       /* note : ensure that platform.h is included first ! */
#include <stdlib.h>     /* malloc, realloc, free */
#include <stdio.h>      /* fprintf */
#include <time.h>       /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
#include <errno.h>
#include <assert.h>

#if defined(__FreeBSD__)
#include <sys/param.h> /* __FreeBSD_version */
#endif /* #ifdef __FreeBSD__ */

#if defined(_WIN32)
#  include <sys/utime.h>  /* utime */
#  include <io.h>         /* _chmod */
#  define ZSTD_USE_UTIMENSAT 0
#else
#  include <unistd.h>     /* chown, stat */
#  include <sys/stat.h>   /* utimensat, st_mtime */
#  if (PLATFORM_POSIX_VERSION >= 200809L && defined(st_mtime)) \
      || (defined(__FreeBSD__) && __FreeBSD_version >= 1100056)
#    define ZSTD_USE_UTIMENSAT 1
#  else
#    define ZSTD_USE_UTIMENSAT 0
#  endif
#  if ZSTD_USE_UTIMENSAT
#    include <fcntl.h>    /* AT_FDCWD */
#  else
#    include <utime.h>    /* utime */
#  endif
#endif

#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
#include <direct.h>     /* needed for _mkdir in windows */
#endif

#if defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L)  /* opendir, readdir require POSIX.1-2001 */
#  include <dirent.h>       /* opendir, readdir */
#  include <string.h>       /* strerror, memcpy */
#endif /* #ifdef _WIN32 */

/*-****************************************
*  Internal Macros
******************************************/

/* CONTROL is almost like an assert(), but is never disabled.
 * It's designed for failures that may happen rarely,
 * but we don't want to maintain a specific error code path for them,
 * such as a malloc() returning NULL for example.
 * Since it's always active, this macro can trigger side effects.
 */
#define CONTROL(c)  {         \
    if (!(c)) {               \
        UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s",  \
                          __FILE__, __LINE__, #c);   \
        exit(1);              \
}   }

/* console log */
#define UTIL_DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }

static int g_traceDepth = 0;
int g_traceFileStat = 0;

#define UTIL_TRACE_CALL(...)                                         \
    {                                                                \
        if (g_traceFileStat) {                                       \
            UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \
            UTIL_DISPLAY(__VA_ARGS__);                               \
            UTIL_DISPLAY("\n");                                      \
            ++g_traceDepth;                                          \
        }                                                            \
    }

#define UTIL_TRACE_RET(ret)                                                     \
    {                                                                           \
        if (g_traceFileStat) {                                                  \
            --g_traceDepth;                                                     \
            UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \
        }                                                                      \
    }

/* A modified version of realloc().
 * If UTIL_realloc() fails the original block is freed.
 */
UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
{
    void *newptr = realloc(ptr, size);
    if (newptr) return newptr;
    free(ptr);
    return NULL;
}

#if defined(_MSC_VER)
    #define chmod _chmod
#endif

#ifndef ZSTD_HAVE_FCHMOD
#if PLATFORM_POSIX_VERSION >= 199309L
#define ZSTD_HAVE_FCHMOD
#endif
#endif

#ifndef ZSTD_HAVE_FCHOWN
#if PLATFORM_POSIX_VERSION >= 200809L
#define ZSTD_HAVE_FCHOWN
#endif
#endif

/*-****************************************
*  Console log
******************************************/
int g_utilDisplayLevel;

int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
                                 const char* acceptableLetters, int hasStdinInput) {
    int ch, result;

    if (hasStdinInput) {
        UTIL_DISPLAY("stdin is an input - not proceeding.\n");
        return 1;
    }

    UTIL_DISPLAY("%s", prompt);
    ch = getchar();
    result = 0;
    if (strchr(acceptableLetters, ch) == NULL) {
        UTIL_DISPLAY("%s \n", abortMsg);
        result = 1;
    }
    /* flush the rest */
    while ((ch!=EOF) && (ch!='\n'))
        ch = getchar();
    return result;
}


/*-*************************************
*  Constants
***************************************/
#define LIST_SIZE_INCREASE   (8*1024)
#define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50


/*-*************************************
*  Functions
***************************************/

void UTIL_traceFileStat(void)
{
    g_traceFileStat = 1;
}

int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf)
{
    int ret;
    UTIL_TRACE_CALL("UTIL_stat(%d, %s)", fd, filename);
#if defined(_MSC_VER)
    if (fd >= 0) {
        ret = !_fstat64(fd, statbuf);
    } else {
        ret = !_stat64(filename, statbuf);
    }
#elif defined(__MINGW32__) && defined (__MSVCRT__)
    if (fd >= 0) {
        ret = !_fstati64(fd, statbuf);
    } else {
        ret = !_stati64(filename, statbuf);
    }
#else
    if (fd >= 0) {
        ret = !fstat(fd, statbuf);
    } else {
        ret = !stat(filename, statbuf);
    }
#endif
    UTIL_TRACE_RET(ret);
    return ret;
}

int UTIL_stat(const char* filename, stat_t* statbuf)
{
    return UTIL_fstat(-1, filename, statbuf);
}

int UTIL_isRegularFile(const char* infilename)
{
    stat_t statbuf;
    int ret;
    UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename);
    ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf);
    UTIL_TRACE_RET(ret);
    return ret;
}

int UTIL_isRegularFileStat(const stat_t* statbuf)
{
#if defined(_MSC_VER)
    return (statbuf->st_mode & S_IFREG) != 0;
#else
    return S_ISREG(statbuf->st_mode) != 0;
#endif
}

/* like chmod, but avoid changing permission of /dev/null */
int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
{
    return UTIL_fchmod(-1, filename, statbuf, permissions);
}

int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions)
{
    stat_t localStatBuf;
    UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions);
    if (statbuf == NULL) {
        if (!UTIL_fstat(fd, filename, &localStatBuf)) {
            UTIL_TRACE_RET(0);
            return 0;
        }
        statbuf = &localStatBuf;
    }
    if (!UTIL_isRegularFileStat(statbuf)) {
        UTIL_TRACE_RET(0);
        return 0; /* pretend success, but don't change anything */
    }
#ifdef ZSTD_HAVE_FCHMOD
    if (fd >= 0) {
        int ret;
        UTIL_TRACE_CALL("fchmod");
        ret = fchmod(fd, permissions);
        UTIL_TRACE_RET(ret);
        UTIL_TRACE_RET(ret);
        return ret;
    } else
#endif
    {
        int ret;
        UTIL_TRACE_CALL("chmod");
        ret = chmod(filename, permissions);
        UTIL_TRACE_RET(ret);
        UTIL_TRACE_RET(ret);
        return ret;
    }
}

/* set access and modification times */
int UTIL_utime(const char* filename, const stat_t *statbuf)
{
    int ret;
    UTIL_TRACE_CALL("UTIL_utime(%s)", filename);
    /* We check that st_mtime is a macro here in order to give us confidence
     * that struct stat has a struct timespec st_mtim member. We need this
     * check because there are some platforms that claim to be POSIX 2008
     * compliant but which do not have st_mtim... */
    /* FreeBSD has implemented POSIX 2008 for a long time but still only
     * advertises support for POSIX 2001. They have a version macro that
     * lets us safely gate them in.
     * See https://docs.freebsd.org/en/books/porters-handbook/versions/.
     */
#if ZSTD_USE_UTIMENSAT
    {
        /* (atime, mtime) */
        struct timespec timebuf[2] = { {0, UTIME_NOW} };
        timebuf[1] = statbuf->st_mtim;
        ret = utimensat(AT_FDCWD, filename, timebuf, 0);
    }
#else
    {
        struct utimbuf timebuf;
        timebuf.actime = time(NULL);
        timebuf.modtime = statbuf->st_mtime;
        ret = utime(filename, &timebuf);
    }
#endif
    errno = 0;
    UTIL_TRACE_RET(ret);
    return ret;
}

int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
{
    return UTIL_setFDStat(-1, filename, statbuf);
}

int UTIL_setFDStat(const int fd, const char *filename, const stat_t *statbuf)
{
    int res = 0;
    stat_t curStatBuf;
    UTIL_TRACE_CALL("UTIL_setFileStat(%d, %s)", fd, filename);

    if (!UTIL_fstat(fd, filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) {
        UTIL_TRACE_RET(-1);
        return -1;
    }

    /* Mimic gzip's behavior:
     *
     * "Change the group first, then the permissions, then the owner.
     * That way, the permissions will be correct on systems that allow
     * users to give away files, without introducing a security hole.
     * Security depends on permissions not containing the setuid or
     * setgid bits." */

#if !defined(_WIN32)
#ifdef ZSTD_HAVE_FCHOWN
    if (fd >= 0) {
        res += fchown(fd, -1, statbuf->st_gid);  /* Apply group ownership */
    } else
#endif
    {
        res += chown(filename, -1, statbuf->st_gid);  /* Apply group ownership */
    }
#endif

    res += UTIL_fchmod(fd, filename, &curStatBuf, statbuf->st_mode & 0777);  /* Copy file permissions */

#if !defined(_WIN32)
#ifdef ZSTD_HAVE_FCHOWN
    if (fd >= 0) {
        res += fchown(fd, statbuf->st_uid, -1);  /* Apply user ownership */
    } else
#endif
    {
        res += chown(filename, statbuf->st_uid, -1);  /* Apply user ownership */
    }
#endif

    errno = 0;
    UTIL_TRACE_RET(-res);
    return -res; /* number of errors is returned */
}

int UTIL_isDirectory(const char* infilename)
{
    stat_t statbuf;
    int ret;
    UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename);
    ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf);
    UTIL_TRACE_RET(ret);
    return ret;
}

int UTIL_isDirectoryStat(const stat_t* statbuf)
{
    int ret;
    UTIL_TRACE_CALL("UTIL_isDirectoryStat()");
#if defined(_MSC_VER)
    ret = (statbuf->st_mode & _S_IFDIR) != 0;
#else
    ret = S_ISDIR(statbuf->st_mode) != 0;
#endif
    UTIL_TRACE_RET(ret);
    return ret;
}

int UTIL_compareStr(const void *p1, const void *p2) {
    return strcmp(* (char * const *) p1, * (char * const *) p2);
}

int UTIL_isSameFile(const char* fName1, const char* fName2)
{
    int ret;
    assert(fName1 != NULL); assert(fName2 != NULL);
    UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2);
#if defined(_MSC_VER) || defined(_WIN32)
    /* note : Visual does not support file identification by inode.
     *        inode does not work on Windows, even with a posix layer, like msys2.
     *        The following work-around is limited to detecting exact name repetition only,
     *        aka `filename` is considered different from `subdir/../filename` */
    ret = !strcmp(fName1, fName2);
#else
    {   stat_t file1Stat;
        stat_t file2Stat;
        ret =  UTIL_stat(fName1, &file1Stat)
            && UTIL_stat(fName2, &file2Stat)
            && UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat);
    }
#endif
    UTIL_TRACE_RET(ret);
    return ret;
}

int UTIL_isSameFileStat(
        const char* fName1, const char* fName2,
        const stat_t* file1Stat, const stat_t* file2Stat)
{
    int ret;
    assert(fName1 != NULL); assert(fName2 != NULL);
    UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2);
#if defined(_MSC_VER) || defined(_WIN32)
    /* note : Visual does not support file identification by inode.
     *        inode does not work on Windows, even with a posix layer, like msys2.
     *        The following work-around is limited to detecting exact name repetition only,
     *        aka `filename` is considered different from `subdir/../filename` */
    (void)file1Stat;
    (void)file2Stat;
    ret = !strcmp(fName1, fName2);
#else
    {
        ret =  (file1Stat->st_dev == file2Stat->st_dev)
            && (file1Stat->st_ino == file2Stat->st_ino);
    }
#endif
    UTIL_TRACE_RET(ret);
    return ret;
}

/* UTIL_isFIFO : distinguish named pipes */
int UTIL_isFIFO(const char* infilename)
{
    UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename);
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
    {
        stat_t statbuf;
        if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) {
            UTIL_TRACE_RET(1);
            return 1;
        }
    }
#endif
    (void)infilename;
    UTIL_TRACE_RET(0);
    return 0;
}

/* UTIL_isFIFO : distinguish named pipes */
int UTIL_isFIFOStat(const stat_t* statbuf)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
    if (S_ISFIFO(statbuf->st_mode)) return 1;
#endif
    (void)statbuf;
    return 0;
}

/* UTIL_isBlockDevStat : distinguish named pipes */
int UTIL_isBlockDevStat(const stat_t* statbuf)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
    if (S_ISBLK(statbuf->st_mode)) return 1;
#endif
    (void)statbuf;
    return 0;
}

int UTIL_isLink(const char* infilename)
{
    UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename);
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
    {
        stat_t statbuf;
        int const r = lstat(infilename, &statbuf);
        if (!r && S_ISLNK(statbuf.st_mode)) {
            UTIL_TRACE_RET(1);
            return 1;
        }
    }
#endif
    (void)infilename;
    UTIL_TRACE_RET(0);
    return 0;
}

static int g_fakeStdinIsConsole = 0;
static int g_fakeStderrIsConsole = 0;
static int g_fakeStdoutIsConsole = 0;

int UTIL_isConsole(FILE* file)
{
    int ret;
    UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file));
    if (file == stdin && g_fakeStdinIsConsole)
        ret = 1;
    else if (file == stderr && g_fakeStderrIsConsole)
        ret = 1;
    else if (file == stdout && g_fakeStdoutIsConsole)
        ret = 1;
    else
        ret = IS_CONSOLE(file);
    UTIL_TRACE_RET(ret);
    return ret;
}

void UTIL_fakeStdinIsConsole(void)
{
    g_fakeStdinIsConsole = 1;
}
void UTIL_fakeStdoutIsConsole(void)
{
    g_fakeStdoutIsConsole = 1;
}
void UTIL_fakeStderrIsConsole(void)
{
    g_fakeStderrIsConsole = 1;
}

U64 UTIL_getFileSize(const char* infilename)
{
    stat_t statbuf;
    UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename);
    if (!UTIL_stat(infilename, &statbuf)) {
        UTIL_TRACE_RET(-1);
        return UTIL_FILESIZE_UNKNOWN;
    }
    {
        U64 const size = UTIL_getFileSizeStat(&statbuf);
        UTIL_TRACE_RET((int)size);
        return size;
    }
}

U64 UTIL_getFileSizeStat(const stat_t* statbuf)
{
    if (!UTIL_isRegularFileStat(statbuf)) return UTIL_FILESIZE_UNKNOWN;
#if defined(_MSC_VER)
    if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
#elif defined(__MINGW32__) && defined (__MSVCRT__)
    if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
#else
    if (!S_ISREG(statbuf->st_mode)) return UTIL_FILESIZE_UNKNOWN;
#endif
    return (U64)statbuf->st_size;
}

UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size)
{
    UTIL_HumanReadableSize_t hrs;

    if (g_utilDisplayLevel > 3) {
        /* In verbose mode, do not scale sizes down, except in the case of
         * values that exceed the integral precision of a double. */
        if (size >= (1ull << 53)) {
            hrs.value = (double)size / (1ull << 20);
            hrs.suffix = " MiB";
            /* At worst, a double representation of a maximal size will be
             * accurate to better than tens of kilobytes. */
            hrs.precision = 2;
        } else {
            hrs.value = (double)size;
            hrs.suffix = " B";
            hrs.precision = 0;
        }
    } else {
        /* In regular mode, scale sizes down and use suffixes. */
        if (size >= (1ull << 60)) {
            hrs.value = (double)size / (1ull << 60);
            hrs.suffix = " EiB";
        } else if (size >= (1ull << 50)) {
            hrs.value = (double)size / (1ull << 50);
            hrs.suffix = " PiB";
        } else if (size >= (1ull << 40)) {
            hrs.value = (double)size / (1ull << 40);
            hrs.suffix = " TiB";
        } else if (size >= (1ull << 30)) {
            hrs.value = (double)size / (1ull << 30);
            hrs.suffix = " GiB";
        } else if (size >= (1ull << 20)) {
            hrs.value = (double)size / (1ull << 20);
            hrs.suffix = " MiB";
        } else if (size >= (1ull << 10)) {
            hrs.value = (double)size / (1ull << 10);
            hrs.suffix = " KiB";
        } else {
            hrs.value = (double)size;
            hrs.suffix = " B";
        }

        if (hrs.value >= 100 || (U64)hrs.value == size) {
            hrs.precision = 0;
        } else if (hrs.value >= 10) {
            hrs.precision = 1;
        } else if (hrs.value > 1) {
            hrs.precision = 2;
        } else {
            hrs.precision = 3;
        }
    }

    return hrs;
}

U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles)
{
    U64 total = 0;
    unsigned n;
    UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles);
    for (n=0; n<nbFiles; n++) {
        U64 const size = UTIL_getFileSize(fileNamesTable[n]);
        if (size == UTIL_FILESIZE_UNKNOWN) {
            UTIL_TRACE_RET(-1);
            return UTIL_FILESIZE_UNKNOWN;
        }
        total += size;
    }
    UTIL_TRACE_RET((int)total);
    return total;
}


/* condition : @file must be valid, and not have reached its end.
 * @return : length of line written into @buf, ended with `\0` instead of '\n',
 *           or 0, if there is no new line */
static size_t readLineFromFile(char* buf, size_t len, FILE* file)
{
    assert(!feof(file));
    if ( fgets(buf, (int) len, file) == NULL ) return 0;
    {   size_t linelen = strlen(buf);
        if (strlen(buf)==0) return 0;
        if (buf[linelen-1] == '\n') linelen--;
        buf[linelen] = '\0';
        return linelen+1;
    }
}

/* Conditions :
 *   size of @inputFileName file must be < @dstCapacity
 *   @dst must be initialized
 * @return : nb of lines
 *       or -1 if there's an error
 */
static int
readLinesFromFile(void* dst, size_t dstCapacity,
            const char* inputFileName)
{
    int nbFiles = 0;
    size_t pos = 0;
    char* const buf = (char*)dst;
    FILE* const inputFile = fopen(inputFileName, "r");

    assert(dst != NULL);

    if(!inputFile) {
        if (g_utilDisplayLevel >= 1) perror("zstd:util:readLinesFromFile");
        return -1;
    }

    while ( !feof(inputFile) ) {
        size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile);
        if (lineLength == 0) break;
        assert(pos + lineLength <= dstCapacity); /* '=' for inputFile not terminated with '\n' */
        pos += lineLength;
        ++nbFiles;
    }

    CONTROL( fclose(inputFile) == 0 );

    return nbFiles;
}

/*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/
FileNamesTable*
UTIL_createFileNamesTable_fromFileName(const char* inputFileName)
{
    size_t nbFiles = 0;
    char* buf;
    size_t bufSize;
    stat_t statbuf;

    if (!UTIL_stat(inputFileName, &statbuf) || !UTIL_isRegularFileStat(&statbuf))
        return NULL;

    {   U64 const inputFileSize = UTIL_getFileSizeStat(&statbuf);
        if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE)
            return NULL;
        bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */
    }

    buf = (char*) malloc(bufSize);
    CONTROL( buf != NULL );

    {   int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName);

        if (ret_nbFiles <= 0) {
          free(buf);
          return NULL;
        }
        nbFiles = (size_t)ret_nbFiles;
    }

    {   const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable));
        CONTROL(filenamesTable != NULL);

        {   size_t fnb, pos = 0;
            for (fnb = 0; fnb < nbFiles; fnb++) {
                filenamesTable[fnb] = buf+pos;
                pos += strlen(buf+pos)+1;  /* +1 for the finishing `\0` */
            }
        assert(pos <= bufSize);
        }

        return UTIL_assembleFileNamesTable(filenamesTable, nbFiles, buf);
    }
}

static FileNamesTable*
UTIL_assembleFileNamesTable2(const char** filenames, size_t tableSize, size_t tableCapacity, char* buf)
{
    FileNamesTable* const table = (FileNamesTable*) malloc(sizeof(*table));
    CONTROL(table != NULL);
    table->fileNames = filenames;
    table->buf = buf;
    table->tableSize = tableSize;
    table->tableCapacity = tableCapacity;
    return table;
}

FileNamesTable*
UTIL_assembleFileNamesTable(const char** filenames, size_t tableSize, char* buf)
{
    return UTIL_assembleFileNamesTable2(filenames, tableSize, tableSize, buf);
}

void UTIL_freeFileNamesTable(FileNamesTable* table)
{
    if (table==NULL) return;
    free((void*)table->fileNames);
    free(table->buf);
    free(table);
}

FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize)
{
    const char** const fnTable = (const char**)malloc(tableSize * sizeof(*fnTable));
    FileNamesTable* fnt;
    if (fnTable==NULL) return NULL;
    fnt = UTIL_assembleFileNamesTable(fnTable, tableSize, NULL);
    fnt->tableSize = 0;   /* the table is empty */
    return fnt;
}

int UTIL_searchFileNamesTable(FileNamesTable* table, char const* name) {
    size_t i;
    for(i=0 ;i < table->tableSize; i++) {
        if(!strcmp(table->fileNames[i], name)) {
            return (int)i;
        }
    }
    return -1;
}

void UTIL_refFilename(FileNamesTable* fnt, const char* filename)
{
    assert(fnt->tableSize < fnt->tableCapacity);
    fnt->fileNames[fnt->tableSize] = filename;
    fnt->tableSize++;
}

static size_t getTotalTableSize(FileNamesTable* table)
{
    size_t fnb, totalSize = 0;
    for(fnb = 0 ; fnb < table->tableSize && table->fileNames[fnb] ; ++fnb) {
        totalSize += strlen(table->fileNames[fnb]) + 1; /* +1 to add '\0' at the end of each fileName */
    }
    return totalSize;
}

FileNamesTable*
UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2)
{
    unsigned newTableIdx = 0;
    size_t pos = 0;
    size_t newTotalTableSize;
    char* buf;

    FileNamesTable* const newTable = UTIL_assembleFileNamesTable(NULL, 0, NULL);
    CONTROL( newTable != NULL );

    newTotalTableSize = getTotalTableSize(table1) + getTotalTableSize(table2);

    buf = (char*) calloc(newTotalTableSize, sizeof(*buf));
    CONTROL ( buf != NULL );

    newTable->buf = buf;
    newTable->tableSize = table1->tableSize + table2->tableSize;
    newTable->fileNames = (const char **) calloc(newTable->tableSize, sizeof(*(newTable->fileNames)));
    CONTROL ( newTable->fileNames != NULL );

    {   unsigned idx1;
        for( idx1=0 ; (idx1 < table1->tableSize) && table1->fileNames[idx1] && (pos < newTotalTableSize); ++idx1, ++newTableIdx) {
            size_t const curLen = strlen(table1->fileNames[idx1]);
            memcpy(buf+pos, table1->fileNames[idx1], curLen);
            assert(newTableIdx <= newTable->tableSize);
            newTable->fileNames[newTableIdx] = buf+pos;
            pos += curLen+1;
    }   }

    {   unsigned idx2;
        for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) {
            size_t const curLen = strlen(table2->fileNames[idx2]);
            memcpy(buf+pos, table2->fileNames[idx2], curLen);
            assert(newTableIdx < newTable->tableSize);
            newTable->fileNames[newTableIdx] = buf+pos;
            pos += curLen+1;
    }   }
    assert(pos <= newTotalTableSize);
    newTable->tableSize = newTableIdx;

    UTIL_freeFileNamesTable(table1);
    UTIL_freeFileNamesTable(table2);

    return newTable;
}

#ifdef _WIN32
static int UTIL_prepareFileList(const char* dirName,
                                char** bufStart, size_t* pos,
                                char** bufEnd, int followLinks)
{
    char* path;
    size_t dirLength, pathLength;
    int nbFiles = 0;
    WIN32_FIND_DATAA cFile;
    HANDLE hFile;

    dirLength = strlen(dirName);
    path = (char*) malloc(dirLength + 3);
    if (!path) return 0;

    memcpy(path, dirName, dirLength);
    path[dirLength] = '\\';
    path[dirLength+1] = '*';
    path[dirLength+2] = 0;

    hFile=FindFirstFileA(path, &cFile);
    if (hFile == INVALID_HANDLE_VALUE) {
        UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
        return 0;
    }
    free(path);

    do {
        size_t const fnameLength = strlen(cFile.cFileName);
        path = (char*) malloc(dirLength + fnameLength + 2);
        if (!path) { FindClose(hFile); return 0; }
        memcpy(path, dirName, dirLength);
        path[dirLength] = '\\';
        memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
        pathLength = dirLength+1+fnameLength;
        path[pathLength] = 0;
        if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
            if ( strcmp (cFile.cFileName, "..") == 0
              || strcmp (cFile.cFileName, ".") == 0 )
                continue;
            /* Recursively call "UTIL_prepareFileList" with the new path. */
            nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks);
            if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
        } else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL)
                 || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE)
                 || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) {
            if (*bufStart + *pos + pathLength >= *bufEnd) {
                ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
                *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
                if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
                *bufEnd = *bufStart + newListSize;
            }
            if (*bufStart + *pos + pathLength < *bufEnd) {
                memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */);
                *pos += pathLength + 1;
                nbFiles++;
        }   }
        free(path);
    } while (FindNextFileA(hFile, &cFile));

    FindClose(hFile);
    return nbFiles;
}

#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L)  /* opendir, readdir require POSIX.1-2001 */

static int UTIL_prepareFileList(const char *dirName,
                                char** bufStart, size_t* pos,
                                char** bufEnd, int followLinks)
{
    DIR* dir;
    struct dirent * entry;
    size_t dirLength;
    int nbFiles = 0;

    if (!(dir = opendir(dirName))) {
        UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
        return 0;
    }

    dirLength = strlen(dirName);
    errno = 0;
    while ((entry = readdir(dir)) != NULL) {
        char* path;
        size_t fnameLength, pathLength;
        if (strcmp (entry->d_name, "..") == 0 ||
            strcmp (entry->d_name, ".") == 0) continue;
        fnameLength = strlen(entry->d_name);
        path = (char*) malloc(dirLength + fnameLength + 2);
        if (!path) { closedir(dir); return 0; }
        memcpy(path, dirName, dirLength);

        path[dirLength] = '/';
        memcpy(path+dirLength+1, entry->d_name, fnameLength);
        pathLength = dirLength+1+fnameLength;
        path[pathLength] = 0;

        if (!followLinks && UTIL_isLink(path)) {
            UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
            free(path);
            continue;
        }

        if (UTIL_isDirectory(path)) {
            nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks);  /* Recursively call "UTIL_prepareFileList" with the new path. */
            if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
        } else {
            if (*bufStart + *pos + pathLength >= *bufEnd) {
                ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
                assert(newListSize >= 0);
                *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
                if (*bufStart != NULL) {
                    *bufEnd = *bufStart + newListSize;
                } else {
                    free(path); closedir(dir); return 0;
                }
            }
            if (*bufStart + *pos + pathLength < *bufEnd) {
                memcpy(*bufStart + *pos, path, pathLength + 1);  /* with final \0 */
                *pos += pathLength + 1;
                nbFiles++;
        }   }
        free(path);
        errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
    }

    if (errno != 0) {
        UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s \n", dirName, strerror(errno));
        free(*bufStart);
        *bufStart = NULL;
    }
    closedir(dir);
    return nbFiles;
}

#else

static int UTIL_prepareFileList(const char *dirName,
                                char** bufStart, size_t* pos,
                                char** bufEnd, int followLinks)
{
    (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
    UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE) \n", dirName);
    return 0;
}

#endif /* #ifdef _WIN32 */

int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
{
  const char* ext = UTIL_getFileExtension(inputName);
  while(*extensionList!=NULL)
  {
    const int isCompressedExtension = strcmp(ext,*extensionList);
    if(isCompressedExtension==0)
      return 1;
    ++extensionList;
  }
   return 0;
}

/*Utility function to get file extension from file */
const char* UTIL_getFileExtension(const char* infilename)
{
   const char* extension = strrchr(infilename, '.');
   if(!extension || extension==infilename) return "";
   return extension;
}

static int pathnameHas2Dots(const char *pathname)
{
    /* We need to figure out whether any ".." present in the path is a whole
     * path token, which is the case if it is bordered on both sides by either
     * the beginning/end of the path or by a directory separator.
     */