/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors

SPDX-License-Identifier: GPL-3.0-or-later

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

Please visit our Website: http://www.httrack.com
*/

/* Parts (inside ARC format routines) by Lars Clausen (lc@statsbiblioteket.dk) */

/* ------------------------------------------------------------ */
/* File: Cache manager for ProxyTrack                           */
/* Author: Xavier Roche                                         */
/* ------------------------------------------------------------ */

#include <limits.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

/* Locking */
#ifdef _WIN32
#include <process.h>            /* _beginthread, _endthread */
#else
#include <pthread.h>
#endif

#define HTSSAFE_ABORT_FUNCTION(A,B,C)
#include "htsglobal.h"

#define HTS_INTERNAL_BYTECODE
#include "coucal.h"
#include "htsmd5.h"
#undef HTS_INTERNAL_BYTECODE
#include "../minizip/mztools.h"
#include "../minizip/zip.h"

#include "htscore.h"
#include "htsback.h"
#include "htslib.h" /* hts_effective_mime */
#include "htsio.h"

#include "store.h"
#include "proxystrings.h"
#include "proxytrack.h"

/* Unlocked functions */

static int PT_LookupCache__New_u(PT_Index index, const char *url);
static PT_Element PT_ReadCache__New_u(PT_Index index, const char *url,
                                      int flags);

static int PT_LookupCache__Old_u(PT_Index index, const char *url);
static PT_Element PT_ReadCache__Old_u(PT_Index index, const char *url,
                                      int flags);

static int PT_LookupCache__Arc_u(PT_Index index, const char *url);
static PT_Element PT_ReadCache__Arc_u(PT_Index index, const char *url,
                                      int flags);

/* Locking */

#ifdef _WIN32
void MutexInit(PT_Mutex * pMutex) {
  *pMutex = CreateMutex(NULL, FALSE, NULL);
}

void MutexLock(PT_Mutex * pMutex) {
  WaitForSingleObject(*pMutex, INFINITE);
}

void MutexUnlock(PT_Mutex * pMutex) {
  ReleaseMutex(*pMutex);
}

void MutexFree(PT_Mutex * pMutex) {
  CloseHandle(*pMutex);
  *pMutex = NULL;
}
#else
void MutexInit(PT_Mutex * pMutex) {
  (void) pthread_mutex_init(pMutex, 0);
}

void MutexLock(PT_Mutex * pMutex) {
  pthread_mutex_lock(pMutex);
}

void MutexUnlock(PT_Mutex * pMutex) {
  pthread_mutex_unlock(pMutex);
}

void MutexFree(PT_Mutex * pMutex) {
  pthread_mutex_destroy(pMutex);
}
#endif

/* Indexes */

typedef struct _PT_Index__New _PT_Index__New;
typedef struct _PT_Index__Old _PT_Index__Old;
typedef struct _PT_Index__Arc _PT_Index__Arc;
typedef struct _PT_Index_Functions _PT_Index_Functions;

typedef struct _PT_Index__New *PT_Index__New;
typedef struct _PT_Index__Old *PT_Index__Old;
typedef struct _PT_Index__Arc *PT_Index__Arc;
typedef struct _PT_Index_Functions *PT_Index_Functions;

enum {
  PT_CACHE_UNDEFINED = -1,
  PT_CACHE_MIN = 0,
  PT_CACHE__NEW = PT_CACHE_MIN,
  PT_CACHE__OLD,
  PT_CACHE__ARC,
  PT_CACHE_MAX = PT_CACHE__ARC
};

static int PT_LoadCache__New(PT_Index index, const char *filename);
static void PT_Index_Delete__New(PT_Index * pindex);
static PT_Element PT_ReadCache__New(PT_Index index, const char *url, int flags);
static int PT_LookupCache__New(PT_Index index, const char *url);
static int PT_SaveCache__New(PT_Indexes indexes, const char *filename);
 /**/ static int PT_LoadCache__Old(PT_Index index, const char *filename);
static void PT_Index_Delete__Old(PT_Index * pindex);
static PT_Element PT_ReadCache__Old(PT_Index index, const char *url, int flags);
static int PT_LookupCache__Old(PT_Index index, const char *url);
 /**/ static int PT_LoadCache__Arc(PT_Index index, const char *filename);
static void PT_Index_Delete__Arc(PT_Index * pindex);
static PT_Element PT_ReadCache__Arc(PT_Index index, const char *url, int flags);
static int PT_LookupCache__Arc(PT_Index index, const char *url);
static int PT_SaveCache__Arc(PT_Indexes indexes, const char *filename);

struct _PT_Index_Functions {
  /* Mandatory services */
  int (*PT_LoadCache) (PT_Index index, const char *filename);
  void (*PT_Index_Delete) (PT_Index * pindex);
    PT_Element(*PT_ReadCache) (PT_Index index, const char *url, int flags);
  int (*PT_LookupCache) (PT_Index index, const char *url);

  /* Optional services */
  int (*PT_SaveCache) (PT_Indexes indexes, const char *filename);
};

static _PT_Index_Functions _IndexFuncts[] = {
  {PT_LoadCache__New, PT_Index_Delete__New, PT_ReadCache__New,
   PT_LookupCache__New, PT_SaveCache__New},
  {PT_LoadCache__Old, PT_Index_Delete__Old, PT_ReadCache__Old,
   PT_LookupCache__Old, NULL},
  {PT_LoadCache__Arc, PT_Index_Delete__Arc, PT_ReadCache__Arc,
   PT_LookupCache__Arc, PT_SaveCache__Arc},
  {NULL, NULL, NULL, NULL}
};

#define PT_INDEX_COMMON_STRUCTURE \
	time_t timestamp;								\
	coucal hash;										\
	char startUrl[1024]

struct _PT_Index__New {
  PT_INDEX_COMMON_STRUCTURE;
  char path[1024];              /* either empty, or must include ending / */
  int fixedPath;
  int safeCache;
  unzFile zFile;
  PT_Mutex zFileLock;
};

struct _PT_Index__Old {
  PT_INDEX_COMMON_STRUCTURE;
  char filenameDat[1024];
  char filenameNdx[1024];
  FILE *dat, *ndx;
  PT_Mutex fileLock;
  int version;
  char lastmodified[1024];
  char path[1024];              /* either empty, or must include ending / */
  int fixedPath;
  int safeCache;
};

struct _PT_Index__Arc {
  PT_INDEX_COMMON_STRUCTURE;
  FILE *file;
  long int fileSize; /* bound for the lengths read out of the file */
  PT_Mutex fileLock;
  int version;
  char lastmodified[1024];
  char line[2048];
  char filenameIndexBuff[2048];
};

struct _PT_Index {
  int type;
  union {
    _PT_Index__New formatNew;
    _PT_Index__Old formatOld;
    _PT_Index__Arc formatArc;
    struct {
      PT_INDEX_COMMON_STRUCTURE;
    } common;
  } slots;
};

struct _PT_Indexes {
  coucal cil;
  struct _PT_Index **index;
  int index_size;
};

PT_Indexes PT_New(void) {
  PT_Indexes index = (PT_Indexes) calloc(sizeof(_PT_Indexes), 1);

  index->cil = coucal_new(0);
  coucal_set_name(index->cil, "index->cil");
  index->index_size = 0;
  index->index = NULL;
  return index;
}

void PT_Delete(PT_Indexes index) {
  if (index != NULL) {
    int i;

    /* PT_IndexMerge took ownership of each index, file handle included */
    for (i = 0; i < index->index_size; i++) {
      PT_Index_Delete(&index->index[i]);
    }
    free(index->index);
    coucal_delete(&index->cil);
    free(index);
  }
}

/* Reads one line into "s" (at most "max" chars plus its NUL) and returns the
   advance. *cut, when asked for, says the line did not fit. */
static int binput(char *buff, char *s, int max, hts_boolean *cut) {
  int count = 0;
  int destCount = 0;

  if (cut != NULL) {
    *cut = HTS_FALSE;
  }
  /* consumed whole: resuming inside a clipped line reads its tail as a field */
  while (buff[count] != '\0' && buff[count] != '\n') {
    if (buff[count] != '\r') {
      if (destCount < max) {
        s[destCount++] = buff[count];
      } else if (cut != NULL) {
        *cut = HTS_TRUE;
      }
    }
    count++;
  }
  s[destCount] = '\0';

  /* only step over a real separator: past the terminating NUL leaves the
     caller's buffer, past a truncated field's next byte silently eats it */
  return buff[count] == '\n' ? count + 1 : count;
}

static time_t file_timestamp(const char *file) {
  struct stat buf;

  if (stat(file, &buf) == 0) {
    time_t tt = buf.st_mtime;

    if (tt != (time_t) 0 && tt != (time_t) - 1) {
      return tt;
    }
  }
  return (time_t) 0;
}

static int PT_Index_Check__(PT_Index index, const char *file, int line) {
  if (index == NULL)
    return 0;
  if (index->type >= PT_CACHE_MIN && index->type <= PT_CACHE_MAX)
    return 1;
  proxytrack_print_log(CRITICAL, "index corrupted in memory at %s:%d", file,
                       line);
  return 0;
}

#define SAFE_INDEX(index) PT_Index_Check__(index, __FILE__, __LINE__)

/* ------------------------------------------------------------ */
/* Generic cache dispatch                                       */
/* ------------------------------------------------------------ */

void PT_Index_Delete(PT_Index * pindex) {
  if (pindex != NULL && (*pindex) != NULL) {
    PT_Index index = *pindex;

    if (SAFE_INDEX(index)) {
      _IndexFuncts[index->type].PT_Index_Delete(pindex);
    }
    free(index);
    *pindex = NULL;
  }
}

static void PT_Index_Delete__New(PT_Index * pindex) {
  if (pindex != NULL && (*pindex) != NULL) {
    PT_Index__New index = &(*pindex)->slots.formatNew;

    if (index->zFile != NULL) {
      unzClose(index->zFile);
      index->zFile = NULL;
    }
    if (index->hash != NULL) {
      coucal_delete(&index->hash);
      index->hash = NULL;
    }
    MutexFree(&index->zFileLock);
  }
}

static void PT_Index_Delete__Old(PT_Index * pindex) {
  if (pindex != NULL && (*pindex) != NULL) {
    PT_Index__Old index = &(*pindex)->slots.formatOld;

    if (index->dat != NULL) {
      fclose(index->dat);
    }
    if (index->ndx != NULL) {
      fclose(index->ndx);
    }
    if (index->hash != NULL) {
      coucal_delete(&index->hash);
      index->hash = NULL;
    }
    MutexFree(&index->fileLock);
  }
}

static void PT_Index_Delete__Arc(PT_Index * pindex) {
  if (pindex != NULL && (*pindex) != NULL) {
    PT_Index__Arc index = &(*pindex)->slots.formatArc;

    if (index->file != NULL) {
      fclose(index->file);
      index->file = NULL;
    }
    if (index->hash != NULL) {
      coucal_delete(&index->hash);
      index->hash = NULL;
    }
    MutexFree(&index->fileLock);
  }
}

int PT_AddIndex(PT_Indexes indexes, const char *path) {
  PT_Index index = PT_LoadCache(path);

  if (index != NULL) {
    int ret = PT_IndexMerge(indexes, &index);

    if (index != NULL) {
      PT_Index_Delete(&index);
    }
    return ret;
  }
  return -1;
}

PT_Element PT_Index_HTML_BuildRootInfo(PT_Indexes indexes) {
  if (indexes != NULL) {
    PT_Element elt = PT_ElementNew();
    int i;
    String html = STRING_EMPTY;

    StringClear(html);
    StringCat(html,
              "<html>" PROXYTRACK_COMMENT_HEADER
              DISABLE_IE_FRIENDLY_HTTP_ERROR_MESSAGES "<head>\r\n"
              "<title>ProxyTrack " PROXYTRACK_VERSION " Catalog</title>"
              "</head>\r\n" "<body>\r\n"
              "<h3>Available sites in this cache:</h3><br />" "<br />");
    StringCat(html, "<ul>\r\n");
    for(i = 0; i < indexes->index_size; i++) {
      if (indexes->index[i] != NULL
          && indexes->index[i]->slots.common.startUrl[0] != '\0') {
        const char *url = indexes->index[i]->slots.common.startUrl;

        StringCat(html, "<li>\r\n");
        StringCat(html, "<a href=\"");
        StringCat(html, url);
        StringCat(html, "\">");
        StringCat(html, url);
        StringCat(html, "</a>\r\n");
        StringCat(html, "</li>\r\n");
      }
    }
    StringCat(html, "</ul>\r\n");
    StringCat(html, "</body></html>\r\n");
    elt->size = StringLength(html);
    elt->adr = StringAcquire(&html);
    elt->statuscode = HTTP_OK;
    strcpybuff(elt->charset, "iso-8859-1");
    strcpybuff(elt->contenttype, "text/html");
    strcpybuff(elt->msg, "OK");
    StringFree(html);
    return elt;
  }
  return NULL;
}

static char *strchr_stop(char *str, char c, char stop) {
  for(; *str != 0 && *str != stop && *str != c; str++) ;
  if (*str == c)
    return str;
  return NULL;
}

char **PT_Enumerate(PT_Indexes indexes, const char *url, int subtree) {
  // should be cached!
  if (indexes != NULL && indexes->cil != NULL) {
    unsigned int urlSize;
    String list = STRING_EMPTY;
    String listindexes = STRING_EMPTY;
    String subitem = STRING_EMPTY;
    unsigned int listCount = 0;
    struct_coucal_enum en = coucal_enum_new(indexes->cil);
    coucal_item *chain;
    coucal hdupes = NULL;

    if (!subtree) {
      hdupes = coucal_new(0);
      coucal_set_name(hdupes, "hdupes");
    }
    StringClear(list);
    StringClear(listindexes);
    StringClear(subitem);
    if (strncmp(url, "http://", 7) == 0)
      url += 7;
    urlSize = (unsigned int) strlen(url);
    while((chain = coucal_enum_next(&en))) {
      long int index = (long int) chain->value.intg;

      if (urlSize == 0 || strncmp(chain->name, url, urlSize) == 0) {
        if (index >= 0 && index < indexes->index_size) {
          char *item = (char*) chain->name + urlSize;

          if (*item == '/')
            item++;
          {
            char *pos = subtree ? 0 : strchr_stop(item, '/', '?');
            unsigned int len =
              pos ? (unsigned int) (pos - item) : (unsigned int) strlen(item);
            if (len > 0 /* default document */  || *item == 0) {
              int isFolder = (item[len] == '/');

              StringClear(subitem);
              if (len > 0)
                StringMemcat(subitem, item, len);
              if (len == 0 || !coucal_exists(hdupes, StringBuff(subitem))) {
                char *ptr = NULL;

                ptr += StringLength(list);
                if (len > 0)
                  StringCat(list, StringBuff(subitem));
                if (isFolder)
                  StringCat(list, "/");
                StringMemcat(list, "\0", 1);    /* NULL terminated strings */
                StringMemcat(listindexes, (char*) &ptr, sizeof(ptr));
                listCount++;
                coucal_write(hdupes, StringBuff(subitem), 0);
              }
            }
          }
        } else {
          proxytrack_print_log(CRITICAL,
                               "PT_Enumerate:Corrupted central index locator");
        }
      }
    }
    StringFree(subitem);
    coucal_delete(&hdupes);
    if (listCount > 0) {
      unsigned int i;
      void *blk;
      char *nullPointer = NULL;
      char *startStrings;

      /* NULL terminated index */
      StringMemcat(listindexes, (char*) &nullPointer, sizeof(nullPointer));
      /* start of all strings (index) */
      startStrings = nullPointer + StringLength(listindexes);
      /* copy list of URLs after indexes */
      StringMemcat(listindexes, StringBuff(list), StringLength(list));
      /* ---- no reallocation beyond this point (fixed addresses) ---- */
      /* start of all strings (pointer) */
      startStrings = (startStrings - nullPointer) + StringBuffRW(listindexes);
      /* transform indexes into references */
      for(i = 0; i < listCount; i++) {
        char *ptr = NULL;
        unsigned int ndx;

        memcpy(&ptr, &StringBuff(listindexes)[i * sizeof(char *)],
               sizeof(char *));
        ndx = (unsigned int) (ptr - nullPointer);
        ptr = startStrings + ndx;
        memcpy(&StringBuffRW(listindexes)[i * sizeof(char *)], &ptr,
               sizeof(char *));
      }
      blk = StringAcquire(&listindexes);
      StringFree(list);
      StringFree(listindexes);
      return (char **) blk;
    }
  }
  return NULL;
}

void PT_Enumerate_Delete(char ***plist) {
  if (plist != NULL && *plist != NULL) {
    free(*plist);
    *plist = NULL;
  }
}

static int PT_GetType(const char *filename) {
  char *dot = strrchr(filename, '.');

  if (dot != NULL) {
    if (strcasecmp(dot, ".zip") == 0) {
      return PT_CACHE__NEW;
    } else if (strcasecmp(dot, ".ndx") == 0 || strcasecmp(dot, ".dat") == 0) {
      return PT_CACHE__OLD;
    } else if (strcasecmp(dot, ".arc") == 0) {
      return PT_CACHE__ARC;
    }
  }
  return PT_CACHE_UNDEFINED;
}

PT_Index PT_LoadCache(const char *filename) {
  int type = PT_GetType(filename);

  if (type != PT_CACHE_UNDEFINED) {
    PT_Index index = calloc(sizeof(_PT_Index), 1);

    if (index != NULL) {
      index->type = type;
      index->slots.common.timestamp = (time_t) time(NULL);
      index->slots.common.startUrl[0] = '\0';
      index->slots.common.hash = coucal_new(0);
      coucal_set_name(index->slots.common.hash, "index->slots.common.hash");
      if (!_IndexFuncts[type].PT_LoadCache(index, filename)) {
        proxytrack_print_log(DEBUG,
                             "reading httrack cache (format #%d) %s : error",
                             type, filename);
        /* the loader may already hold a file handle and entries */
        PT_Index_Delete(&index);
        return NULL;
      } else {
        proxytrack_print_log(DEBUG,
                             "reading httrack cache (format #%d) %s : success",
                             type, filename);
      }
      /* default starting URL is the first hash entry */
      if (index->slots.common.startUrl[0] == '\0') {
        struct_coucal_enum en = coucal_enum_new(index->slots.common.hash);
        coucal_item *chain;

        chain = coucal_enum_next(&en);
        if (chain != NULL && strstr(chain->name, "/robots.txt") != NULL) {
          chain = coucal_enum_next(&en);
        }
        if (chain != NULL) {
          const char *scheme = link_has_authority(chain->name) ? "" : "http://";

          /* dropped rather than truncated: empty already reads as "unset" */
          if (!sprintfbuff(index->slots.common.startUrl, "%s%s", scheme,
                           (const char *) chain->name))
            index->slots.common.startUrl[0] = '\0';
        }
      }
    }
    return index;
  }
  return NULL;
}

static long int filesize(const char *filename) {
  struct stat st;

  memset(&st, 0, sizeof(st));
  if (stat(filename, &st) == 0) {
    return (long int) st.st_size;
  }
  return -1;
}

int PT_LookupCache(PT_Index index, const char *url) {
  if (index != NULL && SAFE_INDEX(index)) {
    return _IndexFuncts[index->type].PT_LookupCache(index, url);
  }
  return 0;
}

int PT_SaveCache(PT_Indexes indexes, const char *filename) {
  int type = PT_GetType(filename);

  if (type != PT_CACHE_UNDEFINED) {
    if (_IndexFuncts[type].PT_SaveCache != NULL) {
      int ret = _IndexFuncts[type].PT_SaveCache(indexes, filename);

      if (ret == 0) {
        (void) set_filetime_time_t(filename, PT_GetTimeIndex(indexes));
        return 0;
      }
    }
  }
  return -1;
}

int PT_EnumCache(PT_Indexes indexes,
                 int (*callback) (void *, const char *url, PT_Element),
                 void *arg) {
  if (indexes != NULL && indexes->cil != NULL) {
    struct_coucal_enum en = coucal_enum_new(indexes->cil);
    coucal_item *chain;

    while((chain = coucal_enum_next(&en))) {
      const long int index_id = (long int) chain->value.intg;
      const char *const url = chain->name;

      if (index_id >= 0 && index_id < indexes->index_size) {
        PT_Element item =
          PT_ReadCache(indexes->index[index_id], url,
                       FETCH_HEADERS | FETCH_BODY);
        if (item != NULL) {
          int ret = callback(arg, url, item);

          PT_Element_Delete(&item);
          if (ret != 0)
            return ret;
        }
      } else {
        proxytrack_print_log(CRITICAL,
                             "PT_ReadCache:Corrupted central index locator");
        return -1;
      }
    }
  }
  return 0;
}

time_t PT_Index_Timestamp(PT_Index index) {
  return index->slots.common.timestamp;
}

static int PT_LookupCache__New(PT_Index index, const char *url) {
  int retCode;

  MutexLock(&index->slots.formatNew.zFileLock);
  {
    retCode = PT_LookupCache__New_u(index, url);
  }
  MutexUnlock(&index->slots.formatNew.zFileLock);
  return retCode;
}

static int PT_LookupCache__New_u(PT_Index index_, const char *url) {
  if (index_ != NULL) {
    PT_Index__New index = &index_->slots.formatNew;

    if (index->hash != NULL && index->zFile != NULL && url != NULL && *url != 0) {
      int hash_pos_return;

      if (strncmp(url, "http://", 7) == 0)
        url += 7;
      hash_pos_return = coucal_read(index->hash, url, NULL);
      if (hash_pos_return)
        return 1;
    }
  }
  return 0;
}

int PT_IndexMerge(PT_Indexes indexes, PT_Index * pindex) {
  if (pindex != NULL && *pindex != NULL && (*pindex)->slots.common.hash != NULL
      && indexes != NULL) {
    PT_Index index = *pindex;
    struct_coucal_enum en = coucal_enum_new(index->slots.common.hash);
    coucal_item *chain;
    /* index_size counts a slot only once the array holds it, so a failed
       realloc leaves neither a phantom entry nor a dropped array */
    PT_Index *const grown = realloc(
        indexes->index, sizeof(*indexes->index) * (indexes->index_size + 1));
    int index_id;
    int nMerged = 0;

    if (grown != NULL) {
      indexes->index = grown;
      index_id = indexes->index_size++;
      indexes->index[index_id] = index;
      *pindex = NULL;
      while((chain = coucal_enum_next(&en)) != NULL) {
        const char *url = chain->name;

        if (url != NULL && url[0] != '\0') {
          intptr_t previous_index_id = 0;

          if (coucal_read(indexes->cil, url, &previous_index_id)) {
            if (previous_index_id >= 0
                && previous_index_id < indexes->index_size) {
              if (indexes->index[previous_index_id]->slots.common.timestamp > index->slots.common.timestamp)    // existing entry is newer
                break;
            } else {
              proxytrack_print_log(CRITICAL,
                                   "PT_IndexMerge:Corrupted central index locator");
            }
          }
          coucal_write(indexes->cil, chain->name, index_id);
          nMerged++;
        }
      }
    } else {
      proxytrack_print_log(CRITICAL, "PT_IndexMerge:Memory exhausted");
    }
    return nMerged;
  }
  return -1;
}

void PT_Element_Delete(PT_Element * pentry) {
  if (pentry != NULL) {
    PT_Element entry = *pentry;

    if (entry != NULL) {
      if (entry->adr != NULL) {
        free(entry->adr);
        entry->adr = NULL;
      }
      if (entry->headers != NULL) {
        free(entry->headers);
        entry->headers = NULL;
      }
      if (entry->location != NULL) {
        free(entry->location);
        entry->location = NULL;
      }
      free(entry);
    }
    *pentry = NULL;
  }
}

PT_Element PT_ReadIndex(PT_Indexes indexes, const char *url, int flags) {
  if (indexes != NULL) {
    intptr_t index_id;

    if (strncmp(url, "http://", 7) == 0)
      url += 7;
    if (coucal_read(indexes->cil, url, &index_id)) {
      if (index_id >= 0 && index_id < indexes->index_size) {
        PT_Element item = PT_ReadCache(indexes->index[index_id], url, flags);

        if (item != NULL) {
          item->indexId = (int) index_id;
          return item;
        }
      } else {
        proxytrack_print_log(CRITICAL,
                             "PT_ReadCache:Corrupted central index locator");
      }
    }
  }
  return NULL;
}

int PT_LookupIndex(PT_Indexes indexes, const char *url) {
  if (indexes != NULL) {
    intptr_t index_id;

    if (strncmp(url, "http://", 7) == 0)
      url += 7;
    if (coucal_read(indexes->cil, url, &index_id)) {
      if (index_id >= 0 && index_id < indexes->index_size) {
        return 1;
      } else {
        proxytrack_print_log(CRITICAL,
                             "PT_ReadCache:Corrupted central index locator");
      }
    }
  }
  return 0;
}

time_t PT_GetTimeIndex(PT_Indexes indexes) {
  if (indexes != NULL && indexes->index_size > 0) {
    int i;
    time_t maxt = indexes->index[0]->slots.common.timestamp;

    for(i = 1; i < indexes->index_size; i++) {
      const time_t currt = indexes->index[i]->slots.common.timestamp;

      if (currt > maxt) {
        maxt = currt;
      }
    }
    return maxt;
  }
  return (time_t) - 1;
}

PT_Index PT_GetIndex(PT_Indexes indexes, int indexId) {
  if (indexes != NULL && indexId >= 0 && indexId < indexes->index_size) {
    return indexes->index[indexId];
  }
  return NULL;
}

PT_Element PT_ElementNew(void) {
  PT_Element r = NULL;

  if ((r = calloc(sizeof(_PT_Element), 1)) == NULL)
    return NULL;
  r->statuscode = STATUSCODE_INVALID;
  r->indexId = -1;
  return r;
}

/* ProxyTrack's htsblk_failf(): a clipped, diagnostic-only failure reason. */
#define PT_Element_failf(R, ...)                                               \
  slprintfbuff_clip((R)->msg, sizeof((R)->msg), __VA_ARGS__)

PT_Element PT_ReadCache(PT_Index index, const char *url, int flags) {
  if (index != NULL && SAFE_INDEX(index)) {
    return _IndexFuncts[index->type].PT_ReadCache(index, url, flags);
  }
  return NULL;
}

static PT_Element PT_ReadCache__New(PT_Index index, const char *url, int flags) {
  PT_Element retCode;

  MutexLock(&index->slots.formatNew.zFileLock);
  {
    retCode = PT_ReadCache__New_u(index, url, flags);
  }
  MutexUnlock(&index->slots.formatNew.zFileLock);
  return retCode;
}

/* ------------------------------------------------------------ */
/* New HTTrack cache (new.zip) format                           */
/* ------------------------------------------------------------ */

/* Values read back off a cache, so the block is bounded rather than trusted to
   the element caps; a field that does not fit is dropped whole and counted,
   since a clipped one reads back as valid. `headers` must be an array. */
#define ZIP_FIELD_STRING(headers, headersSize, dropped, field, value)          \
  do {                                                                         \
    if ((value) != NULL && (value)[0] != '\0' &&                               \
        !slcatprintfbuff(headers, sizeof(headers), &(headersSize),             \
                         "%s: %s\r\n", field, value)) {                        \
      (dropped)++;                                                             \
    }                                                                          \
  } while (0)
#define ZIP_FIELD_INT(headers, headersSize, dropped, field, value)             \
  do {                                                                         \
    if ((value) != 0 &&                                                        \
        !slcatprintfbuff(headers, sizeof(headers), &(headersSize),             \
                         "%s: " LLintP "\r\n", field, (LLint) (value))) {      \
      (dropped)++;                                                             \
    }                                                                          \
  } while (0)
#define ZIP_FIELD_INT_FORCE(headers, headersSize, dropped, field, value)       \
  do {                                                                         \
    if (!slcatprintfbuff(headers, sizeof(headers), &(headersSize),             \
                         "%s: " LLintP "\r\n", field, (LLint) (value))) {      \
      (dropped)++;                                                             \
    }                                                                          \
  } while (0)
/* refvalue_size is mandatory: the cache line is bounded only by the line
   buffer, not by the destination. */
#define ZIP_READFIELD_STRING(line, value, refline, refvalue, refvalue_size)    \
  do {                                                                         \
    if (line[0] != '\0' && strfield2(line, refline)) {                         \
      (void) strclipbuff(refvalue, refvalue_size, value);                      \
      line[0] = '\0';                                                          \
    }                                                                          \
  } while (0)
#define ZIP_READFIELD_INT(line, value, refline, refvalue) do { \
  if (line[0] != '\0' && strfield2(line, refline)) { \
    int intval = 0; \
    sscanf(value, "%d", &intval); \
    (refvalue) = intval; \
    line[0] = '\0'; \
	} \
} while(0)
#define ZIP_READFIELD_LLINT(line, value, refline, refvalue)                    \
  do {                                                                         \
    if (line[0] != '\0' && strfield2(line, refline)) {                         \
      LLint intval = 0;                                                        \
      sscanf(value, LLintP, &intval);                                          \
      (refvalue) = intval;                                                     \
      line[0] = '\0';                                                          \
    }                                                                          \
  } while (0)

/* Set path (capacity size) to filename's parent directory, separator included,
   from an absolute filename. Empty when there is none, or when it would not
   fit: a truncated prefix names a different directory. */
static void index_base_path(char *path, size_t size, const char *filename) {
  const char *abpath;
  int slashes;

  for (slashes = 2, abpath = hts_lastcharptr(filename);
       abpath > filename &&
       ((*abpath != '/' && *abpath != '\\') || --slashes > 0);
       abpath--)
    ;
  path[0] = '\0';
  if (slashes == 0 && *abpath != 0 && (size_t) (abpath - filename) < size - 1) {
    int i;

    strlncatbuff(path, filename, size, (size_t) (abpath - filename) + 1);
    for (i = 0; path[i] != 0; i++) {
      if (path[i] == '\\') {
        path[i] = '/';
      }
    }
  }
}

int PT_LoadCache__New(PT_Index index_, const char *filename) {
  if (index_ != NULL && filename != NULL) {
    PT_Index__New index = &index_->slots.formatNew;
    unzFile zFile = index->zFile = unzOpen(filename);

    index->timestamp = file_timestamp(filename);
    MutexInit(&index->zFileLock);

    // Opened ?
    if (zFile != NULL) {
      coucal hashtable = index->hash;

      index_base_path(index->path, sizeof(index->path), filename);

      /* Ready directory entries */
      if (unzGoToFirstFile(zFile) == Z_OK) {
        char comment[128];
        char filename[HTS_URLMAXSIZE * 4];
        int entries = 0;
        int firstSeen = 0;

        memset(comment, 0, sizeof(comment));    // for truncated reads
        do {
          int readSizeHeader = 0;

          filename[0] = '\0';
          comment[0] = '\0';
          if (unzOpenCurrentFile(zFile) == Z_OK) {
            if ((readSizeHeader =
                 unzGetLocalExtrafield(zFile, comment, sizeof(comment) - 2)) > 0
                && unzGetCurrentFileInfo(zFile, NULL, filename,
                                         sizeof(filename) - 2, NULL, 0, NULL,
                                         0) == Z_OK) {
              long int pos = (long int) unzGetOffset(zFile);

              assertf(readSizeHeader < sizeof(comment));
              comment[readSizeHeader] = '\0';
              entries++;
              if (pos > 0) {
                int dataincache = 0;    // data in cache ?
                char *filenameIndex = filename;

                if (strncmp(filenameIndex, "http://", 7) == 0) {
                  filenameIndex += 7;
                }
                if (comment[0] != '\0') {
                  int maxLine = 2;
                  char *a = comment;

                  while(*a && maxLine-- > 0) {  // parse only few first lines
                    char line[1024];

                    line[0] = '\0';
                    a += binput(a, line, sizeof(line) - 2, NULL);
                    if (strncmp(line, "X-In-Cache:", 11) == 0) {
                      if (strcmp(line, "X-In-Cache: 1") == 0) {
                        dataincache = 1;
                      } else {
                        dataincache = 0;
                      }
                      break;
                    }
                  }
                }
                if (dataincache)
                  coucal_add(hashtable, filenameIndex, pos);
                else
                  coucal_add(hashtable, filenameIndex, -pos);

                /* First link as starting URL */
                if (!firstSeen) {
                  if (strstr(filenameIndex, "/robots.txt") == NULL) {
                    const char *scheme =
                        link_has_authority(filenameIndex) ? "" : "http://";

                    /* dropped rather than truncated; try the next entry */
                    if (sprintfbuff(index->startUrl, "%s%s", scheme,
                                    filenameIndex))
                      firstSeen = 1;
                    else
                      index->startUrl[0] = '\0';
                  }
                }
              } else {
                fprintf(stderr, "Corrupted cache meta entry #%d" LF,
                        (int) entries);
              }
            } else {
              fprintf(stderr, "Corrupted cache entry #%d" LF, (int) entries);
            }
            unzCloseCurrentFile(zFile);
          } else {
            fprintf(stderr, "Corrupted cache entry #%d" LF, (int) entries);
          }
        } while(unzGoToNextFile(zFile) == Z_OK);
        return 1;
      } else {
        coucal_delete(&index->hash);
        index = NULL;
      }
    } else {
      index = NULL;
    }
  }
  return 0;
}

static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
                                      int flags) {
  PT_Index__New index = (PT_Index__New) & index_->slots.formatNew;
  char location_default[HTS_URLMAXSIZE * 2];
  char previous_save[HTS_URLMAXSIZE * 2];
  char previous_save_[HTS_URLMAXSIZE * 2];
  char catbuff[CATBUFF_SIZE];
  intptr_t hash_pos;
  int hash_pos_return;
  PT_Element r = NULL;

  if (index == NULL || index->hash == NULL || index->zFile == NULL
      || url == NULL || *url == 0)
    return NULL;
  if ((r = PT_ElementNew()) == NULL)
    return NULL;
  location_default[0] = '\0';
  previous_save[0] = previous_save_[0] = '\0';
  memset(r, 0, sizeof(_PT_Element));
  r->location = location_default;
  r->location[0] = '\0';
  if (strncmp(url, "http://", 7) == 0)
    url += 7;
  hash_pos_return = coucal_read(index->hash, url, &hash_pos);

  if (hash_pos_return) {
    uLong posInZip;

    if (hash_pos > 0) {
      posInZip = (uLong) hash_pos;
    } else {
      posInZip = (uLong) - hash_pos;
    }
    if (unzSetOffset(index->zFile, posInZip) == Z_OK) {
      /* Read header (Max 8KiB) */
      if (unzOpenCurrentFile(index->zFile) == Z_OK) {
        char headerBuff[8192 + 2];
        int readSizeHeader;
        int dataincache = 0;
        /* signed, so a corrupt value is judged before it becomes a size_t */
        LLint declared_size = 0;

        /* For BIG comments */
        headerBuff[0]
          = headerBuff[sizeof(headerBuff) - 1]
          = headerBuff[sizeof(headerBuff) - 2]
          = headerBuff[sizeof(headerBuff) - 3] = '\0';

        if ((readSizeHeader =
             unzGetLocalExtrafield(index->zFile, headerBuff,
                                   sizeof(headerBuff) - 2)) > 0) {
          int offset = 0;
          /* the longest stored line is "Location: " plus a full-capacity URL;
             a shorter buffer truncates it into a valid-looking wrong target */
          char BIGSTK line[HTS_LOCATION_SIZE + 32];
          int lineEof = 0;

          headerBuff[readSizeHeader] = '\0';
          do {
            char *value;
            hts_boolean cut;

            line[0] = '\0';
            offset += binput(headerBuff + offset, line, sizeof(line) - 2, &cut);
            if (line[0] == '\0') {
              lineEof = 1;
            }
            /* no line HTTrack writes is this long, so a cut one is foreign or
               damaged and its prefix would parse as a wrong value */
            value = cut ? NULL : strchr(line, ':');
            if (value != NULL) {
              *value++ = '\0';
              if (*value == ' ' || *value == '\t')
                value++;
              ZIP_READFIELD_INT(line, value, "X-In-Cache", dataincache);
              ZIP_READFIELD_INT(line, value, "X-Statuscode", r->statuscode);
              ZIP_READFIELD_STRING(line, value, "X-StatusMessage", r->msg,
                                   sizeof(r->msg));
              ZIP_READFIELD_LLINT(line, value, "X-Size", declared_size); // size
              ZIP_READFIELD_STRING(line, value, "Content-Type", r->contenttype,
                                   sizeof(r->contenttype));
              ZIP_READFIELD_STRING(line, value, "X-Charset", r->charset,
                                   sizeof(r->charset));
              ZIP_READFIELD_STRING(line, value, "Last-Modified",
                                   r->lastmodified, sizeof(r->lastmodified));
              ZIP_READFIELD_STRING(line, value, "Etag", r->etag,
                                   sizeof(r->etag));
              ZIP_READFIELD_STRING(line, value, "Location", r->location,
                                   sizeof(location_default));
              ZIP_READFIELD_STRING(line, value, "Content-Disposition",
                                   r->cdispo, sizeof(r->cdispo));
              ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_,
                                   sizeof(previous_save_));
              if (line[0] != '\0') {
                const int len = r->headers ? ((int) strlen(r->headers)) : 0;
                /* one byte more than the four appends below write */
                const int nlen = (int) (strlen(line) + 2 + strlen(value) +
                                        sizeof("\r\n") + 1);
                char *const grown = realloct(r->headers, len + nlen);

                /* keep the headers read so far rather than losing them */
                if (grown != NULL) {
                  r->headers = grown;
                  r->headers[len] = '\0';
                  strcat(r->headers, line);
                  strcat(r->headers, ": ");
                  strcat(r->headers, value);
                  strcat(r->headers, "\r\n");
                }
              }
            }
          } while (offset < readSizeHeader && !lineEof);

          /* Previous entry */
          if (previous_save_[0] != '\0') {
            int pathLen = (int) strlen(index->path);

            if (pathLen > 0 && strncmp(previous_save_, index->path, pathLen) == 0) {    // old (<3.40) buggy format
              strcpybuff(previous_save, previous_save_);
            }
            // relative ? (hack)
            else if (index->safeCache || (previous_save_[0] != '/'      // /home/foo/bar.gif
                                          && (!isalpha(previous_save_[0]) || previous_save_[1] != ':')) // c:/home/foo/bar.gif
              ) {
              index->safeCache = 1;
              snprintf(previous_save, sizeof(previous_save), "%s%s",
                       index->path, previous_save_);
            }
            // bogus format (includes buggy absolute path)
            else {
              /* guess previous path */
              if (index->fixedPath == 0) {
                const char *start = jump_protocol_and_auth(url);
                const char *end = start ? strchr(start, '/') : NULL;
                int len = (int) (end - start);

                if (start != NULL && end != NULL && len > 0 && len < 128) {
                  char piece[128 + 2];
                  const char *where;

                  piece[0] = '\0';
                  strncat(piece, start, len);
                  if ((where = strstr(previous_save_, piece)) != NULL) {
                    index->fixedPath = (int) (where - previous_save_);  // offset to relative path
                  }
                }
              }
              if (index->fixedPath > 0) {
                int saveLen = (int) strlen(previous_save_);

                if (index->fixedPath < saveLen) {
                  snprintf(previous_save, sizeof(previous_save), "%s%s",
                           index->path, previous_save_ + index->fixedPath);
                } else {
                  PT_Element_failf(
                      r, "Bogus fixePath prefix for %s (prefixLen=%d)",
                      previous_save_, (int) index->fixedPath);
                  r->statuscode = STATUSCODE_INVALID;
                }
              } else {
                snprintf(previous_save, sizeof(previous_save), "%s%s",
                         index->path, previous_save_);
              }
            }
          }

          /* Corrupt: negative, or in-cache at or above what the 32-bit zip
             writer could store. A headers-only entry above INT_MAX is
             legitimate; one wider than size_t is refused, not truncated. */
          if (declared_size < 0 ||
              (LLint) (size_t) declared_size != declared_size ||
              (dataincache && declared_size >= INT_MAX)) {
            r->statuscode = STATUSCODE_INVALID;
            strcpybuff(r->msg, "Cache Read Error : Bad Size");
          } else {
            r->size = (size_t) declared_size;
          }

          /* Complete fields */
          r->adr = NULL;
          if (r->statuscode != STATUSCODE_INVALID) {    /* Can continue */
            int ok = 0;

            // Court-circuit:
            // Peut-on stocker le fichier directement sur disque?
            if (ok) {
              if (r->msg[0] == '\0') {
                strcpybuff(r->msg, "Cache Read Error : Unexpected error");
              }
            } else {            // lire en mémoire

              if (!dataincache) {
                /* Read in memory from cache */
                if (flags & FETCH_BODY) {
                  if (strnotempty(previous_save)) {
                    FILE *fp = fopen(file_convert(catbuff, sizeof(catbuff), previous_save), "rb");

                    if (fp != NULL) {
                      r->adr = (char *) malloc(r->size + 1);
                      if (r->adr != NULL) {
                        if (r->size > 0 &&
                            !hts_fread_exact(r->adr, r->size, fp)) {
                          int last_errno = errno;

                          r->statuscode = STATUSCODE_INVALID;
                          PT_Element_failf(r,
                                           "Read error in cache disk data: %s",
                                           strerror(last_errno));
                        }
                        r->adr[r->size] = '\0';
                      } else {
                        r->statuscode = STATUSCODE_INVALID;
                        strcpybuff(r->msg,
                                   "Read error (memory exhausted) from cache");
                      }
                      fclose(fp);
                    } else {
                      r->statuscode = STATUSCODE_INVALID;
                      PT_Element_failf(
                          r, "Read error (can't open '%s') from cache",
                          file_convert(catbuff, sizeof(catbuff),
                                       previous_save));
                    }
                  } else {
                    r->statuscode = STATUSCODE_INVALID;
                    strcpybuff(r->msg, "Cached file name is invalid");
                  }
                }
              } else {
                // lire fichier (d'un coup)
                if (flags & FETCH_BODY) {
                  r->adr = (char *) malloc(r->size + 1);
                  if (r->adr != NULL) {
                    if (unzReadCurrentFile(index->zFile, r->adr, (unsigned int) r->size) != r->size) {  // erreur
                      free(r->adr);
                      r->adr = NULL;
                      r->statuscode = STATUSCODE_INVALID;
                      strcpybuff(r->msg, "Cache Read Error : Read Data");
                    } else
                      *(r->adr + r->size) = '\0';
                  } else {      // erreur
                    r->statuscode = STATUSCODE_INVALID;
                    strcpybuff(r->msg, "Cache Memory Error");
                  }
                }
              }
            }
          }                     // si save==null, ne rien charger (juste en tête)
        } else {
          r->statuscode = STATUSCODE_INVALID;
          strcpybuff(r->msg, "Cache Read Error : Read Header Data");
        }
        unzCloseCurrentFile(index->zFile);
      } else {
        r->statuscode = STATUSCODE_INVALID;
        strcpybuff(r->msg, "Cache Read Error : Open File");
      }

    } else {
      r->statuscode = STATUSCODE_INVALID;
      strcpybuff(r->msg, "Cache Read Error : Bad Offset");
    }
  } else {
    r->statuscode = STATUSCODE_INVALID;
    strcpybuff(r->msg, "File Cache Entry Not Found");
  }
  if (r->location[0] != '\0') {
    r->location = strdup(r->location);
  } else {
    r->location = NULL;
  }
  return r;
}

/* Bytes a writer may take from an element: a reader that could not fetch the
   body still hands back the declared size (#931). */
static size_t PT_Element_BodySize(const PT_Element element) {
  return element->adr != NULL ? element->size : 0;
}

static int PT_SaveCache__New_Fun(void *arg, const char *url, PT_Element element) {
  zipFile zFileOut = (zipFile) arg;
  const size_t body_size = PT_Element_BodySize(element);
  char headers[8192];
  size_t headersSize = 0;
  int headersDropped = 0;
  zip_fileinfo fi;
  int zErr;
  const char *url_adr = "";
  const char *url_fil = "";

  /* Fields */
  headers[0] = '\0';
  /* */
  {
    const char *message;

    if (strlen(element->msg) < 32) {
      message = element->msg;
    } else {
      message = "(See X-StatusMessage)";
    }
    /* 64 characters MAX for first line */
    if (!slcatprintfbuff(headers, sizeof(headers), &headersSize,
                         "HTTP/1.%c %d %s\r\n", '1', element->statuscode,
                         message)) {
      headersDropped++;
    }
  }

  /* Second line MUST ALWAYS be X-In-Cache */
  ZIP_FIELD_INT_FORCE(headers, headersSize, headersDropped, "X-In-Cache", 1);
  ZIP_FIELD_INT(headers, headersSize, headersDropped, "X-StatusCode",
                element->statuscode);
  ZIP_FIELD_STRING(headers, headersSize, headersDropped, "X-StatusMessage",
                   element->msg);
  ZIP_FIELD_INT(headers, headersSize, headersDropped, "X-Size",
                body_size); // size
  ZIP_FIELD_STRING(headers, headersSize, headersDropped, "Content-Type",
                   element->contenttype); // contenttype
  ZIP_FIELD_STRING(headers, headersSize, headersDropped, "X-Charset",
                   element->charset); // contenttype
  ZIP_FIELD_STRING(headers, headersSize, headersDropped, "Last-Modified",
                   element->lastmodified); // last-modified
  ZIP_FIELD_STRING(headers, headersSize, headersDropped, "Etag",
                   element->etag); // Etag
  ZIP_FIELD_STRING(headers, headersSize, headersDropped, "Location",
                   element->location); // 'location' pour moved
  ZIP_FIELD_STRING(headers, headersSize, headersDropped, "Content-Disposition",
                   element->cdispo); // Content-disposition
  ZIP_FIELD_STRING(headers, headersSize, headersDropped, "X-Addr",
                   url_adr); // Original address
  ZIP_FIELD_STRING(headers, headersSize, headersDropped, "X-Fil",
                   url_fil); // Original URI filename
  ZIP_FIELD_STRING(headers, headersSize, headersDropped, "X-Save",
                   ""); // Original save filename
  if (headersDropped != 0) {
    fprintf(stderr, "Headers of %s: %d field(s) dropped" LF, url,
            headersDropped);
  }

  /* Time */
  memset(&fi, 0, sizeof(fi));
  if (element->lastmodified[0] != '\0') {
    struct tm buffer;
    struct tm *tm_s = convert_time_rfc822(&buffer, element->lastmodified);

    if (tm_s) {
      fi.tmz_date.tm_sec = (uInt) tm_s->tm_sec;
      fi.tmz_date.tm_min = (uInt) tm_s->tm_min;
      fi.tmz_date.tm_hour = (uInt) tm_s->tm_hour;
      fi.tmz_date.tm_mday = (uInt) tm_s->tm_mday;
      fi.tmz_date.tm_mon = (uInt) tm_s->tm_mon;
      fi.tmz_date.tm_year = (uInt) tm_s->tm_year;
    }
  }

  /* Open file - NOTE: headers in "comment" */
  if ((zErr = zipOpenNewFileInZip(zFileOut, url, &fi,
                                  /* 
                                     Store headers in realtime in the local file directory as extra field
                                     In case of crash, we'll be able to recover the whole ZIP file by rescanning it
                                   */
                                  headers, (uInt) strlen(headers), NULL, 0, NULL,       /* comment */
                                  Z_DEFLATED, Z_DEFAULT_COMPRESSION)) != Z_OK) {
    assertf(! "zip_zipOpenNewFileInZip_failed");
  }

  /* Write data in cache */
  if (body_size != 0) {
    if ((zErr = zipWriteInFileInZip(zFileOut, element->adr, (int) body_size)) !=
        Z_OK) {
      assertf(! "zip_zipWriteInFileInZip_failed");
    }
  }

  /* Close */
  if ((zErr = zipCloseFileInZip(zFileOut)) != Z_OK) {
    assertf(! "zip_zipCloseFileInZip_failed");
  }

  /* Flush */
  if ((zErr = zipFlush(zFileOut)) != 0) {
    assertf(! "zip_zipFlush_failed");
  }

  return 0;
}

static int PT_SaveCache__New(PT_Indexes indexes, const char *filename) {
  zipFile zFileOut = zipOpen(filename, 0);

  if (zFileOut != NULL) {
    int ret = PT_EnumCache(indexes, PT_SaveCache__New_Fun, (void *) zFileOut);

    zipClose(zFileOut,
             "Created by HTTrack Website Copier/ProxyTrack "
             PROXYTRACK_VERSION);
    zFileOut = NULL;
    if (ret != 0)
      (void) unlink(filename);
    return ret;
  }
  return -1;
}

/* ------------------------------------------------------------ */
/* Old HTTrack cache (dat/ndx) format                           */
/* ------------------------------------------------------------ */

static int cache_brstr(char *adr, char *s, size_t s_size) {
  int i;
  int off;
  char buff[256 + 4];

  off = binput(adr, buff, 256, NULL);
  /* no length-prefixed value follows a field the terminating NUL stopped */
  if (adr[off] == '\0') {
    s[0] = '\0';
    return off;
  }
  /* an empty/non-numeric field leaves i unset: treat as length 0 */
  if (sscanf(buff, "%d", &i) != 1 || i < 0 || i > 32768)
    i = 0;
  if (i > 0) {
    /* a corrupt cache may declare a length past the buffer end; bound the copy
       and the advance to the bytes actually present */
    const size_t avail = strnlen(adr + off, (size_t) i);
    const size_t store = avail < s_size ? avail : s_size - 1;

    memcpy(s, adr + off, store);
    s[store] = '\0';
    off += (int) avail;
  } else {
    s[0] = '\0';
  }
  return off;
}

static void cache_rstr(FILE *fp, char *s, size_t s_size) {
  INTsys i = 0;
  char buff[256 + 4];

  linput(fp, buff, 256);
  /* an unmatched sscanf leaves i untouched: length 0, not a stack value */
  if (sscanf(buff, INTsysP, &i) != 1 || i < 0 || i > 32768)
    i = 0;
  if (i > 0) {
    /* store at most s_size-1 bytes, but consume all i bytes so the next field
       stays aligned when a tampered cache declares more than the destination
       holds */
    const size_t want = (size_t) i;
    const size_t store = want < s_size ? want : s_size - 1;

    if (!hts_fread_exact(s, store, fp)) {
      assertf(! "fread_cache_failed");
    }
    if (want > store && fseek(fp, (long) (want - store), SEEK_CUR) != 0) {
      assertf(!"fseek_cache_failed");
    }
    s[store] = '\0';
  } else {
    s[0] = '\0';
  }
}

static char *cache_rstr_addr(FILE * fp) {
  INTsys i = 0;
  char *addr = NULL;
  char buff[256 + 4];

  linput(fp, buff, 256);
  /* an unmatched sscanf leaves i untouched: length 0, not a stack value */
  if (sscanf(buff, INTsysP, &i) != 1 || i < 0 || i > 32768)
    i = 0;
  if (i > 0) {
    addr = malloc((size_t) i + 1);
    if (addr != NULL) {
      if (!hts_fread_exact(addr, (size_t) i, fp)) {
        assertf(! "fread_cache_failed");
      }
      *(addr + i) = '\0';
    }
  }
  return addr;
}

/* Allocate n+1 bytes for an n-byte cache body, rejecting a hostile/corrupt
   length that would demand an absurd allocation or wrap n+1. NULL on refuse. */
static char *cache_alloc_body(size_t n) {
  if (n >= (size_t) INT32_MAX)
    return NULL;
  return (char *) malloc(n + 1);
}

static void cache_rint(FILE * fp, int *i) {
  char s[256];

  cache_rstr(fp, s, sizeof(s));
  if (sscanf(s, "%d", i) != 1)
    *i = 0;
}

static void cache_rLLint(FILE * fp, unsigned long *i) {
  int l;
  char s[256];

  cache_rstr(fp, s, sizeof(s));
  /* reject a non-numeric or negative field: a negative length cast to unsigned
     would wrap a later malloc(size+1) to a tiny buffer */
  if (sscanf(s, "%d", &l) != 1 || l < 0)
    l = 0;
  *i = (unsigned long) l;
}

static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
  if (index_ != NULL && filename != NULL) {
    char *pos = strrchr(filename, '.');
    PT_Index__Old cache = &index_->slots.formatOld;
    long int ndxSize;

    cache->filenameDat[0] = '\0';
    cache->filenameNdx[0] = '\0';
    /* before the first early return: the readers lock it and the destructor
       frees it, whether or not we get as far as opening a file */
    MutexInit(&cache->fileLock);

    index_base_path(cache->path, sizeof(cache->path), filename);

    /* Index/data filenames */
    if (pos != NULL) {
      const size_t nLen = (size_t) (pos - filename);

      /* a base clipped to fit would name a different pair of files */
      if (nLen > sizeof(cache->filenameDat) - sizeof(".dat")) {
        return 0;
      }
      strlncatbuff(cache->filenameDat, filename, sizeof(cache->filenameDat),
                   nLen);
      strlncatbuff(cache->filenameNdx, filename, sizeof(cache->filenameNdx),
                   nLen);
      strcatbuff(cache->filenameDat, ".dat");
      strcatbuff(cache->filenameNdx, ".ndx");
    }
    ndxSize = filesize(cache->filenameNdx);
    cache->timestamp = file_timestamp(cache->filenameDat);
    cache->dat = fopen(cache->filenameDat, "rb");
    cache->ndx = fopen(cache->filenameNdx, "rb");
    if (cache->dat != NULL && cache->ndx != NULL && ndxSize > 0) {
      char *use = malloc(ndxSize + 1);

      if (hts_fread_exact(use, (size_t) ndxSize, cache->ndx)) {
        char firstline[256];
        char *a = use;

        use[ndxSize] = '\0';
        a += cache_brstr(a, firstline, sizeof(firstline));
        if (strncmp(firstline, "CACHE-", 6) == 0) {     // Nouvelle version du cache
          if (strncmp(firstline, "CACHE-1.", 8) == 0) { // Version 1.1x
            cache->version = (int) (firstline[8] - '0');        // cache 1.x
            if (cache->version <= 5) {
              a += cache_brstr(a, firstline, sizeof(firstline));
              strcpybuff(cache->lastmodified, firstline);
            } else {
              fclose(cache->dat);
              cache->dat = NULL;
              free(use);
              use = NULL;
            }
          } else { // non supporté
            fclose(cache->dat);
            cache->dat = NULL;
            free(use);
            use = NULL;
          }
          /* */
        } else {                // Vieille version du cache
          /* */
          cache->version = 0;   // cache 1.0
          strcpybuff(cache->lastmodified, firstline);
        }

        /* Create hash table for the cache (MUCH FASTER!) */
        if (use) {
          char line[HTS_URLMAXSIZE * 2];
          char linepos[256];
          int pos;
          int firstSeen = 0;

          while((a != NULL) && (a < (use + ndxSize))) {
            a = strchr(a + 1, '\n');    /* start of line */
            if (a) {
              a++;
              /* read "host/file" */
              a += binput(a, line, HTS_URLMAXSIZE, NULL);
              {
                /* binput writes its NUL at s[max], so the second field must be
                   bounded by what is left of line[], not by the same constant
                 */
                const size_t used = strlen(line);

                a += binput(a, line + used, (int) (sizeof(line) - used - 1),
                            NULL);
              }
              /* read position */
              a += binput(a, linepos, 200, NULL);
              /* an unparseable field must not carry the previous entry's
                 offset over, nor read the stack on the first one */
              if (sscanf(linepos, "%d", &pos) != 1)
                pos = 0;

              /* Add entry */
              coucal_add(cache->hash, line, pos);

              /* First link as starting URL */
              if (!firstSeen) {
                if (strstr(line, "/robots.txt") == NULL) {
                  PT_Index__Old index = cache;
                  const char *scheme =
                      link_has_authority(line) ? "" : "http://";

                  /* dropped rather than truncated; try the next entry */
                  if (sprintfbuff(index->startUrl, "%s%s", scheme, line))
                    firstSeen = 1;
                  else
                    index->startUrl[0] = '\0';
                }
              }

            }
          }
          /* Not needed anymore! */
          free(use);
          use = NULL;
          return 1;
        }
      }
    }
  }
  return 0;
}

static String DecodeUrl(const char *url) {
  int i;
  String s = STRING_EMPTY;

  StringClear(s);
  for(i = 0; url[i] != '\0'; i++) {
    if (url[i] == '+') {
      StringAddchar(s, ' ');
    } else if (url[i] == '%') {
      if (url[i + 1] == '%') {
        StringAddchar(s, '%');
        i++;
      } else if (url[i + 1] != 0 && url[i + 2] != 0) {
        char tmp[3];
        int codepoint = 0;

        tmp[0] = url[i + 1];
        tmp[1] = url[i + 2];
        tmp[2] = 0;
        if (sscanf(tmp, "%x", &codepoint) == 1) {
          StringAddchar(s, (char) codepoint);
        }
        i += 2;
      }
    } else {
      StringAddchar(s, url[i]);
    }
  }
  return s;
}

static PT_Element PT_ReadCache__Old(PT_Index index, const char *url, int flags) {
  PT_Element retCode;

  MutexLock(&index->slots.formatOld.fileLock);
  {
    retCode = PT_ReadCache__Old_u(index, url, flags);
  }
  MutexUnlock(&index->slots.formatOld.fileLock);
  return retCode;
}

static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
                                      int flags) {
  PT_Index__Old cache = (PT_Index__Old) & index_->slots.formatOld;
  intptr_t hash_pos;
  int hash_pos_return;
  char location_default[HTS_URLMAXSIZE * 2];
  char previous_save[HTS_URLMAXSIZE * 2];
  char previous_save_[HTS_URLMAXSIZE * 2];
  PT_Element r;
  int ok = 0;

  if (cache == NULL || cache->hash == NULL || url == NULL || *url == 0)
    return NULL;
  if ((r = PT_ElementNew()) == NULL)
    return NULL;
  location_default[0] = '\0';
  previous_save[0] = previous_save_[0] = '\0';
  memset(r, 0, sizeof(_PT_Element));
  r->location = location_default;
  r->location[0] = '\0';
  if (strncmp(url, "http://", 7) == 0)
    url += 7;
  hash_pos_return = coucal_read(cache->hash, url, &hash_pos);

  if (hash_pos_return) {
    int pos = (int) hash_pos;   /* simply */

    if (fseek(cache->dat, (pos > 0) ? pos : (-pos), SEEK_SET) == 0) {
      /* Importer cache1.0 */
      if (cache->version == 0) {
        OLD_htsblk old_r;

        if (hts_fread_exact(
                &old_r, sizeof(old_r),
                cache->dat)) { // lire tout (y compris statuscode etc)
          int i;
          String urlDecoded;

          r->statuscode = old_r.statuscode;
          r->size = old_r.size > 0 ? (size_t) old_r.size : 0; // taille fichier
          /* these fixed fields come straight from fread and carry no guaranteed
             NUL; terminate them before the bounded copy */
          old_r.msg[sizeof(old_r.msg) - 1] = '\0';
          old_r.contenttype[sizeof(old_r.contenttype) - 1] = '\0';
          strcpybuff(r->msg, old_r.msg);
          strcpybuff(r->contenttype, old_r.contenttype);

          /* Guess the destination filename.. this sucks, because this method is not reliable.
             Yes, the old 1.0 cache format was *that* bogus. /rx */
#define FORBIDDEN_CHAR(c) (c == '~' \
	|| c == '\\' \
	|| c == ':' \
	|| c == '*' \
	|| c == '?' \
	|| c == '\"' \
	|| c == '<' \
	|| c == '>' \
	|| c == '|' \
	|| c == '@' \
	|| ((unsigned char) c ) <= 31 \
	|| ((unsigned char) c ) == 127 \
	)
          urlDecoded = DecodeUrl(jump_protocol_and_auth(url));
          strcpy(previous_save_, StringBuff(urlDecoded));
          StringFree(urlDecoded);
          for(i = 0; previous_save_[i] != '\0' && previous_save_[i] != '?'; i++) {
            if (FORBIDDEN_CHAR(previous_save_[i])) {
              previous_save_[i] = '_';
            }
          }
          previous_save_[i] = '\0';
#undef FORBIDDEN_CHAR
          ok = 1;               /* import  ok */
        }
        /* */
        /* Cache 1.1 */
      } else {
        char check[256];
        unsigned long size_read;
        unsigned long int size_;

        check[0] = '\0';
        //
        cache_rint(cache->dat, &r->statuscode);
        cache_rLLint(cache->dat, &size_);
        r->size = (size_t) size_;
        cache_rstr(cache->dat, r->msg, sizeof(r->msg));
        cache_rstr(cache->dat, r->contenttype, sizeof(r->contenttype));
        if (cache->version >= 3)
          cache_rstr(cache->dat, r->charset, sizeof(r->charset));
        cache_rstr(cache->dat, r->lastmodified, sizeof(r->lastmodified));
        cache_rstr(cache->dat, r->etag, sizeof(r->etag));
        cache_rstr(cache->dat, r->location, sizeof(location_default));
        if (cache->version >= 2)
          cache_rstr(cache->dat, r->cdispo, sizeof(r->cdispo));
        if (cache->version >= 4) {
          cache_rstr(cache->dat, previous_save_, sizeof(previous_save_)); // adr
          cache_rstr(cache->dat, previous_save_, sizeof(previous_save_)); // fil
          previous_save[0] = '\0';
          cache_rstr(cache->dat, previous_save_,
                     sizeof(previous_save_)); // save
        }
        if (cache->version >= 5) {
          r->headers = cache_rstr_addr(cache->dat);
        }
        //
        cache_rstr(cache->dat, check, sizeof(check));
        if (strcmp(check, "HTS") == 0) {        /* intégrité OK */
          ok = 1;
        }
        cache_rLLint(cache->dat, &size_read);   /* lire size pour être sûr de la taille déclarée (réécrire) */
        if (size_read > 0) {    /* si inscrite ici */
          r->size = size_read;
        } else {                /* pas de données directement dans le cache, fichier présent? */
          r->size = 0;
        }
      }

      /* Check destination filename */

      {
        PT_Index__Old index = cache;

        /* -------------------- COPY OF THE __New() CODE -------------------- */
        if (previous_save_[0] != '\0') {
          int pathLen = (int) strlen(index->path);

          if (pathLen > 0 && strncmp(previous_save_, index->path, pathLen) == 0) {      // old (<3.40) buggy format
            strcpybuff(previous_save, previous_save_);
          }
          // relative ? (hack)
          else if (index->safeCache || (previous_save_[0] != '/'        // /home/foo/bar.gif
                                        && (!isalpha(previous_save_[0]) || previous_save_[1] != ':'))   // c:/home/foo/bar.gif
            ) {
            index->safeCache = 1;
            snprintf(previous_save, sizeof(previous_save), "%s%s", index->path,
                     previous_save_);
          }
          // bogus format (includes buggy absolute path)
          else {
            /* guess previous path */
            if (index->fixedPath == 0) {
              const char *start = jump_protocol_and_auth(url);
              const char *end = start ? strchr(start, '/') : NULL;
              int len = (int) (end - start);

              if (start != NULL && end != NULL && len > 0 && len < 128) {
                char piece[128 + 2];
                const char *where;

                piece[0] = '\0';
                strncat(piece, start, len);
                if ((where = strstr(previous_save_, piece)) != NULL) {
                  index->fixedPath = (int) (where - previous_save_);    // offset to relative path
                }
              }
            }
            if (index->fixedPath > 0) {
              int saveLen = (int) strlen(previous_save_);

              if (index->fixedPath < saveLen) {
                snprintf(previous_save, sizeof(previous_save), "%s%s",
                         index->path, previous_save_ + index->fixedPath);
              } else {
                PT_Element_failf(r,
                                 "Bogus fixePath prefix for %s (prefixLen=%d)",
                                 previous_save_, (int) index->fixedPath);
                r->statuscode = STATUSCODE_INVALID;
              }
            } else {
              snprintf(previous_save, sizeof(previous_save), "%s%s",
                       index->path, previous_save_);
            }
          }
        }
        /* -------------------- END OF COPY OF THE __New() CODE -------------------- */
      }

      /* Read data */
      if (ok) {
        r->adr = NULL;
        if ((r->statuscode >= 0) && (r->statuscode <= 999)) {
          r->adr = NULL;
          if (pos < 0) {
            if (flags & FETCH_BODY) {
              FILE *fp = fopen(previous_save, "rb");

              if (fp != NULL) {
                r->adr = cache_alloc_body(r->size);
                if (r->adr != NULL) {
                  if (r->size > 0 && !hts_fread_exact(r->adr, r->size, fp)) {
                    r->statuscode = STATUSCODE_INVALID;
                    strcpybuff(r->msg, "Read error in cache disk data");
                  }
                  r->adr[r->size] = '\0';
                } else {
                  r->statuscode = STATUSCODE_INVALID;
                  strcpybuff(r->msg,
                             "Read error (memory exhausted) from cache");
                }
                fclose(fp);
              } else {
                r->statuscode = STATUSCODE_INVALID;
                strcpybuff(r->msg, "Previous cache file not found (2)");
              }
            }
          } else {
            // lire fichier (d'un coup)
            if (flags & FETCH_BODY) {
              r->adr = cache_alloc_body(r->size);
              if (r->adr != NULL) {
                if (!hts_fread_exact(r->adr, r->size, cache->dat)) { // erreur
                  free(r->adr);
                  r->adr = NULL;
                  r->statuscode = STATUSCODE_INVALID;
                  strcpybuff(r->msg, "Cache Read Error : Read Data");
                } else
                  r->adr[r->size] = '\0';
              } else {          // erreur
                r->statuscode = STATUSCODE_INVALID;
                strcpybuff(r->msg, "Cache Memory Error");
              }
            }
          }
        } else {
          r->statuscode = STATUSCODE_INVALID;
          strcpybuff(r->msg, "Cache Read Error : Bad Data");
        }
      } else {                  // erreur
        r->statuscode = STATUSCODE_INVALID;
        strcpybuff(r->msg, "Cache Read Error : Read Header");
      }
    } else {
      r->statuscode = STATUSCODE_INVALID;
      strcpybuff(r->msg, "Cache Read Error : Seek Failed");
    }
  } else {
    r->statuscode = STATUSCODE_INVALID;
    strcpybuff(r->msg, "File Cache Entry Not Found");
  }
  if (r->location[0] != '\0') {
    r->location = strdup(r->location);
  } else {
    r->location = NULL;
  }
  return r;
}

static int PT_LookupCache__Old(PT_Index index, const char *url) {
  int retCode;

  MutexLock(&index->slots.formatOld.fileLock);
  {
    retCode = PT_LookupCache__Old_u(index, url);
  }
  MutexUnlock(&index->slots.formatOld.fileLock);
  return retCode;
}

static int PT_LookupCache__Old_u(PT_Index index_, const char *url) {
  if (index_ != NULL) {
    PT_Index__New cache = (PT_Index__New) & index_->slots.formatNew;

    if (cache == NULL || cache->hash == NULL || url == NULL || *url == 0)
      return 0;
    if (strncmp(url, "http://", 7) == 0)
      url += 7;
    if (coucal_read(cache->hash, url, NULL))
      return 1;
  }
  return 0;
}

/* ------------------------------------------------------------ */
/* Internet Archive Arc 1.0 (arc) format                        */
/* Xavier Roche (roche@httrack.com)                             */
/* Lars Clausen (lc@statsbiblioteket.dk)                        */
/* ------------------------------------------------------------ */

#define ARC_SP ' '

static const char *getArcField(const char *line, int pos) {
  int i;

  for(i = 0; line[i] != '\0' && pos > 0; i++) {
    if (line[i] == ARC_SP)
      pos--;
  }
  if (pos == 0)
    return &line[i];
  return NULL;
}

static char *copyArcField(const char *line, int npos, char *dest, int destMax) {
  const char *pos;

  if ((pos = getArcField(line, npos)) != NULL) {
    int i;

    for(i = 0; pos[i] != '\0' && pos[i] != ARC_SP && (--destMax) > 0; i++) {
      dest[i] = pos[i];
    }
    dest[i] = 0;
    return dest;
  }
  dest[0] = 0;
  return NULL;
}

static int getArcLength(const char *line) {
  const char *pos;

  if ((pos = getArcField(line, 9)) != NULL
      || (pos = getArcField(line, 4)) != NULL
      || (pos = getArcField(line, 2)) != NULL) {
    int length;

    if (sscanf(pos, "%d", &length) == 1) {
      return length;
    }
  }
  return -1;
}

static int skipArcNl(FILE * file) {
  if (fgetc(file) == 0x0a) {
    return 0;
  }
  return -1;
}

/* Stop on the newline opening the first record: archives whose version block
   length swallowed the closing blank line leave only that one. */
static int skipArcVersionNl(FILE *file) {
  long int pos = ftell(file);

  if (pos < 0 || fgetc(file) != 0x0a) {
    return -1;
  }
  if (fgetc(file) == 0x0a) { /* the blank line, when outside the length */
    pos++;
  }
  return fseek(file, pos, SEEK_SET);
}

static int skipArcData(FILE * file, const char *line) {
  int jump = getArcLength(line);

  if (jump != -1) {
    if (fseek(file, jump, SEEK_CUR) == 0 /* && skipArcNl(file) == 0 */ ) {
      return 0;
    }
  }
  return -1;
}

static int getDigit(const char digit) {
  return (int) (digit - '0');
}

static int getDigit2(const char *const pos) {
  return getDigit(pos[0]) * 10 + getDigit(pos[1]);
}

static int getDigit4(const char *const pos) {
  return getDigit(pos[0]) * 1000 + getDigit(pos[1]) * 100 +
    getDigit(pos[2]) * 10 + getDigit(pos[3]);
}

static time_t getGMT(struct tm *tm) {
  time_t t = timegm(tm);

  if (t != (time_t) - 1 && t != (time_t) 0) {
    return t;
  }
  return (time_t) - 1;
}

static time_t getArcTimestamp(const char *const line) {
  const char *pos;

  if ((pos = getArcField(line, 2)) != NULL) {
    int i;

    /* date == YYYYMMDDhhmmss (Greenwich Mean Time) */
    /* example: 20050405154029  */
    for(i = 0; pos[i] >= '0' && pos[i] <= '9'; i++) ;
    if (i == 14) {
      struct tm tm;

      memset(&tm, 0, sizeof(tm));
      tm.tm_year = getDigit4(pos + 0) - 1900;   /* current year minus 1900 */
      tm.tm_mon = getDigit2(pos + 4) - 1;       /* 0 – 11 */
      tm.tm_mday = getDigit2(pos + 6);  /* 1 – 31 */
      tm.tm_hour = getDigit2(pos + 8);  /* 0 – 23 */
      tm.tm_min = getDigit2(pos + 10);  /* 0 – 59 */
      tm.tm_sec = getDigit2(pos + 12);  /* 0 – 59 */
      tm.tm_isdst = 0;
      return getGMT(&tm);
    }
  }
  return (time_t) - 1;
}

static int readArcURLRecord(PT_Index__Arc index) {
  index->line[0] = '\0';
  if (linput(index->file, index->line, sizeof(index->line) - 1)) {
    return 0;
  }
  return -1;
}

#define str_begins(str, sstr) ( strncmp(str, sstr, sizeof(sstr) - 1) == 0 )
static int PT_CompatibleScheme(const char *url) {
  return (str_begins(url, "http:")
          || str_begins(url, "https:")
          || str_begins(url, "ftp:")
          || str_begins(url, "file:"));
}

int PT_LoadCache__Arc(PT_Index index_, const char *filename) {
  if (index_ != NULL && filename != NULL) {
    PT_Index__Arc index = &index_->slots.formatArc;

    index->timestamp = file_timestamp(filename);
    MutexInit(&index->fileLock);
    index->file = fopen(filename, "rb");

    // Opened ?
    if (index->file != NULL) {
      coucal hashtable = index->hash;

      /* past LONG_MAX a 32-bit ftell cannot answer; keep serving the records
         it can still reach, with the bound no longer constraining */
      if (fseek(index->file, 0, SEEK_END) != 0 ||
          (index->fileSize = ftell(index->file)) < 0) {
        index->fileSize = LONG_MAX;
      }
      rewind(index->file);
      if (readArcURLRecord(index) == 0) {
        int entries = 0;

        /* Read first line */
        if (strncmp(index->line, "filedesc://", sizeof("filedesc://") - 1) != 0) {
          fprintf(stderr, "Unexpected bad signature #%s" LF, index->line);
          fclose(index->file);
          index->file = NULL;
          return 0;
        }
        /* Timestamp */
        index->timestamp = getArcTimestamp(index->line);
        /* Skip the version block, leaving the record loop its own separator */
        if (skipArcData(index->file, index->line) != 0 ||
            skipArcVersionNl(index->file) != 0) {
          fprintf(stderr, "Unexpected bad data offset size first entry" LF);
          fclose(index->file);
          index->file = NULL;
          return 0;
        }
        /* Read all meta-entries (not data) */
        while(!feof(index->file)) {
          unsigned long int fpos = ftell(index->file);

          if (skipArcNl(index->file) == 0 && readArcURLRecord(index) == 0) {
            int length = getArcLength(index->line);

            if (length >= 0) {
              const char *filenameIndex = copyArcField(index->line, 0,
                                                       index->filenameIndexBuff, sizeof(index->filenameIndexBuff) - 1); /* can not be NULL */

              if (strncmp(filenameIndex, "http://", 7) == 0) {
                filenameIndex += 7;
              }
              if (*filenameIndex != 0) {
                if (skipArcData(index->file, index->line) != 0) {
                  fprintf(stderr,
                          "Corrupted cache data entry #%d (truncated file?), aborting read"
                          LF, (int) entries);
                }
                /*fprintf(stdout, "adding %s [%d]\n", filenameIndex, (int)fpos); */
                if (PT_CompatibleScheme(index->filenameIndexBuff)) {
                  coucal_add(hashtable, filenameIndex, fpos);  /* position of meta-data */
                  entries++;
                }
              } else {
                fprintf(stderr, "Corrupted cache meta entry #%d" LF,
                        (int) entries);
              }
            } else {
              fprintf(stderr,
                      "Corrupted cache meta entry #%d, aborting read" LF,
                      (int) entries);
              break;
            }
          } else {
            break;
          }
        }

        /* OK */
        return 1;
      } else {
        fprintf(stderr, "Bad file (empty ?)" LF);
      }
    } else {
      fprintf(stderr, "Unable to open file" LF);
      index = NULL;
    }
  } else {
    fprintf(stderr, "Bad arguments" LF);
  }
  return 0;
}

/* Same contract as ZIP_READFIELD_STRING, for the ARC reader's header lines. */
#define HTTP_READFIELD_STRING(line, value, refline, refvalue, refvalue_size)   \
  do {                                                                         \
    if (line[0] != '\0' && strfield2(line, refline)) {                         \
      (void) strclipbuff(refvalue, refvalue_size, value);                      \
      line[0] = '\0';                                                          \
    }                                                                          \
  } while (0)
#define HTTP_READFIELD_INT(line, value, refline, refvalue) do { \
  if (line[0] != '\0' && strfield2(line, refline)) { \
    int intval = 0; \
    sscanf(value, "%d", &intval); \
    (refvalue) = intval; \
    line[0] = '\0'; \
	} \
} while(0)

static PT_Element PT_ReadCache__Arc(PT_Index index, const char *url, int flags) {
  PT_Element retCode;

  MutexLock(&index->slots.formatArc.fileLock);
  {
    retCode = PT_ReadCache__Arc_u(index, url, flags);
  }
  MutexUnlock(&index->slots.formatArc.fileLock);
  return retCode;
}

static PT_Element PT_ReadCache__Arc_u(PT_Index index_, const char *url,
                                      int flags) {
  PT_Index__Arc index = (PT_Index__Arc) & index_->slots.formatArc;
  char location_default[HTS_URLMAXSIZE * 2];
  intptr_t hash_pos;
  int hash_pos_return;
  PT_Element r = NULL;

  if (index == NULL || index->hash == NULL || url == NULL || *url == 0)
    return NULL;
  if ((r = PT_ElementNew()) == NULL)
    return NULL;
  location_default[0] = '\0';
  memset(r, 0, sizeof(_PT_Element));
  r->location = location_default;
  r->location[0] = '\0';
  if (strncmp(url, "http://", 7) == 0)
    url += 7;
  hash_pos_return = coucal_read(index->hash, url, &hash_pos);

  if (hash_pos_return) {
    if (fseek(index->file, (long) hash_pos, SEEK_SET) == 0) {
      if (skipArcNl(index->file) == 0 && readArcURLRecord(index) == 0) {
        long int fposMeta = ftell(index->file);
        int dataLength = getArcLength(index->line);
        const char *pos;

        /* Read HTTP headers */
        /* HTTP/1.1 404 Not Found */
        if (linput(index->file, index->line, sizeof(index->line) - 1)) {
          if ((pos = getArcField(index->line, 1)) != NULL) {
            if (sscanf(pos, "%d", &r->statuscode) != 1) {
              r->statuscode = STATUSCODE_INVALID;
            }
          }
          if ((pos = getArcField(index->line, 2)) != NULL) {
            r->msg[0] = '\0';
            strncatbuff(r->msg, pos, sizeof(r->msg) - 1);
          }
          while(linput(index->file, index->line, sizeof(index->line) - 1)
                && index->line[0] != '\0') {
            char *const line = index->line;
            char *value = strchr(line, ':');

            if (value != NULL) {
              *value = '\0';
              for(value++; *value == ' ' || *value == '\t'; value++) ;
              HTTP_READFIELD_INT(line, value, "Content-Length", r->size);       // size
              HTTP_READFIELD_STRING(line, value, "Content-Type", r->contenttype,
                                    sizeof(r->contenttype));
              HTTP_READFIELD_STRING(line, value, "Last-Modified",
                                    r->lastmodified, sizeof(r->lastmodified));
              HTTP_READFIELD_STRING(line, value, "Etag", r->etag,
                                    sizeof(r->etag));
              HTTP_READFIELD_STRING(line, value, "Location", r->location,
                                    sizeof(location_default));
              HTTP_READFIELD_STRING(line, value, "Content-Disposition",
                                    r->cdispo, sizeof(r->cdispo));
              if (line[0] != '\0') {
                const int len = r->headers ? ((int) strlen(r->headers)) : 0;
                /* one byte more than the four appends below write */
                const int nlen = (int) (strlen(line) + 2 + strlen(value) +
                                        sizeof("\r\n") + 1);
                char *const grown = realloct(r->headers, len + nlen);

                /* keep the headers read so far rather than losing them */
                if (grown != NULL) {
                  r->headers = grown;
                  r->headers[len] = '\0';
                  strcat(r->headers, line);
                  strcat(r->headers, ": ");
                  strcat(r->headers, value);
                  strcat(r->headers, "\r\n");
                }
              }
            }
          }

          /* FIXME charset */
          if (r->contenttype[0] != '\0') {
            char *pos = strchr(r->contenttype, ';');

            if (pos != NULL) {
              /*char *chs = strchr(pos, "charset="); */
              /*HTTP_READFIELD_STRING(line, value, "X-Charset", r->charset); */
              *pos = 0;
              if ((pos = strchr(r->contenttype, ' ')) != NULL) {
                *pos = 0;
              }
            }
          }

          /* Read data */
          if (r->statuscode != STATUSCODE_INVALID) {    /* Can continue */
            if (flags & FETCH_BODY) {
              long int fposCurrent = ftell(index->file);
              long int metaSize = fposCurrent - fposMeta;
              long int fetchSize = (long int) r->size;

              if (fetchSize <= 0) {
                fetchSize = dataLength - metaSize;
              }
              /* the declared body may exceed the archive we allocate it from */
              if (fetchSize < 0 || fetchSize > dataLength - metaSize ||
                  fetchSize > index->fileSize - fposCurrent) {
                r->statuscode = STATUSCODE_INVALID;
                strcpybuff(r->msg, "Cache Read Error : Truncated Data");
              }
              r->size = 0;
              if (r->statuscode != STATUSCODE_INVALID) {
                r->adr = (char *) malloc(fetchSize);
                if (r->adr != NULL) {
                  if (fetchSize > 0
                      && (r->size =
                          (int) fread(r->adr, 1, fetchSize,
                                      index->file)) != fetchSize) {
                    int last_errno = errno;

                    r->statuscode = STATUSCODE_INVALID;
                    PT_Element_failf(r, "Read error in cache disk data: %s",
                                     strerror(last_errno));
                  }
                } else {
                  r->statuscode = STATUSCODE_INVALID;
                  strcpybuff(r->msg,
                             "Read error (memory exhausted) from cache");
                }
              }
            }
          }

        } else {
          r->statuscode = STATUSCODE_INVALID;
          strcpybuff(r->msg, "Cache Read Error : Read Header Error");
        }

      } else {
        r->statuscode = STATUSCODE_INVALID;
        strcpybuff(r->msg, "Cache Read Error : Read Header Error");
      }
    } else {
      r->statuscode = STATUSCODE_INVALID;
      strcpybuff(r->msg, "Cache Read Error : Seek Error");
    }

  } else {
    r->statuscode = STATUSCODE_INVALID;
    strcpybuff(r->msg, "File Cache Entry Not Found");
  }
  if (r->location[0] != '\0') {
    r->location = strdup(r->location);
  } else {
    r->location = NULL;
  }
  return r;
}

static int PT_LookupCache__Arc(PT_Index index, const char *url) {
  int retCode;

  MutexLock(&index->slots.formatArc.fileLock);
  {
    retCode = PT_LookupCache__Arc_u(index, url);
  }
  MutexUnlock(&index->slots.formatArc.fileLock);
  return retCode;
}

static int PT_LookupCache__Arc_u(PT_Index index_, const char *url) {
  if (index_ != NULL) {
    PT_Index__New cache = (PT_Index__New) & index_->slots.formatNew;

    if (cache == NULL || cache->hash == NULL || url == NULL || *url == 0)
      return 0;
    if (strncmp(url, "http://", 7) == 0)
      url += 7;
    if (coucal_read(cache->hash, url, NULL))
      return 1;
  }
  return 0;
}

typedef struct PT_SaveCache__Arc_t {
  PT_Indexes indexes;
  FILE *fp;
  time_t t;
  char filename[64];
  struct tm buff;
  char headers[8192];
  char md5[32 + 2];
} PT_SaveCache__Arc_t;

/* Append src to an .arc header block of capacity size, clipping what does not
   fit: the values come from a cache entry, so shortening beats dropping it.
   HTS_FALSE when it had to clip. */
static hts_boolean arc_headers_cat(char *headers, size_t size,
                                   const char *src) {
  const size_t used = strlen(headers);
  const size_t left = used < size - 1 ? size - used - 1 : 0;

  if (left != 0) {
    strlncatbuff(headers, src, size, left);
  }
  return strlen(src) <= left ? HTS_TRUE : HTS_FALSE;
}

static int PT_SaveCache__Arc_Fun(void *arg, const char *url, PT_Element element) {
  PT_SaveCache__Arc_t *st = (PT_SaveCache__Arc_t *) arg;
  FILE *const fp = st->fp;
  const size_t body_size = PT_Element_BodySize(element);
  struct tm *tm = convert_time_rfc822(&st->buff, element->lastmodified);
  struct tm unknown_date;
  /* the two strcatbuff calls closing the block rely on these 4 bytes */
  const size_t room = sizeof(st->headers) - 4;
  hts_boolean fit;
  int size_headers;

  if (body_size != element->size) {
    fprintf(stderr, "Entry %s stored without its %lu-byte body" LF, url,
            (unsigned long) element->size);
  }

  /* a cached entry with no parseable Last-Modified must not take the writer
     down; the epoch is the conventional "date unknown" */
  if (tm == NULL) {
    memset(&unknown_date, 0, sizeof(unknown_date));
    unknown_date.tm_year = 70;
    unknown_date.tm_mday = 1;
    tm = &unknown_date;
  }

  fit = slprintfbuff(st->headers, room,
                     "HTTP/1.0 %d %s"
                     "\r\n"
                     "X-Server: ProxyTrack " PROXYTRACK_VERSION "\r\n"
                     "Content-type: %s%s%s%s"
                     "\r\n"
                     "Last-modified: %s"
                     "\r\n"
                     "Content-length: %d"
                     "\r\n",
                     element->statuscode, element->msg,
                     /**/ hts_effective_mime(element->contenttype),
                     (element->charset[0] ? "; charset=\"" : ""),
                     (element->charset[0] ? element->charset : ""),
                     (element->charset[0] ? "\"" : ""),
                     /**/ element->lastmodified, (int) body_size);
  if (element->location != NULL && element->location[0] != '\0') {
    if (!arc_headers_cat(st->headers, room, "Location: ") ||
        !arc_headers_cat(st->headers, room, element->location) ||
        !arc_headers_cat(st->headers, room, "\r\n")) {
      fit = HTS_FALSE;
    }
  }
  if (element->headers != NULL &&
      !arc_headers_cat(st->headers, room, element->headers)) {
    fit = HTS_FALSE;
  }
  /* a clip landing mid-line must still end it, or the body reads as a header */
  if (hts_lastchar(st->headers) != '\n') {
    strcatbuff(st->headers, "\r\n");
  }
  strcatbuff(st->headers, "\r\n");
  size_headers = (int) strlen(st->headers);
  if (!fit) {
    fprintf(stderr, "Headers of %s clipped to %d bytes" LF, url, size_headers);
  }

  /* doc == <nl><URL-record><nl><network_doc> */

  /* Format: URL IP date mime result checksum location offset filename length */
  if (body_size != 0) {
    domd5mem(element->adr, body_size, st->md5, 1);
  } else {
    strcpybuff(st->md5, "-");
  }
  fprintf(fp,
          /* nl */
          "\n"
          /* URL-record */
          "%s%s %s %04d%02d%02d%02d%02d%02d %s %d %s %s %ld %s %ld"
          /* nl */
          "\n",
          /* args */
          (link_has_authority(url) ? "" : "http://"), url, "0.0.0.0",
          tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour,
          tm->tm_min, tm->tm_sec, hts_effective_mime(element->contenttype),
          element->statuscode, st->md5,
          (element->location ? element->location : "-"), (long int) ftell(fp),
          st->filename, (long int) (size_headers + body_size));
  /* network_doc */
  if (!hts_fwrite_exact(st->headers, size_headers, fp) ||
      (body_size != 0 && !hts_fwrite_exact(element->adr, body_size, fp))) {
    return 1;                   /* Error */
  }

  return 0;
}

static int PT_SaveCache__Arc(PT_Indexes indexes, const char *filename) {
  FILE *fp = fopen(filename, "wb");

  if (fp != NULL) {
    PT_SaveCache__Arc_t st;
    int ret;
    time_t t = PT_GetTimeIndex(indexes);
    struct tm tm = PT_GetTime(t);

    /* version-2-block ==
       filedesc://<path><sp><ip_address><sp><date><sp>text/plain<sp>200<sp>-<sp>-<sp>0<sp><filename><sp><length><nl>
       2<sp><reserved><sp><origin-code><nl>
       URL<sp>IP-address<sp>Archive-date<sp>Content-type<sp>Result-code<sp>Checksum<sp>Location<sp> Offset<sp>Filename<sp>Archive-length<nl>
       <nl> */
    const char *prefix = "2 0 HTTrack Website Copier"
                         "\n"
                         "URL IP-address Archive-Date Content-Type Result-code "
                         "Checksum Location Offset Filename Archive-length"
                         "\n";
    sprintf(st.filename, "httrack_%d.arc", (int) t);
    fprintf(fp,
            "filedesc://%s 0.0.0.0 %04d%02d%02d%02d%02d%02d text/plain 200 - - 0 %s %d"
            "\n" "%s", st.filename, tm.tm_year + 1900, tm.tm_mon + 1,
            tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, st.filename,
            (int) strlen(prefix), prefix);
    /* the blank line closing the version block is a separator, outside the
       declared length */
    fputc('\n', fp);
    st.fp = fp;
    st.indexes = indexes;
    st.t = t;
    ret = PT_EnumCache(indexes, PT_SaveCache__Arc_Fun, (void *) &st);
    fclose(fp);
    if (ret != 0)
      (void) unlink(filename);
    return ret;
  }
  return -1;
}
