cache.c

   1 /* cache.c: cache management
   2  *
   3  * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com>
   4  *
   5  * Licensed under GNU General Public License v2
   6  *   (see COPYING for full license text)
   7  *
   8  *
   9  * The cache is just a directory structure where each file is a cache slot,
  10  * and each filename is based on the hash of some key (e.g. the cgit url).
  11  * Each file contains the full key followed by the cached content for that
  12  * key.
  13  *
  14  */
  15
  16 #ifdef HAVE_LINUX_SENDFILE
  17 #include <sys/sendfile.h>
  18 #endif
  19 #include "cgit.h"
  20 #include "cache.h"
  21 #include "html.h"
  22
  23 #define CACHE_BUFSIZE (1024 * 4)
  24
  25 struct cache_slot {
  26         const char *key;
  27         int keylen;
  28         int ttl;
  29         cache_fill_fn fn;
  30         int cache_fd;
  31         int lock_fd;
  32         const char *cache_name;
  33         const char *lock_name;
  34         int match;
  35         struct stat cache_st;
  36         int bufsize;
  37         char buf[CACHE_BUFSIZE];
  38 };
  39
  40 /* Open an existing cache slot and fill the cache buffer with
  41  * (part of) the content of the cache file. Return 0 on success
  42  * and errno otherwise.
  43  */
  44 static int open_slot(struct cache_slot *slot)
  45 {
  46         char *bufz;
  47         int bufkeylen = -1;
  48
  49         slot->cache_fd = open(slot->cache_name, O_RDONLY);
  50         if (slot->cache_fd == -1)
  51                 return errno;
  52
  53         if (fstat(slot->cache_fd, &slot->cache_st))
  54                 return errno;
  55
  56         slot->bufsize = xread(slot->cache_fd, slot->buf, sizeof(slot->buf));
  57         if (slot->bufsize < 0)
  58                 return errno;
  59
  60         bufz = memchr(slot->buf, 0, slot->bufsize);
  61         if (bufz)
  62                 bufkeylen = bufz - slot->buf;
  63
  64         slot->match = bufkeylen == slot->keylen &&
  65             !memcmp(slot->key, slot->buf, bufkeylen + 1);
  66
  67         return 0;
  68 }
  69
  70 /* Close the active cache slot */
  71 static int close_slot(struct cache_slot *slot)
  72 {
  73         int err = 0;
  74         if (slot->cache_fd > 0) {
  75                 if (close(slot->cache_fd))
  76                         err = errno;
  77                 else
  78                         slot->cache_fd = -1;
  79         }
  80         return err;
  81 }
  82
  83 /* Print the content of the active cache slot (but skip the key). */
  84 static int print_slot(struct cache_slot *slot)
  85 {
  86 #ifdef HAVE_LINUX_SENDFILE
  87         off_t start_off;
  88         int ret;
  89
  90         start_off = slot->keylen + 1;
  91
  92         do {
  93                 ret = sendfile(STDOUT_FILENO, slot->cache_fd, &start_off,
  94                                 slot->cache_st.st_size - start_off);
  95                 if (ret < 0) {
  96                         if (errno == EAGAIN || errno == EINTR)
  97                                 continue;
  98                         return errno;
  99                 }
 100                 return 0;
 101         } while (1);
 102 #else
 103         ssize_t i, j;
 104
 105         i = lseek(slot->cache_fd, slot->keylen + 1, SEEK_SET);
 106         if (i != slot->keylen + 1)
 107                 return errno;
 108
 109         do {
 110                 i = j = xread(slot->cache_fd, slot->buf, sizeof(slot->buf));
 111                 if (i > 0)
 112                         j = xwrite(STDOUT_FILENO, slot->buf, i);
 113         } while (i > 0 && j == i);
 114
 115         if (i < 0 || j != i)
 116                 return errno;
 117         else
 118                 return 0;
 119 #endif
 120 }
 121
 122 /* Check if the slot has expired */
 123 static int is_expired(struct cache_slot *slot)
 124 {
 125         if (slot->ttl < 0)
 126                 return 0;
 127         else
 128                 return slot->cache_st.st_mtime + slot->ttl * 60 < time(NULL);
 129 }
 130
 131 /* Check if the slot has been modified since we opened it.
 132  * NB: If stat() fails, we pretend the file is modified.
 133  */
 134 static int is_modified(struct cache_slot *slot)
 135 {
 136         struct stat st;
 137
 138         if (stat(slot->cache_name, &st))
 139                 return 1;
 140         return (st.st_ino != slot->cache_st.st_ino ||
 141                 st.st_mtime != slot->cache_st.st_mtime ||
 142                 st.st_size != slot->cache_st.st_size);
 143 }
 144
 145 /* Close an open lockfile */
 146 static int close_lock(struct cache_slot *slot)
 147 {
 148         int err = 0;
 149         if (slot->lock_fd > 0) {
 150                 if (close(slot->lock_fd))
 151                         err = errno;
 152                 else
 153                         slot->lock_fd = -1;
 154         }
 155         return err;
 156 }
 157
 158 /* Create a lockfile used to store the generated content for a cache
 159  * slot, and write the slot key + \0 into it.
 160  * Returns 0 on success and errno otherwise.
 161  */
 162 static int lock_slot(struct cache_slot *slot)
 163 {
 164         slot->lock_fd = open(slot->lock_name, O_RDWR | O_CREAT | O_EXCL,
 165                              S_IRUSR | S_IWUSR);
 166         if (slot->lock_fd == -1)
 167                 return errno;
 168         if (xwrite(slot->lock_fd, slot->key, slot->keylen + 1) < 0)
 169                 return errno;
 170         return 0;
 171 }
 172
 173 /* Release the current lockfile. If `replace_old_slot` is set the
 174  * lockfile replaces the old cache slot, otherwise the lockfile is
 175  * just deleted.
 176  */
 177 static int unlock_slot(struct cache_slot *slot, int replace_old_slot)
 178 {
 179         int err;
 180
 181         if (replace_old_slot)
 182                 err = rename(slot->lock_name, slot->cache_name);
 183         else
 184                 err = unlink(slot->lock_name);
 185
 186         if (err)
 187                 return errno;
 188
 189         return 0;
 190 }
 191
 192 /* Generate the content for the current cache slot by redirecting
 193  * stdout to the lock-fd and invoking the callback function
 194  */
 195 static int fill_slot(struct cache_slot *slot)
 196 {
 197         int tmp;
 198
 199         /* Preserve stdout */
 200         tmp = dup(STDOUT_FILENO);
 201         if (tmp == -1)
 202                 return errno;
 203
 204         /* Redirect stdout to lockfile */
 205         if (dup2(slot->lock_fd, STDOUT_FILENO) == -1)
 206                 return errno;
 207
 208         /* Generate cache content */
 209         slot->fn();
 210
 211         /* update stat info */
 212         if (fstat(slot->lock_fd, &slot->cache_st))
 213                 return errno;
 214
 215         /* Restore stdout */
 216         if (dup2(tmp, STDOUT_FILENO) == -1)
 217                 return errno;
 218
 219         /* Close the temporary filedescriptor */
 220         if (close(tmp))
 221                 return errno;
 222
 223         return 0;
 224 }
 225
 226 /* Crude implementation of 32-bit FNV-1 hash algorithm,
 227  * see http://www.isthe.com/chongo/tech/comp/fnv/ for details
 228  * about the magic numbers.
 229  */
 230 #define FNV_OFFSET 0x811c9dc5
 231 #define FNV_PRIME  0x01000193
 232
 233 unsigned long hash_str(const char *str)
 234 {
 235         unsigned long h = FNV_OFFSET;
 236         unsigned char *s = (unsigned char *)str;
 237
 238         if (!s)
 239                 return h;
 240
 241         while (*s) {
 242                 h *= FNV_PRIME;
 243                 h ^= *s++;
 244         }
 245         return h;
 246 }
 247
 248 static int process_slot(struct cache_slot *slot)
 249 {
 250         int err;
 251
 252         err = open_slot(slot);
 253         if (!err && slot->match) {
 254                 if (is_expired(slot)) {
 255                         if (!lock_slot(slot)) {
 256                                 /* If the cachefile has been replaced between
 257                                  * `open_slot` and `lock_slot`, we'll just
 258                                  * serve the stale content from the original
 259                                  * cachefile. This way we avoid pruning the
 260                                  * newly generated slot. The same code-path
 261                                  * is chosen if fill_slot() fails for some
 262                                  * reason.
 263                                  *
 264                                  * TODO? check if the new slot contains the
 265                                  * same key as the old one, since we would
 266                                  * prefer to serve the newest content.
 267                                  * This will require us to open yet another
 268                                  * file-descriptor and read and compare the
 269                                  * key from the new file, so for now we're
 270                                  * lazy and just ignore the new file.
 271                                  */
 272                                 if (is_modified(slot) || fill_slot(slot)) {
 273                                         unlock_slot(slot, 0);
 274                                         close_lock(slot);
 275                                 } else {
 276                                         close_slot(slot);
 277                                         unlock_slot(slot, 1);
 278                                         slot->cache_fd = slot->lock_fd;
 279                                 }
 280                         }
 281                 }
 282                 if ((err = print_slot(slot)) != 0) {
 283                         cache_log("[cgit] error printing cache %s: %s (%d)\n",
 284                                   slot->cache_name,
 285                                   strerror(err),
 286                                   err);
 287                 }
 288                 close_slot(slot);
 289                 return err;
 290         }
 291
 292         /* If the cache slot does not exist (or its key doesn't match the
 293          * current key), lets try to create a new cache slot for this
 294          * request. If this fails (for whatever reason), lets just generate
 295          * the content without caching it and fool the caller to belive
 296          * everything worked out (but print a warning on stdout).
 297          */
 298
 299         close_slot(slot);
 300         if ((err = lock_slot(slot)) != 0) {
 301                 cache_log("[cgit] Unable to lock slot %s: %s (%d)\n",
 302                           slot->lock_name, strerror(err), err);
 303                 slot->fn();
 304                 return 0;
 305         }
 306
 307         if ((err = fill_slot(slot)) != 0) {
 308                 cache_log("[cgit] Unable to fill slot %s: %s (%d)\n",
 309                           slot->lock_name, strerror(err), err);
 310                 unlock_slot(slot, 0);
 311                 close_lock(slot);
 312                 slot->fn();
 313                 return 0;
 314         }
 315         // We've got a valid cache slot in the lock file, which
 316         // is about to replace the old cache slot. But if we
 317         // release the lockfile and then try to open the new cache
 318         // slot, we might get a race condition with a concurrent
 319         // writer for the same cache slot (with a different key).
 320         // Lets avoid such a race by just printing the content of
 321         // the lock file.
 322         slot->cache_fd = slot->lock_fd;
 323         unlock_slot(slot, 1);
 324         if ((err = print_slot(slot)) != 0) {
 325                 cache_log("[cgit] error printing cache %s: %s (%d)\n",
 326                           slot->cache_name,
 327                           strerror(err),
 328                           err);
 329         }
 330         close_slot(slot);
 331         return err;
 332 }
 333
 334 /* Print cached content to stdout, generate the content if necessary. */
 335 int cache_process(int size, const char *path, const char *key, int ttl,
 336                   cache_fill_fn fn)
 337 {
 338         unsigned long hash;
 339         int i;
 340         struct strbuf filename = STRBUF_INIT;
 341         struct strbuf lockname = STRBUF_INIT;
 342         struct cache_slot slot;
 343         int result;
 344
 345         /* If the cache is disabled, just generate the content */
 346         if (size <= 0 || ttl == 0) {
 347                 fn();
 348                 return 0;
 349         }
 350
 351         /* Verify input, calculate filenames */
 352         if (!path) {
 353                 cache_log("[cgit] Cache path not specified, caching is disabled\n");
 354                 fn();
 355                 return 0;
 356         }
 357         if (!key)
 358                 key = "";
 359         hash = hash_str(key) % size;
 360         strbuf_addstr(&filename, path);
 361         strbuf_ensure_end(&filename, '/');
 362         for (i = 0; i < 8; i++) {
 363                 strbuf_addf(&filename, "%x", (unsigned char)(hash & 0xf));
 364                 hash >>= 4;
 365         }
 366         strbuf_addbuf(&lockname, &filename);
 367         strbuf_addstr(&lockname, ".lock");
 368         slot.fn = fn;
 369         slot.ttl = ttl;
 370         slot.cache_name = filename.buf;
 371         slot.lock_name = lockname.buf;
 372         slot.key = key;
 373         slot.keylen = strlen(key);
 374         result = process_slot(&slot);
 375
 376         strbuf_release(&filename);
 377         strbuf_release(&lockname);
 378         return result;
 379 }
 380
 381 /* Return a strftime formatted date/time
 382  * NB: the result from this function is to shared memory
 383  */
 384 static char *sprintftime(const char *format, time_t time)
 385 {
 386         static char buf[64];
 387         struct tm *tm;
 388
 389         if (!time)
 390                 return NULL;
 391         tm = gmtime(&time);
 392         strftime(buf, sizeof(buf)-1, format, tm);
 393         return buf;
 394 }
 395
 396 int cache_ls(const char *path)
 397 {
 398         DIR *dir;
 399         struct dirent *ent;
 400         int err = 0;
 401         struct cache_slot slot = { 0 };
 402         struct strbuf fullname = STRBUF_INIT;
 403         size_t prefixlen;
 404
 405         if (!path) {
 406                 cache_log("[cgit] cache path not specified\n");
 407                 return -1;
 408         }
 409         dir = opendir(path);
 410         if (!dir) {
 411                 err = errno;
 412                 cache_log("[cgit] unable to open path %s: %s (%d)\n",
 413                           path, strerror(err), err);
 414                 return err;
 415         }
 416         strbuf_addstr(&fullname, path);
 417         strbuf_ensure_end(&fullname, '/');
 418         prefixlen = fullname.len;
 419         while ((ent = readdir(dir)) != NULL) {
 420                 if (strlen(ent->d_name) != 8)
 421                         continue;
 422                 strbuf_setlen(&fullname, prefixlen);
 423                 strbuf_addstr(&fullname, ent->d_name);
 424                 slot.cache_name = fullname.buf;
 425                 if ((err = open_slot(&slot)) != 0) {
 426                         cache_log("[cgit] unable to open path %s: %s (%d)\n",
 427                                   fullname.buf, strerror(err), err);
 428                         continue;
 429                 }
 430                 htmlf("%s %s %10"PRIuMAX" %s\n",
 431                       fullname.buf,
 432                       sprintftime("%Y-%m-%d %H:%M:%S",
 433                                   slot.cache_st.st_mtime),
 434                       (uintmax_t)slot.cache_st.st_size,
 435                       slot.buf);
 436                 close_slot(&slot);
 437         }
 438         closedir(dir);
 439         strbuf_release(&fullname);
 440         return 0;
 441 }
 442
 443 /* Print a message to stdout */
 444 void cache_log(const char *format, ...)
 445 {
 446         va_list args;
 447         va_start(args, format);
 448         vfprintf(stderr, format, args);
 449         va_end(args);
 450 }
 451