-rw-r--r-- | cache.c | 421 | ||||
-rw-r--r-- | cache.h | 38 | ||||
-rw-r--r-- | cgit.c | 166 | ||||
-rw-r--r-- | cgit.h | 1 | ||||
-rw-r--r-- | cmd.c | 21 | ||||
-rwxr-xr-x | tests/setup.sh | 2 | ||||
-rwxr-xr-x | tests/t0020-validate-cache.sh | 67 |
7 files changed, 503 insertions, 213 deletions
@@ -1,120 +1,411 @@ /* cache.c: cache management * * Copyright (C) 2006 Lars Hjemli * * Licensed under GNU General Public License v2 * (see COPYING for full license text) + * + * + * The cache is just a directory structure where each file is a cache slot, + * and each filename is based on the hash of some key (e.g. the cgit url). + * Each file contains the full key followed by the cached content for that + * key. + * */ #include "cgit.h" #include "cache.h" -const int NOLOCK = -1; +#define CACHE_BUFSIZE (1024 * 4) -char *cache_safe_filename(const char *unsafe) +struct cache_slot { + const char *key; + int keylen; + int ttl; + cache_fill_fn fn; + void *cbdata; + int cache_fd; + int lock_fd; + const char *cache_name; + const char *lock_name; + int match; + struct stat cache_st; + struct stat lock_st; + int bufsize; + char buf[CACHE_BUFSIZE]; +}; + +/* Open an existing cache slot and fill the cache buffer with + * (part of) the content of the cache file. Return 0 on success + * and errno otherwise. + */ +static int open_slot(struct cache_slot *slot) { - static char buf[4][PATH_MAX]; - static int bufidx; - char *s; - char c; + char *bufz; + int bufkeylen = -1; - bufidx++; - bufidx &= 3; - s = buf[bufidx]; + slot->cache_fd = open(slot->cache_name, O_RDONLY); + if (slot->cache_fd == -1) + return errno; - while(unsafe && (c = *unsafe++) != 0) { - if (c == '/' || c == ' ' || c == '&' || c == '|' || - c == '>' || c == '<' || c == '.') - c = '_'; - *s++ = c; - } - *s = '\0'; - return buf[bufidx]; + if (fstat(slot->cache_fd, &slot->cache_st)) + return errno; + + slot->bufsize = read(slot->cache_fd, slot->buf, sizeof(slot->buf)); + if (slot->bufsize == 0) + return errno; + + bufz = memchr(slot->buf, 0, slot->bufsize); + if (bufz) + bufkeylen = bufz - slot->buf; + + slot->match = bufkeylen == slot->keylen && + !memcmp(slot->key, slot->buf, bufkeylen + 1); + + return 0; } -int cache_exist(struct cacheitem *item) +/* Close the active cache slot */ +static void close_slot(struct cache_slot *slot) { - if (stat(item->name, &item->st)) { - item->st.st_mtime = 0; - return 0; + if (slot->cache_fd > 0) { + close(slot->cache_fd); + slot->cache_fd = -1; } - return 1; } -int cache_create_dirs() +/* Print the content of the active cache slot (but skip the key). */ +static int print_slot(struct cache_slot *slot) { - char *path; - - path = fmt("%s", ctx.cfg.cache_root); - if (mkdir(path, S_IRWXU) && errno!=EEXIST) - return 0; + ssize_t i, j = 0; - if (!ctx.repo) - return 0; + i = lseek(slot->cache_fd, slot->keylen + 1, SEEK_SET); + if (i != slot->keylen + 1) + return errno; - path = fmt("%s/%s", ctx.cfg.cache_root, - cache_safe_filename(ctx.repo->url)); + while((i=read(slot->cache_fd, slot->buf, sizeof(slot->buf))) > 0) + j = write(STDOUT_FILENO, slot->buf, i); - if (mkdir(path, S_IRWXU) && errno!=EEXIST) + if (j < 0) + return errno; + else return 0; +} - if (ctx.qry.page) { - path = fmt("%s/%s/%s", ctx.cfg.cache_root, - cache_safe_filename(ctx.repo->url), - ctx.qry.page); - if (mkdir(path, S_IRWXU) && errno!=EEXIST) +/* Check if the slot has expired */ +static int is_expired(struct cache_slot *slot) +{ + if (slot->ttl < 0) return 0; + else + return slot->cache_st.st_mtime + slot->ttl*60 < time(NULL); } + +/* Check if the slot has been modified since we opened it. + * NB: If stat() fails, we pretend the file is modified. + */ +static int is_modified(struct cache_slot *slot) +{ + struct stat st; + + if (stat(slot->cache_name, &st)) return 1; + return (st.st_ino != slot->cache_st.st_ino || + st.st_mtime != slot->cache_st.st_mtime || + st.st_size != slot->cache_st.st_size); } -int cache_refill_overdue(const char *lockfile) +/* Close an open lockfile */ +static void close_lock(struct cache_slot *slot) { - struct stat st; + if (slot->lock_fd > 0) { + close(slot->lock_fd); + slot->lock_fd = -1; + } +} - if (stat(lockfile, &st)) +/* Create a lockfile used to store the generated content for a cache + * slot, and write the slot key + \0 into it. + * Returns 0 on success and errno otherwise. + */ +static int lock_slot(struct cache_slot *slot) +{ + slot->lock_fd = open(slot->lock_name, O_RDWR|O_CREAT|O_EXCL, + S_IRUSR|S_IWUSR); + if (slot->lock_fd == -1) + return errno; + write(slot->lock_fd, slot->key, slot->keylen + 1); return 0; +} + +/* Release the current lockfile. If `replace_old_slot` is set the + * lockfile replaces the old cache slot, otherwise the lockfile is + * just deleted. + */ +static int unlock_slot(struct cache_slot *slot, int replace_old_slot) +{ + int err; + + if (replace_old_slot) + err = rename(slot->lock_name, slot->cache_name); else - return (time(NULL) - st.st_mtime > ctx.cfg.cache_max_create_time); + err = unlink(slot->lock_name); + return err; +} + +/* Generate the content for the current cache slot by redirecting + * stdout to the lock-fd and invoking the callback function + */ +static int fill_slot(struct cache_slot *slot) +{ + int tmp; + + /* Preserve stdout */ + tmp = dup(STDOUT_FILENO); + if (tmp == -1) + return errno; + + /* Redirect stdout to lockfile */ + if (dup2(slot->lock_fd, STDOUT_FILENO) == -1) + return errno; + + /* Generate cache content */ + slot->fn(slot->cbdata); + + /* Restore stdout */ + if (dup2(tmp, STDOUT_FILENO) == -1) + return errno; + + /* Close the temporary filedescriptor */ + close(tmp); + return 0; } -int cache_lock(struct cacheitem *item) +/* Crude implementation of 32-bit FNV-1 hash algorithm, + * see http://www.isthe.com/chongo/tech/comp/fnv/ for details + * about the magic numbers. + */ +#define FNV_OFFSET 0x811c9dc5 +#define FNV_PRIME 0x01000193 + +unsigned long hash_str(const char *str) { - int i = 0; - char *lockfile = xstrdup(fmt("%s.lock", item->name)); + unsigned long h = FNV_OFFSET; + unsigned char *s = (unsigned char *)str; - top: - if (++i > ctx.cfg.max_lock_attempts) - die("cache_lock: unable to lock %s: %s", - item->name, strerror(errno)); + if (!s) + return h; - item->fd = open(lockfile, O_WRONLY|O_CREAT|O_EXCL, S_IRUSR|S_IWUSR); + while(*s) { + h *= FNV_PRIME; + h ^= *s++; + } + return h; +} + +static int process_slot(struct cache_slot *slot) +{ + int err; + + err = open_slot(slot); + if (!err && slot->match) { + if (is_expired(slot)) { + if (!lock_slot(slot)) { + /* If the cachefile has been replaced between + * `open_slot` and `lock_slot`, we'll just + * serve the stale content from the original + * cachefile. This way we avoid pruning the + * newly generated slot. The same code-path + * is chosen if fill_slot() fails for some + * reason. + * + * TODO? check if the new slot contains the + * same key as the old one, since we would + * prefer to serve the newest content. + * This will require us to open yet another + * file-descriptor and read and compare the + * key from the new file, so for now we're + * lazy and just ignore the new file. + */ + if (is_modified(slot) || fill_slot(slot)) { + unlock_slot(slot, 0); + close_lock(slot); + } else { + close_slot(slot); + unlock_slot(slot, 1); + slot->cache_fd = slot->lock_fd; + } + } + } + print_slot(slot); + close_slot(slot); + return 0; + } - if (item->fd == NOLOCK && errno == ENOENT && cache_create_dirs()) - goto top; + /* If the cache slot does not exist (or its key doesn't match the + * current key), lets try to create a new cache slot for this + * request. If this fails (for whatever reason), lets just generate + * the content without caching it and fool the caller to belive + * everything worked out (but print a warning on stdout). + */ - if (item->fd == NOLOCK && errno == EEXIST && - cache_refill_overdue(lockfile) && !unlink(lockfile)) - goto top; + close_slot(slot); + if ((err = lock_slot(slot)) != 0) { + cache_log("[cgit] Unable to lock slot %s: %s (%d)\n", + slot->lock_name, strerror(err), err); + slot->fn(slot->cbdata); + return 0; + } - free(lockfile); - return (item->fd > 0); + if ((err = fill_slot(slot)) != 0) { + cache_log("[cgit] Unable to fill slot %s: %s (%d)\n", + slot->lock_name, strerror(err), err); + unlock_slot(slot, 0); + close_lock(slot); + slot->fn(slot->cbdata); + return 0; + } + // We've got a valid cache slot in the lock file, which + // is about to replace the old cache slot. But if we + // release the lockfile and then try to open the new cache + // slot, we might get a race condition with a concurrent + // writer for the same cache slot (with a different key). + // Lets avoid such a race by just printing the content of + // the lock file. + slot->cache_fd = slot->lock_fd; + unlock_slot(slot, 1); + err = print_slot(slot); + close_slot(slot); + return err; } -int cache_unlock(struct cacheitem *item) +/* Print cached content to stdout, generate the content if necessary. */ +int cache_process(int size, const char *path, const char *key, int ttl, + cache_fill_fn fn, void *cbdata) { - close(item->fd); - return (rename(fmt("%s.lock", item->name), item->name) == 0); + unsigned long hash; + int len, i; + char filename[1024]; + char lockname[1024 + 5]; /* 5 = ".lock" */ + struct cache_slot slot; + + /* If the cache is disabled, just generate the content */ + if (size <= 0) { + fn(cbdata); + return 0; } -int cache_cancel_lock(struct cacheitem *item) + /* Verify input, calculate filenames */ + if (!path) { + cache_log("[cgit] Cache path not specified, caching is disabled\n"); + fn(cbdata); + return 0; + } + len = strlen(path); + if (len > sizeof(filename) - 10) { /* 10 = "/01234567\0" */ + cache_log("[cgit] Cache path too long, caching is disabled: %s\n", + path); + fn(cbdata); + return 0; + } + if (!key) + key = ""; + hash = hash_str(key) % size; + strcpy(filename, path); + if (filename[len - 1] != '/') + filename[len++] = '/'; + for(i = 0; i < 8; i++) { + sprintf(filename + len++, "%x", + (unsigned char)(hash & 0xf)); + hash >>= 4; + } + filename[len] = '\0'; + strcpy(lockname, filename); + strcpy(lockname + len, ".lock"); + slot.fn = fn; + slot.cbdata = cbdata; + slot.ttl = ttl; + slot.cache_name = filename; + slot.lock_name = lockname; + slot.key = key; + slot.keylen = strlen(key); + return process_slot(&slot); +} + +/* Return a strftime formatted date/time + * NB: the result from this function is to shared memory + */ +char *sprintftime(const char *format, time_t time) { - return (unlink(fmt("%s.lock", item->name)) == 0); + static char buf[64]; + struct tm *tm; + + if (!time) + return NULL; + tm = gmtime(&time); + strftime(buf, sizeof(buf)-1, format, tm); + return buf; } -int cache_expired(struct cacheitem *item) +int cache_ls(const char *path) { - if (item->ttl < 0) + DIR *dir; + struct dirent *ent; + int err = 0; + struct cache_slot slot; + char fullname[1024]; + char *name; + + if (!path) { + cache_log("[cgit] cache path not specified\n"); + return -1; + } + if (strlen(path) > 1024 - 10) { + cache_log("[cgit] cache path too long: %s\n", + path); + return -1; + } + dir = opendir(path); + if (!dir) { + err = errno; + cache_log("[cgit] unable to open path %s: %s (%d)\n", + path, strerror(err), err); + return err; + } + strcpy(fullname, path); + name = fullname + strlen(path); + if (*(name - 1) != '/') { + *name++ = '/'; + *name = '\0'; + } + slot.cache_name = fullname; + while((ent = readdir(dir)) != NULL) { + if (strlen(ent->d_name) != 8) + continue; + strcpy(name, ent->d_name); + if ((err = open_slot(&slot)) != 0) { + cache_log("[cgit] unable to open path %s: %s (%d)\n", + fullname, strerror(err), err); + continue; + } + printf("%s %s %10lld %s\n", + name, + sprintftime("%Y-%m-%d %H:%M:%S", + slot.cache_st.st_mtime), + slot.cache_st.st_size, + slot.buf); + close_slot(&slot); + } + closedir(dir); return 0; - return item->st.st_mtime + item->ttl * 60 < time(NULL); } + +/* Print a message to stdout */ +void cache_log(const char *format, ...) +{ + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); +} + @@ -1,23 +1,35 @@ /* * Since git has it's own cache.h which we include, * lets test on CGIT_CACHE_H to avoid confusion */ #ifndef CGIT_CACHE_H #define CGIT_CACHE_H -struct cacheitem { - char *name; - struct stat st; - int ttl; - int fd; -}; - -extern char *cache_safe_filename(const char *unsafe); -extern int cache_lock(struct cacheitem *item); -extern int cache_unlock(struct cacheitem *item); -extern int cache_cancel_lock(struct cacheitem *item); -extern int cache_exist(struct cacheitem *item); -extern int cache_expired(struct cacheitem *item); +typedef void (*cache_fill_fn)(void *cbdata); + + +/* Print cached content to stdout, generate the content if necessary. + * + * Parameters + * size max number of cache files + * path directory used to store cache files + * key the key used to lookup cache files + * ttl max cache time in seconds for this key + * fn content generator function for this key + * cbdata user-supplied data to the content generator function + * + * Return value + * 0 indicates success, everyting else is an error + */ +extern int cache_process(int size, const char *path, const char *key, int ttl, + cache_fill_fn fn, void *cbdata); + + +/* List info about all cache entries on stdout */ +extern int cache_ls(const char *path); + +/* Print a message to stdout */ +extern void cache_log(const char *format, ...); #endif /* CGIT_CACHE_H */ @@ -1,475 +1,383 @@ /* cgit.c: cgi for the git scm * * Copyright (C) 2006 Lars Hjemli * * Licensed under GNU General Public License v2 * (see COPYING for full license text) */ #include "cgit.h" #include "cache.h" #include "cmd.h" #include "configfile.h" #include "html.h" #include "ui-shared.h" const char *cgit_version = CGIT_VERSION; void config_cb(const char *name, const char *value) { if (!strcmp(name, "root-title")) ctx.cfg.root_title = xstrdup(value); else if (!strcmp(name, "root-desc")) ctx.cfg.root_desc = xstrdup(value); else if (!strcmp(name, "root-readme")) ctx.cfg.root_readme = xstrdup(value); else if (!strcmp(name, "css")) ctx.cfg.css = xstrdup(value); else if (!strcmp(name, "logo")) ctx.cfg.logo = xstrdup(value); else if (!strcmp(name, "index-header")) ctx.cfg.index_header = xstrdup(value); else if (!strcmp(name, "index-info")) ctx.cfg.index_info = xstrdup(value); else if (!strcmp(name, "logo-link")) ctx.cfg.logo_link = xstrdup(value); else if (!strcmp(name, "module-link")) ctx.cfg.module_link = xstrdup(value); else if (!strcmp(name, "virtual-root")) { ctx.cfg.virtual_root = trim_end(value, '/'); if (!ctx.cfg.virtual_root && (!strcmp(value, "/"))) ctx.cfg.virtual_root = ""; } else if (!strcmp(name, "nocache")) ctx.cfg.nocache = atoi(value); else if (!strcmp(name, "snapshots")) ctx.cfg.snapshots = cgit_parse_snapshots_mask(value); else if (!strcmp(name, "enable-index-links")) ctx.cfg.enable_index_links = atoi(value); else if (!strcmp(name, "enable-log-filecount")) ctx.cfg.enable_log_filecount = atoi(value); else if (!strcmp(name, "enable-log-linecount")) ctx.cfg.enable_log_linecount = atoi(value); + else if (!strcmp(name, "cache-size")) + ctx.cfg.cache_size = atoi(value); else if (!strcmp(name, "cache-root")) ctx.cfg.cache_root = xstrdup(value); else if (!strcmp(name, "cache-root-ttl")) ctx.cfg.cache_root_ttl = atoi(value); else if (!strcmp(name, "cache-repo-ttl")) ctx.cfg.cache_repo_ttl = atoi(value); else if (!strcmp(name, "cache-static-ttl")) ctx.cfg.cache_static_ttl = atoi(value); else if (!strcmp(name, "cache-dynamic-ttl")) ctx.cfg.cache_dynamic_ttl = atoi(value); else if (!strcmp(name, "max-message-length")) ctx.cfg.max_msg_len = atoi(value); else if (!strcmp(name, "max-repodesc-length")) ctx.cfg.max_repodesc_len = atoi(value); else if (!strcmp(name, "max-commit-count")) ctx.cfg.max_commit_count = atoi(value); else if (!strcmp(name, "summary-log")) ctx.cfg.summary_log = atoi(value); else if (!strcmp(name, "summary-branches")) ctx.cfg.summary_branches = atoi(value); else if (!strcmp(name, "summary-tags")) ctx.cfg.summary_tags = atoi(value); else if (!strcmp(name, "agefile")) ctx.cfg.agefile = xstrdup(value); else if (!strcmp(name, "renamelimit")) ctx.cfg.renamelimit = atoi(value); else if (!strcmp(name, "robots")) ctx.cfg.robots = xstrdup(value); else if (!strcmp(name, "clone-prefix")) ctx.cfg.clone_prefix = xstrdup(value); else if (!strcmp(name, "repo.group")) ctx.cfg.repo_group = xstrdup(value); else if (!strcmp(name, "repo.url")) ctx.repo = cgit_add_repo(value); else if (!strcmp(name, "repo.name")) ctx.repo->name = xstrdup(value); else if (ctx.repo && !strcmp(name, "repo.path")) ctx.repo->path = trim_end(value, '/'); else if (ctx.repo && !strcmp(name, "repo.clone-url")) ctx.repo->clone_url = xstrdup(value); else if (ctx.repo && !strcmp(name, "repo.desc")) ctx.repo->desc = xstrdup(value); else if (ctx.repo && !strcmp(name, "repo.owner")) ctx.repo->owner = xstrdup(value); else if (ctx.repo && !strcmp(name, "repo.defbranch")) ctx.repo->defbranch = xstrdup(value); else if (ctx.repo && !strcmp(name, "repo.snapshots")) ctx.repo->snapshots = ctx.cfg.snapshots & cgit_parse_snapshots_mask(value); /* XXX: &? */ else if (ctx.repo && !strcmp(name, "repo.enable-log-filecount")) ctx.repo->enable_log_filecount = ctx.cfg.enable_log_filecount * atoi(value); else if (ctx.repo && !strcmp(name, "repo.enable-log-linecount")) ctx.repo->enable_log_linecount = ctx.cfg.enable_log_linecount * atoi(value); else if (ctx.repo && !strcmp(name, "repo.module-link")) ctx.repo->module_link= xstrdup(value); else if (ctx.repo && !strcmp(name, "repo.readme") && value != NULL) { if (*value == '/') ctx.repo->readme = xstrdup(value); else ctx.repo->readme = xstrdup(fmt("%s/%s", ctx.repo->path, value)); } else if (!strcmp(name, "include")) parse_configfile(value, config_cb); } static void querystring_cb(const char *name, const char *value) { if (!strcmp(name,"r")) { ctx.qry.repo = xstrdup(value); ctx.repo = cgit_get_repoinfo(value); } else if (!strcmp(name, "p")) { ctx.qry.page = xstrdup(value); } else if (!strcmp(name, "url")) { cgit_parse_url(value); } else if (!strcmp(name, "qt")) { ctx.qry.grep = xstrdup(value); } else if (!strcmp(name, "q")) { ctx.qry.search = xstrdup(value); } else if (!strcmp(name, "h")) { ctx.qry.head = xstrdup(value); ctx.qry.has_symref = 1; } else if (!strcmp(name, "id")) { ctx.qry.sha1 = xstrdup(value); ctx.qry.has_sha1 = 1; } else if (!strcmp(name, "id2")) { ctx.qry.sha2 = xstrdup(value); ctx.qry.has_sha1 = 1; } else if (!strcmp(name, "ofs")) { ctx.qry.ofs = atoi(value); } else if (!strcmp(name, "path")) { ctx.qry.path = trim_end(value, '/'); } else if (!strcmp(name, "name")) { ctx.qry.name = xstrdup(value); } } static void prepare_context(struct cgit_context *ctx) { memset(ctx, 0, sizeof(ctx)); ctx->cfg.agefile = "info/web/last-modified"; + ctx->cfg.nocache = 0; + ctx->cfg.cache_size = 0; ctx->cfg.cache_dynamic_ttl = 5; ctx->cfg.cache_max_create_time = 5; ctx->cfg.cache_repo_ttl = 5; ctx->cfg.cache_root = CGIT_CACHE_ROOT; ctx->cfg.cache_root_ttl = 5; ctx->cfg.cache_static_ttl = -1; ctx->cfg.css = "/cgit.css"; ctx->cfg.logo = "/git-logo.png"; ctx->cfg.max_commit_count = 50; ctx->cfg.max_lock_attempts = 5; ctx->cfg.max_msg_len = 60; ctx->cfg.max_repodesc_len = 60; ctx->cfg.module_link = "./?repo=%s&page=commit&id=%s"; ctx->cfg.renamelimit = -1; ctx->cfg.robots = "index, nofollow"; ctx->cfg.root_title = "Git repository browser"; ctx->cfg.root_desc = "a fast webinterface for the git dscm"; ctx->cfg.script_name = CGIT_SCRIPT_NAME; ctx->page.mimetype = "text/html"; ctx->page.charset = PAGE_ENCODING; ctx->page.filename = NULL; -} - -static int cgit_prepare_cache(struct cacheitem *item) -{ - if (!ctx.repo && ctx.qry.repo) { - ctx.page.title = fmt("%s - %s", ctx.cfg.root_title, - "Bad request"); - cgit_print_http_headers(&ctx); - cgit_print_docstart(&ctx); - cgit_print_pageheader(&ctx); - cgit_print_error(fmt("Unknown repo: %s", ctx.qry.repo)); - cgit_print_docend(); - return 0; - } - - if (!ctx.repo) { - item->name = xstrdup(fmt("%s/index.%s.html", - ctx.cfg.cache_root, - cache_safe_filename(ctx.qry.raw))); - item->ttl = ctx.cfg.cache_root_ttl; - return 1; - } - - if (!ctx.qry.page) { - item->name = xstrdup(fmt("%s/%s/index.%s.html", ctx.cfg.cache_root, - cache_safe_filename(ctx.repo->url), - cache_safe_filename(ctx.qry.raw))); - item->ttl = ctx.cfg.cache_repo_ttl; - } else { - item->name = xstrdup(fmt("%s/%s/%s/%s.html", ctx.cfg.cache_root, - cache_safe_filename(ctx.repo->url), - ctx.qry.page, - cache_safe_filename(ctx.qry.raw))); - if (ctx.qry.has_symref) - item->ttl = ctx.cfg.cache_dynamic_ttl; - else if (ctx.qry.has_sha1) - item->ttl = ctx.cfg.cache_static_ttl; - else - item->ttl = ctx.cfg.cache_repo_ttl; - } - return 1; + ctx->page.modified = time(NULL); + ctx->page.expires = ctx->page.modified; } struct refmatch { char *req_ref; char *first_ref; int match; }; int find_current_ref(const char *refname, const unsigned char *sha1, int flags, void *cb_data) { struct refmatch *info; info = (struct refmatch *)cb_data; if (!strcmp(refname, info->req_ref)) info->match = 1; if (!info->first_ref) info->first_ref = xstrdup(refname); return info->match; } char *find_default_branch(struct cgit_repo *repo) { struct refmatch info; info.req_ref = repo->defbranch; info.first_ref = NULL; info.match = 0; for_each_branch_ref(find_current_ref, &info); if (info.match) return info.req_ref; else return info.first_ref; } static int prepare_repo_cmd(struct cgit_context *ctx) { char *tmp; unsigned char sha1[20]; int nongit = 0; setenv("GIT_DIR", ctx->repo->path, 1); setup_git_directory_gently(&nongit); if (nongit) { ctx->page.title = fmt("%s - %s", ctx->cfg.root_title, "config error"); tmp = fmt("Not a git repository: '%s'", ctx->repo->path); ctx->repo = NULL; cgit_print_http_headers(ctx); cgit_print_docstart(ctx); cgit_print_pageheader(ctx); cgit_print_error(tmp); cgit_print_docend(); return 1; } ctx->page.title = fmt("%s - %s", ctx->repo->name, ctx->repo->desc); if (!ctx->qry.head) { ctx->qry.head = xstrdup(find_default_branch(ctx->repo)); ctx->repo->defbranch = ctx->qry.head; } if (!ctx->qry.head) { cgit_print_http_headers(ctx); cgit_print_docstart(ctx); cgit_print_pageheader(ctx); cgit_print_error("Repository seems to be empty"); cgit_print_docend(); return 1; } if (get_sha1(ctx->qry.head, sha1)) { tmp = xstrdup(ctx->qry.head); ctx->qry.head = ctx->repo->defbranch; cgit_print_http_headers(ctx); cgit_print_docstart(ctx); cgit_print_pageheader(ctx); cgit_print_error(fmt("Invalid branch: %s", tmp)); cgit_print_docend(); return 1; } return 0; } -static void process_request(struct cgit_context *ctx) +static void process_request(void *cbdata) { + struct cgit_context *ctx = cbdata; struct cgit_cmd *cmd; cmd = cgit_get_cmd(ctx); if (!cmd) { ctx->page.title = "cgit error"; ctx->repo = NULL; cgit_print_http_headers(ctx); cgit_print_docstart(ctx); cgit_print_pageheader(ctx); cgit_print_error("Invalid request"); cgit_print_docend(); return; } if (cmd->want_repo && !ctx->repo) { cgit_print_http_headers(ctx); cgit_print_docstart(ctx); cgit_print_pageheader(ctx); cgit_print_error(fmt("No repository selected")); cgit_print_docend(); return; } if (ctx->repo && prepare_repo_cmd(ctx)) return; if (cmd->want_layout) { cgit_print_http_headers(ctx); cgit_print_docstart(ctx); cgit_print_pageheader(ctx); } cmd->fn(ctx); if (cmd->want_layout) cgit_print_docend(); } -static long ttl_seconds(long ttl) -{ - if (ttl<0) - return 60 * 60 * 24 * 365; - else - return ttl * 60; -} - -static void cgit_fill_cache(struct cacheitem *item, int use_cache) -{ - int stdout2; - - if (use_cache) { - stdout2 = chk_positive(dup(STDOUT_FILENO), - "Preserving STDOUT"); - chk_zero(close(STDOUT_FILENO), "Closing STDOUT"); - chk_positive(dup2(item->fd, STDOUT_FILENO), "Dup2(cachefile)"); - } - - ctx.page.modified = time(NULL); - ctx.page.expires = ctx.page.modified + ttl_seconds(item->ttl); - process_request(&ctx); - - if (use_cache) { - chk_zero(close(STDOUT_FILENO), "Close redirected STDOUT"); - chk_positive(dup2(stdout2, STDOUT_FILENO), - "Restoring original STDOUT"); - chk_zero(close(stdout2), "Closing temporary STDOUT"); - } -} - -static void cgit_check_cache(struct cacheitem *item) -{ - int i = 0; - - top: - if (++i > ctx.cfg.max_lock_attempts) { - die("cgit_refresh_cache: unable to lock %s: %s", - item->name, strerror(errno)); - } - if (!cache_exist(item)) { - if (!cache_lock(item)) { - sleep(1); - goto top; - } - if (!cache_exist(item)) { - cgit_fill_cache(item, 1); - cache_unlock(item); - } else { - cache_cancel_lock(item); - } - } else if (cache_expired(item) && cache_lock(item)) { - if (cache_expired(item)) { - cgit_fill_cache(item, 1); - cache_unlock(item); - } else { - cache_cancel_lock(item); - } - } -} - -static void cgit_print_cache(struct cacheitem *item) -{ - static char buf[4096]; - ssize_t i; - - int fd = open(item->name, O_RDONLY); - if (fd<0) - die("Unable to open cached file %s", item->name); - - while((i=read(fd, buf, sizeof(buf))) > 0) - write(STDOUT_FILENO, buf, i); - - close(fd); -} - static void cgit_parse_args(int argc, const char **argv) { int i; for (i = 1; i < argc; i++) { if (!strncmp(argv[i], "--cache=", 8)) { ctx.cfg.cache_root = xstrdup(argv[i]+8); } if (!strcmp(argv[i], "--nocache")) { ctx.cfg.nocache = 1; } if (!strncmp(argv[i], "--query=", 8)) { ctx.qry.raw = xstrdup(argv[i]+8); } if (!strncmp(argv[i], "--repo=", 7)) { ctx.qry.repo = xstrdup(argv[i]+7); } if (!strncmp(argv[i], "--page=", 7)) { ctx.qry.page = xstrdup(argv[i]+7); } if (!strncmp(argv[i], "--head=", 7)) { ctx.qry.head = xstrdup(argv[i]+7); ctx.qry.has_symref = 1; } if (!strncmp(argv[i], "--sha1=", 7)) { ctx.qry.sha1 = xstrdup(argv[i]+7); ctx.qry.has_sha1 = 1; } if (!strncmp(argv[i], "--ofs=", 6)) { ctx.qry.ofs = atoi(argv[i]+6); } } } +static int calc_ttl() +{ + if (!ctx.repo) + return ctx.cfg.cache_root_ttl; + + if (!ctx.qry.page) + return ctx.cfg.cache_repo_ttl; + + if (ctx.qry.has_symref) + return ctx.cfg.cache_dynamic_ttl; + + if (ctx.qry.has_sha1) + return ctx.cfg.cache_static_ttl; + + return ctx.cfg.cache_repo_ttl; +} + int main(int argc, const char **argv) { - struct cacheitem item; const char *cgit_config_env = getenv("CGIT_CONFIG"); + int err, ttl; prepare_context(&ctx); - item.st.st_mtime = time(NULL); cgit_repolist.length = 0; cgit_repolist.count = 0; cgit_repolist.repos = NULL; parse_configfile(cgit_config_env ? cgit_config_env : CGIT_CONFIG, config_cb); ctx.repo = NULL; if (getenv("SCRIPT_NAME")) ctx.cfg.script_name = xstrdup(getenv("SCRIPT_NAME")); if (getenv("QUERY_STRING")) ctx.qry.raw = xstrdup(getenv("QUERY_STRING")); cgit_parse_args(argc, argv); http_parse_querystring(ctx.qry.raw, querystring_cb); - if (!cgit_prepare_cache(&item)) - return 0; - if (ctx.cfg.nocache) { - cgit_fill_cache(&item, 0); - } else { - cgit_check_cache(&item); - cgit_print_cache(&item); - } - return 0; + + ttl = calc_ttl(); + ctx.page.expires += ttl*60; + if (ctx.cfg.nocache) + ctx.cfg.cache_size = 0; + err = cache_process(ctx.cfg.cache_size, ctx.cfg.cache_root, + ctx.qry.raw, ttl, process_request, &ctx); + if (err) + cache_log("[cgit] error %d - %s\n", + err, strerror(err)); + return err; } @@ -1,233 +1,234 @@ #ifndef CGIT_H #define CGIT_H #include <git-compat-util.h> #include <cache.h> #include <grep.h> #include <object.h> #include <tree.h> #include <commit.h> #include <tag.h> #include <diff.h> #include <diffcore.h> #include <refs.h> #include <revision.h> #include <log-tree.h> #include <archive.h> #include <xdiff/xdiff.h> #include <utf8.h> /* * Dateformats used on misc. pages */ #define FMT_LONGDATE "%Y-%m-%d %H:%M:%S" #define FMT_SHORTDATE "%Y-%m-%d" /* * Limits used for relative dates */ #define TM_MIN 60 #define TM_HOUR (TM_MIN * 60) #define TM_DAY (TM_HOUR * 24) #define TM_WEEK (TM_DAY * 7) #define TM_YEAR (TM_DAY * 365) #define TM_MONTH (TM_YEAR / 12.0) /* * Default encoding */ #define PAGE_ENCODING "UTF-8" typedef void (*configfn)(const char *name, const char *value); typedef void (*filepair_fn)(struct diff_filepair *pair); typedef void (*linediff_fn)(char *line, int len); struct cgit_repo { char *url; char *name; char *path; char *desc; char *owner; char *defbranch; char *group; char *module_link; char *readme; char *clone_url; int snapshots; int enable_log_filecount; int enable_log_linecount; }; struct cgit_repolist { int length; int count; struct cgit_repo *repos; }; struct commitinfo { struct commit *commit; char *author; char *author_email; unsigned long author_date; char *committer; char *committer_email; unsigned long committer_date; char *subject; char *msg; char *msg_encoding; }; struct taginfo { char *tagger; char *tagger_email; int tagger_date; char *msg; }; struct refinfo { const char *refname; struct object *object; union { struct taginfo *tag; struct commitinfo *commit; }; }; struct reflist { struct refinfo **refs; int alloc; int count; }; struct cgit_query { int has_symref; int has_sha1; char *raw; char *repo; char *page; char *search; char *grep; char *head; char *sha1; char *sha2; char *path; char *name; int ofs; }; struct cgit_config { char *agefile; char *cache_root; char *clone_prefix; char *css; char *index_header; char *index_info; char *logo; char *logo_link; char *module_link; char *repo_group; char *robots; char *root_title; char *root_desc; char *root_readme; char *script_name; char *virtual_root; + int cache_size; int cache_dynamic_ttl; int cache_max_create_time; int cache_repo_ttl; int cache_root_ttl; int cache_static_ttl; int enable_index_links; int enable_log_filecount; int enable_log_linecount; int max_commit_count; int max_lock_attempts; int max_msg_len; int max_repodesc_len; int nocache; int renamelimit; int snapshots; int summary_branches; int summary_log; int summary_tags; }; struct cgit_page { time_t modified; time_t expires; char *mimetype; char *charset; char *filename; char *title; }; struct cgit_context { struct cgit_query qry; struct cgit_config cfg; struct cgit_repo *repo; struct cgit_page page; }; struct cgit_snapshot_format { const char *suffix; const char *mimetype; write_archive_fn_t write_func; int bit; }; extern const char *cgit_version; extern struct cgit_repolist cgit_repolist; extern struct cgit_context ctx; extern const struct cgit_snapshot_format cgit_snapshot_formats[]; extern struct cgit_repo *cgit_add_repo(const char *url); extern struct cgit_repo *cgit_get_repoinfo(const char *url); extern void cgit_repo_config_cb(const char *name, const char *value); extern int chk_zero(int result, char *msg); extern int chk_positive(int result, char *msg); extern int chk_non_negative(int result, char *msg); extern char *trim_end(const char *str, char c); extern char *strlpart(char *txt, int maxlen); extern char *strrpart(char *txt, int maxlen); extern void cgit_add_ref(struct reflist *list, struct refinfo *ref); extern int cgit_refs_cb(const char *refname, const unsigned char *sha1, int flags, void *cb_data); extern void *cgit_free_commitinfo(struct commitinfo *info); extern int cgit_diff_files(const unsigned char *old_sha1, const unsigned char *new_sha1, linediff_fn fn); extern void cgit_diff_tree(const unsigned char *old_sha1, const unsigned char *new_sha1, filepair_fn fn, const char *prefix); extern void cgit_diff_commit(struct commit *commit, filepair_fn fn); extern char *fmt(const char *format,...); extern struct commitinfo *cgit_parse_commit(struct commit *commit); extern struct taginfo *cgit_parse_tag(struct tag *tag); extern void cgit_parse_url(const char *url); extern const char *cgit_repobasename(const char *reponame); extern int cgit_parse_snapshots_mask(const char *str); /* libgit.a either links against or compiles its own implementation of * strcasestr(), and we'd like to reuse it. Simply re-declaring it * seems to do the trick. */ extern char *strcasestr(const char *haystack, const char *needle); #endif /* CGIT_H */ @@ -1,121 +1,132 @@ /* cmd.c: the cgit command dispatcher * * Copyright (C) 2008 Lars Hjemli * * Licensed under GNU General Public License v2 * (see COPYING for full license text) */ #include "cgit.h" #include "cmd.h" +#include "cache.h" +#include "ui-shared.h" #include "ui-blob.h" #include "ui-commit.h" #include "ui-diff.h" #include "ui-log.h" #include "ui-patch.h" #include "ui-refs.h" #include "ui-repolist.h" #include "ui-snapshot.h" #include "ui-summary.h" #include "ui-tag.h" #include "ui-tree.h" static void about_fn(struct cgit_context *ctx) { if (ctx->repo) cgit_print_repo_readme(); else cgit_print_site_readme(); } static void blob_fn(struct cgit_context *ctx) { cgit_print_blob(ctx->qry.sha1, ctx->qry.path); } static void commit_fn(struct cgit_context *ctx) { cgit_print_commit(ctx->qry.sha1); } static void diff_fn(struct cgit_context *ctx) { cgit_print_diff(ctx->qry.sha1, ctx->qry.sha2, ctx->qry.path); } -static void repolist_fn(struct cgit_context *ctx) -{ - cgit_print_repolist(); -} - static void log_fn(struct cgit_context *ctx) { cgit_print_log(ctx->qry.sha1, ctx->qry.ofs, ctx->cfg.max_commit_count, ctx->qry.grep, ctx->qry.search, ctx->qry.path, 1); } +static void ls_cache_fn(struct cgit_context *ctx) +{ + ctx->page.mimetype = "text/plain"; + ctx->page.filename = "ls-cache.txt"; + cgit_print_http_headers(ctx); + cache_ls(ctx->cfg.cache_root); +} + +static void repolist_fn(struct cgit_context *ctx) +{ + cgit_print_repolist(); +} + static void patch_fn(struct cgit_context *ctx) { cgit_print_patch(ctx->qry.sha1); } static void refs_fn(struct cgit_context *ctx) { cgit_print_refs(); } static void snapshot_fn(struct cgit_context *ctx) { cgit_print_snapshot(ctx->qry.head, ctx->qry.sha1, cgit_repobasename(ctx->repo->url), ctx->qry.path, ctx->repo->snapshots); } static void summary_fn(struct cgit_context *ctx) { cgit_print_summary(); } static void tag_fn(struct cgit_context *ctx) { cgit_print_tag(ctx->qry.sha1); } static void tree_fn(struct cgit_context *ctx) { cgit_print_tree(ctx->qry.sha1, ctx->qry.path); } #define def_cmd(name, want_repo, want_layout) \ {#name, name##_fn, want_repo, want_layout} struct cgit_cmd *cgit_get_cmd(struct cgit_context *ctx) { static struct cgit_cmd cmds[] = { def_cmd(about, 0, 1), def_cmd(blob, 1, 0), def_cmd(commit, 1, 1), def_cmd(diff, 1, 1), def_cmd(log, 1, 1), + def_cmd(ls_cache, 0, 0), def_cmd(patch, 1, 0), def_cmd(refs, 1, 1), def_cmd(repolist, 0, 0), def_cmd(snapshot, 1, 0), def_cmd(summary, 1, 1), def_cmd(tag, 1, 1), def_cmd(tree, 1, 1), }; int i; if (ctx->qry.page == NULL) { if (ctx->repo) ctx->qry.page = "summary"; else ctx->qry.page = "repolist"; } for(i = 0; i < sizeof(cmds)/sizeof(*cmds); i++) if (!strcmp(ctx->qry.page, cmds[i].name)) return &cmds[i]; return NULL; } diff --git a/tests/setup.sh b/tests/setup.sh index 66bf406..e37306e 100755 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -1,116 +1,116 @@ # This file should be sourced by all test-scripts # # Main functions: # prepare_tests(description) - setup for testing, i.e. create repos+config # run_test(description, script) - run one test, i.e. eval script # # Helper functions # cgit_query(querystring) - call cgit with the specified querystring # cgit_url(url) - call cgit with the specified virtual url # # Example script: # # . setup.sh # prepare_tests "html validation" # run_test 'repo index' 'cgit_url "/" | tidy -e' # run_test 'repo summary' 'cgit_url "/foo" | tidy -e' mkrepo() { name=$1 count=$2 dir=$PWD test -d $name && return printf "Creating testrepo %s\n" $name mkdir -p $name cd $name git init for ((n=1; n<=count; n++)) do echo $n >file-$n git add file-$n git commit -m "commit $n" done cd $dir } setup_repos() { rm -rf trash/cache mkdir -p trash/cache mkrepo trash/repos/foo 5 >/dev/null mkrepo trash/repos/bar 50 >/dev/null cat >trash/cgitrc <<EOF virtual-root=/ cache-root=$PWD/trash/cache -nocache=0 +cache-size=1021 snapshots=tar.gz tar.bz zip enable-log-filecount=1 enable-log-linecount=1 summary-log=5 summary-branches=5 summary-tags=5 repo.url=foo repo.path=$PWD/trash/repos/foo/.git # Do not specify a description for this repo, as it then will be assigned # the constant value "[no description]" (which actually used to cause a # segfault). repo.url=bar repo.path=$PWD/trash/repos/bar/.git repo.desc=the bar repo EOF } prepare_tests() { setup_repos rm -f test-output.log 2>/dev/null test_count=0 test_failed=0 echo "[$0]" "$@" >test-output.log echo "$@" "($0)" } tests_done() { printf "\n" if test $test_failed -gt 0 then printf "test: *** %s failure(s), logfile=%s\n" \ $test_failed "$(pwd)/test-output.log" false fi } run_test() { desc=$1 script=$2 ((test_count++)) printf "\ntest %d: name='%s'\n" $test_count "$desc" >>test-output.log printf "test %d: eval='%s'\n" $test_count "$2" >>test-output.log eval "$2" >>test-output.log 2>>test-output.log res=$? printf "test %d: exitcode=%d\n" $test_count $res >>test-output.log if test $res = 0 then printf " %2d) %-60s [ok]\n" $test_count "$desc" else ((test_failed++)) printf " %2d) %-60s [failed]\n" $test_count "$desc" fi } cgit_query() { CGIT_CONFIG="$PWD/trash/cgitrc" QUERY_STRING="$1" "$PWD/../cgit" } cgit_url() { CGIT_CONFIG="$PWD/trash/cgitrc" QUERY_STRING="url=$1" "$PWD/../cgit" } diff --git a/tests/t0020-validate-cache.sh b/tests/t0020-validate-cache.sh new file mode 100755 index 0000000..53ec2eb --- a/dev/null +++ b/tests/t0020-validate-cache.sh @@ -0,0 +1,67 @@ +#!/bin/sh + +. ./setup.sh + +prepare_tests 'Validate cache' + +run_test 'verify cache-size=0' ' + + rm -f trash/cache/* && + sed -i -e "s/cache-size=1021$/cache-size=0/" trash/cgitrc && + cgit_url "" && + cgit_url "foo" && + cgit_url "foo/refs" && + cgit_url "foo/tree" && + cgit_url "foo/log" && + cgit_url "foo/diff" && + cgit_url "foo/patch" && + cgit_url "bar" && + cgit_url "bar/refs" && + cgit_url "bar/tree" && + cgit_url "bar/log" && + cgit_url "bar/diff" && + cgit_url "bar/patch" && + test 0 -eq $(ls trash/cache | wc -l) +' + +run_test 'verify cache-size=1' ' + + rm -f trash/cache/* && + sed -i -e "s/cache-size=0$/cache-size=1/" trash/cgitrc && + cgit_url "" && + cgit_url "foo" && + cgit_url "foo/refs" && + cgit_url "foo/tree" && + cgit_url "foo/log" && + cgit_url "foo/diff" && + cgit_url "foo/patch" && + cgit_url "bar" && + cgit_url "bar/refs" && + cgit_url "bar/tree" && + cgit_url "bar/log" && + cgit_url "bar/diff" && + cgit_url "bar/patch" && + test 1 -eq $(ls trash/cache | wc -l) +' + +run_test 'verify cache-size=1021' ' + + rm -f trash/cache/* && + sed -i -e "s/cache-size=1$/cache-size=1021/" trash/cgitrc && + cgit_url "" && + cgit_url "foo" && + cgit_url "foo/refs" && + cgit_url "foo/tree" && + cgit_url "foo/log" && + cgit_url "foo/diff" && + cgit_url "foo/patch" && + cgit_url "bar" && + cgit_url "bar/refs" && + cgit_url "bar/tree" && + cgit_url "bar/log" && + cgit_url "bar/diff" && + cgit_url "bar/patch" && + test 13 -eq $(ls trash/cache | wc -l) +' + +tests_done |