cgit.c: add support for caching autodetected repositories

Signed-off-by: Lars Hjemli <hjemli@gmail.com>

2 files changed, 75 insertions, 1 deletions

diff --git a/cgit.c b/cgit.c
index db96905..594b950 100644
--- a/cgit.c
+++ b/cgit.c
@@ -27,32 +27,34 @@ void add_mimetype(const char *name, const char *value)
 
 struct cgit_filter *new_filter(const char *cmd, int extra_args)
 {
 	struct cgit_filter *f;
 
 	if (!cmd || !cmd[0])
 		return NULL;
 
 	f = xmalloc(sizeof(struct cgit_filter));
 	f->cmd = xstrdup(cmd);
 	f->argv = xmalloc((2 + extra_args) * sizeof(char *));
 	f->argv[0] = f->cmd;
 	f->argv[1] = NULL;
 	return f;
 }
 
+static void process_cached_repolist(const char *path);
+
 void config_cb(const char *name, const char *value)
 {
 	if (!strcmp(name, "root-title"))
 		ctx.cfg.root_title = xstrdup(value);
 	else if (!strcmp(name, "root-desc"))
 		ctx.cfg.root_desc = xstrdup(value);
 	else if (!strcmp(name, "root-readme"))
 		ctx.cfg.root_readme = xstrdup(value);
 	else if (!strcmp(name, "css"))
 		ctx.cfg.css = xstrdup(value);
 	else if (!strcmp(name, "favicon"))
 		ctx.cfg.favicon = xstrdup(value);
 	else if (!strcmp(name, "footer"))
 		ctx.cfg.footer = xstrdup(value);
 	else if (!strcmp(name, "head-include"))
 		ctx.cfg.head_include = xstrdup(value);
@@ -83,32 +85,34 @@ void config_cb(const char *name, const char *value)
 	else if (!strcmp(name, "enable-index-links"))
 		ctx.cfg.enable_index_links = atoi(value);
 	else if (!strcmp(name, "enable-log-filecount"))
 		ctx.cfg.enable_log_filecount = atoi(value);
 	else if (!strcmp(name, "enable-log-linecount"))
 		ctx.cfg.enable_log_linecount = atoi(value);
 	else if (!strcmp(name, "max-stats"))
 		ctx.cfg.max_stats = cgit_find_stats_period(value, NULL);
 	else if (!strcmp(name, "cache-size"))
 		ctx.cfg.cache_size = atoi(value);
 	else if (!strcmp(name, "cache-root"))
 		ctx.cfg.cache_root = xstrdup(value);
 	else if (!strcmp(name, "cache-root-ttl"))
 		ctx.cfg.cache_root_ttl = atoi(value);
 	else if (!strcmp(name, "cache-repo-ttl"))
 		ctx.cfg.cache_repo_ttl = atoi(value);
+	else if (!strcmp(name, "cache-scanrc-ttl"))
+		ctx.cfg.cache_scanrc_ttl = atoi(value);
 	else if (!strcmp(name, "cache-static-ttl"))
 		ctx.cfg.cache_static_ttl = atoi(value);
 	else if (!strcmp(name, "cache-dynamic-ttl"))
 		ctx.cfg.cache_dynamic_ttl = atoi(value);
 	else if (!strcmp(name, "about-filter"))
 		ctx.cfg.about_filter = new_filter(value, 0);
 	else if (!strcmp(name, "commit-filter"))
 		ctx.cfg.commit_filter = new_filter(value, 0);
 	else if (!strcmp(name, "embedded"))
 		ctx.cfg.embedded = atoi(value);
 	else if (!strcmp(name, "max-message-length"))
 		ctx.cfg.max_msg_len = atoi(value);
 	else if (!strcmp(name, "max-repodesc-length"))
 		ctx.cfg.max_repodesc_len = atoi(value);
 	else if (!strcmp(name, "max-repo-count"))
 		ctx.cfg.max_repo_count = atoi(value);
@@ -124,33 +128,36 @@ void config_cb(const char *name, const char *value)
 		ctx.cfg.summary_tags = atoi(value);
 	else if (!strcmp(name, "agefile"))
 		ctx.cfg.agefile = xstrdup(value);
 	else if (!strcmp(name, "renamelimit"))
 		ctx.cfg.renamelimit = atoi(value);
 	else if (!strcmp(name, "robots"))
 		ctx.cfg.robots = xstrdup(value);
 	else if (!strcmp(name, "clone-prefix"))
 		ctx.cfg.clone_prefix = xstrdup(value);
 	else if (!strcmp(name, "local-time"))
 		ctx.cfg.local_time = atoi(value);
 	else if (!prefixcmp(name, "mimetype."))
 		add_mimetype(name + 9, value);
 	else if (!strcmp(name, "repo.group"))
 		ctx.cfg.repo_group = xstrdup(value);
 	else if (!strcmp(name, "repo.scan"))
-		scan_tree(value);
+		if (!ctx.cfg.nocache && ctx.cfg.cache_size)
+			process_cached_repolist(value);
+		else
+			scan_tree(value);
 	else if (!strcmp(name, "repo.url"))
 		ctx.repo = cgit_add_repo(value);
 	else if (!strcmp(name, "repo.name"))
 		ctx.repo->name = xstrdup(value);
 	else if (ctx.repo && !strcmp(name, "repo.path"))
 		ctx.repo->path = trim_end(value, '/');
 	else if (ctx.repo && !strcmp(name, "repo.clone-url"))
 		ctx.repo->clone_url = xstrdup(value);
 	else if (ctx.repo && !strcmp(name, "repo.desc"))
 		ctx.repo->desc = xstrdup(value);
 	else if (ctx.repo && !strcmp(name, "repo.owner"))
 		ctx.repo->owner = xstrdup(value);
 	else if (ctx.repo && !strcmp(name, "repo.defbranch"))
 		ctx.repo->defbranch = xstrdup(value);
 	else if (ctx.repo && !strcmp(name, "repo.snapshots"))
 		ctx.repo->snapshots = ctx.cfg.snapshots & cgit_parse_snapshots_mask(value); /* XXX: &? */
@@ -223,32 +230,33 @@ static void querystring_cb(const char *name, const char *value)
 char *xstrdupn(const char *str)
 {
 	return (str ? xstrdup(str) : NULL);
 }
 
 static void prepare_context(struct cgit_context *ctx)
 {
 	memset(ctx, 0, sizeof(ctx));
 	ctx->cfg.agefile = "info/web/last-modified";
 	ctx->cfg.nocache = 0;
 	ctx->cfg.cache_size = 0;
 	ctx->cfg.cache_dynamic_ttl = 5;
 	ctx->cfg.cache_max_create_time = 5;
 	ctx->cfg.cache_repo_ttl = 5;
 	ctx->cfg.cache_root = CGIT_CACHE_ROOT;
 	ctx->cfg.cache_root_ttl = 5;
+	ctx->cfg.cache_scanrc_ttl = 15;
 	ctx->cfg.cache_static_ttl = -1;
 	ctx->cfg.css = "/cgit.css";
 	ctx->cfg.logo = "/cgit.png";
 	ctx->cfg.local_time = 0;
 	ctx->cfg.max_repo_count = 50;
 	ctx->cfg.max_commit_count = 50;
 	ctx->cfg.max_lock_attempts = 5;
 	ctx->cfg.max_msg_len = 80;
 	ctx->cfg.max_repodesc_len = 80;
 	ctx->cfg.max_stats = 0;
 	ctx->cfg.module_link = "./?repo=%s&page=commit&id=%s";
 	ctx->cfg.renamelimit = -1;
 	ctx->cfg.robots = "index, nofollow";
 	ctx->cfg.root_title = "Git repository browser";
 	ctx->cfg.root_desc = "a fast webinterface for the git dscm";
 	ctx->cfg.script_name = CGIT_SCRIPT_NAME;
@@ -425,32 +433,97 @@ void print_repo(FILE *f, struct cgit_repo *repo)
 		fprintf(f, "repo.owner=%s\n", repo->owner);
 	if (repo->desc)
 		fprintf(f, "repo.desc=%s\n", repo->desc);
 	if (repo->readme)
 		fprintf(f, "repo.readme=%s\n", repo->readme);
 	fprintf(f, "\n");
 }
 
 void print_repolist(FILE *f, struct cgit_repolist *list, int start)
 {
 	int i;
 
 	for(i = start; i < list->count; i++)
 		print_repo(f, &list->repos[i]);
 }
 
+/* Scan 'path' for git repositories, save the resulting repolist in 'cached_rc'
+ * and return 0 on success.
+ */
+static int generate_cached_repolist(const char *path, const char *cached_rc)
+{
+	char *locked_rc;
+	int idx;
+	FILE *f;
+
+	locked_rc = xstrdup(fmt("%s.lock", cached_rc));
+	f = fopen(locked_rc, "wx");
+	if (!f) {
+		/* Inform about the error unless the lockfile already existed,
+		 * since that only means we've got concurrent requests.
+		 */
+		if (errno != EEXIST)
+			fprintf(stderr, "[cgit] Error opening %s: %s (%d)\n",
+				locked_rc, strerror(errno), errno);
+		return errno;
+	}
+	idx = cgit_repolist.count;
+	scan_tree(path);
+	print_repolist(f, &cgit_repolist, idx);
+	if (rename(locked_rc, cached_rc))
+		fprintf(stderr, "[cgit] Error renaming %s to %s: %s (%d)\n",
+			locked_rc, cached_rc, strerror(errno), errno);
+	fclose(f);
+	return 0;
+}
+
+static void process_cached_repolist(const char *path)
+{
+	struct stat st;
+	char *cached_rc;
+	time_t age;
+
+	cached_rc = xstrdup(fmt("%s/rc-%8x", ctx.cfg.cache_root,
+		hash_str(path)));
+
+	if (stat(cached_rc, &st)) {
+		/* Nothing is cached, we need to scan without forking. And
+		 * if we fail to generate a cached repolist, we need to
+		 * invoke scan_tree manually.
+		 */
+		if (generate_cached_repolist(path, cached_rc))
+			scan_tree(path);
+		return;
+	}
+
+	parse_configfile(cached_rc, config_cb);
+
+	/* If the cached configfile hasn't expired, lets exit now */
+	age = time(NULL) - st.st_mtime;
+	if (age <= (ctx.cfg.cache_scanrc_ttl * 60))
+		return;
+
+	/* The cached repolist has been parsed, but it was old. So lets
+	 * rescan the specified path and generate a new cached repolist
+	 * in a child-process to avoid latency for the current request.
+	 */
+	if (fork())
+		return;
+
+	exit(generate_cached_repolist(path, cached_rc));
+}
 
 static void cgit_parse_args(int argc, const char **argv)
 {
 	int i;
 	int scan = 0;
 
 	for (i = 1; i < argc; i++) {
 		if (!strncmp(argv[i], "--cache=", 8)) {
 			ctx.cfg.cache_root = xstrdup(argv[i]+8);
 		}
 		if (!strcmp(argv[i], "--nocache")) {
 			ctx.cfg.nocache = 1;
 		}
 		if (!strcmp(argv[i], "--nohttp")) {
 			ctx.env.no_http = "1";
 		}
diff --git a/cgit.h b/cgit.h
index adb8da4..5659580 100644
--- a/cgit.h
+++ b/cgit.h
@@ -155,32 +155,33 @@ struct cgit_config {
 	char *index_info;
 	char *logo;
 	char *logo_link;
 	char *module_link;
 	char *repo_group;
 	char *robots;
 	char *root_title;
 	char *root_desc;
 	char *root_readme;
 	char *script_name;
 	char *virtual_root;
 	int cache_size;
 	int cache_dynamic_ttl;
 	int cache_max_create_time;
 	int cache_repo_ttl;
 	int cache_root_ttl;
+	int cache_scanrc_ttl;
 	int cache_static_ttl;
 	int embedded;
 	int enable_index_links;
 	int enable_log_filecount;
 	int enable_log_linecount;
 	int local_time;
 	int max_repo_count;
 	int max_commit_count;
 	int max_lock_attempts;
 	int max_msg_len;
 	int max_repodesc_len;
 	int max_stats;
 	int nocache;
 	int noplainemail;
 	int noheader;
 	int renamelimit;