ui-repolist: implement lazy caching of repo->mtime

When sorting the list of repositories by their last modification time,
cgit would (in the worst case) invoke fstat(3) four times and open(3)
twice for each callback from qsort(3). This obviously scales very badly.

Now, the calculated modtime for each repo is saved in repo->mtime, thus
keeping the number of stat/open invocations identical for sorted and
unsorted repo-listings.

Signed-off-by: Lars Hjemli <hjemli@gmail.com>

3 files changed, 14 insertions, 4 deletions

diff --git a/cgit.h b/cgit.h
index ea90833..c99d337 100644
--- a/cgit.h
+++ b/cgit.h
@@ -1,244 +1,245 @@
 #ifndef CGIT_H
 #define CGIT_H
 
 
 #include <git-compat-util.h>
 #include <cache.h>
 #include <grep.h>
 #include <object.h>
 #include <tree.h>
 #include <commit.h>
 #include <tag.h>
 #include <diff.h>
 #include <diffcore.h>
 #include <refs.h>
 #include <revision.h>
 #include <log-tree.h>
 #include <archive.h>
 #include <xdiff/xdiff.h>
 #include <utf8.h>
 
 
 /*
  * Dateformats used on misc. pages
  */
 #define FMT_LONGDATE "%Y-%m-%d %H:%M:%S (%Z)"
 #define FMT_SHORTDATE "%Y-%m-%d"
 #define FMT_ATOMDATE "%Y-%m-%dT%H:%M:%SZ"
 
 
 /*
  * Limits used for relative dates
  */
 #define TM_MIN    60
 #define TM_HOUR  (TM_MIN * 60)
 #define TM_DAY   (TM_HOUR * 24)
 #define TM_WEEK  (TM_DAY * 7)
 #define TM_YEAR  (TM_DAY * 365)
 #define TM_MONTH (TM_YEAR / 12.0)
 
 
 /*
  * Default encoding
  */
 #define PAGE_ENCODING "UTF-8"
 
 typedef void (*configfn)(const char *name, const char *value);
 typedef void (*filepair_fn)(struct diff_filepair *pair);
 typedef void (*linediff_fn)(char *line, int len);
 
 struct cgit_repo {
         char *url;
         char *name;
         char *path;
         char *desc;
         char *owner;
         char *defbranch;
         char *group;
         char *module_link;
         char *readme;
         char *clone_url;
         int snapshots;
         int enable_log_filecount;
         int enable_log_linecount;
+        time_t mtime;
 };
 
 struct cgit_repolist {
         int length;
         int count;
         struct cgit_repo *repos;
 };
 
 struct commitinfo {
         struct commit *commit;
         char *author;
         char *author_email;
         unsigned long author_date;
         char *committer;
         char *committer_email;
         unsigned long committer_date;
         char *subject;
         char *msg;
         char *msg_encoding;
 };
 
 struct taginfo {
         char *tagger;
         char *tagger_email;
         unsigned long tagger_date;
         char *msg;
 };
 
 struct refinfo {
         const char *refname;
         struct object *object;
         union {
                 struct taginfo *tag;
                 struct commitinfo *commit;
         };
 };
 
 struct reflist {
         struct refinfo **refs;
         int alloc;
         int count;
 };
 
 struct cgit_query {
         int has_symref;
         int has_sha1;
         char *raw;
         char *repo;
         char *page;
         char *search;
         char *grep;
         char *head;
         char *sha1;
         char *sha2;
         char *path;
         char *name;
         char *mimetype;
         char *url;
         int   ofs;
         int nohead;
         char *sort;
 };
 
 struct cgit_config {
         char *agefile;
         char *cache_root;
         char *clone_prefix;
         char *css;
         char *favicon;
         char *footer;
         char *index_header;
         char *index_info;
         char *logo;
         char *logo_link;
         char *module_link;
         char *repo_group;
         char *robots;
         char *root_title;
         char *root_desc;
         char *root_readme;
         char *script_name;
         char *virtual_root;
         int cache_size;
         int cache_dynamic_ttl;
         int cache_max_create_time;
         int cache_repo_ttl;
         int cache_root_ttl;
         int cache_static_ttl;
         int enable_index_links;
         int enable_log_filecount;
         int enable_log_linecount;
         int local_time;
         int max_repo_count;
         int max_commit_count;
         int max_lock_attempts;
         int max_msg_len;
         int max_repodesc_len;
         int nocache;
         int renamelimit;
         int snapshots;
         int summary_branches;
         int summary_log;
         int summary_tags;
 };
 
 struct cgit_page {
         time_t modified;
         time_t expires;
         size_t size;
         char *mimetype;
         char *charset;
         char *filename;
         char *title;
 };
 
 struct cgit_context {
         struct cgit_query qry;
         struct cgit_config cfg;
         struct cgit_repo *repo;
         struct cgit_page page;
 };
 
 struct cgit_snapshot_format {
         const char *suffix;
         const char *mimetype;
         write_archive_fn_t write_func;
         int bit;
 };
 
 extern const char *cgit_version;
 
 extern struct cgit_repolist cgit_repolist;
 extern struct cgit_context ctx;
 extern const struct cgit_snapshot_format cgit_snapshot_formats[];
 
 extern struct cgit_repo *cgit_add_repo(const char *url);
 extern struct cgit_repo *cgit_get_repoinfo(const char *url);
 extern void cgit_repo_config_cb(const char *name, const char *value);
 
 extern int chk_zero(int result, char *msg);
 extern int chk_positive(int result, char *msg);
 extern int chk_non_negative(int result, char *msg);
 
 extern char *trim_end(const char *str, char c);
 extern char *strlpart(char *txt, int maxlen);
 extern char *strrpart(char *txt, int maxlen);
 
 extern void cgit_add_ref(struct reflist *list, struct refinfo *ref);
 extern int cgit_refs_cb(const char *refname, const unsigned char *sha1,
                         int flags, void *cb_data);
 
 extern void *cgit_free_commitinfo(struct commitinfo *info);
 
 extern int cgit_diff_files(const unsigned char *old_sha1,
                            const unsigned char *new_sha1,
                            linediff_fn fn);
 
 extern void cgit_diff_tree(const unsigned char *old_sha1,
                            const unsigned char *new_sha1,
                            filepair_fn fn, const char *prefix);
 
 extern void cgit_diff_commit(struct commit *commit, filepair_fn fn);
 
 extern char *fmt(const char *format,...);
 
 extern struct commitinfo *cgit_parse_commit(struct commit *commit);
 extern struct taginfo *cgit_parse_tag(struct tag *tag);
 extern void cgit_parse_url(const char *url);
 
 extern const char *cgit_repobasename(const char *reponame);
 
 extern int cgit_parse_snapshots_mask(const char *str);
 
 /* libgit.a either links against or compiles its own implementation of
  * strcasestr(), and we'd like to reuse it. Simply re-declaring it
  * seems to do the trick.
  */
 extern char *strcasestr(const char *haystack, const char *needle);
 
 
 #endif /* CGIT_H */
diff --git a/shared.c b/shared.c
index f5875e4..89d1bab 100644
--- a/shared.c
+++ b/shared.c
@@ -1,254 +1,255 @@
 /* shared.c: global vars + some callback functions
  *
  * Copyright (C) 2006 Lars Hjemli
  *
  * Licensed under GNU General Public License v2
  *   (see COPYING for full license text)
  */
 
 #include "cgit.h"
 
 struct cgit_repolist cgit_repolist;
 struct cgit_context ctx;
 int cgit_cmd;
 
 int chk_zero(int result, char *msg)
 {
         if (result != 0)
                 die("%s: %s", msg, strerror(errno));
         return result;
 }
 
 int chk_positive(int result, char *msg)
 {
         if (result <= 0)
                 die("%s: %s", msg, strerror(errno));
         return result;
 }
 
 int chk_non_negative(int result, char *msg)
 {
             if (result < 0)
                     die("%s: %s",msg, strerror(errno));
         return result;
 }
 
 struct cgit_repo *cgit_add_repo(const char *url)
 {
         struct cgit_repo *ret;
 
         if (++cgit_repolist.count > cgit_repolist.length) {
                 if (cgit_repolist.length == 0)
                         cgit_repolist.length = 8;
                 else
                         cgit_repolist.length *= 2;
                 cgit_repolist.repos = xrealloc(cgit_repolist.repos,
                                                cgit_repolist.length *
                                                sizeof(struct cgit_repo));
         }
 
         ret = &cgit_repolist.repos[cgit_repolist.count-1];
         ret->url = trim_end(url, '/');
         ret->name = ret->url;
         ret->path = NULL;
         ret->desc = "[no description]";
         ret->owner = NULL;
         ret->group = ctx.cfg.repo_group;
         ret->defbranch = "master";
         ret->snapshots = ctx.cfg.snapshots;
         ret->enable_log_filecount = ctx.cfg.enable_log_filecount;
         ret->enable_log_linecount = ctx.cfg.enable_log_linecount;
         ret->module_link = ctx.cfg.module_link;
         ret->readme = NULL;
+        ret->mtime = -1;
         return ret;
 }
 
 struct cgit_repo *cgit_get_repoinfo(const char *url)
 {
         int i;
         struct cgit_repo *repo;
 
         for (i=0; i<cgit_repolist.count; i++) {
                 repo = &cgit_repolist.repos[i];
                 if (!strcmp(repo->url, url))
                         return repo;
         }
         return NULL;
 }
 
 void *cgit_free_commitinfo(struct commitinfo *info)
 {
         free(info->author);
         free(info->author_email);
         free(info->committer);
         free(info->committer_email);
         free(info->subject);
         free(info->msg);
         free(info->msg_encoding);
         free(info);
         return NULL;
 }
 
 char *trim_end(const char *str, char c)
 {
         int len;
         char *s, *t;
 
         if (str == NULL)
                 return NULL;
         t = (char *)str;
         len = strlen(t);
         while(len > 0 && t[len - 1] == c)
                 len--;
 
         if (len == 0)
                 return NULL;
 
         c = t[len];
         t[len] = '\0';
         s = xstrdup(t);
         t[len] = c;
         return s;
 }
 
 char *strlpart(char *txt, int maxlen)
 {
         char *result;
 
         if (!txt)
                 return txt;
 
         if (strlen(txt) <= maxlen)
                 return txt;
         result = xmalloc(maxlen + 1);
         memcpy(result, txt, maxlen - 3);
         result[maxlen-1] = result[maxlen-2] = result[maxlen-3] = '.';
         result[maxlen] = '\0';
         return result;
 }
 
 char *strrpart(char *txt, int maxlen)
 {
         char *result;
 
         if (!txt)
                 return txt;
 
         if (strlen(txt) <= maxlen)
                 return txt;
         result = xmalloc(maxlen + 1);
         memcpy(result + 3, txt + strlen(txt) - maxlen + 4, maxlen - 3);
         result[0] = result[1] = result[2] = '.';
         return result;
 }
 
 void cgit_add_ref(struct reflist *list, struct refinfo *ref)
 {
         size_t size;
 
         if (list->count >= list->alloc) {
                 list->alloc += (list->alloc ? list->alloc : 4);
                 size = list->alloc * sizeof(struct refinfo *);
                 list->refs = xrealloc(list->refs, size);
         }
         list->refs[list->count++] = ref;
 }
 
 struct refinfo *cgit_mk_refinfo(const char *refname, const unsigned char *sha1)
 {
         struct refinfo *ref;
 
         ref = xmalloc(sizeof (struct refinfo));
         ref->refname = xstrdup(refname);
         ref->object = parse_object(sha1);
         switch (ref->object->type) {
         case OBJ_TAG:
                 ref->tag = cgit_parse_tag((struct tag *)ref->object);
                 break;
         case OBJ_COMMIT:
                 ref->commit = cgit_parse_commit((struct commit *)ref->object);
                 break;
         }
         return ref;
 }
 
 int cgit_refs_cb(const char *refname, const unsigned char *sha1, int flags,
                   void *cb_data)
 {
         struct reflist *list = (struct reflist *)cb_data;
         struct refinfo *info = cgit_mk_refinfo(refname, sha1);
 
         if (info)
                 cgit_add_ref(list, info);
         return 0;
 }
 
 void cgit_diff_tree_cb(struct diff_queue_struct *q,
                        struct diff_options *options, void *data)
 {
         int i;
 
         for (i = 0; i < q->nr; i++) {
                 if (q->queue[i]->status == 'U')
                         continue;
                 ((filepair_fn)data)(q->queue[i]);
         }
 }
 
 static int load_mmfile(mmfile_t *file, const unsigned char *sha1)
 {
         enum object_type type;
 
         if (is_null_sha1(sha1)) {
                 file->ptr = (char *)"";
                 file->size = 0;
         } else {
                 file->ptr = read_sha1_file(sha1, &type, 
                                            (unsigned long *)&file->size);
         }
         return 1;
 }
 
 /*
  * Receive diff-buffers from xdiff and concatenate them as
  * needed across multiple callbacks.
  *
  * This is basically a copy of xdiff-interface.c/xdiff_outf(),
  * ripped from git and modified to use globals instead of
  * a special callback-struct.
  */
 char *diffbuf = NULL;
 int buflen = 0;
 
 int filediff_cb(void *priv, mmbuffer_t *mb, int nbuf)
 {
         int i;
 
         for (i = 0; i < nbuf; i++) {
                 if (mb[i].ptr[mb[i].size-1] != '\n') {
                         /* Incomplete line */
                         diffbuf = xrealloc(diffbuf, buflen + mb[i].size);
                         memcpy(diffbuf + buflen, mb[i].ptr, mb[i].size);
                         buflen += mb[i].size;
                         continue;
                 }
 
                 /* we have a complete line */
                 if (!diffbuf) {
                         ((linediff_fn)priv)(mb[i].ptr, mb[i].size);
                         continue;
                 }
                 diffbuf = xrealloc(diffbuf, buflen + mb[i].size);
                 memcpy(diffbuf + buflen, mb[i].ptr, mb[i].size);
                 ((linediff_fn)priv)(diffbuf, buflen + mb[i].size);
                 free(diffbuf);
                 diffbuf = NULL;
                 buflen = 0;
         }
         if (diffbuf) {
                 ((linediff_fn)priv)(diffbuf, buflen);
                 free(diffbuf);
                 diffbuf = NULL;
                 buflen = 0;
         }
         return 0;
diff --git a/ui-repolist.c b/ui-repolist.c
index cf27cb3..aa743bf 100644
--- a/ui-repolist.c
+++ b/ui-repolist.c
@@ -1,239 +1,247 @@
 /* ui-repolist.c: functions for generating the repolist page
  *
  * Copyright (C) 2006 Lars Hjemli
  *
  * Licensed under GNU General Public License v2
  *   (see COPYING for full license text)
  */
 
 #include <time.h>
 
 #include "cgit.h"
 #include "html.h"
 #include "ui-shared.h"
 
 time_t read_agefile(char *path)
 {
         FILE *f;
         static char buf[64], buf2[64];
 
         if (!(f = fopen(path, "r")))
                 return -1;
         if (fgets(buf, sizeof(buf), f) == NULL)
                 return -1;
         fclose(f);
         if (parse_date(buf, buf2, sizeof(buf2)))
                 return strtoul(buf2, NULL, 10);
         else
                 return 0;
 }
 
 static int get_repo_modtime(const struct cgit_repo *repo, time_t *mtime)
 {
         char *path;
         struct stat s;
+        struct cgit_repo *r = (struct cgit_repo *)repo;
 
+        if (repo->mtime != -1) {
+                *mtime = repo->mtime;
+                return 1;
+        }
         path = fmt("%s/%s", repo->path, ctx.cfg.agefile);
         if (stat(path, &s) == 0) {
                 *mtime = read_agefile(path);
+                r->mtime = *mtime;
                 return 1;
         }
 
         path = fmt("%s/refs/heads/%s", repo->path, repo->defbranch);
-        if (stat(path, &s) == 0) {
+        if (stat(path, &s) == 0)
                 *mtime = s.st_mtime;
-                return 1;
-        }
-        return 0;
+        else
+                *mtime = 0;
+
+        r->mtime = *mtime;
+        return (r->mtime != 0);
 }
 
 static void print_modtime(struct cgit_repo *repo)
 {
         time_t t;
         if (get_repo_modtime(repo, &t))
                 cgit_print_age(t, -1, NULL);
 }
 
 int is_match(struct cgit_repo *repo)
 {
         if (!ctx.qry.search)
                 return 1;
         if (repo->url && strcasestr(repo->url, ctx.qry.search))
                 return 1;
         if (repo->name && strcasestr(repo->name, ctx.qry.search))
                 return 1;
         if (repo->desc && strcasestr(repo->desc, ctx.qry.search))
                 return 1;
         if (repo->owner && strcasestr(repo->owner, ctx.qry.search))
                 return 1;
         return 0;
 }
 
 int is_in_url(struct cgit_repo *repo)
 {
         if (!ctx.qry.url)
                 return 1;
         if (repo->url && !prefixcmp(repo->url, ctx.qry.url))
                 return 1;
         return 0;
 }
 
 void print_sort_header(const char *title, const char *sort)
 {
         htmlf("<th class='left'><a href='./?s=%s", sort);
         if (ctx.qry.search) {
                 html("&q=");
                 html_url_arg(ctx.qry.search);
         }
         htmlf("'>%s</a></th>", title);
 }
 
 void print_header(int columns)
 {
         html("<tr class='nohover'>");
         print_sort_header("Name", "name");
         print_sort_header("Description", "desc");
         print_sort_header("Owner", "owner");
         print_sort_header("Idle", "idle");
         if (ctx.cfg.enable_index_links)
                 html("<th class='left'>Links</th>");
         html("</tr>\n");
 }
 
 
 void print_pager(int items, int pagelen, char *search)
 {
         int i;
         html("<div class='pager'>");
         for(i = 0; i * pagelen < items; i++)
                 cgit_index_link(fmt("[%d]", i+1), fmt("Page %d", i+1), NULL,
                                 search, i * pagelen);
         html("</div>");
 }
 
 static int cmp(const char *s1, const char *s2)
 {
         if (s1 && s2)
                 return strcmp(s1, s2);
         if (s1 && !s2)
                 return -1;
         if (s2 && !s1)
                 return 1;
         return 0;
 }
 
 static int sort_name(const void *a, const void *b)
 {
         const struct cgit_repo *r1 = a;
         const struct cgit_repo *r2 = b;
 
         return cmp(r1->name, r2->name);
 }
 
 static int sort_desc(const void *a, const void *b)
 {
         const struct cgit_repo *r1 = a;
         const struct cgit_repo *r2 = b;
 
         return cmp(r1->desc, r2->desc);
 }
 
 static int sort_owner(const void *a, const void *b)
 {
         const struct cgit_repo *r1 = a;
         const struct cgit_repo *r2 = b;
 
         return cmp(r1->owner, r2->owner);
 }
 
 static int sort_idle(const void *a, const void *b)
 {
         const struct cgit_repo *r1 = a;
         const struct cgit_repo *r2 = b;
         time_t t1, t2;
 
         t1 = t2 = 0;
         get_repo_modtime(r1, &t1);
         get_repo_modtime(r2, &t2);
         return t2 - t1;
 }
 
 struct sortcolumn {
         const char *name;
         int (*fn)(const void *a, const void *b);
 };
 
 struct sortcolumn sortcolumn[] = {
         {"name", sort_name},
         {"desc", sort_desc},
         {"owner", sort_owner},
         {"idle", sort_idle},
         {NULL, NULL}
 };
 
 int sort_repolist(char *field)
 {
         struct sortcolumn *column;
 
         for (column = &sortcolumn[0]; column->name; column++) {
                 if (strcmp(field, column->name))
                         continue;
                 qsort(cgit_repolist.repos, cgit_repolist.count,
                         sizeof(struct cgit_repo), column->fn);
                 return 1;
         }
         return 0;
 }
 
 
 void cgit_print_repolist()
 {
         int i, columns = 4, hits = 0, header = 0;
         char *last_group = NULL;
         int sorted = 0;
 
         if (ctx.cfg.enable_index_links)
                 columns++;
 
         ctx.page.title = ctx.cfg.root_title;
         cgit_print_http_headers(&ctx);
         cgit_print_docstart(&ctx);
         cgit_print_pageheader(&ctx);
 
         if (ctx.cfg.index_header)
                 html_include(ctx.cfg.index_header);
 
         if(ctx.qry.sort)
                 sorted = sort_repolist(ctx.qry.sort);
 
         html("<table summary='repository list' class='list nowrap'>");
         for (i=0; i<cgit_repolist.count; i++) {
                 ctx.repo = &cgit_repolist.repos[i];
                 if (!(is_match(ctx.repo) && is_in_url(ctx.repo)))
                         continue;
                 hits++;
                 if (hits <= ctx.qry.ofs)
                         continue;
                 if (hits > ctx.qry.ofs + ctx.cfg.max_repo_count)
                         continue;
                 if (!header++)
                         print_header(columns);
                 if (!sorted &&
                     ((last_group == NULL && ctx.repo->group != NULL) ||
                     (last_group != NULL && ctx.repo->group == NULL) ||
                     (last_group != NULL && ctx.repo->group != NULL &&
                      strcmp(ctx.repo->group, last_group)))) {
                         htmlf("<tr class='nohover'><td colspan='%d' class='repogroup'>",
                               columns);
                         html_txt(ctx.repo->group);
                         html("</td></tr>");
                         last_group = ctx.repo->group;
                 }
                 htmlf("<tr><td class='%s'>",
                       !sorted && ctx.repo->group ? "sublevel-repo" : "toplevel-repo");
                 cgit_summary_link(ctx.repo->name, ctx.repo->name, NULL, NULL);
                 html("</td><td>");
                 html_link_open(cgit_repourl(ctx.repo->url), NULL, NULL);
                 html_ntxt(ctx.cfg.max_repodesc_len, ctx.repo->desc);
                 html_link_close();
                 html("</td><td>");