Add support for "robots" meta-tag

With this change, cgit will start to generate the "robots" meta-tag, using
a default value of "index, nofollow".

The default value can be modified with a new cgitrc variable, "robots".

Signed-off-by: Lars Hjemli <hjemli@gmail.com>

4 files changed, 11 insertions, 0 deletions

diff --git a/cgit.h b/cgit.h
index 6291c58..ab0efeb 100644
--- a/cgit.h
+++ b/cgit.h
@@ -46,192 +46,193 @@
 #define TM_DAY   (TM_HOUR * 24)
 #define TM_WEEK  (TM_DAY * 7)
 #define TM_YEAR  (TM_DAY * 365)
 #define TM_MONTH (TM_YEAR / 12.0)
 
 
 /*
  * Default encoding
  */
 #define PAGE_ENCODING "UTF-8"
 
 typedef void (*configfn)(const char *name, const char *value);
 typedef void (*filepair_fn)(struct diff_filepair *pair);
 typedef void (*linediff_fn)(char *line, int len);
 
 struct cacheitem {
 	char *name;
 	struct stat st;
 	int ttl;
 	int fd;
 };
 
 struct repoinfo {
 	char *url;
 	char *name;
 	char *path;
 	char *desc;
 	char *owner;
 	char *defbranch;
 	char *group;
 	char *module_link;
 	char *readme;
 	int snapshots;
 	int enable_log_filecount;
 	int enable_log_linecount;
 };
 
 struct repolist {
 	int length;
 	int count;
 	struct repoinfo *repos;
 };
 
 struct commitinfo {
 	struct commit *commit;
 	char *author;
 	char *author_email;
 	unsigned long author_date;
 	char *committer;
 	char *committer_email;
 	unsigned long committer_date;
 	char *subject;
 	char *msg;
 	char *msg_encoding;
 };
 
 struct taginfo {
 	char *tagger;
 	char *tagger_email;
 	int tagger_date;
 	char *msg;
 };
 
 struct refinfo {
 	const char *refname;
 	struct object *object;
 	union {
 		struct taginfo *tag;
 		struct commitinfo *commit;
 	};
 };
 
 struct reflist {
 	struct refinfo **refs;
 	int alloc;
 	int count;
 };
 
 extern const char *cgit_version;
 
 extern struct repolist cgit_repolist;
 extern struct repoinfo *cgit_repo;
 extern int cgit_cmd;
 
 extern char *cgit_root_title;
 extern char *cgit_css;
 extern char *cgit_logo;
 extern char *cgit_index_header;
 extern char *cgit_index_info;
 extern char *cgit_logo_link;
 extern char *cgit_module_link;
 extern char *cgit_agefile;
 extern char *cgit_virtual_root;
 extern char *cgit_script_name;
 extern char *cgit_cache_root;
 extern char *cgit_repo_group;
+extern char *cgit_robots;
 
 extern int cgit_nocache;
 extern int cgit_snapshots;
 extern int cgit_enable_index_links;
 extern int cgit_enable_log_filecount;
 extern int cgit_enable_log_linecount;
 extern int cgit_max_lock_attempts;
 extern int cgit_cache_root_ttl;
 extern int cgit_cache_repo_ttl;
 extern int cgit_cache_dynamic_ttl;
 extern int cgit_cache_static_ttl;
 extern int cgit_cache_max_create_time;
 extern int cgit_summary_log;
 extern int cgit_summary_tags;
 extern int cgit_summary_branches;
 
 extern int cgit_max_msg_len;
 extern int cgit_max_repodesc_len;
 extern int cgit_max_commit_count;
 
 extern int cgit_query_has_symref;
 extern int cgit_query_has_sha1;
 
 extern char *cgit_querystring;
 extern char *cgit_query_repo;
 extern char *cgit_query_page;
 extern char *cgit_query_search;
 extern char *cgit_query_grep;
 extern char *cgit_query_head;
 extern char *cgit_query_sha1;
 extern char *cgit_query_sha2;
 extern char *cgit_query_path;
 extern char *cgit_query_name;
 extern int   cgit_query_ofs;
 
 extern int htmlfd;
 
 extern int cgit_get_cmd_index(const char *cmd);
 extern struct repoinfo *cgit_get_repoinfo(const char *url);
 extern void cgit_global_config_cb(const char *name, const char *value);
 extern void cgit_repo_config_cb(const char *name, const char *value);
 extern void cgit_querystring_cb(const char *name, const char *value);
 
 extern int chk_zero(int result, char *msg);
 extern int chk_positive(int result, char *msg);
 extern int chk_non_negative(int result, char *msg);
 
 extern int hextoint(char c);
 extern char *trim_end(const char *str, char c);
 extern char *strlpart(char *txt, int maxlen);
 extern char *strrpart(char *txt, int maxlen);
 
 extern void cgit_add_ref(struct reflist *list, struct refinfo *ref);
 extern int cgit_refs_cb(const char *refname, const unsigned char *sha1,
 			int flags, void *cb_data);
 
 extern void *cgit_free_commitinfo(struct commitinfo *info);
 
 extern int cgit_diff_files(const unsigned char *old_sha1,
 			   const unsigned char *new_sha1,
 			   linediff_fn fn);
 
 extern void cgit_diff_tree(const unsigned char *old_sha1,
 			   const unsigned char *new_sha1,
 			   filepair_fn fn, const char *prefix);
 
 extern void cgit_diff_commit(struct commit *commit, filepair_fn fn);
 
 extern char *fmt(const char *format,...);
 
 extern void html(const char *txt);
 extern void htmlf(const char *format,...);
 extern void html_txt(char *txt);
 extern void html_ntxt(int len, char *txt);
 extern void html_attr(char *txt);
 extern void html_hidden(char *name, char *value);
 extern void html_option(char *value, char *text, char *selected_value);
 extern void html_link_open(char *url, char *title, char *class);
 extern void html_link_close(void);
 extern void html_filemode(unsigned short mode);
 extern int html_include(const char *filename);
 
 extern int cgit_read_config(const char *filename, configfn fn);
 extern int cgit_parse_query(char *txt, configfn fn);
 extern struct commitinfo *cgit_parse_commit(struct commit *commit);
 extern struct taginfo *cgit_parse_tag(struct tag *tag);
 extern void cgit_parse_url(const char *url);
 
 extern char *cache_safe_filename(const char *unsafe);
 extern int cache_lock(struct cacheitem *item);
 extern int cache_unlock(struct cacheitem *item);
 extern int cache_cancel_lock(struct cacheitem *item);
 extern int cache_exist(struct cacheitem *item);
 extern int cache_expired(struct cacheitem *item);
 
 extern char *cgit_repourl(const char *reponame);
diff --git a/cgitrc b/cgitrc
index 6363c9c..8c616e0 100644
--- a/cgitrc
+++ b/cgitrc
@@ -1,106 +1,111 @@
 ##
 ## cgitrc: template for /etc/cgitrc
 ##
 
 
 ## Uncomment and set to 1 to deactivate caching of generated pages. Mostly
 ## usefull for testing.
 #nocache=0
 
 
+## This variable can be used to override the default value for "robots"
+## meta-tag. If unset, the meta-tag isn't generated.
+#robots=index, nofollow
+
+
 ## Set allowed snapshot types by default. Can be overridden per repo
 # can be any combination of zip/tar.gz/tar.bz2/tar
 #snapshots=0
 
 
 ## Enable/disable extra links to summary/log/tree per repo on index page
 #enable-index-links=0
 
 
 ## Enable/disable display of 'number of files changed' in log view
 #enable-log-filecount=0
 
 
 ## Enable/disable display of 'number of lines changed' in log view
 #enable-log-linecount=0
 
 
 ## Enable/disable display of HEAD shortlog in summary view. Set it to maximum
 ## number of commits that should be displayed
 #summary-log=0
 
 
 ## Restrict the number of branches printed in summary view. Set to 0 to
 ## print all branches.
 #summary-branches=0
 
 
 ## Restrict the number of tags printed in summary view. Set to 0 to
 ## print all tags.
 #summary-tags=0
 
 
 ## The "Idle" column on the repository index page can read a timestamp
 ## from the specified agefile (if this file cannot be found, the mtime
 ## of HEAD is used).
 ## The cgit repo on hjemli.net uses the the following command in it's
 ## post-receive hook to update the age-file:
 ##   git-for-each-ref --format="%(committerdate)" --sort=-committerdate \
 ##     --count=1 > $GIT_DIR/info/web/last-modifie
 ##
 #agefile=info/web/last-modified
 
 
 ## Git detects renames, but with a limit on the number of files to
 ## consider. This option can be used to specify another limit (or -1 to
 ## use the default limit).
 ##
 #renamelimit=-1
 
 
 ## Specify a root for virtual urls. This makes cgit generate urls like
 ##
 ##    http://localhost/git/repo/log/?h=branch
 ##
 ## instead of
 ##
 ##    http://localhost/cgit/cgit.cgi?url=repo/log&h=branch
 ##
 ## For this to work with apache, a rewrite rule must be added to httpd.conf,
 ## possibly looking something like this:
 ##
 ##    RewriteRule ^/git/(.*)$ /cgit/cgit.cgi?url=$1   [L,QSA]
 ##
 ## For this to work with lighttpd, the rewrite rule should look more like this:
 ##
 ##    url.rewrite = (
 ##        "^/git/([^?/]+/[^?]*)?(?:\?(.*))?$" => "/cgit.cgi?url=$1&$2"
 ##    )
 ##
 ## This setting is disabled by default.
 #virtual-root=/git
 
 
 ## Set the title printed on the root page
 #root-title=Git repository browser
 
 
 ## If specified, the file at this path will be included as HTML in the
 ## sidebar on the repository index page
 #index-info=
 
 
 ## If specified, the file at this path will be included as HTML above
 ## the repository index
 #index-header=
 
 
 ## Link to css file
 #css=/cgit/cgit.css
 
 
 ## Link to logo file
 #logo=/cgit/git-logo.png
 
 
 ## Url loaded when clicking the logo
diff --git a/shared.c b/shared.c
index 8cb4808..fd8b1e3 100644
--- a/shared.c
+++ b/shared.c
@@ -1,295 +1,298 @@
 /* shared.c: global vars + some callback functions
  *
  * Copyright (C) 2006 Lars Hjemli
  *
  * Licensed under GNU General Public License v2
  *   (see COPYING for full license text)
  */
 
 #include "cgit.h"
 
 struct repolist cgit_repolist;
 struct repoinfo *cgit_repo;
 int cgit_cmd;
 
 const char *cgit_version = CGIT_VERSION;
 
 char *cgit_root_title   = "Git repository browser";
 char *cgit_css          = "/cgit.css";
 char *cgit_logo         = "/git-logo.png";
 char *cgit_index_header = NULL;
 char *cgit_index_info   = NULL;
 char *cgit_logo_link    = "http://www.kernel.org/pub/software/scm/git/docs/";
 char *cgit_module_link  = "./?repo=%s&page=commit&id=%s";
 char *cgit_agefile      = "info/web/last-modified";
 char *cgit_virtual_root = NULL;
 char *cgit_script_name  = CGIT_SCRIPT_NAME;
 char *cgit_cache_root   = CGIT_CACHE_ROOT;
 char *cgit_repo_group   = NULL;
+char *cgit_robots       = "index, nofollow";
 
 int cgit_nocache               =  0;
 int cgit_snapshots             =  0;
 int cgit_enable_index_links    =  0;
 int cgit_enable_log_filecount  =  0;
 int cgit_enable_log_linecount  =  0;
 int cgit_max_lock_attempts     =  5;
 int cgit_cache_root_ttl        =  5;
 int cgit_cache_repo_ttl        =  5;
 int cgit_cache_dynamic_ttl     =  5;
 int cgit_cache_static_ttl      = -1;
 int cgit_cache_max_create_time =  5;
 int cgit_summary_log           =  0;
 int cgit_summary_tags          =  0;
 int cgit_summary_branches      =  0;
 int cgit_renamelimit           = -1;
 
 int cgit_max_msg_len = 60;
 int cgit_max_repodesc_len = 60;
 int cgit_max_commit_count = 50;
 
 int cgit_query_has_symref = 0;
 int cgit_query_has_sha1   = 0;
 
 char *cgit_querystring  = NULL;
 char *cgit_query_repo   = NULL;
 char *cgit_query_page   = NULL;
 char *cgit_query_head   = NULL;
 char *cgit_query_search = NULL;
 char *cgit_query_grep   = NULL;
 char *cgit_query_sha1   = NULL;
 char *cgit_query_sha2   = NULL;
 char *cgit_query_path   = NULL;
 char *cgit_query_name   = NULL;
 int   cgit_query_ofs    = 0;
 
 int htmlfd = 0;
 
 
 int cgit_get_cmd_index(const char *cmd)
 {
 	static char *cmds[] = {"log", "commit", "diff", "tree", "blob",
 			       "snapshot", "tag", "refs", NULL};
 	int i;
 
 	for(i = 0; cmds[i]; i++)
 		if (!strcmp(cmd, cmds[i]))
 			return i + 1;
 	return 0;
 }
 
 int chk_zero(int result, char *msg)
 {
 	if (result != 0)
 		die("%s: %s", msg, strerror(errno));
 	return result;
 }
 
 int chk_positive(int result, char *msg)
 {
 	if (result <= 0)
 		die("%s: %s", msg, strerror(errno));
 	return result;
 }
 
 int chk_non_negative(int result, char *msg)
 {
     	if (result < 0)
 	    	die("%s: %s",msg, strerror(errno));
 	return result;
 }
 
 struct repoinfo *add_repo(const char *url)
 {
 	struct repoinfo *ret;
 
 	if (++cgit_repolist.count > cgit_repolist.length) {
 		if (cgit_repolist.length == 0)
 			cgit_repolist.length = 8;
 		else
 			cgit_repolist.length *= 2;
 		cgit_repolist.repos = xrealloc(cgit_repolist.repos,
 					       cgit_repolist.length *
 					       sizeof(struct repoinfo));
 	}
 
 	ret = &cgit_repolist.repos[cgit_repolist.count-1];
 	ret->url = trim_end(url, '/');
 	ret->name = ret->url;
 	ret->path = NULL;
 	ret->desc = NULL;
 	ret->owner = NULL;
 	ret->group = cgit_repo_group;
 	ret->defbranch = "master";
 	ret->snapshots = cgit_snapshots;
 	ret->enable_log_filecount = cgit_enable_log_filecount;
 	ret->enable_log_linecount = cgit_enable_log_linecount;
 	ret->module_link = cgit_module_link;
 	ret->readme = NULL;
 	return ret;
 }
 
 struct repoinfo *cgit_get_repoinfo(const char *url)
 {
 	int i;
 	struct repoinfo *repo;
 
 	for (i=0; i<cgit_repolist.count; i++) {
 		repo = &cgit_repolist.repos[i];
 		if (!strcmp(repo->url, url))
 			return repo;
 	}
 	return NULL;
 }
 
 void cgit_global_config_cb(const char *name, const char *value)
 {
 	if (!strcmp(name, "root-title"))
 		cgit_root_title = xstrdup(value);
 	else if (!strcmp(name, "css"))
 		cgit_css = xstrdup(value);
 	else if (!strcmp(name, "logo"))
 		cgit_logo = xstrdup(value);
 	else if (!strcmp(name, "index-header"))
 		cgit_index_header = xstrdup(value);
 	else if (!strcmp(name, "index-info"))
 		cgit_index_info = xstrdup(value);
 	else if (!strcmp(name, "logo-link"))
 		cgit_logo_link = xstrdup(value);
 	else if (!strcmp(name, "module-link"))
 		cgit_module_link = xstrdup(value);
 	else if (!strcmp(name, "virtual-root")) {
 		cgit_virtual_root = trim_end(value, '/');
 		if (!cgit_virtual_root && (!strcmp(value, "/")))
 			cgit_virtual_root = "";
 	} else if (!strcmp(name, "nocache"))
 		cgit_nocache = atoi(value);
 	else if (!strcmp(name, "snapshots"))
 		cgit_snapshots = cgit_parse_snapshots_mask(value);
 	else if (!strcmp(name, "enable-index-links"))
 		cgit_enable_index_links = atoi(value);
 	else if (!strcmp(name, "enable-log-filecount"))
 		cgit_enable_log_filecount = atoi(value);
 	else if (!strcmp(name, "enable-log-linecount"))
 		cgit_enable_log_linecount = atoi(value);
 	else if (!strcmp(name, "cache-root"))
 		cgit_cache_root = xstrdup(value);
 	else if (!strcmp(name, "cache-root-ttl"))
 		cgit_cache_root_ttl = atoi(value);
 	else if (!strcmp(name, "cache-repo-ttl"))
 		cgit_cache_repo_ttl = atoi(value);
 	else if (!strcmp(name, "cache-static-ttl"))
 		cgit_cache_static_ttl = atoi(value);
 	else if (!strcmp(name, "cache-dynamic-ttl"))
 		cgit_cache_dynamic_ttl = atoi(value);
 	else if (!strcmp(name, "max-message-length"))
 		cgit_max_msg_len = atoi(value);
 	else if (!strcmp(name, "max-repodesc-length"))
 		cgit_max_repodesc_len = atoi(value);
 	else if (!strcmp(name, "max-commit-count"))
 		cgit_max_commit_count = atoi(value);
 	else if (!strcmp(name, "summary-log"))
 		cgit_summary_log = atoi(value);
 	else if (!strcmp(name, "summary-branches"))
 		cgit_summary_branches = atoi(value);
 	else if (!strcmp(name, "summary-tags"))
 		cgit_summary_tags = atoi(value);
 	else if (!strcmp(name, "agefile"))
 		cgit_agefile = xstrdup(value);
 	else if (!strcmp(name, "renamelimit"))
 		cgit_renamelimit = atoi(value);
+	else if (!strcmp(name, "robots"))
+		cgit_robots = xstrdup(value);
 	else if (!strcmp(name, "repo.group"))
 		cgit_repo_group = xstrdup(value);
 	else if (!strcmp(name, "repo.url"))
 		cgit_repo = add_repo(value);
 	else if (!strcmp(name, "repo.name"))
 		cgit_repo->name = xstrdup(value);
 	else if (cgit_repo && !strcmp(name, "repo.path"))
 		cgit_repo->path = trim_end(value, '/');
 	else if (cgit_repo && !strcmp(name, "repo.desc"))
 		cgit_repo->desc = xstrdup(value);
 	else if (cgit_repo && !strcmp(name, "repo.owner"))
 		cgit_repo->owner = xstrdup(value);
 	else if (cgit_repo && !strcmp(name, "repo.defbranch"))
 		cgit_repo->defbranch = xstrdup(value);
 	else if (cgit_repo && !strcmp(name, "repo.snapshots"))
 		cgit_repo->snapshots = cgit_snapshots & cgit_parse_snapshots_mask(value); /* XXX: &? */
 	else if (cgit_repo && !strcmp(name, "repo.enable-log-filecount"))
 		cgit_repo->enable_log_filecount = cgit_enable_log_filecount * atoi(value);
 	else if (cgit_repo && !strcmp(name, "repo.enable-log-linecount"))
 		cgit_repo->enable_log_linecount = cgit_enable_log_linecount * atoi(value);
 	else if (cgit_repo && !strcmp(name, "repo.module-link"))
 		cgit_repo->module_link= xstrdup(value);
 	else if (cgit_repo && !strcmp(name, "repo.readme") && value != NULL) {
 		if (*value == '/')
 			cgit_repo->readme = xstrdup(value);
 		else
 			cgit_repo->readme = xstrdup(fmt("%s/%s", cgit_repo->path, value));
 	} else if (!strcmp(name, "include"))
 		cgit_read_config(value, cgit_global_config_cb);
 }
 
 void cgit_querystring_cb(const char *name, const char *value)
 {
 	if (!strcmp(name,"r")) {
 		cgit_query_repo = xstrdup(value);
 		cgit_repo = cgit_get_repoinfo(value);
 	} else if (!strcmp(name, "p")) {
 		cgit_query_page = xstrdup(value);
 		cgit_cmd = cgit_get_cmd_index(value);
 	} else if (!strcmp(name, "url")) {
 		cgit_parse_url(value);
 	} else if (!strcmp(name, "qt")) {
 		cgit_query_grep = xstrdup(value);
 	} else if (!strcmp(name, "q")) {
 		cgit_query_search = xstrdup(value);
 	} else if (!strcmp(name, "h")) {
 		cgit_query_head = xstrdup(value);
 		cgit_query_has_symref = 1;
 	} else if (!strcmp(name, "id")) {
 		cgit_query_sha1 = xstrdup(value);
 		cgit_query_has_sha1 = 1;
 	} else if (!strcmp(name, "id2")) {
 		cgit_query_sha2 = xstrdup(value);
 		cgit_query_has_sha1 = 1;
 	} else if (!strcmp(name, "ofs")) {
 		cgit_query_ofs = atoi(value);
 	} else if (!strcmp(name, "path")) {
 		cgit_query_path = trim_end(value, '/');
 	} else if (!strcmp(name, "name")) {
 		cgit_query_name = xstrdup(value);
 	}
 }
 
 void *cgit_free_commitinfo(struct commitinfo *info)
 {
 	free(info->author);
 	free(info->author_email);
 	free(info->committer);
 	free(info->committer_email);
 	free(info->subject);
 	free(info->msg);
 	free(info->msg_encoding);
 	free(info);
 	return NULL;
 }
 
 int hextoint(char c)
 {
 	if (c >= 'a' && c <= 'f')
 		return 10 + c - 'a';
 	else if (c >= 'A' && c <= 'F')
 		return 10 + c - 'A';
 	else if (c >= '0' && c <= '9')
 		return c - '0';
 	else
 		return -1;
 }
 
 char *trim_end(const char *str, char c)
 {
 	int len;
 	char *s, *t;
 
 	if (str == NULL)
 		return NULL;
 	t = (char *)str;
diff --git a/ui-shared.c b/ui-shared.c
index 5192800..2f771da 100644
--- a/ui-shared.c
+++ b/ui-shared.c
@@ -271,192 +271,194 @@ void cgit_diff_link(char *name, char *title, char *class, char *head,
 	html_txt(name);
 	html("</a>");
 }
 
 void cgit_object_link(struct object *obj)
 {
 	char *page, *arg, *url;
 
 	if (obj->type == OBJ_COMMIT) {
                 cgit_commit_link(fmt("commit %s", sha1_to_hex(obj->sha1)), NULL, NULL,
 				 cgit_query_head, sha1_to_hex(obj->sha1));
 		return;
 	} else if (obj->type == OBJ_TREE) {
 		page = "tree";
 		arg = "id";
 	} else if (obj->type == OBJ_TAG) {
 		page = "tag";
 		arg = "id";
 	} else {
 		page = "blob";
 		arg = "id";
 	}
 
 	url = cgit_pageurl(cgit_query_repo, page,
 			   fmt("%s=%s", arg, sha1_to_hex(obj->sha1)));
 	html_link_open(url, NULL, NULL);
 	htmlf("%s %s", typename(obj->type),
 	      sha1_to_hex(obj->sha1));
 	html_link_close();
 }
 
 void cgit_print_date(time_t secs, char *format)
 {
 	char buf[64];
 	struct tm *time;
 
 	time = gmtime(&secs);
 	strftime(buf, sizeof(buf)-1, format, time);
 	html_txt(buf);
 }
 
 void cgit_print_age(time_t t, time_t max_relative, char *format)
 {
 	time_t now, secs;
 
 	time(&now);
 	secs = now - t;
 
 	if (secs > max_relative && max_relative >= 0) {
 		cgit_print_date(t, format);
 		return;
 	}
 
 	if (secs < TM_HOUR * 2) {
 		htmlf("<span class='age-mins'>%.0f min.</span>",
 		      secs * 1.0 / TM_MIN);
 		return;
 	}
 	if (secs < TM_DAY * 2) {
 		htmlf("<span class='age-hours'>%.0f hours</span>",
 		      secs * 1.0 / TM_HOUR);
 		return;
 	}
 	if (secs < TM_WEEK * 2) {
 		htmlf("<span class='age-days'>%.0f days</span>",
 		      secs * 1.0 / TM_DAY);
 		return;
 	}
 	if (secs < TM_MONTH * 2) {
 		htmlf("<span class='age-weeks'>%.0f weeks</span>",
 		      secs * 1.0 / TM_WEEK);
 		return;
 	}
 	if (secs < TM_YEAR * 2) {
 		htmlf("<span class='age-months'>%.0f months</span>",
 		      secs * 1.0 / TM_MONTH);
 		return;
 	}
 	htmlf("<span class='age-years'>%.0f years</span>",
 	      secs * 1.0 / TM_YEAR);
 }
 
 void cgit_print_docstart(char *title, struct cacheitem *item)
 {
 	html("Content-Type: text/html; charset=" PAGE_ENCODING "\n");
 	htmlf("Last-Modified: %s\n", http_date(item->st.st_mtime));
 	htmlf("Expires: %s\n", http_date(item->st.st_mtime +
 					 ttl_seconds(item->ttl)));
 	html("\n");
 	html(cgit_doctype);
 	html("<html xmlns='http://www.w3.org/1999/xhtml' xml:lang='en' lang='en'>\n");
 	html("<head>\n");
 	html("<title>");
 	html_txt(title);
 	html("</title>\n");
 	htmlf("<meta name='generator' content='cgit %s'/>\n", cgit_version);
+	if (cgit_robots && *cgit_robots)
+		htmlf("<meta name='robots' content='%s'/>\n", cgit_robots);
 	html("<link rel='stylesheet' type='text/css' href='");
 	html_attr(cgit_css);
 	html("'/>\n");
 	html("</head>\n");
 	html("<body>\n");
 }
 
 void cgit_print_docend()
 {
 	html("</td>\n</tr>\n</table>\n</body>\n</html>\n");
 }
 
 int print_branch_option(const char *refname, const unsigned char *sha1,
 			int flags, void *cb_data)
 {
 	char *name = (char *)refname;
 	html_option(name, name, cgit_query_head);
 	return 0;
 }
 
 int print_archive_ref(const char *refname, const unsigned char *sha1,
 		       int flags, void *cb_data)
 {
 	struct tag *tag;
 	struct taginfo *info;
 	struct object *obj;
 	char buf[256], *url;
 	unsigned char fileid[20];
 	int *header = (int *)cb_data;
 
 	if (prefixcmp(refname, "refs/archives"))
 		return 0;
 	strncpy(buf, refname+14, sizeof(buf));
 	obj = parse_object(sha1);
 	if (!obj)
 		return 1;
 	if (obj->type == OBJ_TAG) {
 		tag = lookup_tag(sha1);
 		if (!tag || parse_tag(tag) || !(info = cgit_parse_tag(tag)))
 			return 0;
 		hashcpy(fileid, tag->tagged->sha1);
 	} else if (obj->type != OBJ_BLOB) {
 		return 0;
 	} else {
 		hashcpy(fileid, sha1);
 	}
 	if (!*header) {
 		html("<h1>download</h1>\n");
 		*header = 1;
 	}
 	url = cgit_pageurl(cgit_query_repo, "blob",
 			   fmt("id=%s&amp;path=%s", sha1_to_hex(fileid),
 			       buf));
 	html_link_open(url, NULL, "menu");
 	html_txt(strlpart(buf, 20));
 	html_link_close();
 	return 0;
 }
 
 void add_hidden_formfields(int incl_head, int incl_search, char *page)
 {
 	char *url;
 
 	if (!cgit_virtual_root) {
 		url = fmt("%s/%s", cgit_query_repo, page);
 		if (cgit_query_path)
 			url = fmt("%s/%s", url, cgit_query_path);
 		html_hidden("url", url);
 	}
 
 	if (incl_head && strcmp(cgit_query_head, cgit_repo->defbranch))
 		html_hidden("h", cgit_query_head);
 
 	if (cgit_query_sha1)
 		html_hidden("id", cgit_query_sha1);
 	if (cgit_query_sha2)
 		html_hidden("id2", cgit_query_sha2);
 
 	if (incl_search) {
 		if (cgit_query_grep)
 			html_hidden("qt", cgit_query_grep);
 		if (cgit_query_search)
 			html_hidden("q", cgit_query_search);
 	}
 }
 
 void cgit_print_pageheader(char *title, int show_search)
 {
 	static const char *default_info = "This is cgit, a fast webinterface for git repositories";
 	int header = 0;
 
 	html("<div id='sidebar'>\n");
 	html("<div id='logo'><a href='");
 	html_attr(cgit_rooturl());
 	htmlf("'><img src='%s' alt='cgit'/></a></div>\n",
 	      cgit_logo);