added an identifier normalization utility function

* moved iname leader characters and whitespace characters strings to
  opkele::data namespace
* added opkele::util::normalize_identifier() function

Signed-off-by: Michael Krelin <hacker@klever.net>

5 files changed, 53 insertions, 12 deletions

diff --git a/include/opkele/data.h b/include/opkele/data.h
index d0b0516..904b5ae 100644
--- a/include/opkele/data.h
+++ b/include/opkele/data.h
@@ -1,18 +1,20 @@
 #ifndef __OPKELE_DATA_H
 #define __OPKELE_DATA_H
 
 /**
  * @brief the main opkele namespace
  */
 namespace opkele {
 
     /**
      * @brief internal data opkele namespace
      */
     namespace data {
         extern const char *_default_p;
         extern const char *_default_g;
+        extern const char *_iname_leaders;
+        extern const char *_whitespace_chars;
     }
 }
 
 #endif /* __OPKELE_DATA_H */
diff --git a/include/opkele/util.h b/include/opkele/util.h
index 60955e1..fd974a1 100644
--- a/include/opkele/util.h
+++ b/include/opkele/util.h
@@ -1,114 +1,116 @@
 #ifndef __OPKELE_UTIL_H
 #define __OPKELE_UTIL_H
 
 #include <time.h>
 #include <string>
 #include <vector>
 #include <opkele/types.h>
 
 namespace opkele {
     using std::string;
     using std::vector;
 
     /**
      * @brief opkele utils namespace
      */
     namespace util {
 
         /**
          * Convert internal time representation to w3c format
          * @param t internal representation
          * @return w3c time
          * @throw failed_conversion in case of error
          */
         string time_to_w3c(time_t t);
         /**
          * Convert W3C time representation to  internal time_t
          * @param w w3c representation
          * @return converted time
          * @throw failed_conversion in case of error
          */
         time_t w3c_to_time(const string& w);
 
         /**
          * Encode string to the representation suitable for using in URL.
          * @param str string to encode
          * @return encoded string
          * @throw failed_conversion in case of failure
          */
         string url_encode(const string& str);
 
         /**
          * Make string suitable for using as x(ht)ml attribute.
          * @param str string to escape
          * @return escaped string
          */
         string attr_escape(const string& str);
 
         /**
          * Convert number to string
          * @param l number
          * @return string representation
          * @throw failed_conversion in case of failure
          */
         string long_to_string(long l);
         /**
          * Convert string to number
          * @param s string, containing the number
          * @return the number
          * @throw failed_conversion in case of failure
          */
         long string_to_long(const string& s);
 
         /**
          * Encode binary data using base64.
          * @param data pointer to binary data
          * @param length length of data
          * @return encoded data
          */
         string encode_base64(const void *data,size_t length);
         /**
          * Decode binary data from base64 representation.
          * @param data base64-encoded data
          * @param rv container for decoded binary
          */
         void decode_base64(const string& data,vector<unsigned char>& rv);
 
         /**
          * Normalize http(s) URI according to RFC3986, section 6. URI is
          * expected to have scheme: in front of it.
          * @param uri URI
          * @return normalized URI
          * @throw not_implemented in case of non-httpi(s) URI
          * @throw bad_input in case of malformed URI
          */
         string rfc_3986_normalize_uri(const string& uri);
 
+        string normalize_identifier(const string& usi,bool strip_fragment);
+
         /**
          * Match URI against realm
          * @param uri URI to match
          * @param realm realm to match against
          * @return true if URI matches realm
          */
         bool uri_matches_realm(const string& uri,const string& realm);
 
         /**
          * Strip fragment part from URI
          * @param uri input/output parameter containing the URI
          * @return reference to uri
          */
         string& strip_uri_fragment_part(string& uri);
 
         /**
          * Calculate signature and encode it using base64
          * @param assoc association being used for signing
          * @param om openid message
          * @return base64 representation of the signature
          */
         string base64_signature(const assoc_t& assoc,const basic_openid_message& om);
 
     }
 
 }
 
 #endif /* __OPKELE_UTIL_H */
diff --git a/lib/data.cc b/lib/data.cc
index c040430..f71788f 100644
--- a/lib/data.cc
+++ b/lib/data.cc
@@ -1,11 +1,14 @@
 #include <opkele/data.h>
 
 namespace opkele {
 
     namespace data {
 
         const char *_default_p = "155172898181473697471232257763715539915724801966915404479707795314057629378541917580651227423698188993727816152646631438561595825688188889951272158842675419950341258706556549803580104870537681476726513255747040765857479291291572334510643245094715007229621094194349783925984760375594985848253359305585439638443";
         const char *_default_g = "2";
 
+        const char *_iname_leaders = "=@+$!(";
+        const char *_whitespace_chars = " \t\r\n";
+
     }
 }
diff --git a/lib/discovery.cc b/lib/discovery.cc
index b7f2db6..5913ad4 100644
--- a/lib/discovery.cc
+++ b/lib/discovery.cc
@@ -1,577 +1,576 @@
 #include <list>
 #include <opkele/curl.h>
 #include <opkele/expat.h>
 #include <opkele/uris.h>
 #include <opkele/discovery.h>
 #include <opkele/exception.h>
 #include <opkele/util.h>
 #include <opkele/tidy.h>
+#include <opkele/data.h>
 #include <opkele/debug.h>
 
 #include "config.h"
 
 #define XRDS_HEADER "X-XRDS-Location"
 #define CT_HEADER "Content-Type"
 
 namespace opkele {
     using std::list;
     using xrd::XRD_t;
     using xrd::service_t;
 
     /* TODO: the whole discovery thing needs cleanup and optimization due to
      * many changes of concept. */
 
-    static const char *whitespace = " \t\r\n";
-    static const char *i_leaders = "=@+$!(";
     static const size_t max_html = 16384;
 
     static const struct service_type_t {
         const char *uri;
         const char *forceid;
     } op_service_types[] = {
         { STURI_OPENID20_OP, IDURI_SELECT20 },
         { STURI_OPENID20, 0 },
         { STURI_OPENID11, 0 },
         { STURI_OPENID10, 0 }
     };
     enum {
         st_index_1 = 2, st_index_2 = 1
     };
 
 
     static inline bool is_qelement(const XML_Char *n,const char *qen) {
         return !strcasecmp(n,qen);
     }
     static inline bool is_element(const XML_Char *n,const char *en) {
         if(!strcasecmp(n,en)) return true;
         int nl = strlen(n), enl = strlen(en);
         if( (nl>=(enl+1)) && n[nl-enl-1]=='\t'
                 && !strcasecmp(&n[nl-enl],en) )
             return true;
         return false;
     }
 
     static long element_priority(const XML_Char **a) {
         for(;*a;++a)
             if(!strcasecmp(*(a++),"priority")) {
                 long rv;
                 return (sscanf(*a,"%ld",&rv)==1)?rv:-1;
             }
         return -1;
     }
     /* TODO: ideally all attributes should be
      * retrieved in one run */
     static const char *element_attr(const XML_Char **a, const char *at) {
         for(;*a;++a)
             if(!strcasecmp(*(a++),at)) {
                 return *a;
             }
         return 0;
     }
 
     class idigger_t : public util::curl_t, public util::expat_t {
         public:
             string xri_proxy;
 
             enum {
                 xmode_html = 1, xmode_xrd = 2, xmode_cid = 4,
                 xmode_noredirs = 8
             };
             int xmode;
 
             string xrds_location;
             string http_content_type;
             service_t html_openid1;
             service_t html_openid2;
             string cdata_buf;
             long status_code;
             string status_string;
 
             typedef list<string> pt_stack_t;
             pt_stack_t pt_stack;
             int skipping;
             bool parser_choked;
             string save_html;
 
             XRD_t *xrd;
             service_t *xrd_service;
             string* cdata;
 
             idigger_t()
                 : util::curl_t(easy_init()),
                 util::expat_t(0),
                 xri_proxy(XRI_PROXY_URL) {
                     CURLcode r;
                     (r=misc_sets())
                         || (r=set_write())
                         || (r=set_header())
                         ;
                     if(r)
                         throw exception_curl(OPKELE_CP_ "failed to set curly options",r);
                 }
             ~idigger_t() throw() { }
 
             void yadiscover(endpoint_discovery_iterator oi,const string& yurl,const char **types,bool redirs) {
                 idiscovery_t idis;
                 idis.xri_identity = false;
                 discover_at(idis,yurl,xmode_html|xmode_xrd|(redirs?0:xmode_noredirs));
                 if(!xrds_location.empty()) {
                     idis.clear();
                     discover_at(idis,xrds_location,xmode_xrd);
                 }
                 idis.normalized_id = idis.canonicalized_id = yurl;
                 service_type_t st;
                 for(st.uri=*types;*types;st.uri=*(++types))
                     queue_endpoints(oi,idis,&st);
             }
 
             string discover(endpoint_discovery_iterator& oi,const string& identity) {
                 string rv;
                 idiscovery_t idis;
-                string::size_type fsc = identity.find_first_not_of(whitespace);
+                string::size_type fsc = identity.find_first_not_of(data::_whitespace_chars);
                 if(fsc==string::npos)
                     throw bad_input(OPKELE_CP_ "whitespace-only identity");
-                string::size_type lsc = identity.find_last_not_of(whitespace);
+                string::size_type lsc = identity.find_last_not_of(data::_whitespace_chars);
                 assert(lsc!=string::npos);
                 if(!strncasecmp(identity.c_str()+fsc,"xri://",sizeof("xri://")-1))
                     fsc += sizeof("xri://")-1;
                 if((fsc+1)>=lsc)
                     throw bad_input(OPKELE_CP_ "not a character of importance in identity");
                 string id(identity,fsc,lsc-fsc+1);
                 idis.clear();
-                if(strchr(i_leaders,id[0])) {
+                if(strchr(data::_iname_leaders,id[0])) {
                     /* TODO: further normalize xri identity? Like folding case
                      * or whatever... */
                     rv = id;
                     set<string> cids;
                     for(const struct service_type_t *st=op_service_types;
                             st<&op_service_types[sizeof(op_service_types)/sizeof(*op_service_types)];++st) {
                         idis.clear();
                         discover_at( idis,
                                 xri_proxy + util::url_encode(id)+
                                 "?_xrd_t="+util::url_encode(st->uri)+
                                 "&_xrd_r=application/xrd%2Bxml"
                                 ";sep=true;refs=true",
                                 xmode_xrd );
                         if(status_code==241) continue;
                         if(status_code!=100)
                             throw failed_xri_resolution(OPKELE_CP_
                                     "XRI resolution failed with '"+status_string+"' message"
                                     ", while looking for SEP with type '"+st->uri+"'", status_code);
                         if(idis.xrd.canonical_ids.empty())
                             throw opkele::failed_discovery(OPKELE_CP_ "No CanonicalID for XRI identity found");
                         string cid = idis.xrd.canonical_ids.begin()->second;
                         if(cids.find(cid)==cids.end()) {
                             cids.insert(cid);
                             idis.clear();
                             discover_at( idis,
                                     xri_proxy + util::url_encode(id)+
                                     "?_xrd_t="+util::url_encode(st->uri)+
                                     "&_xrd_r=application/xrd%2Bxml"
                                     ";sep=true;refs=true",
                                     xmode_xrd );
                             if(status_code==241) continue;
                             if(status_code!=100)
                                 throw failed_xri_resolution(OPKELE_CP_
                                         "XRI resolution failed with '"+status_string+"' message"
                                         ", while looking for SEP with type '"+st->uri+"'"
                                         " on canonical id", status_code);
                         }
                         idis.canonicalized_id = cid;
                         idis.normalized_id = rv; idis.xri_identity = true;
                         queue_endpoints(oi,idis,st);
                     }
                 }else{
                     idis.xri_identity = false;
                     if(id.find("://")==string::npos)
                         id.insert(0,"http://");
                     string::size_type fp = id.find('#');
                     if(fp!=string::npos) {
                         string::size_type qp = id.find('?');
                         if(qp==string::npos || qp<fp)
                             id.erase(fp);
                         else if(qp>fp)
                             id.erase(fp,qp-fp);
                     }
                     rv = idis.normalized_id = util::rfc_3986_normalize_uri(id);
                     discover_at(idis,id,xmode_html|xmode_xrd);
                     const char * eu = 0;
                     CURLcode r = easy_getinfo(CURLINFO_EFFECTIVE_URL,&eu);
                     if(r)
                         throw exception_curl(OPKELE_CP_ "failed to get CURLINFO_EFFECTIVE_URL",r);
                     string cid = util::strip_uri_fragment_part( idis.canonicalized_id = util::rfc_3986_normalize_uri(eu) );
                     if(xrds_location.empty()) {
                         html2xrd(oi,idis);
                     }else{
                         idis.clear();
                         idis.canonicalized_id = cid;
                         discover_at(idis,xrds_location,xmode_xrd);
                         if(idis.xrd.empty())
                             html2xrd(oi,idis);
                         else{
                             for(const service_type_t *st=op_service_types;
                                     st<&op_service_types[sizeof(op_service_types)/sizeof(*op_service_types)];++st)
                                 queue_endpoints(oi,idis,st);
                         }
                     }
                 }
                 return rv;
             }
 
             void discover_at(idiscovery_t& idis,const string& url,int xm) {
                 CURLcode r = easy_setopt(CURLOPT_MAXREDIRS, (xm&xmode_noredirs)?0:5);
                 if(r)
                     throw exception_curl(OPKELE_CP_ "failed to set curly maxredirs option");
                 if( (r=easy_setopt(CURLOPT_URL,url.c_str())) )
                     throw exception_curl(OPKELE_CP_ "failed to set curly urlie",r);
 
                 http_content_type.clear();
                 xmode = xm;
                 prepare_to_parse();
                 if(xmode&xmode_html) {
                     xrds_location.clear();
                     save_html.clear();
                     save_html.reserve(max_html);
                 }
                 xrd = &idis.xrd;
 
                 r = easy_perform();
                 if(r && r!=CURLE_WRITE_ERROR)
                     throw exception_curl(OPKELE_CP_ "failed to perform curly request",r);
 
                 if(!parser_choked) {
                     parse(0,0,true);
                 }else if(xmode&xmode_html){
                     /* TODO: do not bother if we've seen xml */
                     try {
                         util::tidy_doc_t td = util::tidy_doc_t::create();
                         if(!td)
                             throw exception_tidy(OPKELE_CP_ "failed to create htmltidy document");
 #ifndef NDEBUG
                         td.opt_set(TidyQuiet,false);
                         td.opt_set(TidyShowWarnings,false);
 #endif /* NDEBUG */
                         td.opt_set(TidyForceOutput,true);
                         td.opt_set(TidyXhtmlOut,true);
                         td.opt_set(TidyDoctypeMode,TidyDoctypeOmit);
                         td.opt_set(TidyMark,false);
                         if(td.parse_string(save_html)<=0)
                             throw exception_tidy(OPKELE_CP_ "tidy failed to parse document");
                         if(td.clean_and_repair()<=0)
                             throw exception_tidy(OPKELE_CP_ "tidy failed to clean and repair");
                         util::tidy_buf_t tide;
                         if(td.save_buffer(tide)<=0)
                             throw exception_tidy(OPKELE_CP_ "tidy failed to save buffer");
                         prepare_to_parse();
                         parse(tide.c_str(),tide.size(),true);
                     }catch(exception_tidy& et) { }
                 }
                 save_html.clear();
             }
 
             void prepare_to_parse() {
                 (*(expat_t*)this) = parser_create_ns();
                 set_user_data(); set_element_handler();
                 set_character_data_handler();
 
                 if(xmode&xmode_html) {
                     html_openid1.clear(); html_openid2.clear();
                     parser_choked = false;
                 }
 
                 cdata = 0; xrd_service = 0; skipping = 0;
                 pt_stack.clear();
                 status_code = 100; status_string.clear();
             }
 
             void html2xrd(endpoint_discovery_iterator& oi,idiscovery_t& id) {
                 XRD_t& x = id.xrd;
                 if(!html_openid2.uris.empty()) {
                     html_openid2.types.insert(STURI_OPENID20);
                     x.services.add(-1,html_openid2);
                     queue_endpoints(oi,id,&op_service_types[st_index_2]);
                 }
                 if(!html_openid1.uris.empty()) {
                     html_openid1.types.insert(STURI_OPENID11);
                     x.services.add(-1,html_openid1);
                     queue_endpoints(oi,id,&op_service_types[st_index_1]);
                 }
             }
 
             size_t write(void *p,size_t s,size_t nm) {
                 /* TODO: limit total size */
                 size_t bytes = s*nm;
                 const char *inbuf = (const char*)p;
                 if(xmode&xmode_html) {
                     size_t mbts = save_html.capacity()-save_html.size();
                     size_t bts = 0;
                     if(mbts>0) {
                         bts = (bytes>mbts)?mbts:bytes;
                         save_html.append(inbuf,bts);
                     }
                     if(skipping<0) return bts;
                 }
                 if(skipping<0) return 0;
                 bool rp = parse(inbuf,bytes,false);
                 if(!rp) {
                     parser_choked = true;
                     skipping = -1;
                     if(!(xmode&xmode_html))
                         bytes = 0;
                 }
                 return bytes;
             }
             size_t header(void *p,size_t s,size_t nm) {
                 size_t bytes = s*nm;
                 const char *h = (const char*)p;
                 const char *colon = (const char*)memchr(p,':',bytes);
                 const char *space = (const char*)memchr(p,' ',bytes);
                 if(space && ( (!colon) || space<colon ) ) {
                     xrds_location.clear(); http_content_type.clear();
                 }else if(colon) {
                     const char *hv = ++colon;
                     size_t hnl = colon-h;
                     int rb;
                     for(rb = bytes-hnl-1;rb>0 && isspace(*hv);++hv,--rb);
                     while(rb>0 && isspace(hv[rb-1])) --rb;
                     if(rb) {
                         if( (hnl>=sizeof(XRDS_HEADER))
                                 && !strncasecmp(h,XRDS_HEADER":",
                                     sizeof(XRDS_HEADER)) ) {
                             xrds_location.assign(hv,rb);
                         }else if( (hnl>=sizeof(CT_HEADER))
                                 && !strncasecmp(h,CT_HEADER":",
                                     sizeof(CT_HEADER)) ) {
                             const char *sc = (const char*)memchr(
                                     hv,';',rb);
                             http_content_type.assign(hv,sc?(sc-hv):rb);
                         }
                     }
                 }
                 return curl_t::header(p,s,nm);
             }
 
             void start_element(const XML_Char *n,const XML_Char **a) {
                 if(skipping<0) return;
                 if(skipping) {
                     if(xmode&xmode_html)
                         html_start_element(n,a);
                     ++skipping; return;
                 }
                 if(pt_stack.empty()) {
                     if(is_qelement(n,NSURI_XRDS "\tXRDS"))
                         return;
                     if(is_qelement(n,NSURI_XRD "\tXRD")) {
                         assert(xrd);
                         xrd->clear();
                         pt_stack.push_back(n);
                     }else if(xmode&xmode_html) {
                         html_start_element(n,a);
                     }else{
                         skipping = -1;
                     }
                 }else{
                     int pt_s = pt_stack.size();
                     if(pt_s==1) {
                         if(is_qelement(n,NSURI_XRD "\tCanonicalID")) {
                             assert(xrd);
                             cdata = &(xrd->canonical_ids.add(element_priority(a),string()));
                         }else if(is_qelement(n,NSURI_XRD "\tLocalID")) {
                             assert(xrd);
                             cdata = &(xrd->local_ids.add(element_priority(a),string()));
                         }else if(is_qelement(n,NSURI_XRD "\tProviderID")) {
                             assert(xrd);
                             cdata = &(xrd->provider_id);
                         }else if(is_qelement(n,NSURI_XRD "\tService")) {
                             assert(xrd);
                             xrd_service = &(xrd->services.add(element_priority(a),
                                         service_t()));
                             pt_stack.push_back(n);
                         }else if(is_qelement(n,NSURI_XRD "\tStatus")) {
                             for(;*a;) {
                                 if(!strcasecmp(*(a++),"code")) {
                                     if(sscanf(*(a++),"%ld",&status_code)==1 && status_code!=100) {
                                         cdata = &status_string;
                                         pt_stack.push_back(n);
                                         break;
                                     }
                                 }else
                                     ++a;
                             }
                         }else if(is_qelement(n,NSURI_XRD "\tExpires")) {
                             assert(xrd);
                             cdata_buf.clear();
                             cdata = &cdata_buf;
                         }else if(xmode&xmode_html) {
                             html_start_element(n,a);
                         }else{
                             skipping = 1;
                         }
                     }else if(pt_s==2) {
                         if(is_qelement(pt_stack.back().c_str(), NSURI_XRD "\tService")) {
                             if(is_qelement(n,NSURI_XRD "\tType")) {
                                 assert(xrd); assert(xrd_service);
                                 cdata_buf.clear();
                                 cdata = &cdata_buf;
                             }else if(is_qelement(n,NSURI_XRD "\tURI")) {
                                 assert(xrd); assert(xrd_service);
                                 const char *append = element_attr(a,"append");
                                 xrd::uri_t& uri = xrd_service->uris.add(element_priority(a),xrd::uri_t("",append?append:""));
                                 cdata = &uri.uri;
                             }else if(is_qelement(n,NSURI_XRD "\tLocalID")
                                     || is_qelement(n,NSURI_OPENID10 "\tDelegate") ) {
                                 assert(xrd); assert(xrd_service);
                                 cdata = &(xrd_service->local_ids.add(element_priority(a),string()));
                             }else if(is_qelement(n,NSURI_XRD "\tProviderID")) {
                                     assert(xrd); assert(xrd_service);
                                     cdata = &(xrd_service->provider_id);
                             }else{
                                 skipping = 1;
                             }
                         }else
                             skipping = 1;
                     }else if(xmode&xmode_html) {
                         html_start_element(n,a);
                     }else{
                         skipping = 1;
                     }
                 }
             }
             void end_element(const XML_Char *n) {
                 if(skipping<0) return;
                 if(skipping) {
                     --skipping; return;
                 }
                 if(is_qelement(n,NSURI_XRD "\tType")) {
                     assert(xrd); assert(xrd_service); assert(cdata==&cdata_buf);
                     xrd_service->types.insert(cdata_buf);
                 }else if(is_qelement(n,NSURI_XRD "\tService")) {
                     assert(xrd); assert(xrd_service);
                     assert(!pt_stack.empty());
                     assert(pt_stack.back()==(NSURI_XRD "\tService"));
                     pt_stack.pop_back();
                     xrd_service = 0;
                 }else if(is_qelement(n,NSURI_XRD "\tStatus")) {
                     assert(xrd);
                     if(is_qelement(pt_stack.back().c_str(),n)) {
                         assert(cdata==&status_string);
                         pt_stack.pop_back();
                         if(status_code!=100)
                             skipping = -1;
                     }
                 }else if(is_qelement(n,NSURI_XRD "\tExpires")) {
                     assert(xrd);
                     xrd->expires = util::w3c_to_time(cdata_buf);
                 }else if((xmode&xmode_html) && is_element(n,"head")) {
                     skipping = -1;
                 }
                 cdata = 0;
             }
             void character_data(const XML_Char *s,int l) {
                 if(skipping) return;
                 if(cdata) cdata->append(s,l);
             }
 
             void html_start_element(const XML_Char *n,const XML_Char **a) {
                 if(is_element(n,"meta")) {
                     bool heq = false;
                     string l;
                     for(;*a;a+=2) {
                         if(!( strcasecmp(a[0],"http-equiv")
                                 || strcasecmp(a[1],XRDS_HEADER) ))
                             heq = true;
                         else if(!strcasecmp(a[0],"content"))
                             l.assign(a[1]);
                     }
                     if(heq)
                         xrds_location = l;
                 }else if(is_element(n,"link")) {
                     string rels;
                     string href;
                     for(;*a;a+=2) {
                         if( !strcasecmp(a[0],"rel") ) {
                             rels.assign(a[1]);
                         }else if( !strcasecmp(a[0],"href") ) {
                             const char *ns = a[1];
                             for(;*ns && isspace(*ns);++ns);
                             href.assign(ns);
-                            string::size_type lns=href.find_last_not_of(whitespace);
+                            string::size_type lns=href.find_last_not_of(data::_whitespace_chars);
                             href.erase(lns+1);
                         }
                     }
-                    for(string::size_type ns=rels.find_first_not_of(whitespace);
-                            ns!=string::npos; ns=rels.find_first_not_of(whitespace,ns)) {
-                        string::size_type s = rels.find_first_of(whitespace,ns);
+                    for(string::size_type ns=rels.find_first_not_of(data::_whitespace_chars);
+                            ns!=string::npos; ns=rels.find_first_not_of(data::_whitespace_chars,ns)) {
+                        string::size_type s = rels.find_first_of(data::_whitespace_chars,ns);
                         string rel;
                         if(s==string::npos) {
                             rel.assign(rels,ns,string::npos);
                             ns = string::npos;
                         }else{
                             rel.assign(rels,ns,s-ns);
                             ns = s;
                         }
                         if(rel=="openid.server")
                             html_openid1.uris.add(-1,xrd::uri_t(href));
                         else if(rel=="openid.delegate")
                             html_openid1.local_ids.add(-1,href);
                         else if(rel=="openid2.provider")
                             html_openid2.uris.add(-1,xrd::uri_t(href));
                         else if(rel=="openid2.local_id")
                             html_openid2.local_ids.add(-1,href);
                     }
                 }else if(is_element(n,"body")) {
                     skipping = -1;
                 }
             }
 
             void queue_endpoints(endpoint_discovery_iterator& oi,
                     const idiscovery_t &id,
                     const service_type_t *st) {
                 openid_endpoint_t ep;
                 ep.claimed_id = id.canonicalized_id;
                 for(xrd::services_t::const_iterator isvc=id.xrd.services.begin();
                         isvc!=id.xrd.services.end(); ++isvc) {
                     const xrd::service_t svc = isvc->second;
                     if(svc.types.find(st->uri)==svc.types.end()) continue;
                     for(xrd::uris_t::const_iterator iu=svc.uris.begin();iu!=svc.uris.end();++iu) {
                         ep.uri = iu->second.uri;
                         if(id.xri_identity) {
                             if(iu->second.append=="qxri") {
                                 ep.uri += id.normalized_id;
                             } /* TODO: else handle other append attribute values */
                         }
                         if(st->forceid) {
                             ep.local_id = ep.claimed_id = st->forceid;
                             *(oi++) = ep;
                         }else{
                             if(svc.local_ids.empty()) {
                                 ep.local_id = ep.claimed_id;
                                 *(oi++) = ep;
                             }else{
                                 for(xrd::local_ids_t::const_iterator ilid=svc.local_ids.begin();
                                         ilid!=svc.local_ids.end(); ++ilid) {
                                     ep.local_id = ilid->second;
                                     *(oi++) = ep;
                                 }
                             }
                         }
                     }
                 }
             }
 
     };
 
     string idiscover(endpoint_discovery_iterator oi,const string& identity) {
         idigger_t idigger;
         return idigger.discover(oi,identity);
     }
 
     void yadiscover(endpoint_discovery_iterator oi,const string& yurl,const char **types,bool redirs) try {
         idigger_t idigger;
         idigger.yadiscover(oi,yurl,types,redirs);
     }catch(exception_curl& ec) {
         if(redirs || ec._error!=CURLE_TOO_MANY_REDIRECTS)
             throw;
     }
 
 }
diff --git a/lib/util.cc b/lib/util.cc
index bb8a2e8..29e6738 100644
--- a/lib/util.cc
+++ b/lib/util.cc
@@ -1,435 +1,470 @@
 #include <errno.h>
 #include <cassert>
 #include <cctype>
 #include <cstring>
 #include <vector>
 #include <string>
 #include <stack>
 #include <algorithm>
 #include <openssl/bio.h>
 #include <openssl/evp.h>
 #include <openssl/sha.h>
 #include <openssl/hmac.h>
 #include <curl/curl.h>
 #include <opkele/util.h>
 #include <opkele/exception.h>
+#include <opkele/data.h>
 #include <opkele/debug.h>
 
 #include <config.h>
 #ifdef HAVE_DEMANGLE
 # include <cxxabi.h>
 #endif
 
 namespace opkele {
     using namespace std;
 
     namespace util {
 
         /*
          * base64
          */
         string encode_base64(const void *data,size_t length) {
             BIO *b64 = 0, *bmem = 0;
             try {
                 b64 = BIO_new(BIO_f_base64());
                 if(!b64)
                     throw exception_openssl(OPKELE_CP_ "failed to BIO_new() base64 encoder");
                 BIO_set_flags(b64,BIO_FLAGS_BASE64_NO_NL);
                 bmem = BIO_new(BIO_s_mem());
                 BIO_set_flags(b64,BIO_CLOSE);
                 if(!bmem)
                     throw exception_openssl(OPKELE_CP_ "failed to BIO_new() memory buffer");
                 BIO_push(b64,bmem);
                 if(((size_t)BIO_write(b64,data,length))!=length)
                     throw exception_openssl(OPKELE_CP_ "failed to BIO_write()");
                 if(BIO_flush(b64)!=1)
                     throw exception_openssl(OPKELE_CP_ "failed to BIO_flush()");
                 char *rvd;
                 long rvl = BIO_get_mem_data(bmem,&rvd);
                 string rv(rvd,rvl);
                 BIO_free_all(b64);
                 return rv;
             }catch(...) {
                 if(b64) BIO_free_all(b64);
                 throw;
             }
         }
 
         void decode_base64(const string& data,vector<unsigned char>& rv) {
             BIO *b64 = 0, *bmem = 0;
             rv.clear();
             try {
                 bmem = BIO_new_mem_buf((void*)data.data(),data.size());
                 if(!bmem)
                     throw exception_openssl(OPKELE_CP_ "failed to BIO_new_mem_buf()");
                 b64 = BIO_new(BIO_f_base64());
                 if(!b64)
                     throw exception_openssl(OPKELE_CP_ "failed to BIO_new() base64 decoder");
                 BIO_set_flags(b64,BIO_FLAGS_BASE64_NO_NL);
                 BIO_push(b64,bmem);
                 unsigned char tmp[512];
                 size_t rb = 0;
                 while((rb=BIO_read(b64,tmp,sizeof(tmp)))>0)
                     rv.insert(rv.end(),tmp,&tmp[rb]);
                 BIO_free_all(b64);
             }catch(...) {
                 if(b64) BIO_free_all(b64);
                 throw;
             }
         }
 
         /*
          * big numerics
          */
 
         BIGNUM *base64_to_bignum(const string& b64) {
             vector<unsigned char> bin;
             decode_base64(b64,bin);
             BIGNUM *rv = BN_bin2bn(&(bin.front()),bin.size(),0);
             if(!rv)
                 throw failed_conversion(OPKELE_CP_ "failed to BN_bin2bn()");
             return rv;
         }
 
         BIGNUM *dec_to_bignum(const string& dec) {
             BIGNUM *rv = 0;
             if(!BN_dec2bn(&rv,dec.c_str()))
                 throw failed_conversion(OPKELE_CP_ "failed to BN_dec2bn()");
             return rv;
         }
 
         string bignum_to_base64(const BIGNUM *bn) {
             vector<unsigned char> bin(BN_num_bytes(bn)+1);
             unsigned char *binptr = &(bin.front())+1;
             int l = BN_bn2bin(bn,binptr);
             if(l && (*binptr)&0x80){
                 (*(--binptr)) = 0; ++l;
             }
             return encode_base64(binptr,l);
         }
 
         /*
          * w3c times
          */
 
         string time_to_w3c(time_t t) {
             struct tm tm_t;
             if(!gmtime_r(&t,&tm_t))
                 throw failed_conversion(OPKELE_CP_ "failed to BN_dec2bn()");
             char rv[25];
             if(!strftime(rv,sizeof(rv)-1,"%Y-%m-%dT%H:%M:%SZ",&tm_t))
                 throw failed_conversion(OPKELE_CP_ "failed to strftime()");
             return rv;
         }
 
         time_t w3c_to_time(const string& w) {
             int fraction;
             struct tm tm_t;
             memset(&tm_t,0,sizeof(tm_t));
             if( (
                         sscanf(
                             w.c_str(),
                             "%04d-%02d-%02dT%02d:%02d:%02dZ",
                             &tm_t.tm_year,&tm_t.tm_mon,&tm_t.tm_mday,
                             &tm_t.tm_hour,&tm_t.tm_min,&tm_t.tm_sec
                             ) != 6
                 ) && (
                     sscanf(
                         w.c_str(),
                         "%04d-%02d-%02dT%02d:%02d:%02d.%03dZ",
                         &tm_t.tm_year,&tm_t.tm_mon,&tm_t.tm_mday,
                         &tm_t.tm_hour,&tm_t.tm_min,&tm_t.tm_sec,
                         &fraction
                         ) != 7
                     ) )
                 throw failed_conversion(OPKELE_CP_ "failed to sscanf()");
             tm_t.tm_mon--;
             tm_t.tm_year-=1900;
             time_t rv = mktime(&tm_t);
             if(rv==(time_t)-1)
                 throw failed_conversion(OPKELE_CP_ "failed to mktime()");
             return rv-timezone;
         }
 
         /*
          *
          */
 
         string url_encode(const string& str) {
             char * t = curl_escape(str.c_str(),str.length());
             if(!t)
                 throw failed_conversion(OPKELE_CP_ "failed to curl_escape()");
             string rv(t);
             curl_free(t);
             return rv;
         }
 
         string attr_escape(const string& str) {
             static const char *unsafechars = "<>&\n\"'";
             string rv;
             string::size_type p=0;
             while(true) {
                 string::size_type us = str.find_first_of(unsafechars,p);
                 if(us==string::npos) {
                     if(p!=str.length())
                         rv.append(str,p,str.length()-p);
                     return rv;
                 }
                 rv.append(str,p,us-p);
                 rv += "&#";
                 rv += long_to_string((long)str[us]);
                 rv += ';';
                 p = us+1;
             }
         }
 
         string long_to_string(long l) {
             char rv[32];
             int r=snprintf(rv,sizeof(rv),"%ld",l);
             if(r<0 || r>=(int)sizeof(rv))
                 throw failed_conversion(OPKELE_CP_ "failed to snprintf()");
             return rv;
         }
 
         long string_to_long(const string& s) {
             char *endptr = 0;
             long rv = strtol(s.c_str(),&endptr,10);
             if((!endptr) || endptr==s.c_str())
                 throw failed_conversion(OPKELE_CP_ "failed to strtol()");
             return rv;
         }
 
         /*
          * Normalize URL according to the rules, described in rfc 3986, section 6
          *
          * - uppercase hex triplets (e.g. %ab -> %AB)
          * - lowercase scheme and host
          * - decode %-encoded characters, specified as unreserved in rfc 3986, section 2.3,
          *   that is - [:alpha:][:digit:]._~-
          * - remove dot segments
          * - remove empty and default ports
          * - if there's no path component, add '/'
          */
          string rfc_3986_normalize_uri(const string& uri) {
-             static const char *whitespace = " \t\r\n";
              string rv;
-             string::size_type ns = uri.find_first_not_of(whitespace);
+             string::size_type ns = uri.find_first_not_of(data::_whitespace_chars);
              if(ns==string::npos)
                  throw bad_input(OPKELE_CP_ "Can't normalize empty URI");
              string::size_type colon = uri.find(':',ns);
              if(colon==string::npos)
                  throw bad_input(OPKELE_CP_ "No scheme specified in URI");
              transform(
                      uri.begin()+ns, uri.begin()+colon+1,
                      back_inserter(rv), ::tolower );
              bool s;
-             string::size_type ul = uri.find_last_not_of(whitespace)+1;
+             string::size_type ul = uri.find_last_not_of(data::_whitespace_chars)+1;
              if(ul <= (colon+3))
                  throw bad_input(OPKELE_CP_ "Unexpected end of URI being normalized encountered");
              if(uri[colon+1]!='/' || uri[colon+2]!='/')
                  throw bad_input(OPKELE_CP_ "Unexpected input in URI being normalized after scheme component");
              if(rv=="http:")
                  s = false;
              else if(rv=="https:")
                  s = true;
              else{
                  /* TODO: support more schemes.  e.g. xri. How do we normalize
                   * xri?
                   */
                  rv.append(uri,colon+1,ul-colon-1);
                  return rv;
              }
              rv += "//";
              string::size_type interesting = uri.find_first_of(":/#?",colon+3);
              if(interesting==string::npos) {
                  transform(
                          uri.begin()+colon+3,uri.begin()+ul,
                          back_inserter(rv), ::tolower );
                  rv += '/'; return rv;
              }
              transform(
                      uri.begin()+colon+3,uri.begin()+interesting,
                      back_inserter(rv), ::tolower );
              bool qf = false;
              char ic = uri[interesting];
              if(ic==':') {
                  string::size_type ni = uri.find_first_of("/#?%",interesting+1);
                  const char *nptr = uri.data()+interesting+1;
                  char *eptr = 0;
                  long port = strtol(nptr,&eptr,10);
                  if( (port>0) && (port<65535) && port!=(s?443:80) ) {
                      char tmp[8];
                      snprintf(tmp,sizeof(tmp),":%ld",port);
                      rv += tmp;
                  }
                  if(ni==string::npos) {
                      rv += '/'; return rv;
                  }
                  interesting = ni;
              }else if(ic!='/') {
                  rv += '/'; rv += ic;
                  qf = true;
                  ++interesting;
              }
              string::size_type n = interesting;
              char tmp[3] = { 0,0,0 };
              stack<string::size_type> psegs; psegs.push(rv.length());
              string pseg;
              for(;n<ul;) {
                  string::size_type unsafe = uri.find_first_of(qf?"%":"%/?#",n);
                  if(unsafe==string::npos) {
                      pseg.append(uri,n,ul-n-1); n = ul-1;
                  }else{
                      pseg.append(uri,n,unsafe-n);
                      n = unsafe;
                  }
                  char c = uri[n++];
                  if(c=='%') {
                      if((n+1)>=ul)
                          throw bad_input(OPKELE_CP_ "Unexpected end of URI encountered while parsing percent-encoded character");
                      tmp[0] = uri[n++];
                      tmp[1] = uri[n++];
                      if(!( isxdigit(tmp[0]) && isxdigit(tmp[1]) ))
                          throw bad_input(OPKELE_CP_ "Invalid percent-encoded character in URI being normalized");
                      int cc = strtol(tmp,0,16);
                      if( isalpha(cc) || isdigit(cc) || strchr("._~-",cc) )
                          pseg += cc;
                      else{
                          pseg += '%';
                          pseg += toupper(tmp[0]); pseg += toupper(tmp[1]);
                      }
                  }else if(qf) {
                      rv += pseg; rv += c;
                      pseg.clear();
                  }else if(n>=ul || strchr("?/#",c)) {
                      if(pseg.empty() || pseg==".") {
                      }else if(pseg=="..") {
                          if(psegs.size()>1) {
                              rv.resize(psegs.top()); psegs.pop();
                          }
                      }else{
                          psegs.push(rv.length());
                          if(c!='/') {
                              pseg += c;
                              qf = true;
                          }
                          rv += '/'; rv += pseg;
                      }
                      if(c=='/' && (n>=ul || strchr("?#",uri[n])) ) {
                          rv += '/';
                          if(n<ul)
                              qf = true;
                      }else if(strchr("?#",c)) {
                          if(psegs.size()==1 && psegs.top()==rv.length())
                              rv += '/';
                          if(pseg.empty())
                              rv += c;
                          qf = true;
                      }
                      pseg.clear();
                  }else{
                      pseg += c;
                  }
              }
              if(!pseg.empty()) {
                  if(!qf) rv += '/';
                  rv += pseg;
              }
              return rv;
          }
 
         string& strip_uri_fragment_part(string& u) {
             string::size_type q = u.find('?'), f = u.find('#');
             if(q==string::npos) {
                 if(f!=string::npos)
                     u.erase(f);
             }else{
                 if(f!=string::npos) {
                     if(f<q)
                         u.erase(f,q-f);
                     else
                         u.erase(f);
                 }
             }
             return u;
         }
 
         bool uri_matches_realm(const string& uri,const string& realm) {
             string nrealm = opkele::util::rfc_3986_normalize_uri(realm);
             string nu = opkele::util::rfc_3986_normalize_uri(uri);
             string::size_type pr = nrealm.find("://");
             string::size_type pu = nu.find("://");
             assert(!(pr==string::npos || pu==string::npos));
             pr += sizeof("://")-1;
             pu += sizeof("://")-1;
             if(!strncmp(nrealm.c_str()+pr,"*.",2)) {
                 pr = nrealm.find('.',pr);
                 pu = nu.find('.',pu);
                 assert(pr!=string::npos);
                 if(pu==string::npos)
                     return false;
                 // TODO: check for overgeneralized realm
             }
             string::size_type lr = nrealm.length();
             string::size_type lu = nu.length();
             if( (lu-pu) < (lr-pr) )
                 return false;
             pair<const char*,const char*> mp = mismatch(
                     nrealm.c_str()+pr,nrealm.c_str()+lr,
                     nu.c_str()+pu);
             if( (*(mp.first-1))!='/'
                     && !strchr("/?#",*mp.second) )
                 return false;
             return true;
         }
 
         string abi_demangle(const char *mn) {
 #ifndef HAVE_DEMANGLE
             return mn;
 #else /* !HAVE_DEMANGLE */
             int dstat;
             char *demangled = abi::__cxa_demangle(mn,0,0,&dstat);
             if(dstat)
                 return mn;
             string rv = demangled;
             free(demangled);
             return rv;
 #endif /* !HAVE_DEMANGLE */
         }
 
         string base64_signature(const assoc_t& assoc,const basic_openid_message& om) {
             const string& slist = om.get_field("signed");
             string kv;
             string::size_type p=0;
             while(true) {
                 string::size_type co = slist.find(',',p);
                 string f = (co==string::npos)
                     ?slist.substr(p):slist.substr(p,co-p);
                 kv += f;
                 kv += ':';
                 kv += om.get_field(f);
                 kv += '\n';
                 if(co==string::npos) break;
                 p = co+1;
             }
             const secret_t& secret = assoc->secret();
             const EVP_MD *evpmd;
             const string& at = assoc->assoc_type();
             if(at=="HMAC-SHA256")
                 evpmd = EVP_sha256();
             else if(at=="HMAC-SHA1")
                 evpmd = EVP_sha1();
             else
                 throw unsupported(OPKELE_CP_ "unknown association type");
             unsigned int md_len = 0;
             unsigned char md[SHA256_DIGEST_LENGTH];
             HMAC(evpmd,
                     &(secret.front()),secret.size(),
                     (const unsigned char*)kv.data(),kv.length(),
                     md,&md_len);
             return encode_base64(md,md_len);
         }
 
+        string normalize_identifier(const string& usi,bool strip_fragment) {
+            if(usi.empty())
+                return usi;
+            string rv;
+            string::size_type fsc = usi.find_first_not_of(data::_whitespace_chars);
+            if(fsc==string::npos)
+                return rv;
+            string::size_type lsc = usi.find_last_not_of(data::_whitespace_chars);
+            assert(lsc!=string::npos);
+            if(!strncasecmp(usi.c_str()+fsc,"xri://",sizeof("xri://")-1))
+                fsc += sizeof("xri://")-1;
+            if( (fsc+1) >= lsc )
+                return rv;
+            rv.assign(usi,fsc,lsc-fsc+1);
+            if(strchr(data::_iname_leaders,rv[0])) {
+                /* TODO: further normalize xri identity, fold case or
+                 * whatever... */
+            }else{
+                if(rv.find("://")==string::npos)
+                    rv.insert(0,"http://");
+                if(strip_fragment) {
+                    string::size_type fp = rv.find('#');
+                    if(fp!=string::npos) {
+                        string::size_type qp = rv.find('?');
+                        if(qp==string::npos || qp<fp)
+                            rv.erase(fp);
+                        else if(qp>fp)
+                            rv.erase(fp,qp-fp);
+                    }
+                }
+                rv = rfc_3986_normalize_uri(rv);
+            }
+            return rv;
+        }
+
     }
 
 }