3 files changed, 14 insertions, 5 deletions
diff --git a/configure.ac b/configure.ac
index 0aa1272..a66e53d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,257 +1,258 @@
 AC_INIT([libopkele], [1.1.99.0], [libopkele-bugs@klever.net])
 AC_CONFIG_SRCDIR([include/opkele/opkele-config.h])
 AC_CONFIG_HEADERS([config.h include/opkele/acconfig.h])
 AM_INIT_AUTOMAKE([dist-bzip2])
 AC_PROG_INSTALL
 AC_PROG_CXX
 AC_PROG_CC
 AC_PROG_LIBTOOL
 PKG_PROG_PKG_CONFIG
 AC_HEADER_STDC
 AC_PATH_PROG([XSLTPROC],[xsltproc],[true])
 AC_MSG_CHECKING([for source tree version])
 if headrev=$(cd $srcdir && git rev-parse --verify HEAD 2>/dev/null) ; then
 PACKAGE_SRC_VERSION="$(cd $srcdir && git describe --tags $headrev)"
 test "$PACKAGE_SRC_VERSION" = "$PACKAGE_VERSION" \
  -o "${PACKAGE_SRC_VERSION#${PACKAGE_VERSION}-}" != "$PACKAGE_SRC_VERSION" || PACKAGE_SRC_VERSION="${PACKAGE_VERSION}:${PACKAGE_SRC_VERSION}"
 ( cd $srcdir && git diff-index $headrev | read dirt ) && PACKAGE_SRC_VERSION="${PACKAGE_SRC_VERSION}-dirty"
 else
 PACKAGE_SRC_VERSION="$PACKAGE_VERSION"
 fi
 AC_MSG_RESULT([$PACKAGE_SRC_VERSION])
 AC_SUBST([PACKAGE_SRC_VERSION])
 AC_DEFINE_UNQUOTED([PACKAGE_SRC_VERSION],["$PACKAGE_SRC_VERSION"],[more or less precise source tree version])
 tr1_mem_std="false"
 tr1_mem_boost="false"
 AC_CHECK_SHAREDPTR(std::tr1,tr1/memory,[ tr1_mem_std=true ])
 AC_CHECK_SHAREDPTR(boost,boost/shared_ptr.hpp,[ tr1_mem_boost=true ])
 tr1_mem=""
 AC_ARG_WITH([tr1-memory],
 AC_HELP_STRING([--with-tr1-memory=<boost|std>],[select tr1/memory (shared_ptr<>) implementation to use]),
 [ tr1_mem="$withval" ]
 )
 AC_MSG_CHECKING([for tr1/memory implementation to use])
 test -z "$tr1_mem" && $tr1_mem_std && tr1_mem=std
 test -z "$tr1_mem" && $tr1_mem_boost && tr1_mem=boost
 if test -z "$tr1_mem" ; then
 AC_MSG_RESULT([none found])
 else
 AC_MSG_RESULT([$tr1_mem])
 fi
 case "$tr1_mem" in
 std)
  $tr1_mem_std || AC_MSG_ERROR([std implementation requested, but not found])
  OPKELE_TR1_MEM_NS=std::tr1
  OPKELE_TR1_MEM_HEADER=tr1/memory
  ;;
 boost)
  $tr1_mem_boost || AC_MSG_ERROR([boost implementation requested, but not found])
  OPKELE_TR1_MEM_NS=boost
  OPKELE_TR1_MEM_HEADER=boost/shared_ptr.hpp
  ;;
 *)
  AC_MSG_ERROR([no shared_ptr<> implementation found])
  ;;
 esac
 AC_SUBST([OPKELE_TR1_MEM_NS])
 AC_SUBST([OPKELE_TR1_MEM_HEADER])
 AC_MSG_CHECKING([for deprecated attribute support])
 AC_COMPILE_IFELSE([
  int __attribute__((deprecated)) deprecated_function();
 ],[
  AC_MSG_RESULT([yes])
  AC_DEFINE([OPKELE_DEPRECATE],[__attribute__((deprecated))],[deprecated function attribute])
 ],[
  AC_MSG_RESULT([no])
  AC_DEFINE([OPKELE_DEPRECATE],,[deprecated function attribute])
 ]
 )
 AC_LANG_PUSH([C++])
 AC_MSG_CHECKING([for abi::__cxa_demangle])
 AC_COMPILE_IFELSE([
  #include <typeinfo>
  using namespace std;
  #include <cxxabi.h>
  int main(int c,char **v) {
   int dstat;
   char *demangled = abi::__cxa_demangle(typeid(dstat).name(),0,0,&dstat);
   return 0;
  }
 ],[
  AC_MSG_RESULT([yes])
  AC_DEFINE([HAVE_DEMANGLE],,[defined if abi::__cxa_demangle is available])
 ],[
  AC_MSG_RESULT([no])
 ]
 )
 AC_LANG_POP([C++])
 PKG_CHECK_MODULES([OPENSSL],[openssl],,[
 AC_MSG_ERROR([no openssl library found. get one from http://www.openssl.org/])
 ])
 WANT_KONFORKA="yes"
 AC_ARG_ENABLE([konforka],
 AC_HELP_STRING([--disable-konforka],[do not use konforka library (default: use if found)]),
 [
  test "${enableval}" = "no" && WANT_KONFORKA="no"
 ]
 )
 if test "${WANT_KONFORKA}" = "yes" ; then
 PKG_CHECK_MODULES([KONFORKA],[konforka],[
  AC_SUBST([KONFORKA_CFLAGS])
  AC_SUBST([KONFORKA_LIBS])
  AC_DEFINE([HAVE_KONFORKA],,[defined in presence of konforka library])
  AC_DEFINE([OPKELE_HAVE_KONFORKA],,[defined in presence of konforka library])
  AC_SUBST([KONFORKA_KONFORKA],[konforka])
 ],[true])
 fi
 WANT_DOXYGEN="yes"
 AC_ARG_ENABLE([doxygen],
 AC_HELP_STRING([--disable-doxygen],[do not generate documentation]),
 [
  test "${enableval}" = "no" && WANT_DOXYGEN="no"
 ]
 )
 if test "${WANT_DOXYGEN}" = "yes" ; then
 AC_WITH_DOXYGEN
 AC_WITH_DOT
 else
 AM_CONDITIONAL([HAVE_DOXYGEN],[false])
 AM_CONDITIONAL([HAVE_DOT],[false])
 fi
 LIBCURL_CHECK_CONFIG(,,,[
 AC_MSG_ERROR([no required libcurl library. get one from http://curl.haxx.se/])
 ])
 AC_CHECK_HEADER([expat.h],[
 AC_CHECK_LIB([expat],[XML_ParserCreate],[
  EXPAT_LIBS=-lexpat
  EXPAT_CFLAGS=
  AC_SUBST([EXPAT_LIBS])
  AC_SUBST([EXPAT_CFLAGS])
 ],[
  AC_MSG_ERROR([no required expat library. get one from http://expat.sourceforge.net/])
 ])
 ],[
 AC_MSG_ERROR([no required expat library. get one from http://expat.sourceforge.net/])
 ])
-AC_CHECK_HEADER([tidy.h],[
+AC_CHECK_HEADERS([tidy.h tidy/tidy.h],[
 AC_CHECK_LIB([tidy],[tidyParseBuffer],[
  TIDY_LIBS=-ltidy
  TIDY_CFLAGS=
  AC_SUBST([TIDY_LIBS])
  AC_SUBST([TIDY_CFLAGS])
 ],[
  AC_MSG_ERROR([no required htmltidy library found. get one from http://tidy.sourceforge.net/])
 ])
 ],[
- AC_MSG_ERROR([no required htmltidy library found. get one from http://tidy.sourceforge.net/])
+ test "$ac_header" = "tidy/tidy.h" \
+ && AC_MSG_ERROR([no required htmltidy library found. get one from http://tidy.sourceforge.net/])
 ])
 if test -n "$PCRE_LIBS" -a -n "$PCRE_CFLAGS" ; then
 AC_SUBST([PCRE_CFLAGS])
 AC_SUBST([PCRE_LIBS])
 :
 else
 PKG_CHECK_MODULES([PCRE],[libpcre],,[
  AC_MSG_ERROR([no libpcre found, go get it at http://www.pcre.org/])
 ])
 fi
 PKG_CHECK_MODULES([SQLITE3],[sqlite3],[have_sqlite3=true],[have_sqlite3=false])
 AM_CONDITIONAL([HAVE_SQLITE3],[$have_sqlite3])
 PKG_CHECK_MODULES([KINGATE],[kingate-plaincgi],[have_kingate=true],[have_kingate=false])
 AM_CONDITIONAL([HAVE_KINGATE],[$have_kingate])
 PKG_CHECK_MODULES([UUID],[uuid],[have_uuid=true],[have_uuid=false])
 AM_CONDITIONAL([HAVE_UUID],[$have_uuid])
 if $have_uuid ; then
 AC_DEFINE([HAVE_LIBUUID],,[defined in presence of libuuid])
 AC_SUBST([UUID_UUID],[uuid])
 fi
 curl_ssl_verify_host="true"
 AC_ARG_ENABLE([ssl-verify-host],
 AC_HELP_STRING([--disable-ssl-verify-host],[disable cURL cert/host relationships verification]),
 [ test "${enableval}" = "no" && curl_ssl_verify_host="false" ]
 )
 ${curl_ssl_verify_host} || AC_DEFINE([DISABLE_CURL_SSL_VERIFYHOST],,[defined if cURL is not to verify cert/host])
 curl_ssl_verify_peer="true"
 AC_ARG_ENABLE([ssl-verify-peer],
 AC_HELP_STRING([--disable-ssl-verify-peer],[disable cURL cert validity verification]),
 [ test "${enableval}" = "no" && curl_ssl_verify_peer="false" ]
 )
 ${curl_ssl_verify_peer} || AC_DEFINE([DISABLE_CURL_SSL_VERIFYPEER],,[defined if cURL is not to verify cert validity])
 postels_law=true
 AC_ARG_ENABLE([postels-law],
 AC_HELP_STRING([--disable-postels-law],[Be strict, do not adhere to Postel's Law ("be conservative in what you do, be liberal in what you accept from others", RFC 793)]),
 [ test "${enableval}" = "no" && postels_law=false ]
 )
 $postels_law && AC_DEFINE([POSTELS_LAW],,[defined if we want to adhere to Postel's Law])
 AC_DEFINE_UNQUOTED([OPKELE_SRC_DIR],["$PWD"],[source directory])
 nitpick=false
 AC_ARG_ENABLE([nitpicking],
 AC_HELP_STRING([--enable-nitpicking],[make compiler somewhat overly fastidious about the code it deals with]),
 [ test "$enableval" = "no" || nitpick=true ]
 )
 if $nitpick ; then
 CPP_NITPICK="-pedantic -Wall -Wextra -Wundef -Wshadow \
  -Wunsafe-loop-optimizations -Wconversion -Wmissing-format-attribute \
  -Wredundant-decls -ansi"
 # -Wlogical-op -Wmissing-noreturn 
 C_NITPICK="$CPP_NITPICK"
 CXX_NITPICK="$C_NITPICK"
 CPPFLAGS="$CPPFLAGS $CPP_NITPICK"
 CFLAGS="$CFLAGS $C_NITPICK"
 CXXFLAGS="$CXXFLAGS $CXX_NITPICK"
 fi
 ndebug=true
 AC_ARG_ENABLE([debug],
 AC_HELP_STRING([--enable-debug],[enable debugging code]),
 [ test "$enableval" = "no" || ndebug=false ]
 )
 if $ndebug ; then
 CPPFLAGS_DEBUG="-DNDEBUG"
 else
 CPPFLAGS_DEBUG=""
 fi
 AC_SUBST([CPPFLAGS_DEBUG])
 xri_proxy_url="https://xri.net/"
 AC_MSG_CHECKING([for XRI resolver proxy])
 AC_ARG_ENABLE([xri-proxy],
 AC_HELP_STRING([--with-xri-proxy=url],[set xri proxy for use when resolving xri identities, default is https://xri.net/]),
 [ xri_proxy_url="$withval" ]
 )
 AC_MSG_RESULT([$xri_proxy_url])
 AC_DEFINE_UNQUOTED([XRI_PROXY_URL],["$xri_proxy_url"],[XRI proxy resolver URL])
 AC_CONFIG_FILES([
 Makefile
  libopkele.pc
  Doxyfile
 include/Makefile
  include/opkele/tr1-mem.h
 lib/Makefile
 test/Makefile
 ])
 AC_OUTPUT
diff --git a/include/opkele/tidy.h b/include/opkele/tidy.h
index 888e7d4..059656d 100644
--- a/include/opkele/tidy.h
+++ b/include/opkele/tidy.h
@@ -1,73 +1,80 @@
 #ifndef __OPKELE_TIDY_H
 #define __OPKELE_TIDY_H
 #include <cassert>
-#include <tidy.h>
+#ifdef HAVE_TIDY_H
-#include <buffio.h>
+# include <tidy.h>
+# include <buffio.h>
+#elif HAVE_TIDY_TIDY_H
+# include <tidy/tidy.h>
+# include <tidy/buffio.h>
+#else
+# error "Don't know where to look for htmltidy headers"
+#endif
 namespace opkele {
    namespace util {
        class tidy_buf_t {
            public:
                TidyBuffer _x;
                tidy_buf_t() { tidyBufInit(&_x); }
                virtual ~tidy_buf_t() throw() {
                    tidyBufFree(&_x); }
                inline operator const TidyBuffer&(void) const { return _x; }
                inline operator TidyBuffer&(void) { return _x; }
                inline operator const char*(void) const { return (const char*)_x.bp; }
                inline operator char*(void) { return (char*)_x.bp; }
                inline const char *c_str() const {
                    return (const char*)_x.bp; }
                inline size_t size() const {
                    return _x.size; }
        };
        class tidy_doc_t {
            public:
                TidyDoc _x;
                tidy_doc_t() : _x(0) { }
                tidy_doc_t(TidyDoc x) : _x(x) { }
                virtual ~tidy_doc_t() throw() {
                    if(_x) tidyRelease(_x); }
                tidy_doc_t& operator=(TidyDoc x) {
                    if(_x) tidyRelease(_x);
                    _x = x;
                    return *this;
                }
                operator const TidyDoc(void) const { return _x; }
                operator TidyDoc(void) { return _x; }
                inline bool opt_set(TidyOptionId o,bool v) {
                    assert(_x);
                    return tidyOptSetBool(_x,o,v?yes:no); }
                inline bool opt_set(TidyOptionId o,int v) {
                    assert(_x);
                    return tidyOptSetInt(_x,o,v); }
                inline int parse_string(const string& s) {
                    assert(_x);
                    return tidyParseString(_x,s.c_str()); }
                inline int clean_and_repair() {
                    assert(_x);
                    return tidyCleanAndRepair(_x); }
                inline int save_buffer(TidyBuffer& ob) {
                    assert(_x);
                    return tidySaveBuffer(_x,&ob); }
                static inline TidyDoc create() {
                    return tidyCreate(); }
        };
    }
 }
 #endif /* __OPKELE_TIDY_H */
diff --git a/lib/discovery.cc b/lib/discovery.cc
index 3b90977..d1989ec 100644
--- a/lib/discovery.cc
+++ b/lib/discovery.cc
@@ -1,583 +1,584 @@
 #include <list>
 #include <opkele/curl.h>
 #include <opkele/expat.h>
 #include <opkele/uris.h>
 #include <opkele/discovery.h>
 #include <opkele/exception.h>
 #include <opkele/util.h>
-#include <opkele/tidy.h>
 #include <opkele/data.h>
 #include <opkele/debug.h>
 #include "config.h"
+#include <opkele/tidy.h>
 #define XRDS_HEADER "X-XRDS-Location"
 #define CT_HEADER "Content-Type"
 namespace opkele {
    using std::list;
    using xrd::XRD_t;
    using xrd::service_t;
    /* TODO: the whole discovery thing needs cleanup and optimization due to
     * many changes of concept. */
    static const size_t max_html = 16384;
    static const struct service_type_t {
        const char *uri;
        const char *forceid;
    } op_service_types[] = {
        { STURI_OPENID20_OP, IDURI_SELECT20 },
        { STURI_OPENID20, 0 },
        { STURI_OPENID11, 0 },
        { STURI_OPENID10, 0 }
    };
    enum {
        st_index_1 = 2, st_index_2 = 1
    };
    static inline bool is_qelement(const XML_Char *n,const char *qen) {
        return !strcasecmp(n,qen);
    }
    static inline bool is_element(const XML_Char *n,const char *en) {
        if(!strcasecmp(n,en)) return true;
        int nl = strlen(n), enl = strlen(en);
        if( (nl>=(enl+1)) && n[nl-enl-1]=='\t'
                && !strcasecmp(&n[nl-enl],en) )
            return true;
        return false;
    }
    static long element_priority(const XML_Char **a) {
        for(;*a;++a)
            if(!strcasecmp(*(a++),"priority")) {
                long rv;
                return (sscanf(*a,"%ld",&rv)==1)?rv:-1;
            }
        return -1;
    }
    /* TODO: ideally all attributes should be
     * retrieved in one run */
    static const char *element_attr(const XML_Char **a, const char *at) {
        for(;*a;++a)
            if(!strcasecmp(*(a++),at)) {
                return *a;
            }
        return 0;
    }
    class idigger_t : public util::curl_t, public util::expat_t {
        public:
            string xri_proxy;
            enum {
                xmode_html = 1, xmode_xrd = 2, xmode_cid = 4,
                xmode_noredirs = 8
            };
            int xmode;
            string xrds_location;
            string http_content_type;
            service_t html_openid1;
            service_t html_openid2;
            string cdata_buf;
            long status_code;
            string status_string;
            typedef list<string> pt_stack_t;
            pt_stack_t pt_stack;
            int skipping;
            bool parser_choked;
            string save_html;
            XRD_t *xrd;
            service_t *xrd_service;
            string* cdata;
            idigger_t()
                : util::curl_t(easy_init()),
                util::expat_t(0),
                xri_proxy(XRI_PROXY_URL) {
                    CURLcode r;
                    (r=misc_sets())
                        || (r=set_write())
                        || (r=set_header())
                        ;
                    if(r)
                        throw exception_curl(OPKELE_CP_ "failed to set curly options",r);
                }
            ~idigger_t() throw() { }
            void yadiscover(endpoint_discovery_iterator oi,const string& yurl,const char **types,bool redirs) {
                idiscovery_t idis;
                idis.xri_identity = false;
                discover_at(idis,yurl,xmode_html|xmode_xrd|(redirs?0:xmode_noredirs));
                if(!xrds_location.empty()) {
                    idis.clear();
                    discover_at(idis,xrds_location,xmode_xrd);
                }
                idis.normalized_id = idis.canonicalized_id = yurl;
                service_type_t st;
                for(st.uri=*types;*types;st.uri=*(++types))
                    queue_endpoints(oi,idis,&st);
            }
            string discover(endpoint_discovery_iterator& oi,const string& identity) {
                string rv;
                idiscovery_t idis;
                string::size_type fsc = identity.find_first_not_of(data::_whitespace_chars);
                if(fsc==string::npos)
                    throw bad_input(OPKELE_CP_ "whitespace-only identity");
                string::size_type lsc = identity.find_last_not_of(data::_whitespace_chars);
                assert(lsc!=string::npos);
                if(!strncasecmp(identity.c_str()+fsc,"xri://",sizeof("xri://")-1))
                    fsc += sizeof("xri://")-1;
                if((fsc+1)>=lsc)
                    throw bad_input(OPKELE_CP_ "not a character of importance in identity");
                string id(identity,fsc,lsc-fsc+1);
                idis.clear();
                if(strchr(data::_iname_leaders,id[0])) {
                    /* TODO: further normalize xri identity? Like folding case
                     * or whatever... */
                    rv = id;
                    set<string> cids;
                    for(const struct service_type_t *st=op_service_types;
                            st<&op_service_types[sizeof(op_service_types)/sizeof(*op_service_types)];++st) {
                        idis.clear();
                        discover_at( idis,
                                xri_proxy + util::url_encode(id)+
                                "?_xrd_t="+util::url_encode(st->uri)+
                                "&_xrd_r=application/xrd%2Bxml"
                                ";sep=true;refs=true",
                                xmode_xrd );
                        if(status_code==241) continue;
                        if(status_code!=100)
                            throw failed_xri_resolution(OPKELE_CP_
                                    "XRI resolution failed with '"+status_string+"' message"
                                    ", while looking for SEP with type '"+st->uri+"'", status_code);
                        if(idis.xrd.canonical_ids.empty())
                            throw opkele::failed_discovery(OPKELE_CP_ "No CanonicalID for XRI identity found");
                        string cid = idis.xrd.canonical_ids.begin()->second;
                        if(cids.find(cid)==cids.end()) {
                            cids.insert(cid);
                            idis.clear();
                            discover_at( idis,
                                    xri_proxy + util::url_encode(id)+
                                    "?_xrd_t="+util::url_encode(st->uri)+
                                    "&_xrd_r=application/xrd%2Bxml"
                                    ";sep=true;refs=true",
                                    xmode_xrd );
                            if(status_code==241) continue;
                            if(status_code!=100)
                                throw failed_xri_resolution(OPKELE_CP_
                                        "XRI resolution failed with '"+status_string+"' message"
                                        ", while looking for SEP with type '"+st->uri+"'"
                                        " on canonical id", status_code);
                        }
                        idis.canonicalized_id = cid;
                        idis.normalized_id = rv; idis.xri_identity = true;
                        queue_endpoints(oi,idis,st);
                    }
                }else{
                    idis.xri_identity = false;
                    if(id.find("://")==string::npos)
                        id.insert(0,"http://");
                    string::size_type fp = id.find('#');
                    if(fp!=string::npos) {
                        string::size_type qp = id.find('?');
                        if(qp==string::npos || qp<fp)
                            id.erase(fp);
                        else if(qp>fp)
                            id.erase(fp,qp-fp);
                    }
                    rv = idis.normalized_id = util::rfc_3986_normalize_uri(id);
                    discover_at(idis,id,xmode_html|xmode_xrd);
                    const char * eu = 0;
                    CURLcode r = easy_getinfo(CURLINFO_EFFECTIVE_URL,&eu);
                    if(r)
                        throw exception_curl(OPKELE_CP_ "failed to get CURLINFO_EFFECTIVE_URL",r);
                    string cid = util::strip_uri_fragment_part( idis.canonicalized_id = util::rfc_3986_normalize_uri(eu) );
                    if(xrds_location.empty()) {
                        if(idis.xrd.empty())
                            html2xrd(oi,idis);
                        else{
                            for(const service_type_t *st=op_service_types;
                                    st<&op_service_types[sizeof(op_service_types)/sizeof(*op_service_types)];++st)
                                queue_endpoints(oi,idis,st);
                        }
                    }else{
                        idis.clear();
                        idis.canonicalized_id = cid;
                        discover_at(idis,xrds_location,xmode_xrd);
                        if(idis.xrd.empty())
                            html2xrd(oi,idis);
                        else{
                            for(const service_type_t *st=op_service_types;
                                    st<&op_service_types[sizeof(op_service_types)/sizeof(*op_service_types)];++st)
                                queue_endpoints(oi,idis,st);
                        }
                    }
                }
                return rv;
            }
            void discover_at(idiscovery_t& idis,const string& url,int xm) {
                CURLcode r = easy_setopt(CURLOPT_MAXREDIRS, (xm&xmode_noredirs)?0:5);
                if(r)
                    throw exception_curl(OPKELE_CP_ "failed to set curly maxredirs option");
                if( (r=easy_setopt(CURLOPT_URL,url.c_str())) )
                    throw exception_curl(OPKELE_CP_ "failed to set curly urlie",r);
                http_content_type.clear();
                xmode = xm;
                prepare_to_parse();
                if(xmode&xmode_html) {
                    xrds_location.clear();
                    save_html.clear();
                    save_html.reserve(max_html);
                }
                xrd = &idis.xrd;
                r = easy_perform();
                if(r && r!=CURLE_WRITE_ERROR)
                    throw exception_curl(OPKELE_CP_ "failed to perform curly request",r);
                if(!parser_choked) {
                    parse(0,0,true);
                }else if(xmode&xmode_html){
                    /* TODO: do not bother if we've seen xml */
                    try {
                        util::tidy_doc_t td = util::tidy_doc_t::create();
                        if(!td)
                            throw exception_tidy(OPKELE_CP_ "failed to create htmltidy document");
 #ifndef NDEBUG
                        td.opt_set(TidyQuiet,false);
                        td.opt_set(TidyShowWarnings,false);
 #endif /* NDEBUG */
                        td.opt_set(TidyForceOutput,true);
                        td.opt_set(TidyXhtmlOut,true);
                        td.opt_set(TidyDoctypeMode,TidyDoctypeOmit);
                        td.opt_set(TidyMark,false);
                        td.opt_set(TidyNumEntities,true);
                        if(td.parse_string(save_html)<=0)
                            throw exception_tidy(OPKELE_CP_ "tidy failed to parse document");
                        if(td.clean_and_repair()<=0)
                            throw exception_tidy(OPKELE_CP_ "tidy failed to clean and repair");
                        util::tidy_buf_t tide;
                        if(td.save_buffer(tide)<=0)
                            throw exception_tidy(OPKELE_CP_ "tidy failed to save buffer");
                        prepare_to_parse();
                        parse(tide.c_str(),tide.size(),true);
                    }catch(exception_tidy& et) { }
                }
                save_html.clear();
            }
            void prepare_to_parse() {
                (*(expat_t*)this) = parser_create_ns();
                set_user_data(); set_element_handler();
                set_character_data_handler();
                if(xmode&xmode_html) {
                    html_openid1.clear(); html_openid2.clear();
                    parser_choked = false;
                }
                cdata = 0; xrd_service = 0; skipping = 0;
                pt_stack.clear();
                status_code = 100; status_string.clear();
            }
            void html2xrd(endpoint_discovery_iterator& oi,idiscovery_t& id) {
                XRD_t& x = id.xrd;
                if(!html_openid2.uris.empty()) {
                    html_openid2.types.insert(STURI_OPENID20);
                    x.services.add(-1,html_openid2);
                    queue_endpoints(oi,id,&op_service_types[st_index_2]);
                }
                if(!html_openid1.uris.empty()) {
                    html_openid1.types.insert(STURI_OPENID11);
                    x.services.add(-1,html_openid1);
                    queue_endpoints(oi,id,&op_service_types[st_index_1]);
                }
            }
            size_t write(void *p,size_t s,size_t nm) {
                /* TODO: limit total size */
                size_t bytes = s*nm;
                const char *inbuf = (const char*)p;
                if(xmode&xmode_html) {
                    size_t mbts = save_html.capacity()-save_html.size();
                    size_t bts = 0;
                    if(mbts>0) {
                        bts = (bytes>mbts)?mbts:bytes;
                        save_html.append(inbuf,bts);
                    }
                    if(skipping<0) return bts;
                }
                if(skipping<0) return 0;
                bool rp = parse(inbuf,bytes,false);
                if(!rp) {
                    parser_choked = true;
                    skipping = -1;
                    if(!(xmode&xmode_html))
                        bytes = 0;
                }
                return bytes;
            }
            size_t header(void *p,size_t s,size_t nm) {
                size_t bytes = s*nm;
                const char *h = (const char*)p;
                const char *colon = (const char*)memchr(p,':',bytes);
                const char *space = (const char*)memchr(p,' ',bytes);
                if(space && ( (!colon) || space<colon ) ) {
                    xrds_location.clear(); http_content_type.clear();
                }else if(colon) {
                    const char *hv = ++colon;
                    size_t hnl = colon-h;
                    int rb;
                    for(rb = bytes-hnl-1;rb>0 && isspace(*hv);++hv,--rb);
                    while(rb>0 && isspace(hv[rb-1])) --rb;
                    if(rb) {
                        if( (hnl>=sizeof(XRDS_HEADER))
                                && !strncasecmp(h,XRDS_HEADER":",
                                    sizeof(XRDS_HEADER)) ) {
                            xrds_location.assign(hv,rb);
                        }else if( (hnl>=sizeof(CT_HEADER))
                                && !strncasecmp(h,CT_HEADER":",
                                    sizeof(CT_HEADER)) ) {
                            const char *sc = (const char*)memchr(
                                    hv,';',rb);
                            http_content_type.assign(hv,sc?(sc-hv):rb);
                        }
                    }
                }
                return curl_t::header(p,s,nm);
            }
            void start_element(const XML_Char *n,const XML_Char **a) {
                if(skipping<0) return;
                if(skipping) {
                    if(xmode&xmode_html)
                        html_start_element(n,a);
                    ++skipping; return;
                }
                if(pt_stack.empty()) {
                    if(is_qelement(n,NSURI_XRDS "\tXRDS"))
                        return;
                    if(is_qelement(n,NSURI_XRD "\tXRD")) {
                        assert(xrd);
                        xrd->clear();
                        pt_stack.push_back(n);
                    }else if(xmode&xmode_html) {
                        html_start_element(n,a);
                    }else{
                        skipping = -1;
                    }
                }else{
                    int pt_s = pt_stack.size();
                    if(pt_s==1) {
                        if(is_qelement(n,NSURI_XRD "\tCanonicalID")) {
                            assert(xrd);
                            cdata = &(xrd->canonical_ids.add(element_priority(a),string()));
                        }else if(is_qelement(n,NSURI_XRD "\tLocalID")) {
                            assert(xrd);
                            cdata = &(xrd->local_ids.add(element_priority(a),string()));
                        }else if(is_qelement(n,NSURI_XRD "\tProviderID")) {
                            assert(xrd);
                            cdata = &(xrd->provider_id);
                        }else if(is_qelement(n,NSURI_XRD "\tService")) {
                            assert(xrd);
                            xrd_service = &(xrd->services.add(element_priority(a),
                                        service_t()));
                            pt_stack.push_back(n);
                        }else if(is_qelement(n,NSURI_XRD "\tStatus")) {
                            for(;*a;) {
                                if(!strcasecmp(*(a++),"code")) {
                                    if(sscanf(*(a++),"%ld",&status_code)==1 && status_code!=100) {
                                        cdata = &status_string;
                                        pt_stack.push_back(n);
                                        break;
                                    }
                                }else
                                    ++a;
                            }
                        }else if(is_qelement(n,NSURI_XRD "\tExpires")) {
                            assert(xrd);
                            cdata_buf.clear();
                            cdata = &cdata_buf;
                        }else if(xmode&xmode_html) {
                            html_start_element(n,a);
                        }else{
                            skipping = 1;
                        }
                    }else if(pt_s==2) {
                        if(is_qelement(pt_stack.back().c_str(), NSURI_XRD "\tService")) {
                            if(is_qelement(n,NSURI_XRD "\tType")) {
                                assert(xrd); assert(xrd_service);
                                cdata_buf.clear();
                                cdata = &cdata_buf;
                            }else if(is_qelement(n,NSURI_XRD "\tURI")) {
                                assert(xrd); assert(xrd_service);
                                const char *append = element_attr(a,"append");
                                xrd::uri_t& uri = xrd_service->uris.add(element_priority(a),xrd::uri_t("",append?append:""));
                                cdata = &uri.uri;
                            }else if(is_qelement(n,NSURI_XRD "\tLocalID")
                                    || is_qelement(n,NSURI_OPENID10 "\tDelegate") ) {
                                assert(xrd); assert(xrd_service);
                                cdata = &(xrd_service->local_ids.add(element_priority(a),string()));
                            }else if(is_qelement(n,NSURI_XRD "\tProviderID")) {
                                    assert(xrd); assert(xrd_service);
                                    cdata = &(xrd_service->provider_id);
                            }else{
                                skipping = 1;
                            }
                        }else
                            skipping = 1;
                    }else if(xmode&xmode_html) {
                        html_start_element(n,a);
                    }else{
                        skipping = 1;
                    }
                }
            }
            void end_element(const XML_Char *n) {
                if(skipping<0) return;
                if(skipping) {
                    --skipping; return;
                }
                if(is_qelement(n,NSURI_XRD "\tType")) {
                    assert(xrd); assert(xrd_service); assert(cdata==&cdata_buf);
                    xrd_service->types.insert(cdata_buf);
                }else if(is_qelement(n,NSURI_XRD "\tService")) {
                    assert(xrd); assert(xrd_service);
                    assert(!pt_stack.empty());
                    assert(pt_stack.back()==(NSURI_XRD "\tService"));
                    pt_stack.pop_back();
                    xrd_service = 0;
                }else if(is_qelement(n,NSURI_XRD "\tStatus")) {
                    assert(xrd);
                    if(is_qelement(pt_stack.back().c_str(),n)) {
                        assert(cdata==&status_string);
                        pt_stack.pop_back();
                        if(status_code!=100)
                            skipping = -1;
                    }
                }else if(is_qelement(n,NSURI_XRD "\tExpires")) {
                    assert(xrd);
                    xrd->expires = util::w3c_to_time(cdata_buf);
                }else if((xmode&xmode_html) && is_element(n,"head")) {
                    skipping = -1;
                }
                cdata = 0;
            }
            void character_data(const XML_Char *s,int l) {
                if(skipping) return;
                if(cdata) cdata->append(s,l);
            }
            void html_start_element(const XML_Char *n,const XML_Char **a) {
                if(is_element(n,"meta")) {
                    bool heq = false;
                    string l;
                    for(;*a;a+=2) {
                        if(!( strcasecmp(a[0],"http-equiv")
                                || strcasecmp(a[1],XRDS_HEADER) ))
                            heq = true;
                        else if(!strcasecmp(a[0],"content"))
                            l.assign(a[1]);
                    }
                    if(heq)
                        xrds_location = l;
                }else if(is_element(n,"link")) {
                    string rels;
                    string href;
                    for(;*a;a+=2) {
                        if( !strcasecmp(a[0],"rel") ) {
                            rels.assign(a[1]);
                        }else if( !strcasecmp(a[0],"href") ) {
                            const char *ns = a[1];
                            for(;*ns && isspace(*ns);++ns);
                            href.assign(ns);
                            string::size_type lns=href.find_last_not_of(data::_whitespace_chars);
                            href.erase(lns+1);
                        }
                    }
                    for(string::size_type ns=rels.find_first_not_of(data::_whitespace_chars);
                            ns!=string::npos; ns=rels.find_first_not_of(data::_whitespace_chars,ns)) {
                        string::size_type s = rels.find_first_of(data::_whitespace_chars,ns);
                        string rel;
                        if(s==string::npos) {
                            rel.assign(rels,ns,string::npos);
                            ns = string::npos;
                        }else{
                            rel.assign(rels,ns,s-ns);
                            ns = s;
                        }
                        if(rel=="openid.server")
                            html_openid1.uris.add(-1,xrd::uri_t(href));
                        else if(rel=="openid.delegate")
                            html_openid1.local_ids.add(-1,href);
                        else if(rel=="openid2.provider")
                            html_openid2.uris.add(-1,xrd::uri_t(href));
                        else if(rel=="openid2.local_id")
                            html_openid2.local_ids.add(-1,href);
                    }
                }else if(is_element(n,"body")) {
                    skipping = -1;
                }
            }
            void queue_endpoints(endpoint_discovery_iterator& oi,
                    const idiscovery_t &id,
                    const service_type_t *st) {
                openid_endpoint_t ep;
                ep.claimed_id = id.canonicalized_id;
                for(xrd::services_t::const_iterator isvc=id.xrd.services.begin();
                        isvc!=id.xrd.services.end(); ++isvc) {
                    const xrd::service_t svc = isvc->second;
                    if(svc.types.find(st->uri)==svc.types.end()) continue;
                    for(xrd::uris_t::const_iterator iu=svc.uris.begin();iu!=svc.uris.end();++iu) {
                        ep.uri = iu->second.uri;
                        if(id.xri_identity) {
                            if(iu->second.append=="qxri") {
                                ep.uri += id.normalized_id;
                            } /* TODO: else handle other append attribute values */
                        }
                        if(st->forceid) {
                            ep.local_id = ep.claimed_id = st->forceid;
                            *(oi++) = ep;
                        }else{
                            if(svc.local_ids.empty()) {
                                ep.local_id = ep.claimed_id;
                                *(oi++) = ep;
                            }else{
                                for(xrd::local_ids_t::const_iterator ilid=svc.local_ids.begin();
                                        ilid!=svc.local_ids.end(); ++ilid) {
                                    ep.local_id = ilid->second;
                                    *(oi++) = ep;
                                }
                            }
                        }
                    }
                }
            }
    };
    string idiscover(endpoint_discovery_iterator oi,const string& identity) {
        idigger_t idigger;
        return idigger.discover(oi,identity);
    }
    void yadiscover(endpoint_discovery_iterator oi,const string& yurl,const char **types,bool redirs) try {
        idigger_t idigger;
        idigger.yadiscover(oi,yurl,types,redirs);
    }catch(exception_curl& ec) {
        if(redirs || ec._error!=CURLE_TOO_MANY_REDIRECTS)
            throw;
    }
 }