12 files changed, 71 insertions, 28 deletions

diff --git a/.gitignore b/.gitignore
index 87771db..16be7d8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,21 +1,16 @@
-configure
+/configure
 Makefile.in
-Doxyfile
-config.log
-depcomp
-config.guess
-config.h
-config.sub
-ltmain.sh
-INSTALL
-NEWS
+/Doxyfile
+/config.log
+/config.h
+/INSTALL
+/NEWS
 Makefile
-config.status
-stamp-h1
-config.h.in
-libtool
-autom4te.cache
-libopkele.pc
-missing
-aclocal.m4
-install-sh
+/config.status
+/stamp-h1
+/config.h.in
+/autom4te.cache
+/libopkele.pc
+/aclocal.m4
+/aclocal.d
+/aux.d
diff --git a/COPYING b/COPYING
index 46d3f30..b9cc74c 100644
--- a/COPYING
+++ b/COPYING
@@ -1,19 +1,19 @@
-Copyright (c) 2005-2008 Klever Group (http://www.klever.net/)
+Copyright (c) 2005-2009 Klever Group (http://www.klever.net/)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
 the Software without restriction, including without limitation the rights to
 use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
 of the Software, and to permit persons to whom the Software is furnished to do
 so, subject to the following conditions:
 
 The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
diff --git a/Makefile.am b/Makefile.am
index 3227bdb..7726dad 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,28 +1,29 @@
-
 SUBDIRS=include lib test
 EXTRA_DIST= NEWS NEWS.xml NEWS.xsl
 
+ACLOCAL_AMFLAGS=-I aclocal.d
+
 pkgconfigdir=${libdir}/pkgconfig
 pkgconfig_DATA=libopkele.pc
 
 all-local: NEWS
 if HAVE_DOXYGEN
 clean-local:
         rm -rf doxydox
 endif
 
 NEWS: NEWS.xsl NEWS.xml
         ${XSLTPROC} -o $@ NEWS.xsl NEWS.xml
 
 if HAVE_DOXYGEN
 dox: Doxyfile
         ${DOXYGEN}
 endif
 
 ISSUEFILES = $$(find ${top_srcdir} -type f '(' \
         -name '*.cc' -or -name '*.h' \
         ')' ) \
         ${top_srcdir}/configure.ac
 issues: todo fixme xxx
 todo fixme xxx:
         @grep --color=auto -in '$@:' ${ISSUEFILES} || true
diff --git a/NEWS.xml b/NEWS.xml
index 160b197..0206018 100644
--- a/NEWS.xml
+++ b/NEWS.xml
@@ -1,65 +1,71 @@
 <?xml version="1.0" encoding="us-ascii"?>
 <news>
+ <version version="2.0.2" date="April 11th, 2009">
+  <ni>Handling of unknown encodings during discovery</ni>
+  <ni>Discovery robustness improvements</ni>
+  <ni>Workaround for OPs (e.g. livejournal.com) breaking specs</ni>
+  <ni>Build fixes and improvements</ni>
+ </version>
  <version version="2.0.1" date="November 22nd, 2008">
   <ni>Compile-time fixes and improvements</ni>
   <ni>Portability improvements for FreeBSD</ni>
   <ni>Really suppress debugging message from htmltidy when --disable-debug is in
   effect</ni>
   <ni>minor bugfixes</ni>
   <ni>thread-safety improvements</ni>
  </version>
  <version version="2.0" date="June 26th, 2008">
   <ni>OpenID 2.0 support</ni>
   <ni>Major rewrite of the whole thing</ni>
   <ni>Support for XRDS (YADIS and XRI/inames) discovery</ni>
   <ni>Sheerly improved html-based discovery (only code using new, 2.0-enabled
   classes benefits from it)</ni>
   <ni>Deprecation of the old api</ni>
   <ni>Added sample RP and OP implementations</ni>
   <ni>Require expat xml stream parser library</ni>
   <ni>Require htmltidy library</ni>
   <ni>Require tr1/memory (shared_ptr) support - either modern gcc or boost
   library</ni>
  </version>
  <version version="0.3.2" date="November 22nd, 2007">
   <ni>code cleanup for stricter compiler</ni>
  </version>
  <version version="0.3.1" date="November 20th, 2007">
   <ni>more robustness improvements in links discovery</ni>
   <ni>removed dependency on pcre c++ bindings, because there are few of them and
   not everyone likes all of them</ni>
   <ni>minor build improvements</ni>
  </version>
  <version version="0.3" date="August 9th, 2007">
   <ni>fixed canonicalization procedure to be specs-compliant. Note, that the old
   consumer_t::canonicalize is now called consumer_t::normalize and the
   canonicalize memeber is now virtual to allow caching layer, not static</ni>
   <ni>robustness improvement in handling associations expiry</ni>
   <ni>minor documentation updates</ni>
  </version>
  <version version="0.2.1" date="June 24th, 2007">
   <ni>open id server invalid signature bugfix</ni>
  </version>
  <version version="0.2" date="June 19th, 2007">
   <ni>A few robustness improvements and optimizations</ni>
   <ni>More liberal key/values messages parsing</ni>
   <ni>Changed unusable --with-pcre++ configure option to --with-pcrepp</ni>
  </version>
  <version version="0.1.1" date="January 16th, 2007">
   <ni>Fixed a bug in curl errors handling</ni>
   <ni>added --disable-ssl-verify-host and --disable-ssl-verify-peer options to
   configure to alter default implementation of consumer's retrieve_links
   behaviour</ni>
   <ni>Build fix that eliminates the need to pass --disable-konforka in the
   absence of it.</ni>
  </version>
  <version version="0.1" date="January 12th, 2007">
   <ni>OpenID simple registration extension implementation</ni>
   <ni>OpenID extensions framework</ni>
   <ni>Canonicalization bugfix</ni>
   <ni>Slightly improved interoperability with buggy implementations</ni>
  </version>
  <version version="0.0" date="July 25th, 2005">
   <ni>Initial release</ni>
  </version>
 </news>
diff --git a/autogen.sh b/autogen.sh
index bf32a35..578206d 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,11 +1,11 @@
 #!/bin/sh
 tool_libtoolize="$(type -P glibtoolize || type -P libtoolize)"
 if test -z "$tool_libtoolize" ; then
  echo "Failed to find libtoolize." ; exit 1;
 fi
    "$tool_libtoolize" -f \
-&& aclocal \
+&& aclocal -I aclocal.d \
 && autoheader \
 && automake -a \
 && autoconf \
 && ./configure "$@"
diff --git a/autoregen.sh b/autoregen.sh
new file mode 100644
index 0000000..ce75a08
--- a/dev/null
+++ b/autoregen.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+eval sh autogen.sh $(./config.status --version | grep '^  with options "'|sed -e 's/^[^"]\+"//' -e 's/"$//') "$@"
diff --git a/configure.ac b/configure.ac
index a7b56ff..2ded490 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,195 +1,197 @@
-AC_INIT([libopkele], [2.0.1], [libopkele-bugs@klever.net])
+AC_INIT([libopkele], [2.0.2], [libopkele-bugs@klever.net])
 AC_CONFIG_SRCDIR([include/opkele/opkele-config.h])
 AC_CONFIG_HEADERS([config.h include/opkele/acconfig.h])
+AC_CONFIG_MACRO_DIR([aclocal.d])
+AC_CONFIG_AUX_DIR([aux.d])
 AM_INIT_AUTOMAKE([dist-bzip2])
 
 AC_PROG_INSTALL
 AC_PROG_CXX
 AC_PROG_CC
 AC_PROG_LIBTOOL
 PKG_PROG_PKG_CONFIG
 
 AC_HEADER_STDC
 AC_CHECK_FUNCS([timegm])
 
 AC_PATH_PROG([XSLTPROC],[xsltproc],[true])
 
 AC_MSG_CHECKING([for source tree version])
 if headrev=$(cd $srcdir && git rev-parse --verify HEAD 2>/dev/null) ; then
  PACKAGE_SRC_VERSION="$(cd $srcdir && git describe --tags $headrev)"
  test "$PACKAGE_SRC_VERSION" = "$PACKAGE_VERSION" \
   -o "${PACKAGE_SRC_VERSION#${PACKAGE_VERSION}-}" != "$PACKAGE_SRC_VERSION" || PACKAGE_SRC_VERSION="${PACKAGE_VERSION}:${PACKAGE_SRC_VERSION}"
  ( cd $srcdir && git diff-index $headrev | read dirt ) && PACKAGE_SRC_VERSION="${PACKAGE_SRC_VERSION}-dirty"
 else
  PACKAGE_SRC_VERSION="$PACKAGE_VERSION"
 fi
 AC_MSG_RESULT([$PACKAGE_SRC_VERSION])
 AC_SUBST([PACKAGE_SRC_VERSION])
 AC_DEFINE_UNQUOTED([PACKAGE_SRC_VERSION],["$PACKAGE_SRC_VERSION"],[more or less precise source tree version])
 
 tr1_mem_std="false"
 tr1_mem_boost="false"
 AC_CHECK_SHAREDPTR(std::tr1,tr1/memory,[ tr1_mem_std=true ])
 AC_CHECK_SHAREDPTR(boost,boost/shared_ptr.hpp,[ tr1_mem_boost=true ])
 tr1_mem=""
 AC_ARG_WITH([tr1-memory],
  AC_HELP_STRING([--with-tr1-memory=<boost|std>],[select tr1/memory (shared_ptr<>) implementation to use]),
  [ tr1_mem="$withval" ]
 )
 AC_MSG_CHECKING([for tr1/memory implementation to use])
 test -z "$tr1_mem" && $tr1_mem_std && tr1_mem=std
 test -z "$tr1_mem" && $tr1_mem_boost && tr1_mem=boost
 if test -z "$tr1_mem" ; then
  AC_MSG_RESULT([none found])
 else
  AC_MSG_RESULT([$tr1_mem])
 fi
 case "$tr1_mem" in
  std)
   $tr1_mem_std || AC_MSG_ERROR([std implementation requested, but not found])
   OPKELE_TR1_MEM_NS=std::tr1
   OPKELE_TR1_MEM_HEADER=tr1/memory
   ;;
  boost)
   $tr1_mem_boost || AC_MSG_ERROR([boost implementation requested, but not found])
   OPKELE_TR1_MEM_NS=boost
   OPKELE_TR1_MEM_HEADER=boost/shared_ptr.hpp
   ;;
  *)
   AC_MSG_ERROR([no shared_ptr<> implementation found])
   ;;
 esac
 AC_SUBST([OPKELE_TR1_MEM_NS])
 AC_SUBST([OPKELE_TR1_MEM_HEADER])
 
 AC_MSG_CHECKING([for deprecated attribute support])
 AC_COMPILE_IFELSE([
   int __attribute__((deprecated)) deprecated_function();
  ],[
   AC_MSG_RESULT([yes])
   AC_DEFINE([OPKELE_DEPRECATE],[__attribute__((deprecated))],[deprecated function attribute])
  ],[
   AC_MSG_RESULT([no])
   AC_DEFINE([OPKELE_DEPRECATE],,[deprecated function attribute])
  ]
 )
 
 AC_LANG_PUSH([C++])
 AC_MSG_CHECKING([for abi::__cxa_demangle])
 AC_COMPILE_IFELSE([
   #include <typeinfo>
   using namespace std;
   #include <cxxabi.h>
   int main(int c,char **v) {
    int dstat;
    char *demangled = abi::__cxa_demangle(typeid(dstat).name(),0,0,&dstat);
    return 0;
   }
  ],[
   AC_MSG_RESULT([yes])
   AC_DEFINE([HAVE_DEMANGLE],,[defined if abi::__cxa_demangle is available])
  ],[
   AC_MSG_RESULT([no])
  ]
 )
 AC_LANG_POP([C++])
 
 
 
 
 PKG_CHECK_MODULES([OPENSSL],[openssl],,[
  AC_MSG_ERROR([no openssl library found. get one from http://www.openssl.org/])
 ])
 
 WANT_KONFORKA="yes"
 AC_ARG_ENABLE([konforka],
  AC_HELP_STRING([--disable-konforka],[do not use konforka library (default: use if found)]),
  [
   test "${enableval}" = "no" && WANT_KONFORKA="no"
  ]
 )
 if test "${WANT_KONFORKA}" = "yes" ; then
  PKG_CHECK_MODULES([KONFORKA],[konforka],[
   AC_SUBST([KONFORKA_CFLAGS])
   AC_SUBST([KONFORKA_LIBS])
   AC_DEFINE([HAVE_KONFORKA],,[defined in presence of konforka library])
   AC_DEFINE([OPKELE_HAVE_KONFORKA],,[defined in presence of konforka library])
   AC_SUBST([KONFORKA_KONFORKA],[konforka])
  ],[true])
 fi
 
 WANT_DOXYGEN="yes"
 AC_ARG_ENABLE([doxygen],
  AC_HELP_STRING([--disable-doxygen],[do not generate documentation]),
  [
   test "${enableval}" = "no" && WANT_DOXYGEN="no"
  ]
 )
 if test "${WANT_DOXYGEN}" = "yes" ; then
  AC_WITH_DOXYGEN
  AC_WITH_DOT
 else
  AM_CONDITIONAL([HAVE_DOXYGEN],[false])
  AM_CONDITIONAL([HAVE_DOT],[false])
 fi
 
 LIBCURL_CHECK_CONFIG(,,,[
  AC_MSG_ERROR([no required libcurl library. get one from http://curl.haxx.se/])
 ])
 
 AC_CHECK_HEADER([expat.h],[
  AC_CHECK_LIB([expat],[XML_ParserCreate],[
   EXPAT_LIBS=-lexpat
   EXPAT_CFLAGS=
   AC_SUBST([EXPAT_LIBS])
   AC_SUBST([EXPAT_CFLAGS])
  ],[
   AC_MSG_ERROR([no required expat library. get one from http://expat.sourceforge.net/])
  ])
 ],[
  AC_MSG_ERROR([no required expat library. get one from http://expat.sourceforge.net/])
 ])
 
 AC_CHECK_HEADERS([tidy.h tidy/tidy.h],[
  AC_CHECK_LIB([tidy],[tidyParseBuffer],[
   TIDY_LIBS=-ltidy
   TIDY_CFLAGS=
   AC_SUBST([TIDY_LIBS])
   AC_SUBST([TIDY_CFLAGS])
  ],[
   AC_MSG_ERROR([no required htmltidy library found. get one from http://tidy.sourceforge.net/])
  ])
 ],[
  test "$ac_header" = "tidy/tidy.h" \
  && AC_MSG_ERROR([no required htmltidy library found. get one from http://tidy.sourceforge.net/])
 ])
 
 if test -n "$PCRE_LIBS" -a -n "$PCRE_CFLAGS" ; then
  AC_SUBST([PCRE_CFLAGS])
  AC_SUBST([PCRE_LIBS])
  :
 else
  PKG_CHECK_MODULES([PCRE],[libpcre],,[
   AC_MSG_ERROR([no libpcre found, go get it at http://www.pcre.org/])
  ])
 fi
 
 PKG_CHECK_MODULES([SQLITE3],[sqlite3],[have_sqlite3=true],[have_sqlite3=false])
 AM_CONDITIONAL([HAVE_SQLITE3],[$have_sqlite3])
 PKG_CHECK_MODULES([KINGATE],[kingate-plaincgi],[have_kingate=true],[have_kingate=false])
 AM_CONDITIONAL([HAVE_KINGATE],[$have_kingate])
 PKG_CHECK_MODULES([UUID],[uuid],[have_uuid=true],[have_uuid=false])
 AM_CONDITIONAL([HAVE_UUID],[$have_uuid])
 if $have_uuid ; then
  AC_DEFINE([HAVE_LIBUUID],,[defined in presence of libuuid])
  AC_SUBST([UUID_UUID],[uuid])
 fi
 
 curl_ssl_verify_host="true"
 AC_ARG_ENABLE([ssl-verify-host],
  AC_HELP_STRING([--disable-ssl-verify-host],[disable cURL cert/host relationships verification]),
  [ test "${enableval}" = "no" && curl_ssl_verify_host="false" ]
 )
 ${curl_ssl_verify_host} || AC_DEFINE([DISABLE_CURL_SSL_VERIFYHOST],,[defined if cURL is not to verify cert/host])
 
 curl_ssl_verify_peer="true"
diff --git a/include/opkele/expat.h b/include/opkele/expat.h
index 3ab1630..21be003 100644
--- a/include/opkele/expat.h
+++ b/include/opkele/expat.h
@@ -1,91 +1,94 @@
 #ifndef __OPKELE_EXPAT_H
 #define __OPKELE_EXPAT_H
 
 #include <cassert>
 #include <expat.h>
 
 namespace opkele {
 
     namespace util {
 
         class expat_t {
             public:
                 XML_Parser _x;
 
                 expat_t() : _x(0) { }
                 expat_t(XML_Parser x) : _x(x) { }
                 virtual ~expat_t() throw();
 
                 expat_t& operator=(XML_Parser x);
 
                 operator const XML_Parser(void) const { return _x; }
                 operator XML_Parser(void) { return _x; }
 
                 inline bool parse(const char *s,int len,bool final=false) {
                     assert(_x);
                     return XML_Parse(_x,s,len,final);
                 }
 
+                virtual int unknown_encoding(const XML_Char * /* n */,XML_Encoding * /* i */) { return XML_STATUS_ERROR; }
+                void set_unknown_encoding_handler();
+
                 virtual void start_element(const XML_Char * /* n */,const XML_Char ** /* a */) { }
                 virtual void end_element(const XML_Char * /* n */) { }
                 void set_element_handler();
 
                 virtual void character_data(const XML_Char * /* s */,int /* l */) { }
                 void set_character_data_handler();
 
                 virtual void processing_instruction(const XML_Char * /* t */,const XML_Char * /* d */) { }
                 void set_processing_instruction_handler();
 
                 virtual void comment(const XML_Char * /* d */) { }
                 void set_comment_handler();
 
                 virtual void start_cdata_section() { }
                 virtual void end_cdata_section() { }
                 void set_cdata_section_handler();
 
                 virtual void default_handler(const XML_Char * /* s */,int /* l */) { }
                 void set_default_handler();
                 void set_default_handler_expand();
 
                 virtual void start_namespace_decl(const XML_Char * /* p */,const XML_Char * /* u */) { }
                 virtual void end_namespace_decl(const XML_Char * /* p */) { }
                 void set_namespace_decl_handler();
 
                 inline enum XML_Error get_error_code() {
                     assert(_x); return XML_GetErrorCode(_x); }
                 static inline const XML_LChar *error_string(XML_Error c) {
                     return XML_ErrorString(c); }
 
                 inline long get_current_byte_index() {
                     assert(_x); return XML_GetCurrentByteIndex(_x); }
                 inline int get_current_line_number() {
                     assert(_x); return XML_GetCurrentLineNumber(_x); }
                 inline int get_current_column_number() {
                     assert(_x); return XML_GetCurrentColumnNumber(_x); }
 
                 inline void set_user_data() {
                     assert(_x); XML_SetUserData(_x,this); }
 
                 inline bool set_base(const XML_Char *b) {
                     assert(_x); return XML_SetBase(_x,b); }
                 inline const XML_Char *get_base() {
                     assert(_x); return XML_GetBase(_x); }
 
                 inline int get_specified_attribute_count() {
                     assert(_x); return XML_GetSpecifiedAttributeCount(_x); }
 
                 inline bool set_param_entity_parsing(enum XML_ParamEntityParsing c) {
                     assert(_x); return XML_SetParamEntityParsing(_x,c); }
 
                 inline static XML_Parser parser_create(const XML_Char *e=0) {
                     return XML_ParserCreate(e); }
                 inline static XML_Parser parser_create_ns(const XML_Char *e=0,XML_Char s='\t') {
                     return XML_ParserCreateNS(e,s); }
 
         };
 
     }
 
 }
 
 #endif /* __OPKELE_EXPAT_H */
diff --git a/lib/basic_rp.cc b/lib/basic_rp.cc
index 3cad71c..9c7113b 100644
--- a/lib/basic_rp.cc
+++ b/lib/basic_rp.cc
@@ -29,298 +29,299 @@ namespace opkele {
             throw non_identity(OPKELE_CP_ "attempting to retrieve identity of non-identity related assertion");
         assert(!claimed_id.empty());
         return identity;
     }
 
     static void dh_get_secret(
             secret_t& secret, const basic_openid_message& om,
             const char *exp_assoc, const char *exp_sess,
             util::dh_t& dh,
             size_t d_len, unsigned char *(*d_fun)(const unsigned char*,size_t,unsigned char*),
             size_t exp_s_len) try {
         if(om.get_field("assoc_type")!=exp_assoc || om.get_field("session_type")!=exp_sess)
             throw bad_input(OPKELE_CP_ "Unexpected associate response");
         util::bignum_t s_pub = util::base64_to_bignum(om.get_field("dh_server_public"));
         vector<unsigned char> ck(DH_size(dh)+1);
         unsigned char *ckptr = &(ck.front())+1;
         int cklen = DH_compute_key(ckptr,s_pub,dh);
         if(cklen<0)
             throw exception_openssl(OPKELE_CP_ "failed to DH_compute_key()");
         if(cklen && (*ckptr)&0x80) {
             (*(--ckptr))=0; ++cklen; }
         assert(d_len<=SHA256_DIGEST_LENGTH);
         unsigned char key_digest[SHA256_DIGEST_LENGTH];
         secret.enxor_from_base64((*d_fun)(ckptr,cklen,key_digest),om.get_field("enc_mac_key"));
         if(secret.size()!=exp_s_len)
             throw bad_input(OPKELE_CP_ "Secret length isn't consistent with association type");
     }catch(opkele::failed_lookup& ofl) {
         throw bad_input(OPKELE_CP_ "Incoherent response from OP");
     } OPKELE_RETHROW
 
     static void direct_request(basic_openid_message& oum,const basic_openid_message& inm,const string& OP) {
         util::curl_pick_t curl = util::curl_pick_t::easy_init();
         if(!curl)
             throw exception_curl(OPKELE_CP_ "failed to initialize curl");
         string request = inm.query_string();
         CURLcode r;
         (r=curl.misc_sets())
             || (r=curl.easy_setopt(CURLOPT_URL,OP.c_str()))
             || (r=curl.easy_setopt(CURLOPT_POST,1))
             || (r=curl.easy_setopt(CURLOPT_POSTFIELDS,request.data()))
             || (r=curl.easy_setopt(CURLOPT_POSTFIELDSIZE,request.length()))
             || (r=curl.set_write());
         if(r)
             throw exception_curl(OPKELE_CP_ "failed to set curly options",r);
         if( (r=curl.easy_perform()) )
             throw exception_curl(OPKELE_CP_ "failed to perform curly request",r);
         oum.from_keyvalues(curl.response);
     }
 
 
     assoc_t basic_RP::associate(const string& OP) {
         util::dh_t dh = DH_new();
         if(!dh)
             throw exception_openssl(OPKELE_CP_ "failed to DH_new()");
         dh->p = util::dec_to_bignum(data::_default_p);
         dh->g = util::dec_to_bignum(data::_default_g);
         if(!DH_generate_key(dh))
             throw exception_openssl(OPKELE_CP_ "failed to DH_generate_key()");
         openid_message_t req;
         req.set_field("ns",OIURI_OPENID20);
         req.set_field("mode","associate");
         req.set_field("dh_modulus",util::bignum_to_base64(dh->p));
         req.set_field("dh_gen",util::bignum_to_base64(dh->g));
         req.set_field("dh_consumer_public",util::bignum_to_base64(dh->pub_key));
         openid_message_t res;
         req.set_field("assoc_type","HMAC-SHA256");
         req.set_field("session_type","DH-SHA256");
         secret_t secret;
         int expires_in;
         try {
             direct_request(res,req,OP);
             dh_get_secret( secret, res,
                     "HMAC-SHA256", "DH-SHA256",
                     dh, SHA256_DIGEST_LENGTH, SHA256, SHA256_DIGEST_LENGTH );
             expires_in = util::string_to_long(res.get_field("expires_in"));
         }catch(exception&) {
             try {
                 req.set_field("assoc_type","HMAC-SHA1");
                 req.set_field("session_type","DH-SHA1");
                 direct_request(res,req,OP);
                 dh_get_secret( secret, res,
                         "HMAC-SHA1", "DH-SHA1",
                         dh, SHA_DIGEST_LENGTH, SHA1, SHA_DIGEST_LENGTH );
                 expires_in = util::string_to_long(res.get_field("expires_in"));
             }catch(bad_input&) {
                 throw dumb_RP(OPKELE_CP_ "OP failed to supply an association");
             }
         }
         return store_assoc(
                 OP, res.get_field("assoc_handle"),
                 res.get_field("assoc_type"), secret,
                 expires_in );
     }
 
     basic_openid_message& basic_RP::checkid_(
             basic_openid_message& rv,
             mode_t mode,
             const string& return_to,const string& realm,
             extension_t *ext) {
         rv.reset_fields();
         rv.set_field("ns",OIURI_OPENID20);
         if(mode==mode_checkid_immediate)
             rv.set_field("mode","checkid_immediate");
         else if(mode==mode_checkid_setup)
             rv.set_field("mode","checkid_setup");
         else
             throw bad_input(OPKELE_CP_ "unknown checkid_* mode");
         if(realm.empty() && return_to.empty())
             throw bad_input(OPKELE_CP_ "At least one of realm and return_to must be non-empty");
         if(!realm.empty()) {
             rv.set_field("realm",realm);
             rv.set_field("trust_root",realm);
         }
         if(!return_to.empty())
             rv.set_field("return_to",return_to);
         const openid_endpoint_t& ep = get_endpoint();
         rv.set_field("claimed_id",ep.claimed_id);
         rv.set_field("identity",ep.local_id);
         try {
             rv.set_field("assoc_handle",find_assoc(ep.uri)->handle());
         }catch(dumb_RP& drp) {
         }catch(failed_lookup& fl) {
             try {
                 rv.set_field("assoc_handle",associate(ep.uri)->handle());
             }catch(dumb_RP& drp) { }
         } OPKELE_RETHROW
         if(ext) ext->rp_checkid_hook(rv);
         return rv;
     }
 
     class signed_part_message_proxy : public basic_openid_message {
         public:
             const basic_openid_message& x;
             set<string> signeds;
 
             signed_part_message_proxy(const basic_openid_message& xx) : x(xx) {
                 const string& slist = x.get_field("signed");
                 string::size_type p = 0;
                 while(true) {
                     string::size_type co = slist.find(',',p);
                     string f = (co==string::npos)
                         ?slist.substr(p):slist.substr(p,co-p);
                     signeds.insert(f);
                     if(co==string::npos) break;
                     p = co+1;
                 }
             }
 
             bool has_field(const string& n) const {
                 return signeds.find(n)!=signeds.end() && x.has_field(n); }
             const string& get_field(const string& n) const {
                 if(signeds.find(n)==signeds.end())
                     throw failed_lookup(OPKELE_CP_ "The field isn't known to be signed");
                 return x.get_field(n); }
 
             fields_iterator fields_begin() const {
                 return signeds.begin(); }
             fields_iterator fields_end() const {
                 return signeds.end(); }
     };
 
     static void parse_query(const string& u,string::size_type q,
             map<string,string>& p) {
         if(q==string::npos)
             return;
         assert(u[q]=='?');
         ++q;
         string::size_type l = u.size();
         while(q<l) {
             string::size_type eq = u.find('=',q);
             string::size_type am = u.find('&',q);
             if(am==string::npos) {
                 if(eq==string::npos) {
                     p[""] = u.substr(q);
                 }else{
                     p[u.substr(q,eq-q)] = u.substr(eq+1);
                 }
                 break;
             }else{
                 if(eq==string::npos || eq>am) {
                     p[""] = u.substr(q,eq-q);
                 }else{
                     p[u.substr(q,eq-q)] = u.substr(eq+1,am-eq-1);
                 }
                 q = ++am;
             }
         }
     }
 
     void basic_RP::id_res(const basic_openid_message& om,extension_t *ext) {
         reset_vars();
         bool o2 = om.has_field("ns")
-            && om.get_field("ns")==OIURI_OPENID20;
+            && om.get_field("ns")==OIURI_OPENID20
+            && om.has_field("op_endpoint") && !om.get_field("op_endpoint").empty();
         if( (!o2) && om.has_field("user_setup_url"))
             throw id_res_setup(OPKELE_CP_ "assertion failed, setup url provided",
                     om.get_field("user_setup_url"));
         string m = om.get_field("mode");
         if(o2 && m=="setup_needed")
             throw id_res_setup(OPKELE_CP_ "setup needed, no setup url provided");
         if(m=="cancel")
             throw id_res_cancel(OPKELE_CP_ "authentication cancelled");
         bool go_dumb=false;
         try {
             string OP = o2
                 ?om.get_field("op_endpoint")
                 :get_endpoint().uri;
             assoc_t assoc = retrieve_assoc(
                     OP,om.get_field("assoc_handle"));
             if(om.get_field("sig")!=util::base64_signature(assoc,om))
                 throw id_res_mismatch(OPKELE_CP_ "signature mismatch");
         }catch(dumb_RP& drp) {
             go_dumb=true;
         }catch(failed_lookup& e) {
             go_dumb=true;
         } OPKELE_RETHROW
         if(go_dumb) {
             try {
                 string OP = o2
                     ?om.get_field("op_endpoint")
                     :get_endpoint().uri;
                 check_authentication(OP,om);
             }catch(failed_check_authentication& fca) {
                 throw id_res_failed(OPKELE_CP_ "failed to check_authentication()");
             } OPKELE_RETHROW
         }
         signed_part_message_proxy signeds(om);
         if(o2) {
             check_nonce(om.get_field("op_endpoint"),
                     om.get_field("response_nonce"));
             static const char *mustsign[] = {
                 "op_endpoint", "return_to", "response_nonce", "assoc_handle",
                 "claimed_id", "identity" };
             for(size_t ms=0;ms<(sizeof(mustsign)/sizeof(*mustsign));++ms) {
                 if(om.has_field(mustsign[ms]) && !signeds.has_field(mustsign[ms]))
                     throw bad_input(OPKELE_CP_ string("Field '")+mustsign[ms]+"' is not signed against the specs");
             }
             if( (
                     (om.has_field("claimed_id")?1:0)
                     ^
                     (om.has_field("identity")?1:0)
                 )&1 )
                 throw bad_input(OPKELE_CP_ "claimed_id and identity must be either both present or both absent");
 
             string turl = util::rfc_3986_normalize_uri(get_this_url());
             util::strip_uri_fragment_part(turl);
             string rurl = util::rfc_3986_normalize_uri(om.get_field("return_to"));
             util::strip_uri_fragment_part(rurl);
             string::size_type
                 tq = turl.find('?'), rq = rurl.find('?');
             if(
                     ((tq==string::npos)?turl:turl.substr(0,tq))
                     !=
                     ((rq==string::npos)?rurl:rurl.substr(0,rq))
               )
                 throw id_res_bad_return_to(OPKELE_CP_ "return_to url doesn't match request url");
             map<string,string> tp; parse_query(turl,tq,tp);
             map<string,string> rp; parse_query(rurl,rq,rp);
             for(map<string,string>::const_iterator rpi=rp.begin();rpi!=rp.end();++rpi) {
                 map<string,string>::const_iterator tpi = tp.find(rpi->first);
                 if(tpi==tp.end())
                     throw id_res_bad_return_to(OPKELE_CP_ string("Parameter '")+rpi->first+"' from return_to is missing from the request");
                 if(tpi->second!=rpi->second)
                     throw id_res_bad_return_to(OPKELE_CP_ string("Parameter '")+rpi->first+"' from return_to doesn't matche the request");
             }
 
             if(om.has_field("claimed_id")) {
                 claimed_id = om.get_field("claimed_id");
                 identity = om.get_field("identity");
                 verify_OP(
                         om.get_field("op_endpoint"),
                         claimed_id, identity );
             }
 
         }else{
             claimed_id = get_endpoint().claimed_id;
             /* TODO: check if this is the identity we asked for */
             identity = om.get_field("identity");
         }
         if(ext) ext->rp_id_res_hook(om,signeds);
     }
 
     void basic_RP::check_authentication(const string& OP,
             const basic_openid_message& om){
         openid_message_t res;
         static const string checkauthmode = "check_authentication";
         direct_request(res,util::change_mode_message_proxy(om,checkauthmode),OP);
         if(res.has_field("is_valid")) {
             if(res.get_field("is_valid")=="true") {
                 if(res.has_field("invalidate_handle"))
                     invalidate_assoc(OP,res.get_field("invalidate_handle"));
                 return;
             }
         }
         throw failed_check_authentication(
                 OPKELE_CP_ "failed to verify response");
     }
 
 }
diff --git a/lib/discovery.cc b/lib/discovery.cc
index bd1f917..b4ed3b6 100644
--- a/lib/discovery.cc
+++ b/lib/discovery.cc
@@ -94,494 +94,517 @@ namespace opkele {
             string save_html;
 
             XRD_t *xrd;
             service_t *xrd_service;
             string* cdata;
 
             idigger_t()
                 : util::curl_t(easy_init()),
                 util::expat_t(0),
                 xri_proxy(XRI_PROXY_URL) {
                     CURLcode r;
                     (r=misc_sets())
                         || (r=set_write())
                         || (r=set_header())
                         ;
                     if(r)
                         throw exception_curl(OPKELE_CP_ "failed to set curly options",r);
                 }
             ~idigger_t() throw() { }
 
             void yadiscover(endpoint_discovery_iterator oi,const string& yurl,const char **types,bool redirs) {
                 idiscovery_t idis;
                 idis.xri_identity = false;
                 discover_at(idis,yurl,xmode_html|xmode_xrd|(redirs?0:xmode_noredirs));
                 if(!xrds_location.empty()) {
                     idis.clear();
                     discover_at(idis,xrds_location,xmode_xrd);
                 }
                 idis.normalized_id = idis.canonicalized_id = yurl;
                 service_type_t st;
                 for(st.uri=*types;*types;st.uri=*(++types))
                     queue_endpoints(oi,idis,&st);
             }
 
             string discover(endpoint_discovery_iterator& oi,const string& identity) {
                 string rv;
                 idiscovery_t idis;
                 string::size_type fsc = identity.find_first_not_of(data::_whitespace_chars);
                 if(fsc==string::npos)
                     throw bad_input(OPKELE_CP_ "whitespace-only identity");
                 string::size_type lsc = identity.find_last_not_of(data::_whitespace_chars);
                 assert(lsc!=string::npos);
                 if(!strncasecmp(identity.c_str()+fsc,"xri://",sizeof("xri://")-1))
                     fsc += sizeof("xri://")-1;
                 if((fsc+1)>=lsc)
                     throw bad_input(OPKELE_CP_ "not a character of importance in identity");
                 string id(identity,fsc,lsc-fsc+1);
                 idis.clear();
                 if(strchr(data::_iname_leaders,id[0])) {
                     /* TODO: further normalize xri identity? Like folding case
                      * or whatever... */
                     rv = id;
                     set<string> cids;
                     for(const struct service_type_t *st=op_service_types;
                             st<&op_service_types[sizeof(op_service_types)/sizeof(*op_service_types)];++st) {
                         idis.clear();
                         discover_at( idis,
                                 xri_proxy + util::url_encode(id)+
                                 "?_xrd_t="+util::url_encode(st->uri)+
                                 "&_xrd_r=application/xrd%2Bxml"
                                 ";sep=true;refs=true",
                                 xmode_xrd );
                         if(status_code==241) continue;
                         if(status_code!=100)
                             throw failed_xri_resolution(OPKELE_CP_
                                     "XRI resolution failed with '"+status_string+"' message"
                                     ", while looking for SEP with type '"+st->uri+"'", status_code);
                         if(idis.xrd.canonical_ids.empty())
                             throw opkele::failed_discovery(OPKELE_CP_ "No CanonicalID for XRI identity found");
                         string cid = idis.xrd.canonical_ids.begin()->second;
                         if(cids.find(cid)==cids.end()) {
                             cids.insert(cid);
                             idis.clear();
                             discover_at( idis,
                                     xri_proxy + util::url_encode(id)+
                                     "?_xrd_t="+util::url_encode(st->uri)+
                                     "&_xrd_r=application/xrd%2Bxml"
                                     ";sep=true;refs=true",
                                     xmode_xrd );
                             if(status_code==241) continue;
                             if(status_code!=100)
                                 throw failed_xri_resolution(OPKELE_CP_
                                         "XRI resolution failed with '"+status_string+"' message"
                                         ", while looking for SEP with type '"+st->uri+"'"
                                         " on canonical id", status_code);
                         }
                         idis.canonicalized_id = cid;
                         idis.normalized_id = rv; idis.xri_identity = true;
                         queue_endpoints(oi,idis,st);
                     }
                 }else{
                     idis.xri_identity = false;
                     if(id.find("://")==string::npos)
                         id.insert(0,"http://");
                     string::size_type fp = id.find('#');
                     if(fp!=string::npos) {
                         string::size_type qp = id.find('?');
                         if(qp==string::npos || qp<fp)
                             id.erase(fp);
                         else if(qp>fp)
                             id.erase(fp,qp-fp);
                     }
                     rv = idis.normalized_id = util::rfc_3986_normalize_uri(id);
                     discover_at(idis,id,xmode_html|xmode_xrd);
                     const char * eu = 0;
                     CURLcode r = easy_getinfo(CURLINFO_EFFECTIVE_URL,&eu);
                     if(r)
                         throw exception_curl(OPKELE_CP_ "failed to get CURLINFO_EFFECTIVE_URL",r);
                     string cid = util::strip_uri_fragment_part( idis.canonicalized_id = util::rfc_3986_normalize_uri(eu) );
                     if(xrds_location.empty()) {
                         if(idis.xrd.empty())
                             html2xrd(oi,idis);
                         else{
                             for(const service_type_t *st=op_service_types;
                                     st<&op_service_types[sizeof(op_service_types)/sizeof(*op_service_types)];++st)
                                 queue_endpoints(oi,idis,st);
                         }
                     }else{
                         idis.clear();
                         idis.canonicalized_id = cid;
                         discover_at(idis,xrds_location,xmode_xrd);
                         if(idis.xrd.empty())
                             html2xrd(oi,idis);
                         else{
                             for(const service_type_t *st=op_service_types;
                                     st<&op_service_types[sizeof(op_service_types)/sizeof(*op_service_types)];++st)
                                 queue_endpoints(oi,idis,st);
                         }
                     }
                 }
                 return rv;
             }
 
             void discover_at(idiscovery_t& idis,const string& url,int xm) {
                 CURLcode r = easy_setopt(CURLOPT_MAXREDIRS, (xm&xmode_noredirs)?0:5);
                 if(r)
                     throw exception_curl(OPKELE_CP_ "failed to set curly maxredirs option");
                 if( (r=easy_setopt(CURLOPT_URL,url.c_str())) )
                     throw exception_curl(OPKELE_CP_ "failed to set curly urlie",r);
 
                 http_content_type.clear();
                 xmode = xm;
                 prepare_to_parse();
                 if(xmode&xmode_html) {
                     xrds_location.clear();
                     save_html.clear();
                     save_html.reserve(max_html);
                 }
                 xrd = &idis.xrd;
 
                 r = easy_perform();
                 if(r && r!=CURLE_WRITE_ERROR)
                     throw exception_curl(OPKELE_CP_ "failed to perform curly request",r);
 
                 if(!parser_choked) {
                     parse(0,0,true);
                 }else if(xmode&xmode_html){
                     /* TODO: do not bother if we've seen xml */
                     try {
                         util::tidy_doc_t td = util::tidy_doc_t::create();
                         if(!td)
                             throw exception_tidy(OPKELE_CP_ "failed to create htmltidy document");
 #ifdef NDEBUG
                         td.opt_set(TidyQuiet,true);
                         td.opt_set(TidyShowWarnings,false);
 #else /* NDEBUG */
                         td.opt_set(TidyQuiet,false);
                         td.opt_set(TidyShowWarnings,true);
 #endif /* NDEBUG */
                         td.opt_set(TidyForceOutput,true);
                         td.opt_set(TidyXhtmlOut,true);
                         td.opt_set(TidyDoctypeMode,TidyDoctypeOmit);
                         td.opt_set(TidyMark,false);
                         td.opt_set(TidyNumEntities,true);
                         if(td.parse_string(save_html)<=0)
                             throw exception_tidy(OPKELE_CP_ "tidy failed to parse document");
                         if(td.clean_and_repair()<=0)
                             throw exception_tidy(OPKELE_CP_ "tidy failed to clean and repair");
                         util::tidy_buf_t tide;
                         if(td.save_buffer(tide)<=0)
                             throw exception_tidy(OPKELE_CP_ "tidy failed to save buffer");
                         prepare_to_parse();
                         parse(tide.c_str(),tide.size(),true);
                     }catch(exception_tidy& et) { }
                 }
                 save_html.clear();
             }
 
             void prepare_to_parse() {
                 (*(expat_t*)this) = parser_create_ns();
                 set_user_data(); set_element_handler();
                 set_character_data_handler();
+                set_unknown_encoding_handler();
 
                 if(xmode&xmode_html) {
                     html_openid1.clear(); html_openid2.clear();
                     parser_choked = false;
                 }
 
                 cdata = 0; xrd_service = 0; skipping = 0;
                 pt_stack.clear();
                 status_code = 100; status_string.clear();
             }
 
             void html2xrd(endpoint_discovery_iterator& oi,idiscovery_t& id) {
                 XRD_t& x = id.xrd;
                 if(!html_openid2.uris.empty()) {
                     html_openid2.types.insert(STURI_OPENID20);
                     x.services.add(-1,html_openid2);
                     queue_endpoints(oi,id,&op_service_types[st_index_2]);
                 }
                 if(!html_openid1.uris.empty()) {
                     html_openid1.types.insert(STURI_OPENID11);
                     x.services.add(-1,html_openid1);
                     queue_endpoints(oi,id,&op_service_types[st_index_1]);
                 }
             }
 
             size_t write(void *p,size_t s,size_t nm) {
                 /* TODO: limit total size */
                 size_t bytes = s*nm;
                 const char *inbuf = (const char*)p;
                 if(xmode&xmode_html) {
                     size_t mbts = save_html.capacity()-save_html.size();
                     size_t bts = 0;
                     if(mbts>0) {
                         bts = (bytes>mbts)?mbts:bytes;
                         save_html.append(inbuf,bts);
                     }
                     if(skipping<0) return bts;
                 }
                 if(skipping<0) return 0;
                 bool rp = parse(inbuf,bytes,false);
                 if(!rp) {
                     parser_choked = true;
                     skipping = -1;
                     if(!(xmode&xmode_html))
                         bytes = 0;
                 }
                 return bytes;
             }
             size_t header(void *p,size_t s,size_t nm) {
                 size_t bytes = s*nm;
                 const char *h = (const char*)p;
                 const char *colon = (const char*)memchr(p,':',bytes);
                 const char *space = (const char*)memchr(p,' ',bytes);
                 if(space && ( (!colon) || space<colon ) ) {
                     xrds_location.clear(); http_content_type.clear();
                 }else if(colon) {
                     const char *hv = ++colon;
                     size_t hnl = colon-h;
                     int rb;
                     for(rb = bytes-hnl-1;rb>0 && isspace(*hv);++hv,--rb) ;
                     while(rb>0 && isspace(hv[rb-1])) --rb;
                     if(rb) {
                         if( (hnl>=sizeof(XRDS_HEADER))
                                 && !strncasecmp(h,XRDS_HEADER":",
                                     sizeof(XRDS_HEADER)) ) {
                             xrds_location.assign(hv,rb);
                         }else if( (hnl>=sizeof(CT_HEADER))
                                 && !strncasecmp(h,CT_HEADER":",
                                     sizeof(CT_HEADER)) ) {
                             const char *sc = (const char*)memchr(
                                     hv,';',rb);
                             http_content_type.assign(hv,sc?(sc-hv):rb);
                         }
                     }
                 }
                 return curl_t::header(p,s,nm);
             }
 
             void start_element(const XML_Char *n,const XML_Char **a) {
                 if(skipping<0) return;
                 if(skipping) {
                     if(xmode&xmode_html)
                         html_start_element(n,a);
                     ++skipping; return;
                 }
                 if(pt_stack.empty()) {
                     if(is_qelement(n,NSURI_XRDS "\tXRDS"))
                         return;
                     if(is_qelement(n,NSURI_XRD "\tXRD")) {
                         assert(xrd);
                         xrd->clear();
                         pt_stack.push_back(n);
                     }else if(xmode&xmode_html) {
                         html_start_element(n,a);
                     }else{
                         skipping = -1;
                     }
                 }else{
                     int pt_s = pt_stack.size();
                     if(pt_s==1) {
                         if(is_qelement(n,NSURI_XRD "\tCanonicalID")) {
                             assert(xrd);
                             cdata = &(xrd->canonical_ids.add(element_priority(a),string()));
                         }else if(is_qelement(n,NSURI_XRD "\tLocalID")) {
                             assert(xrd);
                             cdata = &(xrd->local_ids.add(element_priority(a),string()));
                         }else if(is_qelement(n,NSURI_XRD "\tProviderID")) {
                             assert(xrd);
                             cdata = &(xrd->provider_id);
                         }else if(is_qelement(n,NSURI_XRD "\tService")) {
                             assert(xrd);
                             xrd_service = &(xrd->services.add(element_priority(a),
                                         service_t()));
                             pt_stack.push_back(n);
                         }else if(is_qelement(n,NSURI_XRD "\tStatus")) {
                             for(;*a;) {
                                 if(!strcasecmp(*(a++),"code")) {
                                     if(sscanf(*(a++),"%ld",&status_code)==1 && status_code!=100) {
                                         cdata = &status_string;
                                         pt_stack.push_back(n);
                                         break;
                                     }
                                 }else
                                     ++a;
                             }
                         }else if(is_qelement(n,NSURI_XRD "\tExpires")) {
                             assert(xrd);
                             cdata_buf.clear();
                             cdata = &cdata_buf;
                         }else if(xmode&xmode_html) {
                             html_start_element(n,a);
                         }else{
                             skipping = 1;
                         }
                     }else if(pt_s==2) {
                         if(is_qelement(pt_stack.back().c_str(), NSURI_XRD "\tService")) {
                             if(is_qelement(n,NSURI_XRD "\tType")) {
                                 assert(xrd); assert(xrd_service);
                                 cdata_buf.clear();
                                 cdata = &cdata_buf;
                             }else if(is_qelement(n,NSURI_XRD "\tURI")) {
                                 assert(xrd); assert(xrd_service);
                                 const char *append = element_attr(a,"append");
                                 xrd::uri_t& uri = xrd_service->uris.add(element_priority(a),xrd::uri_t("",append?append:""));
                                 cdata = &uri.uri;
                             }else if(is_qelement(n,NSURI_XRD "\tLocalID")
                                     || is_qelement(n,NSURI_OPENID10 "\tDelegate") ) {
                                 assert(xrd); assert(xrd_service);
                                 cdata = &(xrd_service->local_ids.add(element_priority(a),string()));
                             }else if(is_qelement(n,NSURI_XRD "\tProviderID")) {
                                     assert(xrd); assert(xrd_service);
                                     cdata = &(xrd_service->provider_id);
                             }else{
                                 skipping = 1;
                             }
                         }else
                             skipping = 1;
                     }else if(xmode&xmode_html) {
                         html_start_element(n,a);
                     }else{
                         skipping = 1;
                     }
                 }
             }
             void end_element(const XML_Char *n) {
                 if(skipping<0) return;
                 if(skipping) {
                     --skipping; return;
                 }
                 if(is_qelement(n,NSURI_XRD "\tType")) {
-                    assert(xrd); assert(xrd_service); assert(cdata==&cdata_buf);
+                    if(xrd && xrd_service) {
+                        assert(cdata==&cdata_buf);
                     xrd_service->types.insert(cdata_buf);
+                    }
                 }else if(is_qelement(n,NSURI_XRD "\tService")) {
-                    assert(xrd); assert(xrd_service);
+                    if(!(xrd && xrd_service)) {
+                        skipping = -1;
+                    }else{
                     assert(!pt_stack.empty());
                     assert(pt_stack.back()==(NSURI_XRD "\tService"));
                     pt_stack.pop_back();
                     xrd_service = 0;
+                    }
                 }else if(is_qelement(n,NSURI_XRD "\tStatus")) {
-                    assert(xrd);
+                    if(!xrd) {
+                        skipping=-1;
+                    }else{
                     if(is_qelement(pt_stack.back().c_str(),n)) {
                         assert(cdata==&status_string);
                         pt_stack.pop_back();
                         if(status_code!=100)
                             skipping = -1;
                     }
+                    }
                 }else if(is_qelement(n,NSURI_XRD "\tExpires")) {
-                    assert(xrd);
+                    if(!xrd) {
+                        skipping=-1;
+                    }else{
                     xrd->expires = util::w3c_to_time(cdata_buf);
+                    }
+                }else if(is_qelement(n,NSURI_XRD "\tXRD")) {
+                    assert(!pt_stack.empty());
+                    assert(pt_stack.back()==(NSURI_XRD "\tXRD"));
+                    pt_stack.pop_back();
                 }else if((xmode&xmode_html) && is_element(n,"head")) {
                     skipping = -1;
                 }
                 cdata = 0;
             }
             void character_data(const XML_Char *s,int l) {
                 if(skipping) return;
                 if(cdata) cdata->append(s,l);
             }
 
             void html_start_element(const XML_Char *n,const XML_Char **a) {
                 if(is_element(n,"meta")) {
                     bool heq = false;
                     string l;
                     for(;*a;a+=2) {
                         if(!( strcasecmp(a[0],"http-equiv")
                                 || strcasecmp(a[1],XRDS_HEADER) ))
                             heq = true;
                         else if(!strcasecmp(a[0],"content"))
                             l.assign(a[1]);
                     }
                     if(heq)
                         xrds_location = l;
                 }else if(is_element(n,"link")) {
                     string rels;
                     string href;
                     for(;*a;a+=2) {
                         if( !strcasecmp(a[0],"rel") ) {
                             rels.assign(a[1]);
                         }else if( !strcasecmp(a[0],"href") ) {
                             const char *ns = a[1];
                             for(;*ns && isspace(*ns);++ns) ;
                             href.assign(ns);
                             string::size_type lns=href.find_last_not_of(data::_whitespace_chars);
                             href.erase(lns+1);
                         }
                     }
                     for(string::size_type ns=rels.find_first_not_of(data::_whitespace_chars);
                             ns!=string::npos; ns=rels.find_first_not_of(data::_whitespace_chars,ns)) {
                         string::size_type s = rels.find_first_of(data::_whitespace_chars,ns);
                         string rel;
                         if(s==string::npos) {
                             rel.assign(rels,ns,string::npos);
                             ns = string::npos;
                         }else{
                             rel.assign(rels,ns,s-ns);
                             ns = s;
                         }
                         if(rel=="openid.server")
                             html_openid1.uris.add(-1,xrd::uri_t(href));
                         else if(rel=="openid.delegate")
                             html_openid1.local_ids.add(-1,href);
                         else if(rel=="openid2.provider")
                             html_openid2.uris.add(-1,xrd::uri_t(href));
                         else if(rel=="openid2.local_id")
                             html_openid2.local_ids.add(-1,href);
                     }
                 }else if(is_element(n,"body")) {
                     skipping = -1;
                 }
             }
 
             void queue_endpoints(endpoint_discovery_iterator& oi,
                     const idiscovery_t &id,
                     const service_type_t *st) {
                 openid_endpoint_t ep;
                 ep.claimed_id = id.canonicalized_id;
                 for(xrd::services_t::const_iterator isvc=id.xrd.services.begin();
                         isvc!=id.xrd.services.end(); ++isvc) {
                     const xrd::service_t svc = isvc->second;
                     if(svc.types.find(st->uri)==svc.types.end()) continue;
                     for(xrd::uris_t::const_iterator iu=svc.uris.begin();iu!=svc.uris.end();++iu) {
                         ep.uri = iu->second.uri;
                         if(id.xri_identity) {
                             if(iu->second.append=="qxri") {
                                 ep.uri += id.normalized_id;
                             } /* TODO: else handle other append attribute values */
                         }
                         if(st->forceid) {
                             ep.local_id = ep.claimed_id = st->forceid;
                             *(oi++) = ep;
                         }else{
                             if(svc.local_ids.empty()) {
                                 ep.local_id = ep.claimed_id;
                                 *(oi++) = ep;
                             }else{
                                 for(xrd::local_ids_t::const_iterator ilid=svc.local_ids.begin();
                                         ilid!=svc.local_ids.end(); ++ilid) {
                                     ep.local_id = ilid->second;
                                     *(oi++) = ep;
                                 }
                             }
                         }
                     }
                 }
             }
 
+            int unknown_encoding(const XML_Char* /* n */,XML_Encoding *i) {
+                for(unsigned int ii=0;ii < sizeof(i->map)/sizeof(i->map[0]);++ii)
+                    i->map[ii] = ii;
+                i->convert = 0; i->release = 0;
+                return XML_STATUS_OK;
+            }
+
     };
 
     string idiscover(endpoint_discovery_iterator oi,const string& identity) {
         idigger_t idigger;
         return idigger.discover(oi,identity);
     }
 
     void yadiscover(endpoint_discovery_iterator oi,const string& yurl,const char **types,bool redirs) try {
         idigger_t idigger;
         idigger.yadiscover(oi,yurl,types,redirs);
     }catch(exception_curl& ec) {
         if(redirs || ec._error!=CURLE_TOO_MANY_REDIRECTS)
             throw;
     }
 
 }
diff --git a/lib/expat.cc b/lib/expat.cc
index c4dab7e..fb58a9a 100644
--- a/lib/expat.cc
+++ b/lib/expat.cc
@@ -1,97 +1,106 @@
 #include <opkele/expat.h>
 
 namespace opkele {
 
     namespace util {
 
         expat_t::~expat_t() throw() {
             if(_x)
                 XML_ParserFree(_x);
         }
 
         expat_t& expat_t::operator=(XML_Parser x) {
             if(_x)
                 XML_ParserFree(_x);
             _x = x;
             return *this;
         }
 
         static void _start_element(void* ud,const XML_Char *n,const XML_Char **a) {
             ((expat_t*)ud)->start_element(n,a);
         }
         static void _end_element(void *ud,const XML_Char *n) {
             ((expat_t*)ud)->end_element(n);
         }
 
         void expat_t::set_element_handler() {
             assert(_x);
             XML_SetElementHandler(_x,_start_element,_end_element);
         }
 
         static void _character_data(void *ud,const XML_Char *s,int l) {
             ((expat_t*)ud)->character_data(s,l);
         }
 
         void expat_t::set_character_data_handler() {
             assert(_x);
             XML_SetCharacterDataHandler(_x,_character_data);
         }
 
         static void _processing_instruction(void *ud,const XML_Char *t,const XML_Char *d) {
             ((expat_t*)ud)->processing_instruction(t,d);
         }
 
         void expat_t::set_processing_instruction_handler() {
             assert(_x);
             XML_SetProcessingInstructionHandler(_x,_processing_instruction);
         }
 
         static void _comment(void *ud,const XML_Char *d) {
             ((expat_t*)ud)->comment(d);
         }
 
         void expat_t::set_comment_handler() {
             assert(_x);
             XML_SetCommentHandler(_x,_comment);
         }
 
         static void _start_cdata_section(void *ud) {
             ((expat_t*)ud)->start_cdata_section();
         }
         static void _end_cdata_section(void *ud) {
             ((expat_t*)ud)->end_cdata_section();
         }
 
         void expat_t::set_cdata_section_handler() {
             assert(_x);
             XML_SetCdataSectionHandler(_x,_start_cdata_section,_end_cdata_section);
         }
 
         static void _default_handler(void *ud,const XML_Char *s,int l) {
             ((expat_t*)ud)->default_handler(s,l);
         }
 
         void expat_t::set_default_handler() {
             assert(_x);
             XML_SetDefaultHandler(_x,_default_handler);
         }
         void expat_t::set_default_handler_expand() {
             assert(_x);
             XML_SetDefaultHandlerExpand(_x,_default_handler);
         }
 
         static void _start_namespace_decl(void *ud,const XML_Char *p,const XML_Char *u) {
             ((expat_t*)ud)->start_namespace_decl(p,u);
         }
         static void _end_namespace_decl(void *ud,const XML_Char *p) {
             ((expat_t*)ud)->end_namespace_decl(p);
         }
 
         void expat_t::set_namespace_decl_handler() {
             assert(_x);
             XML_SetNamespaceDeclHandler(_x,_start_namespace_decl,_end_namespace_decl);
         }
 
+        static int _unknown_encoding(void *ehd,const XML_Char *n,XML_Encoding *i) {
+            return ((expat_t*)ehd)->unknown_encoding(n,i);
+        }
+
+        void expat_t::set_unknown_encoding_handler() {
+            assert(_x);
+            XML_SetUnknownEncodingHandler(_x,_unknown_encoding,this);
+        }
+
     }
 
 }
diff --git a/test/RP.cc b/test/RP.cc
index 35ee71d..f015723 100644
--- a/test/RP.cc
+++ b/test/RP.cc
@@ -1,195 +1,196 @@
 #include <uuid/uuid.h>
 #include <iostream>
 #include <cassert>
+#include <cstdlib>
 #include <stdexcept>
 #include <string>
 #include <set>
 #include <iterator>
 using namespace std;
 #include <kingate/exception.h>
 #include <kingate/plaincgi.h>
 #include <kingate/cgi_gateway.h>
 #include <opkele/exception.h>
 #include <opkele/types.h>
 #include <opkele/util.h>
 #include <opkele/uris.h>
 #include <opkele/discovery.h>
 #include <opkele/association.h>
 #include <opkele/sreg.h>
 using namespace opkele;
 #include <opkele/prequeue_rp.h>
 #include <opkele/debug.h>
 
 #include "sqlite.h"
 #include "kingate_openid_message.h"
 
 #undef DUMB_RP
 
 #ifdef DUMB_RP
 # define DUMBTHROW throw opkele::dumb_RP(OPKELE_CP_ "This RP is dumb")
 #else
 # define DUMBTHROW (void)0
 #endif
 
 class rpdb_t : public sqlite3_t {
     public:
         rpdb_t()
             : sqlite3_t("/tmp/RP.db") {
                 assert(_D);
                 char **resp; int nrow,ncol; char *errm;
                 if(sqlite3_get_table(
                         _D,"SELECT a_op FROM assoc LIMIT 0",
                         &resp,&nrow,&ncol,&errm)!=SQLITE_OK) {
                     extern const char *__RP_db_bootstrap;
                     DOUT_("Bootstrapping DB");
                     if(sqlite3_exec(_D,__RP_db_bootstrap,NULL,NULL,&errm)!=SQLITE_OK)
                         throw opkele::exception(OPKELE_CP_ string("Failed to bootstrap SQLite database: ")+errm);
                 }else
                     sqlite3_free_table(resp);
 
             }
 };
 
 class example_rp_t : public opkele::prequeue_RP {
     public:
         mutable rpdb_t db;
         kingate::cookie htc;
         long as_id;
         int ordinal;
         kingate::cgi_gateway& gw;
 
         example_rp_t(kingate::cgi_gateway& g)
         : as_id(-1), ordinal(0), gw(g), have_eqtop(false)  {
             try {
                 htc = gw.cookies.get_cookie("ht_session");
                 as_id = opkele::util::string_to_long(gw.get_param("asid"));
             }catch(kingate::exception_notfound& kenf) {
                 uuid_t uuid; uuid_generate(uuid);
                 htc = kingate::cookie("ht_session",util::encode_base64(uuid,sizeof(uuid)));
                 sqlite3_mem_t<char*> S = sqlite3_mprintf(
                         "INSERT INTO ht_sessions (hts_id) VALUES (%Q)",
                         htc.get_value().c_str());
                 db.exec(S);
             }
         }
 
         /* Global persistent store */
 
         opkele::assoc_t store_assoc(
                 const string& OP,const string& handle,
                 const string& type,const secret_t& secret,
                 int expires_in) {
             DUMBTHROW;
             DOUT_("Storing '" << handle << "' assoc with '" << OP << "'");
             time_t exp = time(0)+expires_in;
             sqlite3_mem_t<char*>
                 S = sqlite3_mprintf(
                         "INSERT INTO assoc"
                         " (a_op,a_handle,a_type,a_ctime,a_etime,a_secret)"
                         " VALUES ("
                         "  %Q,%Q,%Q,"
                         "  datetime('now'), datetime('now','+%d seconds'),"
                         "  %Q"
                         " );", OP.c_str(), handle.c_str(), type.c_str(),
                         expires_in,
                         util::encode_base64(&(secret.front()),secret.size()).c_str() );
             db.exec(S);
             return opkele::assoc_t(new opkele::association(
                         OP, handle, type, secret, exp, false ));
         }
 
         opkele::assoc_t find_assoc(
                 const string& OP) {
             DUMBTHROW;
             DOUT_("Looking for an assoc with '" << OP << '\'');
             sqlite3_mem_t<char*>
                 S = sqlite3_mprintf(
                         "SELECT"
                         "  a_op,a_handle,a_type,a_secret,"
                         "  strftime('%%s',a_etime) AS a_etime"
                         " FROM assoc"
                         " WHERE a_op=%Q AND a_itime IS NULL AND NOT a_stateless"
                         "  AND ( a_etime > datetime('now','-30 seconds') )"
                         " LIMIT 1",
                         OP.c_str());
             sqlite3_table_t T;
             int nr,nc;
             db.get_table(S,T,&nr,&nc);
             if(nr<1)
                 throw opkele::failed_lookup(OPKELE_CP_ "Couldn't find unexpired handle");
             assert(nr==1);
             assert(nc==5);
             secret_t secret;
             util::decode_base64(T.get(1,3,nc),secret);
             DOUT_(" found '" << T.get(1,1,nc) << '\'');
             return opkele::assoc_t(new opkele::association(
                         T.get(1,0,nc), T.get(1,1,nc), T.get(1,2,nc),
                         secret, strtol(T.get(1,4,nc),0,0), false ));
         }
 
         opkele::assoc_t retrieve_assoc(
                 const string& OP,const string& handle) {
             DUMBTHROW;
             DOUT_("Retrieving assoc '" << handle << "' with '" << OP << '\'');
             sqlite3_mem_t<char*>
                 S = sqlite3_mprintf(
                         "SELECT"
                         "  a_op,a_handle,a_type,a_secret,"
                         "  strftime('%%s',a_etime) AS a_etime"
                         " FROM assoc"
                         " WHERE a_op=%Q AND a_handle=%Q"
                         "  AND a_itime IS NULL AND NOT a_stateless"
                         " LIMIT 1",
                         OP.c_str(),handle.c_str());
             sqlite3_table_t T;
             int nr,nc;
             db.get_table(S,T,&nr,&nc);
             if(nr<1)
                 throw opkele::failed_lookup(OPKELE_CP_ "couldn't retrieve valid association");
             assert(nr==1); assert(nc==5);
             secret_t secret; util::decode_base64(T.get(1,3,nc),secret);
             DOUT_(" found. type=" << T.get(1,2,nc) << '\'');
             return opkele::assoc_t(new opkele::association(
                         T.get(1,0,nc), T.get(1,1,nc), T.get(1,2,nc),
                         secret, strtol(T.get(1,4,nc),0,0), false ));
         }
 
         void invalidate_assoc(
                 const string& OP,const string& handle) {
             DUMBTHROW;
             DOUT_("Invalidating assoc '" << handle << "' with '" << OP << '\'');
             sqlite3_mem_t<char*>
                 S = sqlite3_mprintf(
                         "UPDATE assoc SET a_itime=datetime('now')"
                         " WHERE a_op=%Q AND a_handle=%Q",
                         OP.c_str(), handle.c_str() );
             db.exec(S);
         }
 
         void check_nonce(const string& OP,const string& nonce) {
             DOUT_("Checking nonce '" << nonce << "' from '" << OP << '\'');
             sqlite3_mem_t<char*>
                 S = sqlite3_mprintf(
                         "SELECT 1 FROM nonces WHERE n_op=%Q AND n_once=%Q",
                         OP.c_str(), nonce.c_str());
             sqlite3_table_t T;
             int nr,nc;
             db.get_table(S,T,&nr,&nc);
             if(nr)
                 throw opkele::id_res_bad_nonce(OPKELE_CP_ "already seen that nonce");
             sqlite3_mem_t<char*>
                 SS = sqlite3_mprintf(
                         "INSERT INTO nonces (n_op,n_once) VALUES (%Q,%Q)",
                         OP.c_str(), nonce.c_str());
             db.exec(SS);
         }
 
         /* Session perisistent store */
 
         void begin_queueing() {
             assert(as_id>=0);
             DOUT_("Resetting queue for session '" << htc.get_value() << "'/" << as_id);
             sqlite3_mem_t<char*> S = sqlite3_mprintf(
                     "DELETE FROM endpoints_queue"
                     " WHERE as_id=%ld",
                     as_id);