fix for discovery failing on some entities coming before the tag of interest

One of the numerous opportunities to express my gratitude by means of commit
message to Joseph Smarr of Plaxo for spotting it. Thanks, Joseph!

Signed-off-by: Michael Krelin <hacker@klever.net>

1 files changed, 1 insertions, 0 deletions

diff --git a/lib/discovery.cc b/lib/discovery.cc
index c118c80..3b90977 100644
--- a/lib/discovery.cc
+++ b/lib/discovery.cc
@@ -167,192 +167,193 @@ namespace opkele {
 				    xri_proxy + util::url_encode(id)+
 				    "?_xrd_t="+util::url_encode(st->uri)+
 				    "&_xrd_r=application/xrd%2Bxml"
 				    ";sep=true;refs=true",
 				    xmode_xrd );
 			    if(status_code==241) continue;
 			    if(status_code!=100)
 				throw failed_xri_resolution(OPKELE_CP_
 					"XRI resolution failed with '"+status_string+"' message"
 					", while looking for SEP with type '"+st->uri+"'"
 					" on canonical id", status_code);
 			}
 			idis.canonicalized_id = cid;
 			idis.normalized_id = rv; idis.xri_identity = true;
 			queue_endpoints(oi,idis,st);
 		    }
 		}else{
 		    idis.xri_identity = false;
 		    if(id.find("://")==string::npos)
 			id.insert(0,"http://");
 		    string::size_type fp = id.find('#');
 		    if(fp!=string::npos) {
 			string::size_type qp = id.find('?');
 			if(qp==string::npos || qp<fp)
 			    id.erase(fp);
 			else if(qp>fp)
 			    id.erase(fp,qp-fp);
 		    }
 		    rv = idis.normalized_id = util::rfc_3986_normalize_uri(id);
 		    discover_at(idis,id,xmode_html|xmode_xrd);
 		    const char * eu = 0;
 		    CURLcode r = easy_getinfo(CURLINFO_EFFECTIVE_URL,&eu);
 		    if(r)
 			throw exception_curl(OPKELE_CP_ "failed to get CURLINFO_EFFECTIVE_URL",r);
 		    string cid = util::strip_uri_fragment_part( idis.canonicalized_id = util::rfc_3986_normalize_uri(eu) );
 		    if(xrds_location.empty()) {
 			if(idis.xrd.empty())
 			    html2xrd(oi,idis);
 			else{
 			    for(const service_type_t *st=op_service_types;
 				    st<&op_service_types[sizeof(op_service_types)/sizeof(*op_service_types)];++st)
 				queue_endpoints(oi,idis,st);
 			}
 		    }else{
 			idis.clear();
 			idis.canonicalized_id = cid;
 			discover_at(idis,xrds_location,xmode_xrd);
 			if(idis.xrd.empty())
 			    html2xrd(oi,idis);
 			else{
 			    for(const service_type_t *st=op_service_types;
 				    st<&op_service_types[sizeof(op_service_types)/sizeof(*op_service_types)];++st)
 				queue_endpoints(oi,idis,st);
 			}
 		    }
 		}
 		return rv;
 	    }
 
 	    void discover_at(idiscovery_t& idis,const string& url,int xm) {
 		CURLcode r = easy_setopt(CURLOPT_MAXREDIRS, (xm&xmode_noredirs)?0:5);
 		if(r)
 		    throw exception_curl(OPKELE_CP_ "failed to set curly maxredirs option");
 		if( (r=easy_setopt(CURLOPT_URL,url.c_str())) )
 		    throw exception_curl(OPKELE_CP_ "failed to set curly urlie",r);
 
 		http_content_type.clear();
 		xmode = xm;
 		prepare_to_parse();
 		if(xmode&xmode_html) {
 		    xrds_location.clear();
 		    save_html.clear();
 		    save_html.reserve(max_html);
 		}
 		xrd = &idis.xrd;
 
 		r = easy_perform();
 		if(r && r!=CURLE_WRITE_ERROR)
 		    throw exception_curl(OPKELE_CP_ "failed to perform curly request",r);
 
 		if(!parser_choked) {
 		    parse(0,0,true);
 		}else if(xmode&xmode_html){
 		    /* TODO: do not bother if we've seen xml */
 		    try {
 			util::tidy_doc_t td = util::tidy_doc_t::create();
 			if(!td)
 			    throw exception_tidy(OPKELE_CP_ "failed to create htmltidy document");
 #ifndef NDEBUG
 			td.opt_set(TidyQuiet,false);
 			td.opt_set(TidyShowWarnings,false);
 #endif /* NDEBUG */
 			td.opt_set(TidyForceOutput,true);
 			td.opt_set(TidyXhtmlOut,true);
 			td.opt_set(TidyDoctypeMode,TidyDoctypeOmit);
 			td.opt_set(TidyMark,false);
+			td.opt_set(TidyNumEntities,true);
 			if(td.parse_string(save_html)<=0)
 			    throw exception_tidy(OPKELE_CP_ "tidy failed to parse document");
 			if(td.clean_and_repair()<=0)
 			    throw exception_tidy(OPKELE_CP_ "tidy failed to clean and repair");
 			util::tidy_buf_t tide;
 			if(td.save_buffer(tide)<=0)
 			    throw exception_tidy(OPKELE_CP_ "tidy failed to save buffer");
 			prepare_to_parse();
 			parse(tide.c_str(),tide.size(),true);
 		    }catch(exception_tidy& et) { }
 		}
 		save_html.clear();
 	    }
 
 	    void prepare_to_parse() {
 		(*(expat_t*)this) = parser_create_ns();
 		set_user_data(); set_element_handler();
 		set_character_data_handler();
 
 		if(xmode&xmode_html) {
 		    html_openid1.clear(); html_openid2.clear();
 		    parser_choked = false;
 		}
 
 		cdata = 0; xrd_service = 0; skipping = 0;
 		pt_stack.clear();
 		status_code = 100; status_string.clear();
 	    }
 
 	    void html2xrd(endpoint_discovery_iterator& oi,idiscovery_t& id) {
 		XRD_t& x = id.xrd;
 		if(!html_openid2.uris.empty()) {
 		    html_openid2.types.insert(STURI_OPENID20);
 		    x.services.add(-1,html_openid2);
 		    queue_endpoints(oi,id,&op_service_types[st_index_2]);
 		}
 		if(!html_openid1.uris.empty()) {
 		    html_openid1.types.insert(STURI_OPENID11);
 		    x.services.add(-1,html_openid1);
 		    queue_endpoints(oi,id,&op_service_types[st_index_1]);
 		}
 	    }
 
 	    size_t write(void *p,size_t s,size_t nm) {
 		/* TODO: limit total size */
 		size_t bytes = s*nm;
 		const char *inbuf = (const char*)p;
 		if(xmode&xmode_html) {
 		    size_t mbts = save_html.capacity()-save_html.size();
 		    size_t bts = 0;
 		    if(mbts>0) {
 			bts = (bytes>mbts)?mbts:bytes;
 			save_html.append(inbuf,bts);
 		    }
 		    if(skipping<0) return bts;
 		}
 		if(skipping<0) return 0;
 		bool rp = parse(inbuf,bytes,false);
 		if(!rp) {
 		    parser_choked = true;
 		    skipping = -1;
 		    if(!(xmode&xmode_html))
 			bytes = 0;
 		}
 		return bytes;
 	    }
 	    size_t header(void *p,size_t s,size_t nm) {
 		size_t bytes = s*nm;
 		const char *h = (const char*)p;
 		const char *colon = (const char*)memchr(p,':',bytes);
 		const char *space = (const char*)memchr(p,' ',bytes);
 		if(space && ( (!colon) || space<colon ) ) {
 		    xrds_location.clear(); http_content_type.clear();
 		}else if(colon) {
 		    const char *hv = ++colon;
 		    size_t hnl = colon-h;
 		    int rb;
 		    for(rb = bytes-hnl-1;rb>0 && isspace(*hv);++hv,--rb);
 		    while(rb>0 && isspace(hv[rb-1])) --rb;
 		    if(rb) {
 			if( (hnl>=sizeof(XRDS_HEADER))
 				&& !strncasecmp(h,XRDS_HEADER":",
 				    sizeof(XRDS_HEADER)) ) {
 			    xrds_location.assign(hv,rb);
 			}else if( (hnl>=sizeof(CT_HEADER))
 				&& !strncasecmp(h,CT_HEADER":",
 				    sizeof(CT_HEADER)) ) {
 			    const char *sc = (const char*)memchr(
 				    hv,';',rb);
 			    http_content_type.assign(hv,sc?(sc-hv):rb);
 			}
 		    }
 		}
 		return curl_t::header(p,s,nm);
 	    }