summaryrefslogtreecommitdiffabout
Unidiff
Diffstat (more/less context) (ignore whitespace changes)
-rw-r--r--include/opkele/util.h10
-rw-r--r--lib/util.cc131
-rw-r--r--test/test.cc67
3 files changed, 208 insertions, 0 deletions
diff --git a/include/opkele/util.h b/include/opkele/util.h
index edc1859..085c9e6 100644
--- a/include/opkele/util.h
+++ b/include/opkele/util.h
@@ -127,4 +127,14 @@ namespace opkele {
127 */ 127 */
128 void decode_base64(const string& data,vector<unsigned char>& rv); 128 void decode_base64(const string& data,vector<unsigned char>& rv);
129
130 /**
131 * Normalize http(s) URI according to RFC3986, section 6. URI is
132 * expected to have scheme: in front of it.
133 * @param uri URI
134 * @return normalized URI
135 * @throw not_implemented in case of non-httpi(s) URI
136 * @throw bad_input in case of malformed URI
137 */
138 string rfc_3986_normalize_uri(const string& uri);
129 } 139 }
130 140
diff --git a/lib/util.cc b/lib/util.cc
index 26be66a..eacf6d7 100644
--- a/lib/util.cc
+++ b/lib/util.cc
@@ -1,7 +1,9 @@
1#include <errno.h> 1#include <errno.h>
2#include <cassert> 2#include <cassert>
3#include <cctype>
3#include <cstring> 4#include <cstring>
4#include <vector> 5#include <vector>
5#include <string> 6#include <string>
7#include <stack>
6#include <openssl/bio.h> 8#include <openssl/bio.h>
7#include <openssl/evp.h> 9#include <openssl/evp.h>
@@ -160,4 +162,133 @@ namespace opkele {
160 } 162 }
161 163
164 /*
165 * Normalize URL according to the rules, described in rfc 3986, section 6
166 *
167 * - uppercase hext triplets (e.g. %ab -> %AB)
168 * - lowercase scheme and host
169 * - decode %-encoded characters, specified as unreserved in rfc 3986, section 2.3,
170 * that is - [:alpha:][:digit:]._~-
171 * - remove dot segments
172 * - remove empty and default ports
173 * - if there's no path component, add '/'
174 */
175 string rfc_3986_normalize_uri(const string& uri) {
176 string rv;
177 string::size_type colon = uri.find(':');
178 if(colon==string::npos)
179 throw bad_input(OPKELE_CP_ "No scheme specified in URI");
180 transform(
181 uri.begin(), uri.begin()+colon+1,
182 back_inserter(rv), ::tolower );
183 bool s;
184 if(rv=="http:")
185 s = false;
186 else if(rv=="https:")
187 s = true;
188 else
189 throw not_implemented(OPKELE_CP_ "Only http(s) URIs can be normalized here");
190 string::size_type ul = uri.length();
191 if(ul <= (colon+3))
192 throw bad_input(OPKELE_CP_ "Unexpected end of URI being normalized encountered");
193 if(uri[colon+1]!='/' || uri[colon+2]!='/')
194 throw bad_input(OPKELE_CP_ "Unexpected input in URI being normalized after scheme component");
195 rv += "//";
196 string::size_type interesting = uri.find_first_of(":/#?",colon+3);
197 if(interesting==string::npos) {
198 transform(
199 uri.begin()+colon+3,uri.end(),
200 back_inserter(rv), ::tolower );
201 rv += '/'; return rv;
202 }
203 transform(
204 uri.begin()+colon+3,uri.begin()+interesting,
205 back_inserter(rv), ::tolower );
206 bool qf = false;
207 char ic = uri[interesting];
208 if(ic==':') {
209 string::size_type ni = uri.find_first_of("/#?%",interesting+1);
210 const char *nptr = uri.data()+interesting+1;
211 char *eptr = 0;
212 long port = strtol(nptr,&eptr,10);
213 if( (port>0) && (port<65535) && port!=(s?443:80) ) {
214 char tmp[6];
215 snprintf(tmp,sizeof(tmp),"%d",port);
216 rv += ':'; rv += tmp;
217 }
218 if(ni==string::npos) {
219 rv += '/'; return rv;
220 }
221 interesting = ni;
222 }else if(ic!='/') {
223 rv += '/'; rv += ic;
224 qf = true;
225 ++interesting;
226 }
227 string::size_type n = interesting;
228 char tmp[3] = { 0,0,0 };
229 stack<string::size_type> psegs; psegs.push(rv.length());
230 string pseg;
231 for(;n<ul;) {
232 string::size_type unsafe = uri.find_first_of(qf?"%":"%/?#",n);
233 if(unsafe==string::npos) {
234 pseg.append(uri,n,ul-n-1); n = ul-1;
235 }else{
236 pseg.append(uri,n,unsafe-n);
237 n = unsafe;
238 }
239 char c = uri[n++];
240 if(c=='%') {
241 if((n+1)>=ul)
242 throw bad_input(OPKELE_CP_ "Unexpected end of URI encountered while parsing percent-encoded character");
243 tmp[0] = uri[n++];
244 tmp[1] = uri[n++];
245 if(!( isxdigit(tmp[0]) && isxdigit(tmp[1]) ))
246 throw bad_input(OPKELE_CP_ "Invalid percent-encoded character in URI being normalized");
247 int cc = strtol(tmp,0,16);
248 if( isalpha(cc) || isdigit(cc) || strchr("._~-",cc) )
249 pseg += cc;
250 else{
251 pseg += '%';
252 pseg += toupper(tmp[0]); pseg += toupper(tmp[1]);
253 }
254 }else if(qf) {
255 rv += pseg; rv += c;
256 pseg.clear();
257 }else if(n>=ul || strchr("?/#",c)) {
258 if(pseg.empty() || pseg==".") {
259 }else if(pseg=="..") {
260 if(psegs.size()>1) {
261 rv.resize(psegs.top()); psegs.pop();
262 }
263 }else{
264 psegs.push(rv.length());
265 if(c!='/') {
266 pseg += c;
267 qf = true;
268 }
269 rv += '/'; rv += pseg;
270 }
271 if(c=='/' && (n>=ul || strchr("?#",uri[n])) ) {
272 rv += '/';
273 if(n<ul)
274 qf = true;
275 }else if(strchr("?#",c)) {
276 if(psegs.size()==1 && psegs.top()==rv.length())
277 rv += '/';
278 if(pseg.empty())
279 rv += c;
280 qf = true;
281 }
282 pseg.clear();
283 }else{
284 pseg += c;
285 }
286 }
287 if(!pseg.empty()) {
288 rv += '/'; rv += pseg;
289 }
290 return rv;
291 }
292
162 } 293 }
163 294
diff --git a/test/test.cc b/test/test.cc
index f92284c..1a012b5 100644
--- a/test/test.cc
+++ b/test/test.cc
@@ -4,4 +4,5 @@ using namespace std;
4#include <opkele/exception.h> 4#include <opkele/exception.h>
5#include <opkele/consumer.h> 5#include <opkele/consumer.h>
6#include <opkele/util.h>
6 7
7#include "config.h" 8#include "config.h"
@@ -60,6 +61,72 @@ void test_retrieve_links() {
60} 61}
61 62
63void test_rfc_3986_normalize_uri(const string &ouri,bool success,const string& nuri="") {
64 try {
65 string n = opkele::util::rfc_3986_normalize_uri(ouri);
66 if(!success)
67 throw failed_test(OPKELE_CP_ "Normalized URI when it shouldn't");
68 if(n!=nuri)
69 throw failed_test(OPKELE_CP_ "rfc_3986_test_failed for '"+ouri+"' failed, expected '"+nuri+"', got '"+n+"'");
70 }catch(opkele::bad_input& obi) {
71 if(success)
72 throw failed_test(OPKELE_CP_ "Test '"+ouri+"' failed due to 'bad_input'["+obi.what()+"]");
73 }catch(opkele::not_implemented& oni) {
74 if(success)
75 throw failed_test(OPKELE_CP_ "Test '"+ouri+"' failed due to 'not_implemented'["+oni.what()+"]");
76 }
77}
78
79void test_rfc_3986_normalize_uri() {
80 test_rfc_3986_normalize_uri(
81 "invalid", false );
82 test_rfc_3986_normalize_uri(
83 "ftp://hacker.klever.net/", false );
84 test_rfc_3986_normalize_uri(
85 "http://", false );
86 test_rfc_3986_normalize_uri(
87 "http:/hacker.klever.net/", false );
88 test_rfc_3986_normalize_uri(
89 "hTTp://hacker.klever.net#uh?oh", true, "http://hacker.klever.net/#uh?oh" );
90 test_rfc_3986_normalize_uri(
91 "http://hacker.klever.net?uh#oh", true, "http://hacker.klever.net/?uh#oh" );
92 test_rfc_3986_normalize_uri(
93 "http://hacker.klever.net:80/", true, "http://hacker.klever.net/" );
94 test_rfc_3986_normalize_uri(
95 "http://hacker.klever.net:80?uh", true, "http://hacker.klever.net/?uh" );
96 test_rfc_3986_normalize_uri(
97 "http://hacker.klever.net:80#uh", true, "http://hacker.klever.net/#uh" );
98 test_rfc_3986_normalize_uri(
99 "https://hacker.klever.net:443", true, "https://hacker.klever.net/" );
100 test_rfc_3986_normalize_uri(
101 "http://hacker.klever.net:?oh", true, "http://hacker.klever.net/?oh" );
102 test_rfc_3986_normalize_uri(
103 "http://hacker.klever.net/ah%2E", true, "http://hacker.klever.net/ah." );
104 test_rfc_3986_normalize_uri(
105 "http://hacker.klever.net/ah/%2E/", true, "http://hacker.klever.net/ah/" );
106 test_rfc_3986_normalize_uri(
107 "http://hacker.klever.net/ah/%2b/", true, "http://hacker.klever.net/ah/%2B/" );
108 test_rfc_3986_normalize_uri(
109 "http://hacker.klever.net/ah/./oh?eh", true, "http://hacker.klever.net/ah/oh?eh" );
110 test_rfc_3986_normalize_uri(
111 "http://hacker.klever.net/ah/../oh?", true, "http://hacker.klever.net/oh?" );
112 test_rfc_3986_normalize_uri(
113 "http://hacker.klever.net/ah//oh?", true, "http://hacker.klever.net/ah/oh?" );
114 test_rfc_3986_normalize_uri(
115 "http://hacker.klever.net/ah/?", true, "http://hacker.klever.net/ah/?" );
116 test_rfc_3986_normalize_uri(
117 "http://hacker.klever.net/ah/%", false );
118 test_rfc_3986_normalize_uri(
119 "http://hacker.klever.net/ah/%a", false );
120 test_rfc_3986_normalize_uri(
121 "http://hacker.klever.net/ah/%zx", false );
122 test_rfc_3986_normalize_uri(
123 "http://hacker.klever.net/ah/%5x", false );
124 test_rfc_3986_normalize_uri(
125 "Http://Hacker.Klever.Net:", true, "http://hacker.klever.net/" );
126}
127
62int main() { 128int main() {
63 try { 129 try {
130 test_rfc_3986_normalize_uri();
64 test_retrieve_links(); 131 test_retrieve_links();
65 }catch(failed_test& ft) { 132 }catch(failed_test& ft) {