Diffstat (limited to 'noncore/unsupported/qpdf/xpdf/Decrypt.cc') (more/less context) (ignore whitespace changes)
-rw-r--r-- | noncore/unsupported/qpdf/xpdf/Decrypt.cc | 385 |
1 files changed, 385 insertions, 0 deletions
diff --git a/noncore/unsupported/qpdf/xpdf/Decrypt.cc b/noncore/unsupported/qpdf/xpdf/Decrypt.cc new file mode 100644 index 0000000..2def802 --- a/dev/null +++ b/noncore/unsupported/qpdf/xpdf/Decrypt.cc | |||
@@ -0,0 +1,385 @@ | |||
1 | //======================================================================== | ||
2 | // | ||
3 | // Decrypt.cc | ||
4 | // | ||
5 | // Copyright 1996 Derek B. Noonburg | ||
6 | // | ||
7 | //======================================================================== | ||
8 | |||
9 | #ifdef __GNUC__ | ||
10 | #pragma implementation | ||
11 | #endif | ||
12 | |||
13 | #include <aconf.h> | ||
14 | #include "gmem.h" | ||
15 | #include "Decrypt.h" | ||
16 | |||
17 | static void rc4InitKey(Guchar *key, int keyLen, Guchar *state); | ||
18 | static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c); | ||
19 | static void md5(Guchar *msg, int msgLen, Guchar *digest); | ||
20 | |||
21 | static Guchar passwordPad[32] = { | ||
22 | 0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41, | ||
23 | 0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08, | ||
24 | 0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80, | ||
25 | 0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a | ||
26 | }; | ||
27 | |||
28 | //------------------------------------------------------------------------ | ||
29 | // Decrypt | ||
30 | //------------------------------------------------------------------------ | ||
31 | |||
32 | Decrypt::Decrypt(Guchar *fileKey, int keyLength, int objNum, int objGen) { | ||
33 | int i; | ||
34 | |||
35 | // construct object key | ||
36 | for (i = 0; i < keyLength; ++i) { | ||
37 | objKey[i] = fileKey[i]; | ||
38 | } | ||
39 | objKey[keyLength] = objNum & 0xff; | ||
40 | objKey[keyLength + 1] = (objNum >> 8) & 0xff; | ||
41 | objKey[keyLength + 2] = (objNum >> 16) & 0xff; | ||
42 | objKey[keyLength + 3] = objGen & 0xff; | ||
43 | objKey[keyLength + 4] = (objGen >> 8) & 0xff; | ||
44 | md5(objKey, keyLength + 5, objKey); | ||
45 | |||
46 | // set up for decryption | ||
47 | x = y = 0; | ||
48 | if ((objKeyLength = keyLength + 5) > 16) { | ||
49 | objKeyLength = 16; | ||
50 | } | ||
51 | rc4InitKey(objKey, objKeyLength, state); | ||
52 | } | ||
53 | |||
54 | void Decrypt::reset() { | ||
55 | x = y = 0; | ||
56 | rc4InitKey(objKey, objKeyLength, state); | ||
57 | } | ||
58 | |||
59 | Guchar Decrypt::decryptByte(Guchar c) { | ||
60 | return rc4DecryptByte(state, &x, &y, c); | ||
61 | } | ||
62 | |||
63 | GBool Decrypt::makeFileKey(int encVersion, int encRevision, int keyLength, | ||
64 | GString *ownerKey, GString *userKey, | ||
65 | int permissions, GString *fileID, | ||
66 | GString *ownerPassword, GString *userPassword, | ||
67 | Guchar *fileKey, GBool *ownerPasswordOk) { | ||
68 | Guchar test[32]; | ||
69 | GString *userPassword2; | ||
70 | Guchar fState[256]; | ||
71 | Guchar fx, fy; | ||
72 | int len, i; | ||
73 | |||
74 | // try using the supplied owner password to generate the user password | ||
75 | if (ownerPassword) { | ||
76 | len = ownerPassword->getLength(); | ||
77 | if (len < 32) { | ||
78 | memcpy(test, ownerPassword->getCString(), len); | ||
79 | memcpy(test + len, passwordPad, 32 - len); | ||
80 | } else { | ||
81 | memcpy(test, ownerPassword->getCString(), 32); | ||
82 | } | ||
83 | } else { | ||
84 | memcpy(test, passwordPad, 32); | ||
85 | } | ||
86 | md5(test, 32, test); | ||
87 | if (encRevision == 3) { | ||
88 | for (i = 0; i < 50; ++i) { | ||
89 | md5(test, 16, test); | ||
90 | } | ||
91 | } | ||
92 | rc4InitKey(test, keyLength, fState); | ||
93 | fx = fy = 0; | ||
94 | for (i = 0; i < 32; ++i) { | ||
95 | test[i] = rc4DecryptByte(fState, &fx, &fy, ownerKey->getChar(i)); | ||
96 | } | ||
97 | userPassword2 = new GString((char *)test, 32); | ||
98 | if (makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey, | ||
99 | permissions, fileID, userPassword2, fileKey)) { | ||
100 | *ownerPasswordOk = gTrue; | ||
101 | delete userPassword2; | ||
102 | return gTrue; | ||
103 | } | ||
104 | *ownerPasswordOk = gFalse; | ||
105 | delete userPassword2; | ||
106 | |||
107 | // try using the supplied user password | ||
108 | return makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey, | ||
109 | permissions, fileID, userPassword, fileKey); | ||
110 | } | ||
111 | |||
112 | GBool Decrypt::makeFileKey2(int encVersion, int encRevision, int keyLength, | ||
113 | GString *ownerKey, GString *userKey, | ||
114 | int permissions, GString *fileID, | ||
115 | GString *userPassword, Guchar *fileKey) { | ||
116 | Guchar *buf; | ||
117 | Guchar test[32]; | ||
118 | Guchar fState[256]; | ||
119 | Guchar tmpKey[16]; | ||
120 | Guchar fx, fy; | ||
121 | int len, i, j; | ||
122 | GBool ok; | ||
123 | |||
124 | // generate file key | ||
125 | buf = (Guchar *)gmalloc(68 + fileID->getLength()); | ||
126 | if (userPassword) { | ||
127 | len = userPassword->getLength(); | ||
128 | if (len < 32) { | ||
129 | memcpy(buf, userPassword->getCString(), len); | ||
130 | memcpy(buf + len, passwordPad, 32 - len); | ||
131 | } else { | ||
132 | memcpy(buf, userPassword->getCString(), 32); | ||
133 | } | ||
134 | } else { | ||
135 | memcpy(buf, passwordPad, 32); | ||
136 | } | ||
137 | memcpy(buf + 32, ownerKey->getCString(), 32); | ||
138 | buf[64] = permissions & 0xff; | ||
139 | buf[65] = (permissions >> 8) & 0xff; | ||
140 | buf[66] = (permissions >> 16) & 0xff; | ||
141 | buf[67] = (permissions >> 24) & 0xff; | ||
142 | memcpy(buf + 68, fileID->getCString(), fileID->getLength()); | ||
143 | md5(buf, 68 + fileID->getLength(), fileKey); | ||
144 | if (encRevision == 3) { | ||
145 | for (i = 0; i < 50; ++i) { | ||
146 | md5(fileKey, 16, fileKey); | ||
147 | } | ||
148 | } | ||
149 | |||
150 | // test user password | ||
151 | if (encRevision == 2) { | ||
152 | rc4InitKey(fileKey, keyLength, fState); | ||
153 | fx = fy = 0; | ||
154 | for (i = 0; i < 32; ++i) { | ||
155 | test[i] = rc4DecryptByte(fState, &fx, &fy, userKey->getChar(i)); | ||
156 | } | ||
157 | ok = memcmp(test, passwordPad, 32) == 0; | ||
158 | } else if (encRevision == 3) { | ||
159 | memcpy(test, userKey->getCString(), 32); | ||
160 | for (i = 19; i >= 0; --i) { | ||
161 | for (j = 0; j < keyLength; ++j) { | ||
162 | tmpKey[j] = fileKey[j] ^ i; | ||
163 | } | ||
164 | rc4InitKey(tmpKey, keyLength, fState); | ||
165 | fx = fy = 0; | ||
166 | for (j = 0; j < 32; ++j) { | ||
167 | test[j] = rc4DecryptByte(fState, &fx, &fy, test[j]); | ||
168 | } | ||
169 | } | ||
170 | memcpy(buf, passwordPad, 32); | ||
171 | memcpy(buf + 32, fileID->getCString(), fileID->getLength()); | ||
172 | md5(buf, 32 + fileID->getLength(), buf); | ||
173 | ok = memcmp(test, buf, 16) == 0; | ||
174 | } else { | ||
175 | ok = gFalse; | ||
176 | } | ||
177 | |||
178 | gfree(buf); | ||
179 | return ok; | ||
180 | } | ||
181 | |||
182 | //------------------------------------------------------------------------ | ||
183 | // RC4-compatible decryption | ||
184 | //------------------------------------------------------------------------ | ||
185 | |||
186 | static void rc4InitKey(Guchar *key, int keyLen, Guchar *state) { | ||
187 | Guchar index1, index2; | ||
188 | Guchar t; | ||
189 | int i; | ||
190 | |||
191 | for (i = 0; i < 256; ++i) | ||
192 | state[i] = i; | ||
193 | index1 = index2 = 0; | ||
194 | for (i = 0; i < 256; ++i) { | ||
195 | index2 = (key[index1] + state[i] + index2) % 256; | ||
196 | t = state[i]; | ||
197 | state[i] = state[index2]; | ||
198 | state[index2] = t; | ||
199 | index1 = (index1 + 1) % keyLen; | ||
200 | } | ||
201 | } | ||
202 | |||
203 | static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c) { | ||
204 | Guchar x1, y1, tx, ty; | ||
205 | |||
206 | x1 = *x = (*x + 1) % 256; | ||
207 | y1 = *y = (state[*x] + *y) % 256; | ||
208 | tx = state[x1]; | ||
209 | ty = state[y1]; | ||
210 | state[x1] = ty; | ||
211 | state[y1] = tx; | ||
212 | return c ^ state[(tx + ty) % 256]; | ||
213 | } | ||
214 | |||
215 | //------------------------------------------------------------------------ | ||
216 | // MD5 message digest | ||
217 | //------------------------------------------------------------------------ | ||
218 | |||
219 | // this works around a bug in older Sun compilers | ||
220 | static inline Gulong rotateLeft(Gulong x, int r) { | ||
221 | x &= 0xffffffff; | ||
222 | return ((x << r) | (x >> (32 - r))) & 0xffffffff; | ||
223 | } | ||
224 | |||
225 | static inline Gulong md5Round1(Gulong a, Gulong b, Gulong c, Gulong d, | ||
226 | Gulong Xk, Gulong s, Gulong Ti) { | ||
227 | return b + rotateLeft((a + ((b & c) | (~b & d)) + Xk + Ti), s); | ||
228 | } | ||
229 | |||
230 | static inline Gulong md5Round2(Gulong a, Gulong b, Gulong c, Gulong d, | ||
231 | Gulong Xk, Gulong s, Gulong Ti) { | ||
232 | return b + rotateLeft((a + ((b & d) | (c & ~d)) + Xk + Ti), s); | ||
233 | } | ||
234 | |||
235 | static inline Gulong md5Round3(Gulong a, Gulong b, Gulong c, Gulong d, | ||
236 | Gulong Xk, Gulong s, Gulong Ti) { | ||
237 | return b + rotateLeft((a + (b ^ c ^ d) + Xk + Ti), s); | ||
238 | } | ||
239 | |||
240 | static inline Gulong md5Round4(Gulong a, Gulong b, Gulong c, Gulong d, | ||
241 | Gulong Xk, Gulong s, Gulong Ti) { | ||
242 | return b + rotateLeft((a + (c ^ (b | ~d)) + Xk + Ti), s); | ||
243 | } | ||
244 | |||
245 | static void md5(Guchar *msg, int msgLen, Guchar *digest) { | ||
246 | Gulong x[16]; | ||
247 | Gulong a, b, c, d, aa, bb, cc, dd; | ||
248 | int n64; | ||
249 | int i, j, k; | ||
250 | |||
251 | // compute number of 64-byte blocks | ||
252 | // (length + pad byte (0x80) + 8 bytes for length) | ||
253 | n64 = (msgLen + 1 + 8 + 63) / 64; | ||
254 | |||
255 | // initialize a, b, c, d | ||
256 | a = 0x67452301; | ||
257 | b = 0xefcdab89; | ||
258 | c = 0x98badcfe; | ||
259 | d = 0x10325476; | ||
260 | |||
261 | // loop through blocks | ||
262 | k = 0; | ||
263 | for (i = 0; i < n64; ++i) { | ||
264 | |||
265 | // grab a 64-byte block | ||
266 | for (j = 0; j < 16 && k < msgLen - 3; ++j, k += 4) | ||
267 | x[j] = (((((msg[k+3] << 8) + msg[k+2]) << 8) + msg[k+1]) << 8) + msg[k]; | ||
268 | if (i == n64 - 1) { | ||
269 | if (k == msgLen - 3) | ||
270 | x[j] = 0x80000000 + (((msg[k+2] << 8) + msg[k+1]) << 8) + msg[k]; | ||
271 | else if (k == msgLen - 2) | ||
272 | x[j] = 0x800000 + (msg[k+1] << 8) + msg[k]; | ||
273 | else if (k == msgLen - 1) | ||
274 | x[j] = 0x8000 + msg[k]; | ||
275 | else | ||
276 | x[j] = 0x80; | ||
277 | ++j; | ||
278 | while (j < 16) | ||
279 | x[j++] = 0; | ||
280 | x[14] = msgLen << 3; | ||
281 | } | ||
282 | |||
283 | // save a, b, c, d | ||
284 | aa = a; | ||
285 | bb = b; | ||
286 | cc = c; | ||
287 | dd = d; | ||
288 | |||
289 | // round 1 | ||
290 | a = md5Round1(a, b, c, d, x[0], 7, 0xd76aa478); | ||
291 | d = md5Round1(d, a, b, c, x[1], 12, 0xe8c7b756); | ||
292 | c = md5Round1(c, d, a, b, x[2], 17, 0x242070db); | ||
293 | b = md5Round1(b, c, d, a, x[3], 22, 0xc1bdceee); | ||
294 | a = md5Round1(a, b, c, d, x[4], 7, 0xf57c0faf); | ||
295 | d = md5Round1(d, a, b, c, x[5], 12, 0x4787c62a); | ||
296 | c = md5Round1(c, d, a, b, x[6], 17, 0xa8304613); | ||
297 | b = md5Round1(b, c, d, a, x[7], 22, 0xfd469501); | ||
298 | a = md5Round1(a, b, c, d, x[8], 7, 0x698098d8); | ||
299 | d = md5Round1(d, a, b, c, x[9], 12, 0x8b44f7af); | ||
300 | c = md5Round1(c, d, a, b, x[10], 17, 0xffff5bb1); | ||
301 | b = md5Round1(b, c, d, a, x[11], 22, 0x895cd7be); | ||
302 | a = md5Round1(a, b, c, d, x[12], 7, 0x6b901122); | ||
303 | d = md5Round1(d, a, b, c, x[13], 12, 0xfd987193); | ||
304 | c = md5Round1(c, d, a, b, x[14], 17, 0xa679438e); | ||
305 | b = md5Round1(b, c, d, a, x[15], 22, 0x49b40821); | ||
306 | |||
307 | // round 2 | ||
308 | a = md5Round2(a, b, c, d, x[1], 5, 0xf61e2562); | ||
309 | d = md5Round2(d, a, b, c, x[6], 9, 0xc040b340); | ||
310 | c = md5Round2(c, d, a, b, x[11], 14, 0x265e5a51); | ||
311 | b = md5Round2(b, c, d, a, x[0], 20, 0xe9b6c7aa); | ||
312 | a = md5Round2(a, b, c, d, x[5], 5, 0xd62f105d); | ||
313 | d = md5Round2(d, a, b, c, x[10], 9, 0x02441453); | ||
314 | c = md5Round2(c, d, a, b, x[15], 14, 0xd8a1e681); | ||
315 | b = md5Round2(b, c, d, a, x[4], 20, 0xe7d3fbc8); | ||
316 | a = md5Round2(a, b, c, d, x[9], 5, 0x21e1cde6); | ||
317 | d = md5Round2(d, a, b, c, x[14], 9, 0xc33707d6); | ||
318 | c = md5Round2(c, d, a, b, x[3], 14, 0xf4d50d87); | ||
319 | b = md5Round2(b, c, d, a, x[8], 20, 0x455a14ed); | ||
320 | a = md5Round2(a, b, c, d, x[13], 5, 0xa9e3e905); | ||
321 | d = md5Round2(d, a, b, c, x[2], 9, 0xfcefa3f8); | ||
322 | c = md5Round2(c, d, a, b, x[7], 14, 0x676f02d9); | ||
323 | b = md5Round2(b, c, d, a, x[12], 20, 0x8d2a4c8a); | ||
324 | |||
325 | // round 3 | ||
326 | a = md5Round3(a, b, c, d, x[5], 4, 0xfffa3942); | ||
327 | d = md5Round3(d, a, b, c, x[8], 11, 0x8771f681); | ||
328 | c = md5Round3(c, d, a, b, x[11], 16, 0x6d9d6122); | ||
329 | b = md5Round3(b, c, d, a, x[14], 23, 0xfde5380c); | ||
330 | a = md5Round3(a, b, c, d, x[1], 4, 0xa4beea44); | ||
331 | d = md5Round3(d, a, b, c, x[4], 11, 0x4bdecfa9); | ||
332 | c = md5Round3(c, d, a, b, x[7], 16, 0xf6bb4b60); | ||
333 | b = md5Round3(b, c, d, a, x[10], 23, 0xbebfbc70); | ||
334 | a = md5Round3(a, b, c, d, x[13], 4, 0x289b7ec6); | ||
335 | d = md5Round3(d, a, b, c, x[0], 11, 0xeaa127fa); | ||
336 | c = md5Round3(c, d, a, b, x[3], 16, 0xd4ef3085); | ||
337 | b = md5Round3(b, c, d, a, x[6], 23, 0x04881d05); | ||
338 | a = md5Round3(a, b, c, d, x[9], 4, 0xd9d4d039); | ||
339 | d = md5Round3(d, a, b, c, x[12], 11, 0xe6db99e5); | ||
340 | c = md5Round3(c, d, a, b, x[15], 16, 0x1fa27cf8); | ||
341 | b = md5Round3(b, c, d, a, x[2], 23, 0xc4ac5665); | ||
342 | |||
343 | // round 4 | ||
344 | a = md5Round4(a, b, c, d, x[0], 6, 0xf4292244); | ||
345 | d = md5Round4(d, a, b, c, x[7], 10, 0x432aff97); | ||
346 | c = md5Round4(c, d, a, b, x[14], 15, 0xab9423a7); | ||
347 | b = md5Round4(b, c, d, a, x[5], 21, 0xfc93a039); | ||
348 | a = md5Round4(a, b, c, d, x[12], 6, 0x655b59c3); | ||
349 | d = md5Round4(d, a, b, c, x[3], 10, 0x8f0ccc92); | ||
350 | c = md5Round4(c, d, a, b, x[10], 15, 0xffeff47d); | ||
351 | b = md5Round4(b, c, d, a, x[1], 21, 0x85845dd1); | ||
352 | a = md5Round4(a, b, c, d, x[8], 6, 0x6fa87e4f); | ||
353 | d = md5Round4(d, a, b, c, x[15], 10, 0xfe2ce6e0); | ||
354 | c = md5Round4(c, d, a, b, x[6], 15, 0xa3014314); | ||
355 | b = md5Round4(b, c, d, a, x[13], 21, 0x4e0811a1); | ||
356 | a = md5Round4(a, b, c, d, x[4], 6, 0xf7537e82); | ||
357 | d = md5Round4(d, a, b, c, x[11], 10, 0xbd3af235); | ||
358 | c = md5Round4(c, d, a, b, x[2], 15, 0x2ad7d2bb); | ||
359 | b = md5Round4(b, c, d, a, x[9], 21, 0xeb86d391); | ||
360 | |||
361 | // increment a, b, c, d | ||
362 | a += aa; | ||
363 | b += bb; | ||
364 | c += cc; | ||
365 | d += dd; | ||
366 | } | ||
367 | |||
368 | // break digest into bytes | ||
369 | digest[0] = a & 0xff; | ||
370 | digest[1] = (a >>= 8) & 0xff; | ||
371 | digest[2] = (a >>= 8) & 0xff; | ||
372 | digest[3] = (a >>= 8) & 0xff; | ||
373 | digest[4] = b & 0xff; | ||
374 | digest[5] = (b >>= 8) & 0xff; | ||
375 | digest[6] = (b >>= 8) & 0xff; | ||
376 | digest[7] = (b >>= 8) & 0xff; | ||
377 | digest[8] = c & 0xff; | ||
378 | digest[9] = (c >>= 8) & 0xff; | ||
379 | digest[10] = (c >>= 8) & 0xff; | ||
380 | digest[11] = (c >>= 8) & 0xff; | ||
381 | digest[12] = d & 0xff; | ||
382 | digest[13] = (d >>= 8) & 0xff; | ||
383 | digest[14] = (d >>= 8) & 0xff; | ||
384 | digest[15] = (d >>= 8) & 0xff; | ||
385 | } | ||