Add support for mechanisms with no integrity
[openssh.git] / acss.c
1 /*      $Id: acss.c,v 1.4 2006/07/24 04:51:01 djm Exp $ */
2 /*
3  * Copyright (c) 2004 The OpenBSD project
4  *
5  * Permission to use, copy, modify, and distribute this software for any
6  * purpose with or without fee is hereby granted, provided that the above
7  * copyright notice and this permission notice appear in all copies.
8  *
9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16  */
17
18 #include "includes.h"
19
20 #include <string.h>
21
22 #include <openssl/evp.h>
23
24 #if !defined(EVP_CTRL_SET_ACSS_MODE) && (OPENSSL_VERSION_NUMBER >= 0x00906000L)
25
26 #include "acss.h"
27
28 /* decryption sbox */
29 static unsigned char sboxdec[] = {
30         0x33, 0x73, 0x3b, 0x26, 0x63, 0x23, 0x6b, 0x76,
31         0x3e, 0x7e, 0x36, 0x2b, 0x6e, 0x2e, 0x66, 0x7b,
32         0xd3, 0x93, 0xdb, 0x06, 0x43, 0x03, 0x4b, 0x96,
33         0xde, 0x9e, 0xd6, 0x0b, 0x4e, 0x0e, 0x46, 0x9b,
34         0x57, 0x17, 0x5f, 0x82, 0xc7, 0x87, 0xcf, 0x12,
35         0x5a, 0x1a, 0x52, 0x8f, 0xca, 0x8a, 0xc2, 0x1f,
36         0xd9, 0x99, 0xd1, 0x00, 0x49, 0x09, 0x41, 0x90,
37         0xd8, 0x98, 0xd0, 0x01, 0x48, 0x08, 0x40, 0x91,
38         0x3d, 0x7d, 0x35, 0x24, 0x6d, 0x2d, 0x65, 0x74,
39         0x3c, 0x7c, 0x34, 0x25, 0x6c, 0x2c, 0x64, 0x75,
40         0xdd, 0x9d, 0xd5, 0x04, 0x4d, 0x0d, 0x45, 0x94,
41         0xdc, 0x9c, 0xd4, 0x05, 0x4c, 0x0c, 0x44, 0x95,
42         0x59, 0x19, 0x51, 0x80, 0xc9, 0x89, 0xc1, 0x10,
43         0x58, 0x18, 0x50, 0x81, 0xc8, 0x88, 0xc0, 0x11,
44         0xd7, 0x97, 0xdf, 0x02, 0x47, 0x07, 0x4f, 0x92,
45         0xda, 0x9a, 0xd2, 0x0f, 0x4a, 0x0a, 0x42, 0x9f,
46         0x53, 0x13, 0x5b, 0x86, 0xc3, 0x83, 0xcb, 0x16,
47         0x5e, 0x1e, 0x56, 0x8b, 0xce, 0x8e, 0xc6, 0x1b,
48         0xb3, 0xf3, 0xbb, 0xa6, 0xe3, 0xa3, 0xeb, 0xf6,
49         0xbe, 0xfe, 0xb6, 0xab, 0xee, 0xae, 0xe6, 0xfb,
50         0x37, 0x77, 0x3f, 0x22, 0x67, 0x27, 0x6f, 0x72,
51         0x3a, 0x7a, 0x32, 0x2f, 0x6a, 0x2a, 0x62, 0x7f,
52         0xb9, 0xf9, 0xb1, 0xa0, 0xe9, 0xa9, 0xe1, 0xf0,
53         0xb8, 0xf8, 0xb0, 0xa1, 0xe8, 0xa8, 0xe0, 0xf1,
54         0x5d, 0x1d, 0x55, 0x84, 0xcd, 0x8d, 0xc5, 0x14,
55         0x5c, 0x1c, 0x54, 0x85, 0xcc, 0x8c, 0xc4, 0x15,
56         0xbd, 0xfd, 0xb5, 0xa4, 0xed, 0xad, 0xe5, 0xf4,
57         0xbc, 0xfc, 0xb4, 0xa5, 0xec, 0xac, 0xe4, 0xf5,
58         0x39, 0x79, 0x31, 0x20, 0x69, 0x29, 0x61, 0x70,
59         0x38, 0x78, 0x30, 0x21, 0x68, 0x28, 0x60, 0x71,
60         0xb7, 0xf7, 0xbf, 0xa2, 0xe7, 0xa7, 0xef, 0xf2,
61         0xba, 0xfa, 0xb2, 0xaf, 0xea, 0xaa, 0xe2, 0xff
62 };
63
64 /* encryption sbox */
65 static unsigned char sboxenc[] = {
66         0x33, 0x3b, 0x73, 0x15, 0x53, 0x5b, 0x13, 0x75,
67         0x3d, 0x35, 0x7d, 0x1b, 0x5d, 0x55, 0x1d, 0x7b,
68         0x67, 0x6f, 0x27, 0x81, 0xc7, 0xcf, 0x87, 0x21,
69         0x69, 0x61, 0x29, 0x8f, 0xc9, 0xc1, 0x89, 0x2f,
70         0xe3, 0xeb, 0xa3, 0x05, 0x43, 0x4b, 0x03, 0xa5,
71         0xed, 0xe5, 0xad, 0x0b, 0x4d, 0x45, 0x0d, 0xab,
72         0xea, 0xe2, 0xaa, 0x00, 0x4a, 0x42, 0x0a, 0xa0,
73         0xe8, 0xe0, 0xa8, 0x02, 0x48, 0x40, 0x08, 0xa2,
74         0x3e, 0x36, 0x7e, 0x14, 0x5e, 0x56, 0x1e, 0x74,
75         0x3c, 0x34, 0x7c, 0x16, 0x5c, 0x54, 0x1c, 0x76,
76         0x6a, 0x62, 0x2a, 0x80, 0xca, 0xc2, 0x8a, 0x20,
77         0x68, 0x60, 0x28, 0x82, 0xc8, 0xc0, 0x88, 0x22,
78         0xee, 0xe6, 0xae, 0x04, 0x4e, 0x46, 0x0e, 0xa4,
79         0xec, 0xe4, 0xac, 0x06, 0x4c, 0x44, 0x0c, 0xa6,
80         0xe7, 0xef, 0xa7, 0x01, 0x47, 0x4f, 0x07, 0xa1,
81         0xe9, 0xe1, 0xa9, 0x0f, 0x49, 0x41, 0x09, 0xaf,
82         0x63, 0x6b, 0x23, 0x85, 0xc3, 0xcb, 0x83, 0x25,
83         0x6d, 0x65, 0x2d, 0x8b, 0xcd, 0xc5, 0x8d, 0x2b,
84         0x37, 0x3f, 0x77, 0x11, 0x57, 0x5f, 0x17, 0x71,
85         0x39, 0x31, 0x79, 0x1f, 0x59, 0x51, 0x19, 0x7f,
86         0xb3, 0xbb, 0xf3, 0x95, 0xd3, 0xdb, 0x93, 0xf5,
87         0xbd, 0xb5, 0xfd, 0x9b, 0xdd, 0xd5, 0x9d, 0xfb,
88         0xba, 0xb2, 0xfa, 0x90, 0xda, 0xd2, 0x9a, 0xf0,
89         0xb8, 0xb0, 0xf8, 0x92, 0xd8, 0xd0, 0x98, 0xf2,
90         0x6e, 0x66, 0x2e, 0x84, 0xce, 0xc6, 0x8e, 0x24,
91         0x6c, 0x64, 0x2c, 0x86, 0xcc, 0xc4, 0x8c, 0x26,
92         0x3a, 0x32, 0x7a, 0x10, 0x5a, 0x52, 0x1a, 0x70,
93         0x38, 0x30, 0x78, 0x12, 0x58, 0x50, 0x18, 0x72,
94         0xbe, 0xb6, 0xfe, 0x94, 0xde, 0xd6, 0x9e, 0xf4,
95         0xbc, 0xb4, 0xfc, 0x96, 0xdc, 0xd4, 0x9c, 0xf6,
96         0xb7, 0xbf, 0xf7, 0x91, 0xd7, 0xdf, 0x97, 0xf1,
97         0xb9, 0xb1, 0xf9, 0x9f, 0xd9, 0xd1, 0x99, 0xff
98 };
99
100 static unsigned char reverse[] = {
101         0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0,
102         0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0,
103         0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8,
104         0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8,
105         0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4,
106         0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4,
107         0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec,
108         0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc,
109         0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2,
110         0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2,
111         0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea,
112         0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa,
113         0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6,
114         0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6,
115         0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee,
116         0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe,
117         0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1,
118         0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1,
119         0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9,
120         0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9,
121         0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5,
122         0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5,
123         0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed,
124         0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd,
125         0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3,
126         0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3,
127         0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb,
128         0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb,
129         0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7,
130         0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7,
131         0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef,
132         0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff
133 };
134
135 /*
136  * Two linear feedback shift registers are used:
137  *
138  * lfsr17:  polynomial of degree 17, primitive modulo 2 (listed in Schneier)
139  *          x^15 + x + 1
140  * lfsr25:  polynomial of degree 25, not know if primitive modulo 2
141  *          x^13 + x^5 + x^4 + x^1 + 1
142  *
143  * Output bits are discarded, instead the feedback bits are added to produce
144  * the cipher stream.  Depending on the mode, feedback bytes may be inverted
145  * bit-wise before addition.
146  *
147  * The lfsrs are seeded with bytes from the raw key:
148  *
149  * lfsr17:  byte 0[0:7] at bit 9
150  *          byte 1[0:7] at bit 0
151  *
152  * lfsr25:  byte 2[0:4] at bit 16
153  *          byte 2[5:7] at bit 22
154  *          byte 3[0:7] at bit 8
155  *          byte 4[0:7] at bit 0
156  *
157  * To prevent 0 cycles, 1's are inject at bit 8 in lfrs17 and bit 21 in
158  * lfsr25.
159  *
160  */
161
162 int
163 acss(ACSS_KEY *key, unsigned long len, const unsigned char *in,
164     unsigned char *out)
165 {
166         unsigned long i;
167         unsigned long lfsr17tmp, lfsr25tmp, lfsrsumtmp;
168
169         lfsrsumtmp = lfsr17tmp = lfsr25tmp = 0;
170
171         /* keystream is sum of lfsrs */
172         for (i = 0; i < len; i++) {
173                 lfsr17tmp = key->lfsr17 ^ (key->lfsr17 >> 14);
174                 key->lfsr17 = (key->lfsr17 >> 8)
175                         ^ (lfsr17tmp << 9)
176                         ^ (lfsr17tmp << 12)
177                         ^ (lfsr17tmp << 15);
178                 key->lfsr17 &= 0x1ffff; /* 17 bit LFSR */
179
180                 lfsr25tmp = key->lfsr25
181                         ^ (key->lfsr25 >> 3)
182                         ^ (key->lfsr25 >> 4)
183                         ^ (key->lfsr25 >> 12);
184                 key->lfsr25 = (key->lfsr25 >> 8) ^ (lfsr25tmp << 17);
185                 key->lfsr25 &= 0x1ffffff;       /* 25 bit LFSR */
186
187                 lfsrsumtmp = key->lfsrsum;
188
189                 /* addition */
190                 switch (key->mode) {
191                 case ACSS_AUTHENTICATE:
192                 case ACSS_DATA:
193                         key->lfsrsum = 0xff & ~(key->lfsr17 >> 9);
194                         key->lfsrsum += key->lfsr25 >> 17;
195                         break;
196                 case ACSS_SESSIONKEY:
197                         key->lfsrsum = key->lfsr17 >> 9;
198                         key->lfsrsum += key->lfsr25 >> 17;
199                         break;
200                 case ACSS_TITLEKEY:
201                         key->lfsrsum = key->lfsr17 >> 9;
202                         key->lfsrsum += 0xff & ~(key->lfsr25 >> 17);
203                         break;
204                 default:
205                         return 1;
206                 }
207                 key->lfsrsum += (lfsrsumtmp >> 8);
208
209                 if (key->encrypt) {
210                         out[i] = sboxenc[(in[i] ^ key->lfsrsum) & 0xff];
211                 } else {
212                         out[i] = (sboxdec[in[i]] ^ key->lfsrsum) & 0xff;
213                 }
214         }
215
216         return 0;
217 }
218
219 static void
220 acss_seed(ACSS_KEY *key)
221 {
222         int i;
223
224         /* if available, mangle with subkey */
225         if (key->subkey_avilable) {
226                 for (i = 0; i < ACSS_KEYSIZE; i++)
227                         key->seed[i] = reverse[key->data[i] ^ key->subkey[i]];
228         } else {
229                 for (i = 0; i < ACSS_KEYSIZE; i++)
230                         key->seed[i] = reverse[key->data[i]];
231         }
232
233         /* seed lfsrs */
234         key->lfsr17 = key->seed[1]
235                 | (key->seed[0] << 9)
236                 | (1 << 8);     /* inject 1 at bit 9 */
237         key->lfsr25 = key->seed[4]
238                 | (key->seed[3] << 8)
239                 | ((key->seed[2] & 0x1f) << 16)
240                 | ((key->seed[2] & 0xe0) << 17)
241                         | (1 << 21);    /* inject 1 at bit 22 */
242
243         key->lfsrsum = 0;
244 }
245
246 void
247 acss_setkey(ACSS_KEY *key, const unsigned char *data, int enc, int mode)
248 {
249         memcpy(key->data, data, sizeof(key->data));
250         memset(key->subkey, 0, sizeof(key->subkey));
251
252         if (enc != -1)
253                 key->encrypt = enc;
254         key->mode = mode;
255         key->subkey_avilable = 0;
256
257         acss_seed(key);
258 }
259
260 void
261 acss_setsubkey(ACSS_KEY *key, const unsigned char *subkey)
262 {
263         memcpy(key->subkey, subkey, sizeof(key->subkey));
264         key->subkey_avilable = 1;
265         acss_seed(key);
266 }
267 #endif