sslencode.c (13244B)
1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 /* 3 * This file is PRIVATE to SSL. 4 * 5 * This Source Code Form is subject to the terms of the Mozilla Public 6 * License, v. 2.0. If a copy of the MPL was not distributed with this 7 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 8 9 #include "nss.h" 10 #include "prnetdb.h" 11 #include "ssl.h" 12 #include "sslimpl.h" 13 #include "sslproto.h" 14 15 /* Helper function to encode an unsigned integer into a buffer. */ 16 static void 17 ssl_EncodeUintX(PRUint8 *to, PRUint64 value, unsigned int bytes) 18 { 19 PRUint64 encoded; 20 21 PORT_Assert(bytes > 0 && bytes <= sizeof(encoded)); 22 23 encoded = PR_htonll(value); 24 PORT_Memcpy(to, ((unsigned char *)(&encoded)) + (sizeof(encoded) - bytes), 25 bytes); 26 } 27 28 /* Grow a buffer to hold newLen bytes of data. When used for recv/xmit buffers, 29 * the caller must hold xmitBufLock or recvBufLock, as appropriate. */ 30 SECStatus 31 sslBuffer_Grow(sslBuffer *b, unsigned int newLen) 32 { 33 PORT_Assert(b); 34 if (b->fixed) { 35 PORT_Assert(newLen <= b->space); 36 if (newLen > b->space) { 37 PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); 38 return SECFailure; 39 } 40 return SECSuccess; 41 } 42 43 /* If buf is non-NULL, space must be non-zero; 44 * if buf is NULL, space must be zero. */ 45 PORT_Assert((b->buf && b->space) || (!b->buf && !b->space)); 46 if (newLen > b->space) { 47 newLen = PR_MAX(newLen, b->space + 2048); 48 unsigned char *newBuf; 49 if (b->buf) { 50 newBuf = (unsigned char *)PORT_Realloc(b->buf, newLen); 51 } else { 52 newBuf = (unsigned char *)PORT_Alloc(newLen); 53 } 54 if (!newBuf) { 55 return SECFailure; 56 } 57 b->buf = newBuf; 58 b->space = newLen; 59 } 60 return SECSuccess; 61 } 62 63 /* Appends len copies of c to b */ 64 SECStatus 65 sslBuffer_Fill(sslBuffer *b, PRUint8 c, size_t len) 66 { 67 PORT_Assert(b); 68 SECStatus rv = sslBuffer_Grow(b, b->len + len); 69 if (rv != SECSuccess) { 70 return SECFailure; 71 } 72 if (len > 0) { 73 memset(SSL_BUFFER_NEXT(b), c, len); 74 } 75 b->len += len; 76 return SECSuccess; 77 } 78 79 SECStatus 80 sslBuffer_Append(sslBuffer *b, const void *data, unsigned int len) 81 { 82 SECStatus rv = sslBuffer_Grow(b, b->len + len); 83 if (rv != SECSuccess) { 84 return SECFailure; /* Code already set. */ 85 } 86 if (len > 0) { 87 PORT_Assert(data); 88 PORT_Memcpy(SSL_BUFFER_NEXT(b), data, len); 89 } 90 b->len += len; 91 return SECSuccess; 92 } 93 94 SECStatus 95 sslBuffer_AppendNumber(sslBuffer *b, PRUint64 v, unsigned int size) 96 { 97 SECStatus rv = sslBuffer_Grow(b, b->len + size); 98 if (rv != SECSuccess) { 99 return SECFailure; 100 } 101 ssl_EncodeUintX(SSL_BUFFER_NEXT(b), v, size); 102 b->len += size; 103 return SECSuccess; 104 } 105 106 SECStatus 107 sslBuffer_AppendVariable(sslBuffer *b, const PRUint8 *data, unsigned int len, 108 unsigned int size) 109 { 110 PORT_Assert(size <= 4 && size > 0); 111 PORT_Assert(b); 112 if (len >= (1ULL << (8 * size))) { 113 PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); 114 return SECFailure; 115 } 116 117 if (sslBuffer_Grow(b, b->len + len + size) != SECSuccess) { 118 return SECFailure; 119 } 120 121 ssl_EncodeUintX(SSL_BUFFER_NEXT(b), len, size); 122 b->len += size; 123 if (len != 0) { 124 PORT_Assert(data); 125 /* We sometimes pass NULL, 0 and memcpy() doesn't want NULL. */ 126 PORT_Memcpy(SSL_BUFFER_NEXT(b), data, len); 127 } 128 b->len += len; 129 return SECSuccess; 130 } 131 132 SECStatus 133 sslBuffer_AppendBuffer(sslBuffer *b, const sslBuffer *append) 134 { 135 return sslBuffer_Append(b, append->buf, append->len); 136 } 137 138 SECStatus 139 sslBuffer_AppendBufferVariable(sslBuffer *b, const sslBuffer *append, 140 unsigned int size) 141 { 142 return sslBuffer_AppendVariable(b, append->buf, append->len, size); 143 } 144 145 SECStatus 146 sslBuffer_Skip(sslBuffer *b, unsigned int size, unsigned int *savedOffset) 147 { 148 if (sslBuffer_Grow(b, b->len + size) != SECSuccess) { 149 return SECFailure; 150 } 151 152 if (savedOffset) { 153 *savedOffset = b->len; 154 } 155 b->len += size; 156 return SECSuccess; 157 } 158 159 /* A common problem is that a buffer is used to construct a variable length 160 * structure of unknown length. The length field for that structure is then 161 * populated afterwards. This function makes this process a little easier. 162 * 163 * To use this, before encoding the variable length structure, skip the spot 164 * where the length would be using sslBuffer_Skip(). After encoding the 165 * structure, and before encoding anything else, call this function passing the 166 * value returned from sslBuffer_Skip() as |at| to have the length inserted. 167 */ 168 SECStatus 169 sslBuffer_InsertLength(sslBuffer *b, unsigned int at, unsigned int size) 170 { 171 unsigned int len; 172 173 PORT_Assert(b->len >= at + size); 174 PORT_Assert(b->space >= at + size); 175 len = b->len - (at + size); 176 177 PORT_Assert(size <= 4 && size > 0); 178 if (len >= (1ULL << (8 * size))) { 179 PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); 180 return SECFailure; 181 } 182 183 ssl_EncodeUintX(SSL_BUFFER_BASE(b) + at, len, size); 184 return SECSuccess; 185 } 186 187 SECStatus 188 sslBuffer_InsertNumber(sslBuffer *b, unsigned int at, 189 PRUint64 v, unsigned int size) 190 { 191 PORT_Assert(b->len >= at + size); 192 PORT_Assert(b->space >= at + size); 193 194 PORT_Assert(size <= 4 && size > 0); 195 if (v >= (1ULL << (8 * size))) { 196 PORT_SetError(SEC_ERROR_LIBRARY_FAILURE); 197 return SECFailure; 198 } 199 200 ssl_EncodeUintX(SSL_BUFFER_BASE(b) + at, v, size); 201 return SECSuccess; 202 } 203 204 void 205 sslBuffer_Clear(sslBuffer *b) 206 { 207 if (!b->fixed) { 208 if (b->buf) { 209 PORT_Free(b->buf); 210 b->buf = NULL; 211 } 212 b->space = 0; 213 } 214 b->len = 0; 215 } 216 217 SECStatus 218 sslRead_Read(sslReader *reader, unsigned int count, sslReadBuffer *out) 219 { 220 if (!reader || !out) { 221 PORT_SetError(SEC_ERROR_INVALID_ARGS); 222 return SECFailure; 223 } 224 if (reader->buf.len < reader->offset || 225 count > SSL_READER_REMAINING(reader)) { 226 PORT_SetError(SEC_ERROR_BAD_DATA); 227 return SECFailure; 228 } 229 230 out->buf = SSL_READER_CURRENT(reader); 231 out->len = count; 232 reader->offset += count; 233 234 return SECSuccess; 235 } 236 237 SECStatus 238 sslRead_ReadVariable(sslReader *reader, unsigned int sizeLen, sslReadBuffer *out) 239 { 240 PRUint64 variableLen = 0; 241 SECStatus rv = sslRead_ReadNumber(reader, sizeLen, &variableLen); 242 if (rv != SECSuccess) { 243 PORT_SetError(SEC_ERROR_BAD_DATA); 244 return SECFailure; 245 } 246 if (!variableLen) { 247 // It is ok to have an empty variable. 248 out->len = variableLen; 249 return SECSuccess; 250 } 251 return sslRead_Read(reader, variableLen, out); 252 } 253 254 SECStatus 255 sslRead_ReadNumber(sslReader *reader, unsigned int bytes, PRUint64 *num) 256 { 257 if (!reader || !num) { 258 PORT_SetError(SEC_ERROR_INVALID_ARGS); 259 return SECFailure; 260 } 261 if (reader->buf.len < reader->offset || 262 bytes > SSL_READER_REMAINING(reader) || 263 bytes > 8) { 264 PORT_SetError(SEC_ERROR_BAD_DATA); 265 return SECFailure; 266 } 267 unsigned int i; 268 PRUint64 number = 0; 269 for (i = 0; i < bytes; i++) { 270 number = (number << 8) + reader->buf.buf[i + reader->offset]; 271 } 272 273 reader->offset = reader->offset + bytes; 274 *num = number; 275 return SECSuccess; 276 } 277 278 /************************************************************************** 279 * Append Handshake functions. 280 * All these functions set appropriate error codes. 281 * Most rely on ssl3_AppendHandshake to set the error code. 282 **************************************************************************/ 283 #define MAX_SEND_BUF_LENGTH 32000 /* watch for 16-bit integer overflow */ 284 #define MIN_SEND_BUF_LENGTH 4000 285 286 static SECStatus 287 ssl3_AppendHandshakeInternal(sslSocket *ss, const void *void_src, unsigned int bytes, PRBool suppressHash) 288 { 289 unsigned char *src = (unsigned char *)void_src; 290 int room = ss->sec.ci.sendBuf.space - ss->sec.ci.sendBuf.len; 291 SECStatus rv; 292 293 PORT_Assert(ss->opt.noLocks || ssl_HaveSSL3HandshakeLock(ss)); /* protects sendBuf. */ 294 295 if (!bytes) 296 return SECSuccess; 297 if (ss->sec.ci.sendBuf.space < MAX_SEND_BUF_LENGTH && room < bytes) { 298 rv = sslBuffer_Grow(&ss->sec.ci.sendBuf, PR_MAX(MIN_SEND_BUF_LENGTH, 299 PR_MIN(MAX_SEND_BUF_LENGTH, ss->sec.ci.sendBuf.len + bytes))); 300 if (rv != SECSuccess) 301 return SECFailure; /* sslBuffer_Grow sets a memory error code. */ 302 room = ss->sec.ci.sendBuf.space - ss->sec.ci.sendBuf.len; 303 } 304 305 PRINT_BUF(60, (ss, "Append to Handshake", (unsigned char *)void_src, bytes)); 306 // TODO: Move firstHsDone and version check into callers as a suppression. 307 if (!suppressHash && (!ss->firstHsDone || ss->version < SSL_LIBRARY_VERSION_TLS_1_3)) { 308 rv = ssl3_UpdateHandshakeHashes(ss, src, bytes); 309 if (rv != SECSuccess) 310 return SECFailure; /* error code set by ssl3_UpdateHandshakeHashes */ 311 } 312 313 while (bytes > room) { 314 if (room > 0) 315 PORT_Memcpy(ss->sec.ci.sendBuf.buf + ss->sec.ci.sendBuf.len, src, 316 room); 317 ss->sec.ci.sendBuf.len += room; 318 rv = ssl3_FlushHandshake(ss, ssl_SEND_FLAG_FORCE_INTO_BUFFER); 319 if (rv != SECSuccess) { 320 return SECFailure; /* error code set by ssl3_FlushHandshake */ 321 } 322 bytes -= room; 323 src += room; 324 room = ss->sec.ci.sendBuf.space; 325 PORT_Assert(ss->sec.ci.sendBuf.len == 0); 326 } 327 PORT_Memcpy(ss->sec.ci.sendBuf.buf + ss->sec.ci.sendBuf.len, src, bytes); 328 ss->sec.ci.sendBuf.len += bytes; 329 return SECSuccess; 330 } 331 332 SECStatus 333 ssl3_AppendHandshakeSuppressHash(sslSocket *ss, const void *void_src, unsigned int bytes) 334 { 335 return ssl3_AppendHandshakeInternal(ss, void_src, bytes, PR_TRUE); 336 } 337 338 SECStatus 339 ssl3_AppendHandshake(sslSocket *ss, const void *void_src, unsigned int bytes) 340 { 341 return ssl3_AppendHandshakeInternal(ss, void_src, bytes, PR_FALSE); 342 } 343 344 SECStatus 345 ssl3_AppendHandshakeNumberSuppressHash(sslSocket *ss, PRUint64 num, unsigned int lenSize, PRBool suppressHash) 346 { 347 if ((lenSize > 8) || ((lenSize < 8) && (num >= (1ULL << (8 * lenSize))))) { 348 PORT_SetError(SEC_ERROR_INVALID_ARGS); 349 return SECFailure; 350 } 351 352 PRUint8 b[sizeof(num)]; 353 SSL_TRC(60, ("%d: number:", SSL_GETPID())); 354 ssl_EncodeUintX(b, num, lenSize); 355 return ssl3_AppendHandshakeInternal(ss, b, lenSize, suppressHash); 356 } 357 358 SECStatus 359 ssl3_AppendHandshakeNumber(sslSocket *ss, PRUint64 num, unsigned int lenSize) 360 { 361 return ssl3_AppendHandshakeNumberSuppressHash(ss, num, lenSize, PR_FALSE); 362 } 363 364 SECStatus 365 ssl3_AppendHandshakeVariable(sslSocket *ss, const PRUint8 *src, 366 unsigned int bytes, unsigned int lenSize) 367 { 368 SECStatus rv; 369 370 PORT_Assert((bytes < (1 << 8) && lenSize == 1) || 371 (bytes < (1L << 16) && lenSize == 2) || 372 (bytes < (1L << 24) && lenSize == 3)); 373 374 SSL_TRC(60, ("%d: append variable:", SSL_GETPID())); 375 rv = ssl3_AppendHandshakeNumber(ss, bytes, lenSize); 376 if (rv != SECSuccess) { 377 return SECFailure; /* error code set by AppendHandshake. */ 378 } 379 SSL_TRC(60, ("data:")); 380 return ssl3_AppendHandshake(ss, src, bytes); 381 } 382 383 SECStatus 384 ssl3_AppendBufferToHandshake(sslSocket *ss, sslBuffer *buf) 385 { 386 return ssl3_AppendHandshake(ss, buf->buf, buf->len); 387 } 388 389 SECStatus 390 ssl3_AppendBufferToHandshakeVariable(sslSocket *ss, sslBuffer *buf, 391 unsigned int lenSize) 392 { 393 return ssl3_AppendHandshakeVariable(ss, buf->buf, buf->len, lenSize); 394 } 395 396 SECStatus 397 ssl3_MaybeUpdateHashWithSavedRecord(sslSocket *ss) 398 { 399 SECStatus rv; 400 /* dtls13ClientMessageBuffer is not empty if ClientHello has sent DTLS1.3 */ 401 if (ss->ssl3.hs.dtls13ClientMessageBuffer.len == 0) { 402 return SECSuccess; 403 } 404 405 size_t offset = 0; 406 407 /* the first clause checks the version that was received in ServerHello: 408 * only if it's DTLS1.3, we remove the necessary fields. 409 * the second clause checks if we send 0rtt (see TestTls13ZeroRttDowngrade). 410 */ 411 if ((ss->version == ss->ssl3.cwSpec->version || ss->ssl3.hs.zeroRttState == ssl_0rtt_sent)) { 412 if (ss->ssl3.hs.dtls13ClientMessageBuffer.len < 12) { 413 PORT_SetError(SEC_ERROR_INVALID_ARGS); 414 return SECFailure; 415 } 416 417 rv = ssl3_UpdateHandshakeHashes(ss, ss->ssl3.hs.dtls13ClientMessageBuffer.buf, 4); 418 if (rv != SECSuccess) { 419 return SECFailure; 420 } 421 offset = 12; 422 } 423 424 PORT_Assert(offset < ss->ssl3.hs.dtls13ClientMessageBuffer.len); 425 rv = ssl3_UpdateHandshakeHashes(ss, ss->ssl3.hs.dtls13ClientMessageBuffer.buf + offset, 426 ss->ssl3.hs.dtls13ClientMessageBuffer.len - offset); 427 if (rv != SECSuccess) { 428 return SECFailure; 429 } 430 431 sslBuffer_Clear(&ss->ssl3.hs.dtls13ClientMessageBuffer); 432 ss->ssl3.hs.dtls13ClientMessageBuffer.len = 0; 433 return SECSuccess; 434 } 435 436 SECStatus 437 ssl3_CopyToSECItem(sslBuffer *buf, SECItem *i) 438 { 439 return SECITEM_MakeItem(NULL, i, buf->buf, buf->len); 440 }