tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

secsign.c (38269B)


      1 /*
      2 * Signature stuff.
      3 *
      4 * This Source Code Form is subject to the terms of the Mozilla Public
      5 * License, v. 2.0. If a copy of the MPL was not distributed with this
      6 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      7 
      8 #include <stdio.h>
      9 #include "cryptohi.h"
     10 #include "sechash.h"
     11 #include "secder.h"
     12 #include "keyhi.h"
     13 #include "secoid.h"
     14 #include "secdig.h"
     15 #include "pk11func.h"
     16 #include "secerr.h"
     17 #include "keyi.h"
     18 #include "nss.h"
     19 
     20 struct SGNContextStr {
     21    SECOidTag signalg;
     22    SECOidTag hashalg;
     23    CK_MECHANISM_TYPE mech;
     24    void *hashcx;
     25    /* if we are using explicitly hashing, this value will be non-null */
     26    const SECHashObject *hashobj;
     27    /* if we are using the combined mechanism, this value will be non-null */
     28    PK11Context *signcx;
     29    SECKEYPrivateKey *key;
     30    SECItem mechparams;
     31 };
     32 
     33 static SGNContext *
     34 sgn_NewContext(SECOidTag alg, SECItem *params, SECKEYPrivateKey *key)
     35 {
     36    SGNContext *cx;
     37    SECOidTag hashalg, signalg;
     38    CK_MECHANISM_TYPE mech;
     39    SECItem mechparams;
     40    KeyType keyType;
     41    PRUint32 policyFlags;
     42    PRInt32 optFlags;
     43    SECStatus rv;
     44 
     45    /* OK, map a PKCS #7 hash and encrypt algorithm into
     46     * a standard hashing algorithm. Why did we pass in the whole
     47     * PKCS #7 algTag if we were just going to change here you might
     48     * ask. Well the answer is for some cards we may have to do the
     49     * hashing on card. It may not support CKM_RSA_PKCS sign algorithm,
     50     * it may just support CKM_SHA1_RSA_PKCS and/or CKM_MD5_RSA_PKCS.
     51     */
     52    /* we have a private key, not a public key, so don't pass it in */
     53    rv = sec_DecodeSigAlg(NULL, alg, params, &signalg, &hashalg, &mech,
     54                          &mechparams);
     55    if (rv != SECSuccess) {
     56        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
     57        return NULL;
     58    }
     59    keyType = seckey_GetKeyType(signalg);
     60 
     61    /* verify our key type */
     62    if (key->keyType != keyType &&
     63        !((key->keyType == dsaKey) && (keyType == fortezzaKey)) &&
     64        !((key->keyType == rsaKey) && (keyType == rsaPssKey))) {
     65        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
     66        goto loser;
     67    }
     68    if (NSS_OptionGet(NSS_KEY_SIZE_POLICY_FLAGS, &optFlags) != SECFailure) {
     69        if (optFlags & NSS_KEY_SIZE_POLICY_SIGN_FLAG) {
     70            rv = SECKEY_EnforceKeySize(key->keyType,
     71                                       SECKEY_PrivateKeyStrengthInBits(key),
     72                                       SEC_ERROR_SIGNATURE_ALGORITHM_DISABLED);
     73            if (rv != SECSuccess) {
     74                goto loser;
     75            }
     76        }
     77    }
     78    /* check the policy on the hash algorithm */
     79    if ((NSS_GetAlgorithmPolicy(hashalg, &policyFlags) == SECFailure) ||
     80        !(policyFlags & NSS_USE_ALG_IN_ANY_SIGNATURE)) {
     81        PORT_SetError(SEC_ERROR_SIGNATURE_ALGORITHM_DISABLED);
     82        goto loser;
     83    }
     84    /* check the policy on the encryption algorithm */
     85    if ((NSS_GetAlgorithmPolicy(signalg, &policyFlags) == SECFailure) ||
     86        !(policyFlags & NSS_USE_ALG_IN_ANY_SIGNATURE)) {
     87        PORT_SetError(SEC_ERROR_SIGNATURE_ALGORITHM_DISABLED);
     88        goto loser;
     89    }
     90 
     91    cx = (SGNContext *)PORT_ZAlloc(sizeof(SGNContext));
     92    if (!cx) {
     93        goto loser;
     94    }
     95    cx->hashalg = hashalg;
     96    cx->signalg = signalg;
     97    cx->mech = mech;
     98    cx->key = key;
     99    cx->mechparams = mechparams;
    100    return cx;
    101 loser:
    102    SECITEM_FreeItem(&mechparams, PR_FALSE);
    103    return NULL;
    104 }
    105 
    106 SGNContext *
    107 SGN_NewContext(SECOidTag alg, SECKEYPrivateKey *key)
    108 {
    109    return sgn_NewContext(alg, NULL, key);
    110 }
    111 
    112 SGNContext *
    113 SGN_NewContextWithAlgorithmID(SECAlgorithmID *alg, SECKEYPrivateKey *key)
    114 {
    115    SECOidTag tag = SECOID_GetAlgorithmTag(alg);
    116    return sgn_NewContext(tag, &alg->parameters, key);
    117 }
    118 
    119 void
    120 SGN_DestroyContext(SGNContext *cx, PRBool freeit)
    121 {
    122    if (cx) {
    123        if (cx->hashcx != NULL) {
    124            (*cx->hashobj->destroy)(cx->hashcx, PR_TRUE);
    125            cx->hashcx = NULL;
    126        }
    127        if (cx->signcx != NULL) {
    128            PK11_DestroyContext(cx->signcx, PR_TRUE);
    129            cx->signcx = NULL;
    130        }
    131        SECITEM_FreeItem(&cx->mechparams, PR_FALSE);
    132        if (freeit) {
    133            PORT_ZFree(cx, sizeof(SGNContext));
    134        }
    135    }
    136 }
    137 
    138 static PK11Context *
    139 sgn_CreateCombinedContext(SGNContext *cx)
    140 {
    141    /* the particular combination of hash and signature doesn't have a combined
    142     * mechanism, fall back to hand hash & sign */
    143    if (cx->mech == CKM_INVALID_MECHANISM) {
    144        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
    145        return NULL;
    146    }
    147    /* the token the private key resides in doesn't support the combined
    148     * mechanism, fall back to hand hash & sign */
    149    if (!PK11_DoesMechanismFlag(cx->key->pkcs11Slot, cx->mech, CKF_SIGN)) {
    150        PORT_SetError(SEC_ERROR_NO_TOKEN);
    151        return NULL;
    152    }
    153    return PK11_CreateContextByPrivKey(cx->mech, CKA_SIGN, cx->key,
    154                                       &cx->mechparams);
    155 }
    156 
    157 SECStatus
    158 SGN_Begin(SGNContext *cx)
    159 {
    160    PK11Context *signcx = NULL;
    161 
    162    if (cx->hashcx != NULL) {
    163        (*cx->hashobj->destroy)(cx->hashcx, PR_TRUE);
    164        cx->hashcx = NULL;
    165    }
    166    if (cx->signcx != NULL) {
    167        (void)PK11_DestroyContext(cx->signcx, PR_TRUE);
    168        cx->signcx = NULL;
    169    }
    170    /* if we can get a combined context, we'll use that */
    171    signcx = sgn_CreateCombinedContext(cx);
    172    if (signcx != NULL) {
    173        cx->signcx = signcx;
    174        return SECSuccess;
    175    }
    176 
    177    cx->hashobj = HASH_GetHashObjectByOidTag(cx->hashalg);
    178    if (!cx->hashobj)
    179        return SECFailure; /* error code is already set */
    180 
    181    cx->hashcx = (*cx->hashobj->create)();
    182    if (cx->hashcx == NULL)
    183        return SECFailure;
    184 
    185    (*cx->hashobj->begin)(cx->hashcx);
    186    return SECSuccess;
    187 }
    188 
    189 SECStatus
    190 SGN_Update(SGNContext *cx, const unsigned char *input, unsigned int inputLen)
    191 {
    192    if (cx->hashcx == NULL) {
    193        if (cx->signcx == NULL) {
    194            PORT_SetError(SEC_ERROR_INVALID_ARGS);
    195            return SECFailure;
    196        }
    197        return PK11_DigestOp(cx->signcx, input, inputLen);
    198    }
    199    (*cx->hashobj->update)(cx->hashcx, input, inputLen);
    200    return SECSuccess;
    201 }
    202 
    203 /* XXX Old template; want to expunge it eventually. */
    204 static DERTemplate SECAlgorithmIDTemplate[] = {
    205    { DER_SEQUENCE,
    206      0, NULL, sizeof(SECAlgorithmID) },
    207    { DER_OBJECT_ID,
    208      offsetof(SECAlgorithmID, algorithm) },
    209    { DER_OPTIONAL | DER_ANY,
    210      offsetof(SECAlgorithmID, parameters) },
    211    { 0 }
    212 };
    213 
    214 /*
    215 * XXX OLD Template.  Once all uses have been switched over to new one,
    216 * remove this.
    217 */
    218 static DERTemplate SGNDigestInfoTemplate[] = {
    219    { DER_SEQUENCE,
    220      0, NULL, sizeof(SGNDigestInfo) },
    221    { DER_INLINE,
    222      offsetof(SGNDigestInfo, digestAlgorithm),
    223      SECAlgorithmIDTemplate },
    224    { DER_OCTET_STRING,
    225      offsetof(SGNDigestInfo, digest) },
    226    { 0 }
    227 };
    228 
    229 static SECStatus
    230 sgn_allocateSignatureItem(SECKEYPrivateKey *privKey, SECItem *sigitem)
    231 {
    232    int signatureLen;
    233    signatureLen = PK11_SignatureLen(privKey);
    234    if (signatureLen <= 0) {
    235        PORT_SetError(SEC_ERROR_INVALID_KEY);
    236        return SECFailure;
    237    }
    238    sigitem->len = signatureLen;
    239    sigitem->data = (unsigned char *)PORT_Alloc(signatureLen);
    240 
    241    if (sigitem->data == NULL) {
    242        PORT_SetError(SEC_ERROR_NO_MEMORY);
    243        return SECFailure;
    244    }
    245    return SECSuccess;
    246 }
    247 
    248 /* Sometimes the DER signature format for the signature is different than
    249 * The PKCS #11 format for the signature. This code handles the correction
    250 * from PKCS #11 to DER */
    251 static SECStatus
    252 sgn_PKCS11ToX509Sig(SGNContext *cx, SECItem *sigitem)
    253 {
    254    SECStatus rv;
    255    SECItem result = { siBuffer, NULL, 0 };
    256 
    257    if ((cx->signalg == SEC_OID_ANSIX9_DSA_SIGNATURE) ||
    258        (cx->signalg == SEC_OID_ANSIX962_EC_PUBLIC_KEY)) {
    259        /* DSAU_EncodeDerSigWithLen works for DSA and ECDSA */
    260        rv = DSAU_EncodeDerSigWithLen(&result, sigitem, sigitem->len);
    261        /* we are done with sigItem. In case of failure, we want to free
    262         * it anyway */
    263        SECITEM_FreeItem(sigitem, PR_FALSE);
    264        if (rv != SECSuccess) {
    265            return rv;
    266        }
    267        *sigitem = result;
    268    }
    269    return SECSuccess;
    270 }
    271 
    272 SECStatus
    273 SGN_End(SGNContext *cx, SECItem *result)
    274 {
    275    unsigned char digest[HASH_LENGTH_MAX];
    276    unsigned part1;
    277    SECStatus rv;
    278    SECItem digder, sigitem;
    279    PLArenaPool *arena = 0;
    280    SECKEYPrivateKey *privKey = cx->key;
    281    SGNDigestInfo *di = 0;
    282 
    283    result->data = 0;
    284    digder.data = 0;
    285    sigitem.data = 0;
    286 
    287    if (cx->hashcx == NULL) {
    288        if (cx->signcx == NULL) {
    289            PORT_SetError(SEC_ERROR_INVALID_ARGS);
    290            return SECFailure;
    291        }
    292        /* if we are doing the combined hash/sign function, just finalize
    293         * the signature */
    294        rv = sgn_allocateSignatureItem(privKey, result);
    295        if (rv != SECSuccess) {
    296            return rv;
    297        }
    298        rv = PK11_DigestFinal(cx->signcx, result->data, &result->len,
    299                              result->len);
    300        if (rv != SECSuccess) {
    301            SECITEM_ZfreeItem(result, PR_FALSE);
    302            result->data = NULL;
    303            return rv;
    304        }
    305        return sgn_PKCS11ToX509Sig(cx, result);
    306    }
    307    /* Finish up digest function */
    308    (*cx->hashobj->end)(cx->hashcx, digest, &part1, sizeof(digest));
    309 
    310    if (privKey->keyType == rsaKey &&
    311        cx->signalg != SEC_OID_PKCS1_RSA_PSS_SIGNATURE) {
    312 
    313        arena = PORT_NewArena(DER_DEFAULT_CHUNKSIZE);
    314        if (!arena) {
    315            rv = SECFailure;
    316            goto loser;
    317        }
    318 
    319        /* Construct digest info */
    320        di = SGN_CreateDigestInfo(cx->hashalg, digest, part1);
    321        if (!di) {
    322            rv = SECFailure;
    323            goto loser;
    324        }
    325 
    326        /* Der encode the digest as a DigestInfo */
    327        rv = DER_Encode(arena, &digder, SGNDigestInfoTemplate,
    328                        di);
    329        if (rv != SECSuccess) {
    330            goto loser;
    331        }
    332    } else {
    333        digder.data = digest;
    334        digder.len = part1;
    335    }
    336 
    337    /*
    338    ** Encrypt signature after constructing appropriate PKCS#1 signature
    339    ** block
    340    */
    341    rv = sgn_allocateSignatureItem(privKey, &sigitem);
    342    if (rv != SECSuccess) {
    343        return rv;
    344    }
    345 
    346    if (cx->signalg == SEC_OID_PKCS1_RSA_PSS_SIGNATURE) {
    347        rv = PK11_SignWithMechanism(privKey, CKM_RSA_PKCS_PSS, &cx->mechparams,
    348                                    &sigitem, &digder);
    349        if (rv != SECSuccess) {
    350            goto loser;
    351        }
    352    } else {
    353        rv = PK11_Sign(privKey, &sigitem, &digder);
    354        if (rv != SECSuccess) {
    355            goto loser;
    356        }
    357    }
    358    rv = sgn_PKCS11ToX509Sig(cx, &sigitem);
    359    *result = sigitem;
    360    if (rv != SECSuccess) {
    361        goto loser;
    362    }
    363    return SECSuccess;
    364 
    365 loser:
    366    if (rv != SECSuccess) {
    367        SECITEM_FreeItem(&sigitem, PR_FALSE);
    368    }
    369    SGN_DestroyDigestInfo(di);
    370    if (arena != NULL) {
    371        PORT_FreeArena(arena, PR_FALSE);
    372    }
    373    return rv;
    374 }
    375 
    376 static SECStatus
    377 sgn_SingleShot(SGNContext *cx, const unsigned char *input,
    378               unsigned int inputLen, SECItem *result)
    379 {
    380    SECStatus rv;
    381 
    382    result->data = 0;
    383    /* if we have the combined mechanism, just do the single shot
    384     * version */
    385    if ((cx->mech != CKM_INVALID_MECHANISM) &&
    386        PK11_DoesMechanismFlag(cx->key->pkcs11Slot, cx->mech, CKF_SIGN)) {
    387        SECItem data = { siBuffer, (unsigned char *)input, inputLen };
    388        rv = sgn_allocateSignatureItem(cx->key, result);
    389        if (rv != SECSuccess) {
    390            return rv;
    391        }
    392        rv = PK11_SignWithMechanism(cx->key, cx->mech, &cx->mechparams,
    393                                    result, &data);
    394        if (rv != SECSuccess) {
    395            SECITEM_ZfreeItem(result, PR_FALSE);
    396            return rv;
    397        }
    398        return sgn_PKCS11ToX509Sig(cx, result);
    399    }
    400    /* fall back to the stream version */
    401    rv = SGN_Begin(cx);
    402    if (rv != SECSuccess)
    403        return rv;
    404 
    405    rv = SGN_Update(cx, input, inputLen);
    406    if (rv != SECSuccess)
    407        return rv;
    408 
    409    return SGN_End(cx, result);
    410 }
    411 
    412 /************************************************************************/
    413 
    414 static SECStatus
    415 sec_SignData(SECItem *res, const unsigned char *buf, int len,
    416             SECKEYPrivateKey *pk, SECOidTag algid, SECItem *params)
    417 {
    418    SECStatus rv;
    419    SGNContext *sgn;
    420 
    421    sgn = sgn_NewContext(algid, params, pk);
    422 
    423    if (sgn == NULL)
    424        return SECFailure;
    425 
    426    rv = sgn_SingleShot(sgn, buf, len, res);
    427    if (rv != SECSuccess)
    428        goto loser;
    429 
    430 loser:
    431    SGN_DestroyContext(sgn, PR_TRUE);
    432    return rv;
    433 }
    434 
    435 /*
    436 ** Sign a block of data returning in result a bunch of bytes that are the
    437 ** signature. Returns zero on success, an error code on failure.
    438 */
    439 SECStatus
    440 SEC_SignData(SECItem *res, const unsigned char *buf, int len,
    441             SECKEYPrivateKey *pk, SECOidTag algid)
    442 {
    443    return sec_SignData(res, buf, len, pk, algid, NULL);
    444 }
    445 
    446 SECStatus
    447 SEC_SignDataWithAlgorithmID(SECItem *res, const unsigned char *buf, int len,
    448                            SECKEYPrivateKey *pk, SECAlgorithmID *algid)
    449 {
    450    SECOidTag tag = SECOID_GetAlgorithmTag(algid);
    451    return sec_SignData(res, buf, len, pk, tag, &algid->parameters);
    452 }
    453 
    454 /************************************************************************/
    455 
    456 DERTemplate CERTSignedDataTemplate[] = {
    457    { DER_SEQUENCE,
    458      0, NULL, sizeof(CERTSignedData) },
    459    { DER_ANY,
    460      offsetof(CERTSignedData, data) },
    461    { DER_INLINE,
    462      offsetof(CERTSignedData, signatureAlgorithm),
    463      SECAlgorithmIDTemplate },
    464    { DER_BIT_STRING,
    465      offsetof(CERTSignedData, signature) },
    466    { 0 }
    467 };
    468 
    469 SEC_ASN1_MKSUB(SECOID_AlgorithmIDTemplate)
    470 
    471 const SEC_ASN1Template CERT_SignedDataTemplate[] = {
    472    { SEC_ASN1_SEQUENCE,
    473      0, NULL, sizeof(CERTSignedData) },
    474    { SEC_ASN1_ANY,
    475      offsetof(CERTSignedData, data) },
    476    { SEC_ASN1_INLINE | SEC_ASN1_XTRN,
    477      offsetof(CERTSignedData, signatureAlgorithm),
    478      SEC_ASN1_SUB(SECOID_AlgorithmIDTemplate) },
    479    { SEC_ASN1_BIT_STRING,
    480      offsetof(CERTSignedData, signature) },
    481    { 0 }
    482 };
    483 
    484 SEC_ASN1_CHOOSER_IMPLEMENT(CERT_SignedDataTemplate)
    485 
    486 static SECStatus
    487 sec_DerSignData(PLArenaPool *arena, SECItem *result,
    488                const unsigned char *buf, int len, SECKEYPrivateKey *pk,
    489                SECOidTag algID, SECItem *params)
    490 {
    491    SECItem it;
    492    CERTSignedData sd;
    493    SECStatus rv;
    494 
    495    it.data = 0;
    496 
    497    /* XXX We should probably have some asserts here to make sure the key type
    498     * and algID match
    499     */
    500 
    501    if (algID == SEC_OID_UNKNOWN) {
    502        switch (pk->keyType) {
    503            case rsaKey:
    504                algID = SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION;
    505                break;
    506            case dsaKey:
    507                /* get Signature length (= q_len*2) and work from there */
    508                switch (PK11_SignatureLen(pk)) {
    509                    case 320:
    510                        algID = SEC_OID_ANSIX9_DSA_SIGNATURE_WITH_SHA1_DIGEST;
    511                        break;
    512                    case 448:
    513                        algID = SEC_OID_NIST_DSA_SIGNATURE_WITH_SHA224_DIGEST;
    514                        break;
    515                    case 512:
    516                    default:
    517                        algID = SEC_OID_NIST_DSA_SIGNATURE_WITH_SHA256_DIGEST;
    518                        break;
    519                }
    520                break;
    521            case ecKey:
    522                algID = SEC_OID_ANSIX962_ECDSA_SHA256_SIGNATURE;
    523                break;
    524            case mldsaKey:
    525                algID = seckey_GetParameterSet(pk);
    526                break;
    527            default:
    528                algID = SEC_OID_UNKNOWN;
    529                break;
    530        }
    531    }
    532    if (algID == SEC_OID_UNKNOWN) {
    533        PORT_SetError(SEC_ERROR_INVALID_KEY);
    534        return SECFailure;
    535    }
    536 
    537    /* Sign input buffer */
    538    rv = sec_SignData(&it, buf, len, pk, algID, params);
    539    if (rv)
    540        goto loser;
    541 
    542    /* Fill out SignedData object */
    543    PORT_Memset(&sd, 0, sizeof(sd));
    544    sd.data.data = (unsigned char *)buf;
    545    sd.data.len = len;
    546    sd.signature.data = it.data;
    547    sd.signature.len = it.len << 3; /* convert to bit string */
    548    rv = SECOID_SetAlgorithmID(arena, &sd.signatureAlgorithm, algID, params);
    549    if (rv)
    550        goto loser;
    551 
    552    /* DER encode the signed data object */
    553    rv = DER_Encode(arena, result, CERTSignedDataTemplate, &sd);
    554    /* FALL THROUGH */
    555 
    556 loser:
    557    PORT_Free(it.data);
    558    return rv;
    559 }
    560 
    561 SECStatus
    562 SEC_DerSignData(PLArenaPool *arena, SECItem *result,
    563                const unsigned char *buf, int len, SECKEYPrivateKey *pk,
    564                SECOidTag algID)
    565 {
    566    return sec_DerSignData(arena, result, buf, len, pk, algID, NULL);
    567 }
    568 
    569 SECStatus
    570 SEC_DerSignDataWithAlgorithmID(PLArenaPool *arena, SECItem *result,
    571                               const unsigned char *buf, int len,
    572                               SECKEYPrivateKey *pk,
    573                               SECAlgorithmID *algID)
    574 {
    575    SECOidTag tag = SECOID_GetAlgorithmTag(algID);
    576    return sec_DerSignData(arena, result, buf, len, pk, tag, &algID->parameters);
    577 }
    578 
    579 SECStatus
    580 SGN_Digest(SECKEYPrivateKey *privKey,
    581           SECOidTag algtag, SECItem *result, SECItem *digest)
    582 {
    583    int modulusLen;
    584    SECStatus rv;
    585    SECItem digder;
    586    PLArenaPool *arena = 0;
    587    SGNDigestInfo *di = 0;
    588    SECOidTag enctag;
    589    PRUint32 policyFlags;
    590    PRInt32 optFlags;
    591 
    592    result->data = 0;
    593 
    594    if (NSS_OptionGet(NSS_KEY_SIZE_POLICY_FLAGS, &optFlags) != SECFailure) {
    595        if (optFlags & NSS_KEY_SIZE_POLICY_SIGN_FLAG) {
    596            rv = SECKEY_EnforceKeySize(privKey->keyType,
    597                                       SECKEY_PrivateKeyStrengthInBits(privKey),
    598                                       SEC_ERROR_SIGNATURE_ALGORITHM_DISABLED);
    599            if (rv != SECSuccess) {
    600                return SECFailure;
    601            }
    602        }
    603    }
    604    /* check the policy on the hash algorithm */
    605    if ((NSS_GetAlgorithmPolicy(algtag, &policyFlags) == SECFailure) ||
    606        !(policyFlags & NSS_USE_ALG_IN_ANY_SIGNATURE)) {
    607        PORT_SetError(SEC_ERROR_SIGNATURE_ALGORITHM_DISABLED);
    608        return SECFailure;
    609    }
    610 
    611    if (privKey->keyType == mldsaKey) {
    612        /* don't allow digest sign for mldsa. May be possible if mu
    613         * is enabled, in that case the hash must be the special mldsa
    614         * hash, which we don't have defined yet */
    615        PORT_SetError(SEC_ERROR_UNSUPPORTED_KEYALG);
    616        return SECFailure;
    617    }
    618    /* check the policy on the encryption algorithm */
    619    enctag = sec_GetEncAlgFromSigAlg(
    620        SEC_GetSignatureAlgorithmOidTagByKey(privKey, NULL, algtag));
    621    if ((enctag == SEC_OID_UNKNOWN) ||
    622        (NSS_GetAlgorithmPolicy(enctag, &policyFlags) == SECFailure) ||
    623        !(policyFlags & NSS_USE_ALG_IN_ANY_SIGNATURE)) {
    624        PORT_SetError(SEC_ERROR_SIGNATURE_ALGORITHM_DISABLED);
    625        return SECFailure;
    626    }
    627 
    628    if (privKey->keyType == rsaKey) {
    629 
    630        arena = PORT_NewArena(DER_DEFAULT_CHUNKSIZE);
    631        if (!arena) {
    632            rv = SECFailure;
    633            goto loser;
    634        }
    635 
    636        /* Construct digest info */
    637        di = SGN_CreateDigestInfo(algtag, digest->data, digest->len);
    638        if (!di) {
    639            rv = SECFailure;
    640            goto loser;
    641        }
    642 
    643        /* Der encode the digest as a DigestInfo */
    644        rv = DER_Encode(arena, &digder, SGNDigestInfoTemplate,
    645                        di);
    646        if (rv != SECSuccess) {
    647            goto loser;
    648        }
    649    } else {
    650        digder.data = digest->data;
    651        digder.len = digest->len;
    652    }
    653 
    654    /*
    655    ** Encrypt signature after constructing appropriate PKCS#1 signature
    656    ** block
    657    */
    658    modulusLen = PK11_SignatureLen(privKey);
    659    if (modulusLen <= 0) {
    660        PORT_SetError(SEC_ERROR_INVALID_KEY);
    661        rv = SECFailure;
    662        goto loser;
    663    }
    664    result->len = modulusLen;
    665    result->data = (unsigned char *)PORT_Alloc(modulusLen);
    666    result->type = siBuffer;
    667 
    668    if (result->data == NULL) {
    669        rv = SECFailure;
    670        goto loser;
    671    }
    672 
    673    rv = PK11_Sign(privKey, result, &digder);
    674    if (rv != SECSuccess) {
    675        PORT_Free(result->data);
    676        result->data = NULL;
    677    }
    678 
    679 loser:
    680    SGN_DestroyDigestInfo(di);
    681    if (arena != NULL) {
    682        PORT_FreeArena(arena, PR_FALSE);
    683    }
    684    return rv;
    685 }
    686 
    687 SECOidTag
    688 SEC_GetSignatureAlgorithmOidTag(KeyType keyType, SECOidTag hashAlgTag)
    689 {
    690    SECOidTag sigTag = SEC_OID_UNKNOWN;
    691 
    692    switch (keyType) {
    693        case rsaPssKey:
    694            sigTag = SEC_OID_PKCS1_RSA_PSS_SIGNATURE;
    695            break;
    696        case rsaKey:
    697            switch (hashAlgTag) {
    698                case SEC_OID_MD2:
    699                    sigTag = SEC_OID_PKCS1_MD2_WITH_RSA_ENCRYPTION;
    700                    break;
    701                case SEC_OID_MD5:
    702                    sigTag = SEC_OID_PKCS1_MD5_WITH_RSA_ENCRYPTION;
    703                    break;
    704                case SEC_OID_SHA1:
    705                    sigTag = SEC_OID_PKCS1_SHA1_WITH_RSA_ENCRYPTION;
    706                    break;
    707                case SEC_OID_SHA224:
    708                    sigTag = SEC_OID_PKCS1_SHA224_WITH_RSA_ENCRYPTION;
    709                    break;
    710                case SEC_OID_UNKNOWN: /* default for RSA if not specified */
    711                case SEC_OID_SHA256:
    712                    sigTag = SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION;
    713                    break;
    714                case SEC_OID_SHA384:
    715                    sigTag = SEC_OID_PKCS1_SHA384_WITH_RSA_ENCRYPTION;
    716                    break;
    717                case SEC_OID_SHA512:
    718                    sigTag = SEC_OID_PKCS1_SHA512_WITH_RSA_ENCRYPTION;
    719                    break;
    720                default:
    721                    break;
    722            }
    723            break;
    724        case dsaKey:
    725            switch (hashAlgTag) {
    726                case SEC_OID_SHA1:
    727                    sigTag = SEC_OID_ANSIX9_DSA_SIGNATURE_WITH_SHA1_DIGEST;
    728                    break;
    729                case SEC_OID_SHA224:
    730                    sigTag = SEC_OID_NIST_DSA_SIGNATURE_WITH_SHA224_DIGEST;
    731                    break;
    732                case SEC_OID_UNKNOWN: /* default for DSA if not specified */
    733                case SEC_OID_SHA256:
    734                    sigTag = SEC_OID_NIST_DSA_SIGNATURE_WITH_SHA256_DIGEST;
    735                    break;
    736                default:
    737                    break;
    738            }
    739            break;
    740        case mldsaKey:
    741            switch (hashAlgTag) {
    742                case SEC_OID_ML_DSA_44:
    743                case SEC_OID_ML_DSA_65:
    744                case SEC_OID_ML_DSA_87:
    745                    /* only accept known mldsa values. For mldsa, the hash must
    746                     * match the underlying signature algorithm */
    747                    sigTag = hashAlgTag;
    748                    break;
    749                default:
    750                    break;
    751            }
    752            break;
    753        case ecKey:
    754            switch (hashAlgTag) {
    755                case SEC_OID_SHA1:
    756                    sigTag = SEC_OID_ANSIX962_ECDSA_SHA1_SIGNATURE;
    757                    break;
    758                case SEC_OID_SHA224:
    759                    sigTag = SEC_OID_ANSIX962_ECDSA_SHA224_SIGNATURE;
    760                    break;
    761                case SEC_OID_UNKNOWN: /* default for ECDSA if not specified */
    762                case SEC_OID_SHA256:
    763                    sigTag = SEC_OID_ANSIX962_ECDSA_SHA256_SIGNATURE;
    764                    break;
    765                case SEC_OID_SHA384:
    766                    sigTag = SEC_OID_ANSIX962_ECDSA_SHA384_SIGNATURE;
    767                    break;
    768                case SEC_OID_SHA512:
    769                    sigTag = SEC_OID_ANSIX962_ECDSA_SHA512_SIGNATURE;
    770                    break;
    771                default:
    772                    break;
    773            }
    774        default:
    775            break;
    776    }
    777    return sigTag;
    778 }
    779 
    780 SECOidTag
    781 SEC_GetSignatureAlgorithmOidTagByKey(const SECKEYPrivateKey *privKey, const SECKEYPublicKey *pubKey, SECOidTag hashAlgTag)
    782 {
    783    KeyType keyType = nullKey;
    784    /* make sure we have a key */
    785    if ((privKey && pubKey) || (!privKey && !pubKey)) {
    786        PORT_SetError(SEC_ERROR_INVALID_ARGS);
    787        return SEC_OID_UNKNOWN;
    788    }
    789    /* make sure we have only one key */
    790    if (privKey) {
    791        keyType = privKey->keyType;
    792        /* for mldsa, the hash has to match the paramset anyway, so
    793         * pass in the param set as the hash */
    794        if (keyType == mldsaKey) {
    795            hashAlgTag = seckey_GetParameterSet(privKey);
    796        }
    797    } else {
    798        /* the logic above should guarentee the following assert. */
    799        PORT_Assert(pubKey != NULL);
    800        PORT_Assert(privKey == NULL);
    801        keyType = pubKey->keyType;
    802        /* for mldsa, the hash has to match the paramset anyway, so
    803         * pass in the param set as the hash */
    804        if (keyType == mldsaKey) {
    805            hashAlgTag = pubKey->u.mldsa.paramSet;
    806        }
    807    }
    808    return SEC_GetSignatureAlgorithmOidTag(keyType, hashAlgTag);
    809 }
    810 
    811 SECItem *
    812 sec_CreateRSAPSSParameters(PLArenaPool *arena,
    813                           SECItem *result,
    814                           SECOidTag hashAlgTag,
    815                           const SECItem *params,
    816                           int modBytes)
    817 {
    818    SECKEYRSAPSSParams pssParams;
    819    int hashLength;
    820    unsigned long saltLength;
    821    PRBool defaultSHA1 = PR_FALSE;
    822    SECStatus rv;
    823 
    824    PORT_Memset(&pssParams, 0, sizeof(pssParams));
    825 
    826    if (params && params->data) {
    827        /* The parameters field should either be empty or contain
    828         * valid RSA-PSS parameters */
    829        PORT_Assert(!(params->len == 2 &&
    830                      params->data[0] == SEC_ASN1_NULL &&
    831                      params->data[1] == 0));
    832        rv = SEC_QuickDERDecodeItem(arena, &pssParams,
    833                                    SECKEY_RSAPSSParamsTemplate,
    834                                    params);
    835        if (rv != SECSuccess) {
    836            return NULL;
    837        }
    838        defaultSHA1 = PR_TRUE;
    839    }
    840 
    841    if (pssParams.trailerField.data) {
    842        unsigned long trailerField;
    843 
    844        rv = SEC_ASN1DecodeInteger((SECItem *)&pssParams.trailerField,
    845                                   &trailerField);
    846        if (rv != SECSuccess) {
    847            return NULL;
    848        }
    849        if (trailerField != 1) {
    850            PORT_SetError(SEC_ERROR_INVALID_ARGS);
    851            return NULL;
    852        }
    853    }
    854 
    855    /* Determine the hash algorithm to use, based on hashAlgTag and
    856     * pssParams.hashAlg; there are 6  cases.
    857     *  case:
    858     *  1) We have params and params.hashAlg and we have a specified hashAlgTag,
    859     *  make sure that hashAlgTag specified by the appication matches.
    860     *  2) We have params, but no params.hashAlg and we have a specified
    861     *  hashAlg, make sure the hashAlgTag matches SEC_OID_SHA1.
    862     *  3) we did not specify any parameters but we did specified
    863     *  a hashAlgTag. Use the specified hash algtag.
    864     *  4) We have params and params.hashAlg and we did not specify a
    865     *  hashAlgTag, use the hashAlg from the parameter.
    866     *  5) We have params, but no params.hashAlg and we did not specify a
    867     *  hashAlgTag, use the SEC_OID_SHA1
    868     *  6) We did not specify any parameters, nor did we specify a
    869     *  hashAlgTag, use the key size to select an appropriate hashAlg.
    870     */
    871    if (hashAlgTag != SEC_OID_UNKNOWN) {
    872        SECOidTag tag = SEC_OID_UNKNOWN;
    873 
    874        if (pssParams.hashAlg) {
    875            tag = SECOID_GetAlgorithmTag(pssParams.hashAlg);
    876        } else if (defaultSHA1) {
    877            tag = SEC_OID_SHA1;
    878        }
    879 
    880        if (tag != SEC_OID_UNKNOWN && tag != hashAlgTag) {
    881            PORT_SetError(SEC_ERROR_INVALID_ARGS);
    882            return NULL;
    883        }
    884    } else if (hashAlgTag == SEC_OID_UNKNOWN) {
    885        if (pssParams.hashAlg) {
    886            hashAlgTag = SECOID_GetAlgorithmTag(pssParams.hashAlg);
    887        } else if (defaultSHA1) {
    888            hashAlgTag = SEC_OID_SHA1;
    889        } else {
    890            /* Find a suitable hash algorithm based on the NIST recommendation */
    891            if (modBytes <= 384) { /* 128, in NIST 800-57, Part 1 */
    892                hashAlgTag = SEC_OID_SHA256;
    893            } else if (modBytes <= 960) { /* 192, NIST 800-57, Part 1 */
    894                hashAlgTag = SEC_OID_SHA384;
    895            } else {
    896                hashAlgTag = SEC_OID_SHA512;
    897            }
    898        }
    899    }
    900 
    901    /* explicitly restrict hashAlg to SHA2 variants */
    902    if (hashAlgTag != SEC_OID_SHA1 && hashAlgTag != SEC_OID_SHA224 &&
    903        hashAlgTag != SEC_OID_SHA256 && hashAlgTag != SEC_OID_SHA384 &&
    904        hashAlgTag != SEC_OID_SHA512) {
    905        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
    906        return NULL;
    907    }
    908 
    909    /* Now that the hash algorithm is decided, check if it matches the
    910     * existing parameters if any */
    911    if (pssParams.maskAlg) {
    912        SECAlgorithmID maskHashAlg;
    913 
    914        if (SECOID_GetAlgorithmTag(pssParams.maskAlg) != SEC_OID_PKCS1_MGF1) {
    915            PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
    916            return NULL;
    917        }
    918 
    919        if (pssParams.maskAlg->parameters.data == NULL) {
    920            PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
    921            return NULL;
    922        }
    923 
    924        PORT_Memset(&maskHashAlg, 0, sizeof(maskHashAlg));
    925        rv = SEC_QuickDERDecodeItem(arena, &maskHashAlg,
    926                                    SEC_ASN1_GET(SECOID_AlgorithmIDTemplate),
    927                                    &pssParams.maskAlg->parameters);
    928        if (rv != SECSuccess) {
    929            return NULL;
    930        }
    931 
    932        /* Following the recommendation in RFC 4055, assume the hash
    933         * algorithm identical to pssParam.hashAlg */
    934        if (SECOID_GetAlgorithmTag(&maskHashAlg) != hashAlgTag) {
    935            PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
    936            return NULL;
    937        }
    938    } else if (defaultSHA1) {
    939        if (hashAlgTag != SEC_OID_SHA1) {
    940            PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
    941            return NULL;
    942        }
    943    }
    944 
    945    hashLength = HASH_ResultLenByOidTag(hashAlgTag);
    946 
    947    if (pssParams.saltLength.data) {
    948        rv = SEC_ASN1DecodeInteger((SECItem *)&pssParams.saltLength,
    949                                   &saltLength);
    950        if (rv != SECSuccess) {
    951            return NULL;
    952        }
    953 
    954        /* The specified salt length is too long */
    955        if (saltLength > (unsigned long)(modBytes - hashLength - 2)) {
    956            PORT_SetError(SEC_ERROR_INVALID_ARGS);
    957            return NULL;
    958        }
    959    } else if (defaultSHA1) {
    960        saltLength = 20;
    961    }
    962 
    963    /* Fill in the parameters */
    964    if (pssParams.hashAlg) {
    965        if (hashAlgTag == SEC_OID_SHA1) {
    966            /* Omit hashAlg if the the algorithm is SHA-1 (default) */
    967            pssParams.hashAlg = NULL;
    968        }
    969    } else {
    970        if (hashAlgTag != SEC_OID_SHA1) {
    971            pssParams.hashAlg = PORT_ArenaZAlloc(arena, sizeof(SECAlgorithmID));
    972            if (!pssParams.hashAlg) {
    973                return NULL;
    974            }
    975            rv = SECOID_SetAlgorithmID(arena, pssParams.hashAlg, hashAlgTag,
    976                                       NULL);
    977            if (rv != SECSuccess) {
    978                return NULL;
    979            }
    980        }
    981    }
    982 
    983    if (pssParams.maskAlg) {
    984        if (hashAlgTag == SEC_OID_SHA1) {
    985            /* Omit maskAlg if the the algorithm is SHA-1 (default) */
    986            pssParams.maskAlg = NULL;
    987        }
    988    } else {
    989        if (hashAlgTag != SEC_OID_SHA1) {
    990            SECItem *hashAlgItem;
    991 
    992            PORT_Assert(pssParams.hashAlg != NULL);
    993 
    994            hashAlgItem = SEC_ASN1EncodeItem(arena, NULL, pssParams.hashAlg,
    995                                             SEC_ASN1_GET(SECOID_AlgorithmIDTemplate));
    996            if (!hashAlgItem) {
    997                return NULL;
    998            }
    999            pssParams.maskAlg = PORT_ArenaZAlloc(arena, sizeof(SECAlgorithmID));
   1000            if (!pssParams.maskAlg) {
   1001                return NULL;
   1002            }
   1003            rv = SECOID_SetAlgorithmID(arena, pssParams.maskAlg,
   1004                                       SEC_OID_PKCS1_MGF1, hashAlgItem);
   1005            if (rv != SECSuccess) {
   1006                return NULL;
   1007            }
   1008        }
   1009    }
   1010 
   1011    if (pssParams.saltLength.data) {
   1012        if (saltLength == 20) {
   1013            /* Omit the salt length if it is the default */
   1014            pssParams.saltLength.data = NULL;
   1015        }
   1016    } else {
   1017        /* Find a suitable length from the hash algorithm and modulus bits */
   1018        saltLength = PR_MIN(hashLength, modBytes - hashLength - 2);
   1019 
   1020        if (saltLength != 20 &&
   1021            !SEC_ASN1EncodeInteger(arena, &pssParams.saltLength, saltLength)) {
   1022            return NULL;
   1023        }
   1024    }
   1025 
   1026    if (pssParams.trailerField.data) {
   1027        /* Omit trailerField if the value is 1 (default) */
   1028        pssParams.trailerField.data = NULL;
   1029    }
   1030 
   1031    return SEC_ASN1EncodeItem(arena, result,
   1032                              &pssParams, SECKEY_RSAPSSParamsTemplate);
   1033 }
   1034 
   1035 SECItem *
   1036 SEC_CreateRSAPSSParameters(PLArenaPool *arena,
   1037                           SECItem *result,
   1038                           SECOidTag hashAlgTag,
   1039                           const SECItem *params,
   1040                           const SECKEYPrivateKey *privKey,
   1041                           const SECKEYPublicKey *pubKey)
   1042 {
   1043    /* if no keys are provided arrange for the saltLength to be hashLen */
   1044    int modBytes = (HASH_LENGTH_MAX * 2) + 2;
   1045 
   1046    /* we don't need all these parameters, but we should have at least
   1047     * one of these */
   1048    if (!privKey && !pubKey && !params && (hashAlgTag == SEC_OID_UNKNOWN)) {
   1049        PORT_SetError(SEC_ERROR_INVALID_ARGS);
   1050        return NULL;
   1051    }
   1052    /* only allow one key */
   1053    if (privKey && pubKey) {
   1054        PORT_SetError(SEC_ERROR_INVALID_ARGS);
   1055        return NULL;
   1056    }
   1057    if (privKey) {
   1058        if (privKey->keyType != rsaKey && privKey->keyType != rsaPssKey) {
   1059            PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
   1060            return NULL;
   1061        }
   1062        /* cast away the const, even though it's still logically a const
   1063         * function */
   1064        modBytes = PK11_GetPrivateModulusLen((SECKEYPrivateKey *)privKey);
   1065    } else if (pubKey) {
   1066        if (pubKey->keyType != rsaKey && pubKey->keyType != rsaPssKey) {
   1067            PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
   1068            return NULL;
   1069        }
   1070        modBytes = pubKey->u.rsa.modulus.len;
   1071    }
   1072    return sec_CreateRSAPSSParameters(arena, result, hashAlgTag,
   1073                                      params, modBytes);
   1074 }
   1075 
   1076 SECItem *
   1077 SEC_CreateSignatureAlgorithmParameters(PLArenaPool *arena,
   1078                                       SECItem *result,
   1079                                       SECOidTag signAlgTag,
   1080                                       SECOidTag hashAlgTag,
   1081                                       const SECItem *params,
   1082                                       const SECKEYPrivateKey *key)
   1083 {
   1084    PORT_SetError(0);
   1085    switch (signAlgTag) {
   1086        case SEC_OID_PKCS1_RSA_PSS_SIGNATURE:
   1087            return SEC_CreateRSAPSSParameters(arena, result,
   1088                                              hashAlgTag, params, key, NULL);
   1089 
   1090        default:
   1091            if (params == NULL)
   1092                return NULL;
   1093            if (result == NULL)
   1094                result = SECITEM_AllocItem(arena, NULL, 0);
   1095            if (result == NULL) {
   1096                return NULL;
   1097            }
   1098            if (SECITEM_CopyItem(arena, result, params) != SECSuccess)
   1099                return NULL;
   1100            return result;
   1101    }
   1102 }
   1103 
   1104 SECItem *
   1105 SEC_CreateVerifyAlgorithmParameters(PLArenaPool *arena,
   1106                                    SECItem *result,
   1107                                    SECOidTag signAlgTag,
   1108                                    SECOidTag hashAlgTag,
   1109                                    const SECItem *params,
   1110                                    const SECKEYPublicKey *key)
   1111 {
   1112    PORT_SetError(0);
   1113    switch (signAlgTag) {
   1114        case SEC_OID_PKCS1_RSA_PSS_SIGNATURE:
   1115            return SEC_CreateRSAPSSParameters(arena, result,
   1116                                              hashAlgTag, params, NULL, key);
   1117 
   1118        default:
   1119            if (params == NULL)
   1120                return NULL;
   1121            if (result == NULL)
   1122                result = SECITEM_AllocItem(arena, NULL, 0);
   1123            if (result == NULL) {
   1124                return NULL;
   1125            }
   1126            if (SECITEM_CopyItem(arena, result, params) != SECSuccess)
   1127                return NULL;
   1128            return result;
   1129    }
   1130 }
   1131 
   1132 SECStatus
   1133 SEC_CreateSignatureAlgorithmID(PLArenaPool *arena,
   1134                               SECAlgorithmID *signAlgID,
   1135                               SECOidTag signAlgTag,
   1136                               SECOidTag hashAlgTag,
   1137                               const SECItem *params,
   1138                               const SECKEYPrivateKey *privKey,
   1139                               const SECKEYPublicKey *pubKey)
   1140 {
   1141    SECItem *newParams = NULL;
   1142 
   1143    if (signAlgTag == SEC_OID_UNKNOWN) {
   1144        signAlgTag = SEC_GetSignatureAlgorithmOidTagByKey(privKey, pubKey,
   1145                                                          hashAlgTag);
   1146    } else {
   1147        /* SEC_GetSignatureAlgorithm already checks if privKey and pubKey
   1148         * is present and the only case */
   1149        if ((privKey && pubKey) || (!privKey && !pubKey)) {
   1150            PORT_SetError(SEC_ERROR_INVALID_ARGS);
   1151            return SECFailure;
   1152        }
   1153    }
   1154 
   1155    if (signAlgTag == SEC_OID_UNKNOWN) {
   1156        /* error already set by SEC_GetSignatureAlgorithmOidTagByKey */
   1157        return SECFailure;
   1158    }
   1159 
   1160    if (privKey) {
   1161        newParams = SEC_CreateSignatureAlgorithmParameters(arena, NULL,
   1162                                                           signAlgTag,
   1163                                                           hashAlgTag,
   1164                                                           params,
   1165                                                           privKey);
   1166    } else {
   1167        /* must be pubKey */
   1168        newParams = SEC_CreateVerifyAlgorithmParameters(arena, NULL,
   1169                                                        signAlgTag,
   1170                                                        hashAlgTag,
   1171                                                        params,
   1172                                                        pubKey);
   1173    }
   1174 
   1175    /* It's legal (and common) for params to be NULL; look at the error
   1176     * code to see if there was a failure */
   1177    if (!newParams && PORT_GetError() != 0) {
   1178        return SECFailure;
   1179    }
   1180 
   1181    return SECOID_SetAlgorithmID(arena, signAlgID, signAlgTag, newParams);
   1182 }