rsassa_pss.c

/*
* Copyright (c) 2025 Emil Lenngren
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
*    list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
*    this list of conditions and the following disclaimer in the documentation
*    and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include <string.h>

#include "rsassa_pss.h"

#include "bignum_high_level.h"

static int mgf1(uint8_t *mask, uint8_t *mgf_seed_plus_four_temp_bytes, size_t mask_len, hash_fn hash_function, void *hash_function_user_data, size_t hash_length_bytes, uint8_t *temp_area) {
    uint32_t counter = 0;
    while (mask_len > 0) {
        uint32_t counter_bswapped = __builtin_bswap32(counter++);
        memcpy(mgf_seed_plus_four_temp_bytes + hash_length_bytes, &counter_bswapped, sizeof(uint32_t));
        int res = hash_function(hash_function_user_data, temp_area, mgf_seed_plus_four_temp_bytes, hash_length_bytes + sizeof(uint32_t));
        if (res != 0) {
            return res;
        }
        size_t chunk_size = mask_len;
        if (chunk_size > hash_length_bytes) {
            chunk_size = hash_length_bytes;
        }
        memcpy(mask, temp_area, chunk_size);
        mask += chunk_size;
        mask_len -= chunk_size;
    }
    return 0;
}

int rsassa_pss_verify_hash(
    const uint8_t *public_key_e, size_t public_key_e_length_bytes,
    const uint8_t *public_key_n, size_t public_key_n_length_bytes,
    const uint8_t *message_hash,
    const uint8_t *signature, size_t signature_length_bytes,
    hash_fn hash_function,
    void *hash_function_user_data,
    size_t hash_length_bytes,
    uint32_t *temp_area)
{
    while (public_key_e_length_bytes >= 2 && public_key_e[0] == 0) {
        ++public_key_e;
        --public_key_e_length_bytes;
    }
    if (public_key_e_length_bytes == 0 || (public_key_e_length_bytes == 1 && public_key_e[0] < 3)) {
        return -1;
    }

    if (public_key_n_length_bytes != signature_length_bytes) {
        // "invalid signature"
        return -1;
    }

    int ret = bignum_modexp_public_exponent_big_endian_input(
        signature, public_key_e, public_key_n,
        signature_length_bytes, public_key_e_length_bytes, public_key_n_length_bytes,
        temp_area);
    
    if (ret != 0) {
        return -1;
    }
    
    size_t aligned_length = (public_key_n_length_bytes + 31) & ~(size_t)31;
    
    // Code for PSS:
    uint8_t *encoded_message = (uint8_t *)temp_area;
    if (public_key_n[0] == 0) {
        return -1;
    }
    uint32_t initial_zero_bits = __builtin_clz(public_key_n[0] << 24);
    size_t em_len = public_key_n_length_bytes - (initial_zero_bits == 7);
    
    if (em_len < public_key_n_length_bytes && encoded_message[em_len] != 0) {
        // "integer too large"
        return -1;
    }
    
    // 3. If emLen < hLen + sLen + 2, output "inconsistent" and stop.
    if (em_len < 2 * hash_length_bytes + 2) {
        return -1;
    }
    
    // 4. If the rightmost octet of EM does not have hexadecimal value
    //    0xbc, output "inconsistent" and stop.
    if (encoded_message[0] != 0xbc) {
        return -1;
    }
    
    bignum_endian_reverse(encoded_message, em_len);
    
    // 5. Let maskedDB be the leftmost emLen - hLen - 1 octets of EM,
    //    and let H be the next hLen octets.
    uint8_t *masked_db = encoded_message;
    size_t masked_db_len = em_len - hash_length_bytes - 1;
    uint8_t *H = masked_db + masked_db_len;
    
    // 6. If the leftmost 8emLen - emBits bits of the leftmost octet in
    //    maskedDB are not all equal to zero, output "inconsistent" and
    //    stop.
    size_t num_high_zero_bits = ((initial_zero_bits + 1) % 8);
    if ((masked_db[0] >> (8 - num_high_zero_bits)) != 0) {
        return -1;
    }
    
    // 7. Let dbMask = MGF(H, emLen - hLen - 1).
    // Note: mgf1 writes four temporary bytes after H, which is safe at this point.
    uint8_t *db_mask = (uint8_t *)temp_area + 2 * aligned_length;
    uint8_t *temp_area_hash = (uint8_t *)temp_area + 3 * aligned_length;
    if (mgf1(db_mask, H, masked_db_len, hash_function, hash_function_user_data, hash_length_bytes, temp_area_hash) != 0) {
        return -2;
    }
    
    // 8. Let DB = maskedDB \xor dbMask.
    uint8_t *db = masked_db;
    for (size_t i = 0; i < masked_db_len; i++) {
        db[i] = masked_db[i] ^ db_mask[i];
    }
    
    // 9. Set the leftmost 8emLen - emBits bits of the leftmost octet
    //    in DB to zero.
    db[0] = (uint8_t)(((uint32_t)db[0] << (24 + num_high_zero_bits)) >> (24 + num_high_zero_bits));
    
    // 10. If the emLen - hLen - sLen - 2 leftmost octets of DB are not
    //     zero or if the octet at position emLen - hLen - sLen - 1 (the
    //     leftmost position is "position 1") does not have hexadecimal
    //     value 0x01, output "inconsistent" and stop.
    for (size_t i = 0; i < em_len - 2 * hash_length_bytes - 2; i++) {
        if (db[i] != 0x00) {
            return -1;
        }
    }
    if (db[em_len - 2 * hash_length_bytes - 2] != 0x01) {
        return -1;
    }

    // 11. Let salt be the last sLen octets of DB.
    uint8_t *salt = db + masked_db_len - hash_length_bytes;
    
    // 12. Let
    //
    //        M' = (0x)00 00 00 00 00 00 00 00 || mHash || salt ;
    //
    //     M' is an octet string of length 8 + hLen + sLen with eight
    //     initial zero octets.
    memset(temp_area_hash, 0x00, 8);
    memcpy(temp_area_hash + 8, message_hash, hash_length_bytes);
    memcpy(temp_area_hash + 8 + hash_length_bytes, salt, hash_length_bytes);
    
    // 13. Let H' = Hash(M'), an octet string of length hLen.
    uint8_t *h_prim = (uint8_t *)temp_area + 2 * aligned_length;
    if (hash_function(hash_function_user_data, h_prim, temp_area_hash, 8 + 2 * hash_length_bytes) != 0) {
        return -2;
    }
    
    // 14. If H = H', output "consistent".  Otherwise, output
    //     "inconsistent".
    return memcmp(H, h_prim, hash_length_bytes) == 0 ? 0 : -1;
}

int rsassa_pss_sign_hash(
    uint8_t *signature,
    const uint8_t *private_key_n, size_t private_key_n_length_bytes,
    const uint8_t *private_key_p, size_t private_key_p_length_bytes,
    const uint8_t *private_key_q, size_t private_key_q_length_bytes,
    const uint8_t *private_key_q_inv, size_t private_key_q_inv_length_bytes,
    const uint8_t *private_key_dp, size_t private_key_dp_length_bytes,
    const uint8_t *private_key_dq, size_t private_key_dq_length_bytes,
    const uint8_t *message_hash,
    const uint8_t *randomly_generated_salt,
    hash_fn hash_function,
    void *hash_function_user_data,
    size_t hash_length_bytes,
    uint32_t *temp_area // at least 25 max(p, q) size
)
{
    size_t p_q_len = private_key_p_length_bytes;
    if (private_key_q_length_bytes > p_q_len) {
        p_q_len = private_key_q_length_bytes;
    }
    p_q_len = (p_q_len + 31) & ~(size_t)31;
    size_t double_p_q_len = 2 * p_q_len;
    
    uint32_t initial_zero_bits = __builtin_clz(private_key_n[0] << 24);
    size_t em_len = private_key_n_length_bytes - (initial_zero_bits == 7);
    
    uint8_t *encoded_message = (uint8_t *)temp_area;
    
    // 3. If emLen < hLen + sLen + 2, output "encoding error" and stop.
    if (em_len < 2 * hash_length_bytes + 2) {
        return -1;
    }
    
    // 4. Generate a random octet string salt of length sLen; if sLen =
    //    0, then salt is the empty string.

    // 5. Let
    //
    //       M' = (0x)00 00 00 00 00 00 00 00 || mHash || salt;
    //
    //    M' is an octet string of length 8 + hLen + sLen with eight
    //    initial zero octets.
    uint8_t *m_prim = (uint8_t *)temp_area + double_p_q_len;
    memset(m_prim, 0x00, 8);
    memcpy(m_prim + 8, message_hash, hash_length_bytes);
    memcpy(m_prim + 8 + hash_length_bytes, randomly_generated_salt, hash_length_bytes);

    // 6. Let H = Hash(M'), an octet string of length hLen.
    uint8_t *H = encoded_message + em_len - 1 - hash_length_bytes;
    if (hash_function(hash_function_user_data, H, m_prim, 8 + 2 * hash_length_bytes) != 0) {
        return -2;
    }
    
    // 7. Generate an octet string PS consisting of emLen - sLen - hLen
    //    - 2 zero octets.  The length of PS may be 0.
    memset(encoded_message, 0x00, em_len - 2 * hash_length_bytes - 2);
    
    // 8. Let DB = PS || 0x01 || salt; DB is an octet string of length
    //    emLen - hLen - 1.
    encoded_message[em_len - 2 * hash_length_bytes - 2] = 0x01;
    memcpy(encoded_message + em_len - 2 * hash_length_bytes - 1, randomly_generated_salt, hash_length_bytes);
    size_t db_len = em_len - hash_length_bytes - 1;
    
    // 9. Let dbMask = MGF(H, emLen - hLen - 1).
    // Note: mgf1 writes four temporary bytes after H, which is safe at this point.
    uint8_t *db_mask = (uint8_t *)temp_area + 2 * double_p_q_len;
    uint8_t *mgf_temp = (uint8_t *)temp_area + 3 * double_p_q_len;
    if (mgf1(db_mask, H, db_len, hash_function, hash_function_user_data, hash_length_bytes, mgf_temp) != 0) {
        return -2;
    }
    
    // 10. Let maskedDB = DB \xor dbMask.
    for (size_t i = 0; i < db_len; i++) {
        encoded_message[i] ^= db_mask[i];
    }
    
    // 11. Set the leftmost 8emLen - emBits bits of the leftmost octet
    //     in maskedDB to zero.
    size_t num_high_zero_bits = ((initial_zero_bits + 1) % 8);
    encoded_message[0] = (uint8_t)(((uint32_t)encoded_message[0] << (24 + num_high_zero_bits)) >> (24 + num_high_zero_bits));
    
    // 12. Let EM = maskedDB || H || 0xbc.
    encoded_message[em_len - 1] = 0xbc;
    
    bignum_endian_reverse(encoded_message, em_len);
    
    rsa_private_exp_crt_big_endian_key(
        private_key_n_length_bytes,
        private_key_p,
        private_key_p_length_bytes,
        private_key_q,
        private_key_q_length_bytes,
        private_key_q_inv,
        private_key_q_inv_length_bytes,
        private_key_dp,
        private_key_dp_length_bytes,
        private_key_dq,
        private_key_dq_length_bytes,
        p_q_len,
        temp_area);
    
    bignum_little_to_big_endian(signature, private_key_n_length_bytes, temp_area);
    
    return 0;
}