Add mu generator support for ML-DSA

Add support for SHAKE-128 and SHAKE-256 digests along with XOF output length
configuration.

Implement `MuGenerator` and the `mldsa_with_mu` parameter helper to compute
and supply the FIPS 204 `mu` value directly for ML-DSA signatures and
verification, and add corresponding test vectors.

Assisted-by: Gemini:Gemini 3.7 Flash
Signed-off-by: Simo Sorce <simo@redhat.com>

Author: simo5

ossl/src/digest.rs

@@ -187,6 +187,8 @@ pub enum DigestAlg {
     Sha3_256,
     Sha3_384,
     Sha3_512,
+    Shake128,
+    Shake256,
     #[cfg(feature = "rfc9580")]
     Md5,
     #[cfg(feature = "rfc9580")]
@@ -206,6 +208,8 @@ pub(crate) fn digest_to_string(digest: DigestAlg) -> &'static CStr {
         DigestAlg::Sha3_256 => cstr!(OSSL_DIGEST_NAME_SHA3_256),
         DigestAlg::Sha3_384 => cstr!(OSSL_DIGEST_NAME_SHA3_384),
         DigestAlg::Sha3_512 => cstr!(OSSL_DIGEST_NAME_SHA3_512),
+        DigestAlg::Shake128 => c"SHAKE-128",
+        DigestAlg::Shake256 => c"SHAKE-256",
         #[cfg(feature = "rfc9580")]
         DigestAlg::Md5 => cstr!(OSSL_DIGEST_NAME_MD5),
         #[cfg(feature = "rfc9580")]
@@ -226,6 +230,8 @@ pub(crate) fn string_to_digest(digest: &CStr) -> Result<DigestAlg, Error> {
         b"SHA3-256" => Ok(DigestAlg::Sha3_256),
         b"SHA3-384" => Ok(DigestAlg::Sha3_384),
         b"SHA3-512" => Ok(DigestAlg::Sha3_512),
+        b"SHAKE-128" => Ok(DigestAlg::Shake128),
+        b"SHAKE-256" => Ok(DigestAlg::Shake256),
         #[cfg(feature = "rfc9580")]
         b"MD5" => Ok(DigestAlg::Md5),
         #[cfg(feature = "rfc9580")]
@@ -335,6 +341,22 @@ impl OsslDigest {
         Ok(usize::try_from(retlen)?)
     }
 
+    /// Sets the requested output length for an XOF algorithm like SHAKE.
+    #[cfg(ossl_v350)]
+    pub fn set_xoflen(&mut self, len: usize) -> Result<(), Error> {
+        let mut params_builder = crate::OsslParamBuilder::with_capacity(1);
+        params_builder
+            .add_owned_uint(cstr!(OSSL_DIGEST_PARAM_XOFLEN), len as c_uint)?;
+        let params = params_builder.finalize();
+        if unsafe {
+            EVP_MD_CTX_set_params(self.ctx.as_mut_ptr(), params.as_ptr())
+        } != 1
+        {
+            return Err(Error::new(ErrorKind::OsslError));
+        }
+        Ok(())
+    }
+
     /// Provides the size of the expected output digest
     pub fn size(&self) -> usize {
         self.size

ossl/src/non-openssl-sys-bindings.rs

@@ -5,6 +5,7 @@ pub const OSSL_PARAM_UTF8_STRING: u32 = 4;
 // Signature + AsymCipher
 pub const OSSL_SIGNATURE_PARAM_CONTEXT_STRING: &[u8; 15] = b"context-string\0";
 pub const OSSL_SIGNATURE_PARAM_DETERMINISTIC: &[u8; 14] = b"deterministic\0";
+pub const OSSL_SIGNATURE_PARAM_MU: &[u8; 3] = b"mu\0";
 pub const OSSL_SIGNATURE_PARAM_PSS_SALTLEN: &[u8; 8] = b"saltlen\0";
 pub const OSSL_SIGNATURE_PARAM_MESSAGE_ENCODING: &[u8; 17] =
     b"message-encoding\0";
@@ -36,6 +37,7 @@ pub const OSSL_DIGEST_NAME_SHA3_224: &[u8; 9] = b"SHA3-224\0";
 pub const OSSL_DIGEST_NAME_SHA3_256: &[u8; 9] = b"SHA3-256\0";
 pub const OSSL_DIGEST_NAME_SHA3_384: &[u8; 9] = b"SHA3-384\0";
 pub const OSSL_DIGEST_NAME_SHA3_512: &[u8; 9] = b"SHA3-512\0";
+pub const OSSL_DIGEST_PARAM_XOFLEN: &[u8; 7] = b"xoflen\0";
 
 // Rand
 pub const OSSL_RAND_PARAM_STRENGTH: &[u8; 9] = b"strength\0";

ossl/src/signature.rs

@@ -650,6 +650,126 @@ pub fn mldsa_params<'a>(
     return Ok(Some(params));
 }
 
+/// Generator for the exact `mu` (μ) value required for ML-DSA signatures
+/// according to FIPS 204.
+#[cfg(ossl_v350)]
+pub struct MuGenerator {
+    shake_mu: crate::digest::OsslDigest,
+    tr: Vec<u8>,
+    context: Option<Vec<u8>>,
+}
+
+#[cfg(ossl_v350)]
+impl MuGenerator {
+    pub fn new(
+        libctx: &OsslContext,
+        pkey: &EvpPkey,
+        context: Option<&[u8]>,
+    ) -> Result<Self, Error> {
+        if let Some(c) = context {
+            if c.len() > 255 {
+                return Err(Error::new(ErrorKind::BadArg));
+            }
+        }
+
+        let pkey_data = pkey.export_public()?;
+        let pk_bytes = match pkey_data {
+            crate::pkey::PkeyData::Mlkey(mlk) => {
+                mlk.pubkey.clone().ok_or(Error::new(ErrorKind::NullPtr))?
+            }
+            _ => return Err(Error::new(ErrorKind::WrapperError)),
+        };
+
+        let mut tr = vec![0u8; 64];
+        let mut shake_tr =
+            crate::digest::OsslDigest::new(libctx, DigestAlg::Shake256, None)?;
+        shake_tr.update(&pk_bytes)?;
+        shake_tr.set_xoflen(64)?;
+        shake_tr.finalize(&mut tr)?;
+
+        let mut mugen = MuGenerator {
+            shake_mu: crate::digest::OsslDigest::new(
+                libctx,
+                DigestAlg::Shake256,
+                None,
+            )?,
+            tr,
+            context: context.map(|c| c.to_vec()),
+        };
+        mugen._init()?;
+        Ok(mugen)
+    }
+
+    fn _init(&mut self) -> Result<(), Error> {
+        self.shake_mu.update(&self.tr)?;
+
+        let ctx_len = match &self.context {
+            Some(c) => c.len() as u8,
+            None => 0,
+        };
+
+        let mut m_prime = Vec::with_capacity(2 + ctx_len as usize);
+        m_prime.push(0x00);
+        m_prime.push(ctx_len);
+        if let Some(c) = &self.context {
+            m_prime.extend_from_slice(c);
+        }
+        self.shake_mu.update(&m_prime)?;
+
+        Ok(())
+    }
+
+    pub fn reinit(&mut self) -> Result<(), Error> {
+        self.shake_mu.reset(None)?;
+        self._init()
+    }
+
+    pub fn update(&mut self, data: &[u8]) -> Result<(), Error> {
+        self.shake_mu.update(data)
+    }
+
+    pub fn finalize(
+        &mut self,
+        data: Option<&[u8]>,
+        out_mu: &mut [u8],
+    ) -> Result<(), Error> {
+        if out_mu.len() != 64 {
+            return Err(Error::new(ErrorKind::BadArg));
+        }
+        if let Some(d) = data {
+            self.shake_mu.update(d)?;
+        }
+        self.shake_mu.set_xoflen(64)?;
+        self.shake_mu.finalize(out_mu)?;
+        Ok(())
+    }
+}
+
+/// Helper to generate OsslParam arrays for Mldsa initialization with mu.
+/// When this helper is used, the signature will only be one shot and only a
+/// properly calculated MU buffer (which must be exactly 64 bytes in length)
+/// can be used as data for the sign()/verify() calls.
+#[cfg(ossl_v350)]
+pub fn mldsa_with_mu<'a>(
+    context: Option<&'a Vec<u8>>,
+    deterministic: bool,
+) -> Result<Option<OsslParam<'a>>, Error> {
+    let mut params_builder = OsslParamBuilder::with_capacity(3);
+    params_builder.add_owned_int(cstr!(OSSL_SIGNATURE_PARAM_MU), 1)?;
+    if let Some(ctx) = context {
+        params_builder.add_octet_string(
+            cstr!(OSSL_SIGNATURE_PARAM_CONTEXT_STRING),
+            ctx,
+        )?;
+    }
+    if deterministic {
+        params_builder
+            .add_owned_int(cstr!(OSSL_SIGNATURE_PARAM_DETERMINISTIC), 1)?;
+    }
+    let params = params_builder.finalize();
+    return Ok(Some(params));
+}
+
 /// Helper to generate OsslParam arrays for SLH-DSA initialization
 /// FIXME: The same as mldsa_params?
 #[cfg(ossl_v350)]