Fix VFS signature verification, add v3 cert derivation

Author: LukeFZ

LibConeshell/ConeshellV2.cs

@@ -279,18 +279,18 @@ private byte[] DecryptMessageInternal(byte[] encryptedBody, byte[] key, byte[] i
         0x1C3AF78Eu
     };
 
-    public virtual byte[] DecryptVfs(byte[] dbData, bool skipVerification = true)
+    public virtual byte[] DecryptVfs(byte[] dbData, bool skipVerification = false)
     {
         var inputStream = new MemoryStream(dbData);
         var inputReader = new BinaryReader(inputStream);
 
         if (inputReader.ReadUInt32() != VfsHeaderMagic)
             throw new InvalidDataException("Invalid database header.");
 
-        return DecryptVfsInternal(dbData, inputReader, skipVerification);
+        return DecryptVfsInternal(dbData, inputReader, skipVerification, !skipVerification ? DeriveVfsPublicKey(VfsCertConstants) : "");
     }
 
-    protected byte[] DecryptVfsInternal(byte[] dbData, BinaryReader inputReader, bool skipVerification, int headerOffset = 0)
+    protected static byte[] DecryptVfsInternal(byte[] dbData, BinaryReader inputReader, bool skipVerification, string publicKey = "", int headerOffset = 0)
     {
         const int fullHeaderSize = 0x4 + 0x4 + 0x10 + 0x10 + 0x4 + 0x10 + 0x100;
         var headerSize = fullHeaderSize - headerOffset;
@@ -303,27 +303,10 @@ protected byte[] DecryptVfsInternal(byte[] dbData, BinaryReader inputReader, boo
         var gcmIv = inputReader.ReadBytes(0x10);
         var gcmAdd2 = inputReader.ReadUInt32();
         var gcmTag = inputReader.ReadBytes(0x10);
-        var verifySig = inputReader.ReadBytes(0x100);
+        var signature = inputReader.ReadBytes(0x100);
 
         var gcmAdd = BitConverter.GetBytes(gcmAdd1).Concat(BitConverter.GetBytes(gcmAdd2)).ToArray();
 
-        if (!skipVerification)
-        {
-            var rsaEncPublicKey = Encoding.UTF8.GetString(VfsDerivePublicKey()[..^1]);
-            using var rsaPublicKeyStringReader = new StringReader(rsaEncPublicKey);
-            var rsaPublicKeyPemReader = new PemReader(rsaPublicKeyStringReader);
-            if (rsaPublicKeyPemReader.ReadObject() is not RsaKeyParameters rsaPublicKey)
-                throw new InvalidDataException("Failed to parse derived VFS public key.");
-
-            var signer = new RsaDigestSigner(new Sha1Digest());
-            signer.Init(false, rsaPublicKey);
-            signer.BlockUpdate(gcmTag, 0, gcmTag.Length);
-            var sigResult = signer.VerifySignature(verifySig);
-
-            if (!sigResult)
-                throw new InvalidDataException("Failed to verify VFS signature.");
-        }
-
         var encryptedLength = dbData.Length - headerSize;
         var encryptedData = new byte[encryptedLength + gcmTag.Length];
         if (inputReader.Read(encryptedData, 0, encryptedLength) != encryptedLength)
@@ -333,6 +316,17 @@ protected byte[] DecryptVfsInternal(byte[] dbData, BinaryReader inputReader, boo
 
         inputReader.Dispose();
 
+        if (!skipVerification)
+        {
+            var signedData = gcmTag.Concat(BitConverter.GetBytes(encryptedLength - 4)).ToArray();
+
+            var rsa = RSA.Create();
+            rsa.ImportFromPem(publicKey);
+            var sigResult = rsa.VerifyHash(signedData, signature, HashAlgorithmName.SHA1, RSASignaturePadding.Pkcs1);
+            if (!sigResult)
+                throw new InvalidDataException("Failed to verify VFS signature.");
+        }
+
         var gcm = new GcmBlockCipher(new AesEngine());
         gcm.Init(false, new AeadParameters(new KeyParameter(gcmKey), gcmTag.Length * 8, gcmIv, gcmAdd));
 
@@ -356,12 +350,9 @@ protected byte[] DecryptVfsInternal(byte[] dbData, BinaryReader inputReader, boo
             : bodyData;
     }
 
-    private byte[] VfsDerivePublicKey()
+    protected static string DeriveVfsPublicKey(uint[] encCert, ulong seed = 0x8BE53A46A921AF07, ulong add = 0x31D7038E3C2AB8B)
     {
-        const ulong roundConstant1 = 0x5851F42D4C957F2D;
-        const ulong roundConstant2 = 0x31D7038E3C2AB8B;
-
-        var round = 0x8BE53A46A921AF07;
+        var round = seed;
         var result = new byte[0x1c4];
         using var outputStream = new MemoryStream(result);
         using var outputWriter = new BinaryWriter(outputStream);
@@ -370,15 +361,15 @@ private byte[] VfsDerivePublicKey()
         {
             for (int i = 0; i < 0x1c4 / 4; i++)
             {
-                var rk = VfsCertConstants[i];
+                var rk = encCert[i];
                 var rg = (round ^ (round >> 18)) >> 27;
                 var rv = BitOperations.RotateRight((uint)rg, (int)(round >> 59)) ^ rk;
                 outputWriter.Write(rv);
                 round = TransformConstant * round + add;
             }
         }
 
-        return result;
+        return Encoding.UTF8.GetString(result[..^1]);
     }
 
     #endregion

LibConeshell/ConeshellV3.cs

@@ -95,8 +95,14 @@ protected override byte[] DeriveDeviceSecret(byte[] sharedSecret)
 
     protected override uint VfsHeaderMagic => 0x03007ADA;
 
+    private static ulong ReadBigEndianULong(BinaryReader reader)
+        => BinaryPrimitives.ReadUInt64BigEndian(reader.ReadBytes(8));
+
     public override byte[] DecryptVfs(byte[] dbData, bool skipVerification = false /* TODO: Not implemented */)
     {
+        if (!skipVerification && _vfsCert == null)
+            throw new ArgumentException("You need to supply a VFS certificate for signature verification.", nameof(skipVerification));
+
         var inputStream = new MemoryStream(dbData);
         var inputReader = new BinaryReader(inputStream);
 
@@ -110,16 +116,34 @@ public override byte[] DecryptVfs(byte[] dbData, bool skipVerification = false /
         inputStream = new MemoryStream(processedData);
         inputReader = new BinaryReader(inputStream);
 
-        return DecryptVfsInternal(processedData, inputReader, true, 4);
+        var publicKey = "";
+        if (!skipVerification)
+        {
+            using var encCertStream = new MemoryStream(_vfsCert!);
+            using var encCertReader = new BinaryReader(encCertStream);
+            encCertReader.BaseStream.Position += 4;
+
+            var transformConstant1 = ReadBigEndianULong(encCertReader);
+            var transformConstant2 = ReadBigEndianULong(encCertReader);
+
+            var transformMixed1 = (2 * transformConstant2) | 1;
+            var transformMixed2 = transformMixed1 + TransformConstant * (transformMixed1 + transformConstant1);
+
+            var encCert = new uint[0x1c4 / 4];
+            for (int i = 0; i < 0x1c4 / 4; i++)
+                encCert[i] = encCertReader.ReadUInt32();
+
+            publicKey = DeriveVfsPublicKey(encCert, transformMixed2, transformMixed1);
+
+        }
+
+        return DecryptVfsInternal(processedData, inputReader, skipVerification, publicKey, 4);
     }
 
     private static byte[] PreprocessVfs(BinaryReader dbReader, int remainingLength)
     {
         const int transformLength = 16;
 
-        ulong ReadBigEndianULong(BinaryReader reader)
-            => BinaryPrimitives.ReadUInt64BigEndian(reader.ReadBytes(8));
-
         unchecked
         {
             var transformInputConstant1 = ReadBigEndianULong(dbReader);