Fix deterministic MVID and add PdbChecksum (#810)

* Fix deterministic MVID and add PdbChecksum (#31)

* Fix how pdb path is calculated in the tests

* Fix portable PDB stamp in CodeView header (#32)

* Introduce ISymbolWriter.Write

This mostly cleans up the code to make it easier to understand. `ISymbolWriter.GetDebugHeader` no longer actually writes the symbols, there's a new `Write` method for just that.

The assembly writer calls `Write` first and then the image writer calls `GetDebugHeader` when it's needed.

This is partially taken from #617.

Author: vitek-karas

Mono.Cecil.Cil/PortablePdb.cs

@@ -12,7 +12,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.IO.Compression;
-
+using System.Security.Cryptography;
 using Mono.Cecil.Metadata;
 using Mono.Cecil.PE;
 
@@ -39,7 +39,7 @@ public ISymbolReader GetSymbolReader (ModuleDefinition module, Stream symbolStre
 
 		ISymbolReader GetSymbolReader (ModuleDefinition module, Disposable<Stream> symbolStream, string fileName)
 		{
-			return new PortablePdbReader (ImageReader.ReadPortablePdb (symbolStream, fileName), module);
+			return new PortablePdbReader (ImageReader.ReadPortablePdb (symbolStream, fileName, out _), module);
 		}
 	}
 
@@ -234,24 +234,27 @@ public ISymbolWriter GetSymbolWriter (ModuleDefinition module, string fileName)
 			Mixin.CheckModule (module);
 			Mixin.CheckFileName (fileName);
 
-			var file = File.OpenWrite (Mixin.GetPdbFileName (fileName));
-			return GetSymbolWriter (module, Disposable.Owned (file as Stream));
+			var file = File.Open (Mixin.GetPdbFileName (fileName), FileMode.OpenOrCreate, FileAccess.ReadWrite);
+			return GetSymbolWriter (module, Disposable.Owned (file as Stream), Disposable.NotOwned ((Stream)null));
 		}
 
 		public ISymbolWriter GetSymbolWriter (ModuleDefinition module, Stream symbolStream)
 		{
 			Mixin.CheckModule (module);
 			Mixin.CheckStream (symbolStream);
 
-			return GetSymbolWriter (module, Disposable.NotOwned (symbolStream));
+			// In order to compute the PDB checksum, the stream we're writing to needs to be able to
+			// seek and read as well. We can't assume this about a stream provided by the user.
+			// So in this case, create a memory stream to cache the PDB.
+			return GetSymbolWriter (module, Disposable.Owned (new MemoryStream() as Stream), Disposable.NotOwned (symbolStream));
 		}
 
-		ISymbolWriter GetSymbolWriter (ModuleDefinition module, Disposable<Stream> stream)
+		ISymbolWriter GetSymbolWriter (ModuleDefinition module, Disposable<Stream> stream, Disposable<Stream> final_stream)
 		{
 			var metadata = new MetadataBuilder (module, this);
 			var writer = ImageWriter.CreateDebugWriter (module, metadata, stream);
 
-			return new PortablePdbWriter (metadata, module, writer);
+			return new PortablePdbWriter (metadata, module, writer, final_stream);
 		}
 	}
 
@@ -260,9 +263,14 @@ public sealed class PortablePdbWriter : ISymbolWriter {
 		readonly MetadataBuilder pdb_metadata;
 		readonly ModuleDefinition module;
 		readonly ImageWriter writer;
+		readonly Disposable<Stream> final_stream;
 
 		MetadataBuilder module_metadata;
 
+		internal byte [] pdb_checksum;
+		internal Guid pdb_id_guid;
+		internal uint pdb_id_stamp;
+
 		bool IsEmbedded { get { return writer == null; } }
 
 		internal PortablePdbWriter (MetadataBuilder pdb_metadata, ModuleDefinition module)
@@ -278,56 +286,100 @@ internal PortablePdbWriter (MetadataBuilder pdb_metadata, ModuleDefinition modul
 			pdb_metadata.AddCustomDebugInformations (module);
 		}
 
-		internal PortablePdbWriter (MetadataBuilder pdb_metadata, ModuleDefinition module, ImageWriter writer)
+		internal PortablePdbWriter (MetadataBuilder pdb_metadata, ModuleDefinition module, ImageWriter writer, Disposable<Stream> final_stream)
 			: this (pdb_metadata, module)
 		{
 			this.writer = writer;
+			this.final_stream = final_stream;
 		}
 
 		public ISymbolReaderProvider GetReaderProvider ()
 		{
 			return new PortablePdbReaderProvider ();
 		}
 
+		public void Write (MethodDebugInformation info)
+		{
+			CheckMethodDebugInformationTable ();
+
+			pdb_metadata.AddMethodDebugInformation (info);
+		}
+
+		public void Write ()
+		{
+			if (IsEmbedded)
+				return;
+
+			WritePdbFile ();
+
+			if (final_stream.value != null) {
+				writer.BaseStream.Seek (0, SeekOrigin.Begin);
+				var buffer = new byte [8192];
+				CryptoService.CopyStreamChunk (writer.BaseStream, final_stream.value, buffer, (int)writer.BaseStream.Length);
+			}
+		}
+
 		public ImageDebugHeader GetDebugHeader ()
 		{
 			if (IsEmbedded)
 				return new ImageDebugHeader ();
 
-			var directory = new ImageDebugDirectory () {
-				MajorVersion = 256,
-				MinorVersion = 20557,
-				Type = ImageDebugType.CodeView,
-				TimeDateStamp = (int) module.timestamp,
-			};
-
-			var buffer = new ByteBuffer ();
-			// RSDS
-			buffer.WriteUInt32 (0x53445352);
-			// Module ID
-			buffer.WriteBytes (module.Mvid.ToByteArray ());
-			// PDB Age
-			buffer.WriteUInt32 (1);
-			// PDB Path
-			var fileName = writer.BaseStream.GetFileName ();
-			if (string.IsNullOrEmpty (fileName)) {
-				fileName = module.Assembly.Name.Name + ".pdb";
+			ImageDebugHeaderEntry codeViewEntry;
+			{
+				var codeViewDirectory = new ImageDebugDirectory () {
+					MajorVersion = 256,
+					MinorVersion = 20557,
+					Type = ImageDebugType.CodeView,
+					TimeDateStamp = (int)pdb_id_stamp,
+				};
+
+				var buffer = new ByteBuffer ();
+				// RSDS
+				buffer.WriteUInt32 (0x53445352);
+				// Module ID
+				buffer.WriteBytes (pdb_id_guid.ToByteArray ());
+				// PDB Age
+				buffer.WriteUInt32 (1);
+				// PDB Path
+				var fileName = writer.BaseStream.GetFileName ();
+				if (string.IsNullOrEmpty (fileName)) {
+					fileName = module.Assembly.Name.Name + ".pdb";
+				}
+				buffer.WriteBytes (System.Text.Encoding.UTF8.GetBytes (fileName));
+				buffer.WriteByte (0);
+
+				var data = new byte [buffer.length];
+				Buffer.BlockCopy (buffer.buffer, 0, data, 0, buffer.length);
+				codeViewDirectory.SizeOfData = data.Length;
+
+				codeViewEntry = new ImageDebugHeaderEntry (codeViewDirectory, data);
 			}
-			buffer.WriteBytes (System.Text.Encoding.UTF8.GetBytes (fileName));
-			buffer.WriteByte (0);
 
-			var data = new byte [buffer.length];
-			Buffer.BlockCopy (buffer.buffer, 0, data, 0, buffer.length);
-			directory.SizeOfData = data.Length;
+			ImageDebugHeaderEntry pdbChecksumEntry;
+			{
+				var pdbChecksumDirectory = new ImageDebugDirectory () {
+					MajorVersion = 1,
+					MinorVersion = 0,
+					Type = ImageDebugType.PdbChecksum,
+					TimeDateStamp = 0
+				};
 
-			return new ImageDebugHeader (new ImageDebugHeaderEntry (directory, data));
-		}
+				var buffer = new ByteBuffer ();
+				// SHA256 - Algorithm name
+				buffer.WriteBytes (System.Text.Encoding.UTF8.GetBytes ("SHA256"));
+				buffer.WriteByte (0);
 
-		public void Write (MethodDebugInformation info)
-		{
-			CheckMethodDebugInformationTable ();
+				// Checksum - 32 bytes
+				buffer.WriteBytes (pdb_checksum);
 
-			pdb_metadata.AddMethodDebugInformation (info);
+				var data = new byte [buffer.length];
+				Buffer.BlockCopy (buffer.buffer, 0, data, 0, buffer.length);
+				pdbChecksumDirectory.SizeOfData = data.Length;
+
+				pdbChecksumEntry = new ImageDebugHeaderEntry (pdbChecksumDirectory, data);
+			}
+
+			return new ImageDebugHeader (new ImageDebugHeaderEntry [] { codeViewEntry, pdbChecksumEntry });
 		}
 
 		void CheckMethodDebugInformationTable ()
@@ -343,10 +395,8 @@ void CheckMethodDebugInformationTable ()
 
 		public void Dispose ()
 		{
-			if (IsEmbedded)
-				return;
-
-			WritePdbFile ();
+			writer.stream.Dispose ();
+			final_stream.Dispose ();
 		}
 
 		void WritePdbFile ()
@@ -360,15 +410,18 @@ void WritePdbFile ()
 			writer.WriteMetadata ();
 
 			writer.Flush ();
-			writer.stream.Dispose ();
+
+			ComputeChecksumAndPdbId ();
+
+			WritePdbId ();
 		}
 
 		void WritePdbHeap ()
 		{
 			var pdb_heap = pdb_metadata.pdb_heap;
 
-			pdb_heap.WriteBytes (module.Mvid.ToByteArray ());
-			pdb_heap.WriteUInt32 (module_metadata.timestamp);
+			// PDB ID ( GUID + TimeStamp ) are left zeroed out for now. Will be filled at the end with a hash.
+			pdb_heap.WriteBytes (20);
 
 			pdb_heap.WriteUInt32 (module_metadata.entry_point.ToUInt32 ());
 
@@ -399,6 +452,32 @@ void WriteTableHeap ()
 			pdb_metadata.table_heap.ComputeTableInformations ();
 			pdb_metadata.table_heap.WriteTableHeap ();
 		}
+
+		void ComputeChecksumAndPdbId ()
+		{
+			var buffer = new byte [8192];
+
+			// Compute the has of the entire file - PDB ID is zeroes still
+			writer.BaseStream.Seek (0, SeekOrigin.Begin);
+			var sha256 = SHA256.Create ();
+			using (var crypto_stream = new CryptoStream (Stream.Null, sha256, CryptoStreamMode.Write)) {
+				CryptoService.CopyStreamChunk (writer.BaseStream, crypto_stream, buffer, (int)writer.BaseStream.Length);
+			}
+
+			pdb_checksum = sha256.Hash;
+
+			var hashBytes = new ByteBuffer (pdb_checksum);
+			pdb_id_guid = new Guid (hashBytes.ReadBytes (16));
+			pdb_id_stamp = hashBytes.ReadUInt32 ();
+		}
+
+		void WritePdbId ()
+		{
+			// PDB ID is the first 20 bytes of the PdbHeap
+			writer.MoveToRVA (TextSegment.PdbHeap);
+			writer.WriteBytes (pdb_id_guid.ToByteArray ());
+			writer.WriteUInt32 (pdb_id_stamp);
+		}
 	}
 
 	public sealed class EmbeddedPortablePdbWriterProvider : ISymbolWriterProvider {
@@ -435,9 +514,14 @@ public ISymbolReaderProvider GetReaderProvider ()
 			return new EmbeddedPortablePdbReaderProvider ();
 		}
 
+		public void Write (MethodDebugInformation info)
+		{
+			writer.Write (info);
+		}
+
 		public ImageDebugHeader GetDebugHeader ()
 		{
-			writer.Dispose ();
+			ImageDebugHeader pdbDebugHeader = writer.GetDebugHeader ();
 
 			var directory = new ImageDebugDirectory {
 				Type = ImageDebugType.EmbeddedPortablePdb,
@@ -462,19 +546,22 @@ public ImageDebugHeader GetDebugHeader ()
 
 			directory.SizeOfData = (int) data.Length;
 
-			return new ImageDebugHeader (new [] {
-				writer.GetDebugHeader ().Entries [0],
-				new ImageDebugHeaderEntry (directory, data.ToArray ())
-			});
+			var debugHeaderEntries = new ImageDebugHeaderEntry [pdbDebugHeader.Entries.Length + 1];
+			for (int i = 0; i < pdbDebugHeader.Entries.Length; i++)
+				debugHeaderEntries [i] = pdbDebugHeader.Entries [i];
+			debugHeaderEntries [debugHeaderEntries.Length - 1] = new ImageDebugHeaderEntry (directory, data.ToArray ());
+
+			return new ImageDebugHeader (debugHeaderEntries);
 		}
 
-		public void Write (MethodDebugInformation info)
+		public void Write ()
 		{
-			writer.Write (info);
+			writer.Write ();
 		}
 
 		public void Dispose ()
 		{
+			writer.Dispose ();
 		}
 	}
 

Mono.Cecil.Cil/Symbols.cs

@@ -39,6 +39,7 @@ public enum ImageDebugType {
 		CodeView = 2,
 		Deterministic = 16,
 		EmbeddedPortablePdb = 17,
+		PdbChecksum = 19,
 	}
 
 	public sealed class ImageDebugHeader {
@@ -1114,6 +1115,7 @@ public interface ISymbolWriter : IDisposable {
 		ISymbolReaderProvider GetReaderProvider ();
 		ImageDebugHeader GetDebugHeader ();
 		void Write (MethodDebugInformation info);
+		void Write ();
 	}
 
 	public interface ISymbolWriterProvider {
@@ -1174,6 +1176,11 @@ public static ImageDebugHeaderEntry GetEmbeddedPortablePdbEntry (this ImageDebug
 			return GetEntry (header, ImageDebugType.EmbeddedPortablePdb);
 		}
 
+		public static ImageDebugHeaderEntry GetPdbChecksumEntry (this ImageDebugHeader header)
+		{
+			return GetEntry (header, ImageDebugType.PdbChecksum);
+		}
+
 		private static ImageDebugHeaderEntry GetEntry (this ImageDebugHeader header, ImageDebugType type)
 		{
 			if (!header.HasEntries)

Mono.Cecil.PE/ImageReader.cs

@@ -27,6 +27,7 @@ sealed class ImageReader : BinaryStreamReader {
 		DataDirectory metadata;
 
 		uint table_heap_offset;
+		uint pdb_heap_offset;
 
 		public ImageReader (Disposable<Stream> stream, string file_name)
 			: base (stream.value)
@@ -400,6 +401,7 @@ void ReadMetadataStream (Section section)
 				break;
 			case "#Pdb":
 				image.PdbHeap = new PdbHeap (data);
+				pdb_heap_offset = offset;
 				break;
 			}
 		}
@@ -768,7 +770,7 @@ public static Image ReadImage (Disposable<Stream> stream, string file_name)
 			}
 		}
 
-		public static Image ReadPortablePdb (Disposable<Stream> stream, string file_name)
+		public static Image ReadPortablePdb (Disposable<Stream> stream, string file_name, out uint pdb_heap_offset)
 		{
 			try {
 				var reader = new ImageReader (stream, file_name);
@@ -785,6 +787,7 @@ public static Image ReadPortablePdb (Disposable<Stream> stream, string file_name
 
 				reader.metadata = new DataDirectory (0, length);
 				reader.ReadMetadata ();
+				pdb_heap_offset = reader.pdb_heap_offset;
 				return reader.image;
 			} catch (EndOfStreamException e) {
 				throw new BadImageFormatException (stream.value.GetFileName (), e);