Added SoundFonts support and sustain pedal button state

ImAxel0 · ImAxel0 · commit 6d728add3bd7 · 2024-11-18T17:47:17.000+01:00
diff --git a/Openthesia/Drawings.cs b/Openthesia/Drawings.cs
@@ -9,6 +9,8 @@ public class Drawings
     public static IntPtr C;
     public static IntPtr CSharp;
     public static IntPtr CSharpWhite;
+    public static IntPtr SustainPedalOff;
+    public static IntPtr SustainPedalOn;
 
     public static void RenderMatrixBackground()
     {
diff --git a/Openthesia/IOHandle.cs b/Openthesia/IOHandle.cs
@@ -1,5 +1,4 @@
-﻿using Melanchall.DryWetMidi.Common;
-using Melanchall.DryWetMidi.Core;
+﻿using Melanchall.DryWetMidi.Core;
 using Melanchall.DryWetMidi.Multimedia;
 
 namespace Openthesia;
@@ -9,6 +8,12 @@ public class IOHandle
     public static List<int> PressedKeys { get; private set; } = new();
 
     public static List<NoteRect> NoteRects = new();
+
+    private static HashSet<int> _sustainedNotes = new(); // Keeps track of sustained notes
+
+    private static bool _sustainPedalActive = false;
+    public static bool SustainPedalActive => _sustainPedalActive;
+
     public struct NoteRect
     {
         public int KeyNum;
@@ -22,6 +27,13 @@ public struct NoteRect
 
     private static void OnKeyPress(NoteOnEvent ev)
     {
+        // Check if sustain pedal is active
+        if (_sustainPedalActive)
+        {
+            // add to sustained notes
+            _sustainedNotes.Add(ev.NoteNumber);
+        }
+
         if (Router.Route == Router.Routes.PlayMode)
         {
             bool isBlack = ev.GetNoteName().ToString().EndsWith("Sharp");
@@ -36,26 +48,55 @@ private static void OnKeyPress(NoteOnEvent ev)
             };
             NoteRects.Add(note);
         }
-
+        //MidiPlayer.RealTimeSoundFontPlayer.Synthesizer.ProcessMidiMessage(0, 144 /*NOTE ON*/, ev.NoteNumber, ev.Velocity);
+        MidiPlayer.SoundFontEngine?.PlayNote(0, ev.NoteNumber, ev.Velocity);
         PressedKeys.Add(ev.NoteNumber);
     }
 
     private static void OnKeyRelease(NoteOffEvent ev)
     {
+        if (_sustainPedalActive)
+        {
+            // If sustain pedal is active, don't stop the note immediately
+            _sustainedNotes.Add(ev.NoteNumber);
+        }
+        else
+        {
+            // If sustain pedal is not active, stop the note immediately
+            //MidiPlayer.RealTimeSoundFontPlayer.Synthesizer.ProcessMidiMessage(0, 128, ev.NoteNumber, ev.Velocity);
+            MidiPlayer.SoundFontEngine?.StopNote(0, ev.NoteNumber);
+        }
+
         if (Router.Route == Router.Routes.PlayMode)
         {
             int index = NoteRects.FindIndex(x => x.KeyNum == ev.NoteNumber && !x.WasReleased);
             var n = NoteRects[index];
             //var n = NoteRects.Find(x => x.KeyNum == ev.NoteNumber && !x.WasReleased);
             //var n = NoteRects[NoteRects.Count - 1];
             n.WasReleased = true;
-            n.FinalTime = n.Time;
+            n.FinalTime = n.Time;         
             NoteRects[index] = n;
         }
-
+        
         PressedKeys.Remove(ev.NoteNumber);
     }
 
+    private static void OnSustainPedalOn()
+    {
+        _sustainPedalActive = true;
+    }
+
+    private static void OnSustainPedalOff()
+    {
+        _sustainPedalActive = false;
+        // Stop all sustained notes when the sustain pedal is released
+        foreach (var note in _sustainedNotes)
+        {
+            MidiPlayer.SoundFontEngine?.StopNote(0, note);
+        }
+        _sustainedNotes.Clear();
+    }
+
     public static void OnEventReceived(object sender, MidiEventReceivedEventArgs e)
     {
         var eType = e.Event.EventType;
@@ -68,6 +109,20 @@ public static void OnEventReceived(object sender, MidiEventReceivedEventArgs e)
             case MidiEventType.NoteOff:
                 OnKeyRelease((NoteOffEvent)e.Event);
                 break;
+            case MidiEventType.ControlChange:
+                var controlChangeEvent = (ControlChangeEvent)e.Event;
+                if (controlChangeEvent.ControlNumber == 64) // 64 is the sustain pedal
+                {
+                    if (controlChangeEvent.ControlValue > 63)  // Sustain pedal ON (value greater than 63)
+                    {
+                        OnSustainPedalOn();
+                    }
+                    else  // Sustain pedal OFF (value <= 63)
+                    {
+                        OnSustainPedalOff();
+                    }
+                }
+                break;
         }
     }
 
@@ -87,6 +142,20 @@ public static void OnEventReceived(object sender, MidiEventPlayedEventArgs e)
             case MidiEventType.NoteOff:
                 OnKeyRelease((NoteOffEvent)e.Event);
                 break;
+            case MidiEventType.ControlChange:
+                var controlChangeEvent = (ControlChangeEvent)e.Event;
+                if (controlChangeEvent.ControlNumber == 64) // 64 is the sustain pedal
+                {
+                    if (controlChangeEvent.ControlValue > 63)  // Sustain pedal ON (value greater than 63)
+                    {
+                        OnSustainPedalOn();
+                    }
+                    else  // Sustain pedal OFF (value <= 63)
+                    {
+                        OnSustainPedalOff();
+                    }
+                }
+                break;
         }
     }
 
diff --git a/Openthesia/ImGuiController.cs b/Openthesia/ImGuiController.cs
@@ -174,6 +174,26 @@ public static void LoadImages(GraphicsDevice _gd, ImGuiController _controller)
         var img4 = new ImageSharpTexture(stream4);
         var dimg4 = img4.CreateDeviceTexture(_gd, _gd.ResourceFactory);
         Drawings.CSharpWhite = _controller.GetOrCreateImGuiBinding(_gd.ResourceFactory, dimg4);
+
+        // Sustain pedal off
+        TryGetEmbeddedResourceBytes("SustainPedalOff", out var sustainPedalOff);
+        Stream stream5 = new MemoryStream();
+        stream5.Write(sustainPedalOff);
+        stream5.Position = 0;
+
+        var img5 = new ImageSharpTexture(stream5);
+        var dimg5 = img5.CreateDeviceTexture(_gd, _gd.ResourceFactory);
+        Drawings.SustainPedalOff = _controller.GetOrCreateImGuiBinding(_gd.ResourceFactory, dimg5);
+
+        // Sustain pedal on
+        TryGetEmbeddedResourceBytes("SustainPedalOn", out var sustainPedalOn);
+        Stream stream6 = new MemoryStream();
+        stream6.Write(sustainPedalOn);
+        stream6.Position = 0;
+
+        var img6 = new ImageSharpTexture(stream6);
+        var dimg6 = img6.CreateDeviceTexture(_gd, _gd.ResourceFactory);
+        Drawings.SustainPedalOn = _controller.GetOrCreateImGuiBinding(_gd.ResourceFactory, dimg6);
     }
 
     public void WindowResized(int width, int height)
diff --git a/Openthesia/LeftRightData.cs b/Openthesia/LeftRightData.cs
@@ -1,10 +1,4 @@
-﻿using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using System.Xml.Serialization;
-using Vanara.PInvoke;
+﻿using System.Xml.Serialization;
 
 namespace Openthesia;
 
diff --git a/Openthesia/MidiEditing.cs b/Openthesia/MidiEditing.cs
@@ -1,11 +1,4 @@
-﻿using Melanchall.DryWetMidi.Interaction;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Text.Json.Nodes;
-using System.Threading.Tasks;
-using System.Xml.Serialization;
+﻿using System.Xml.Serialization;
 using Vanara.PInvoke;
 
 namespace Openthesia;
diff --git a/Openthesia/MidiFileView.cs b/Openthesia/MidiFileView.cs
@@ -1,13 +1,6 @@
 ﻿using IconFonts;
 using ImGuiNET;
-using Melanchall.DryWetMidi.Interaction;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Numerics;
-using System.Text;
-using System.Threading.Tasks;
-using Vulkan.Xlib;
 
 namespace Openthesia;
 
diff --git a/Openthesia/MidiList.cs b/Openthesia/MidiList.cs
@@ -1,7 +1,5 @@
 ﻿using IconFonts;
 using ImGuiNET;
-using Melanchall.DryWetMidi.Core;
-using Melanchall.DryWetMidi.Interaction;
 using System.Numerics;
 
 namespace Openthesia;
diff --git a/Openthesia/MidiPlayer.cs b/Openthesia/MidiPlayer.cs
@@ -5,6 +5,7 @@ namespace Openthesia;
 
 public class MidiPlayer
 {
+    public static SoundFontPlayer SoundFontEngine;
     public static Playback Playback;
     public static MetricTimeSpan Time;
     public static float Seconds;
diff --git a/Openthesia/MidiSampleProvider.cs b/Openthesia/MidiSampleProvider.cs
@@ -0,0 +1,22 @@
+﻿using MeltySynth;
+using NAudio.Wave;
+
+namespace Openthesia;
+
+public class MidiSampleProvider : ISampleProvider
+{
+    private static WaveFormat format = WaveFormat.CreateIeeeFloatWaveFormat(44100, 2);
+    private Synthesizer _synthesizer;
+
+    public MidiSampleProvider(Synthesizer synthesizer)
+    {
+        _synthesizer = synthesizer;
+    }
+
+    public int Read(float[] buffer, int offset, int count)
+    {
+        _synthesizer.RenderInterleaved(buffer.AsSpan(offset, count));
+        return count;
+    }
+    public WaveFormat WaveFormat => format;
+}
diff --git a/Openthesia/Openthesia.csproj b/Openthesia/Openthesia.csproj
@@ -18,6 +18,8 @@
     <None Remove="Resources\imgui-vertex.hlsl.bytes" />
     <None Remove="Resources\Keys\wsharp.png" />
     <None Remove="Resources\logoimg.png" />
+    <None Remove="Resources\SustainPedalOff.png" />
+    <None Remove="Resources\SustainPedalOn.png" />
   </ItemGroup>
 
   <ItemGroup>
@@ -33,11 +35,15 @@
     <EmbeddedResource Include="Resources\imgui-vertex.hlsl.bytes" />
     <EmbeddedResource Include="Resources\Keys\wsharp.png" />
     <EmbeddedResource Include="Resources\logoimg.png" />
+    <EmbeddedResource Include="Resources\SustainPedalOff.png" />
+    <EmbeddedResource Include="Resources\SustainPedalOn.png" />
   </ItemGroup>
 
   <ItemGroup>
     <PackageReference Include="ImGui.NET" Version="1.90.1.1" />
     <PackageReference Include="Melanchall.DryWetMidi" Version="7.1.0" />
+    <PackageReference Include="MeltySynth" Version="2.4.1" />
+    <PackageReference Include="NAudio" Version="2.2.1" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="Syroot.Windows.IO.KnownFolders" Version="1.3.0" />
     <PackageReference Include="Vanara.PInvoke.User32" Version="3.4.17" />
diff --git a/Openthesia/PianoRenderer.cs b/Openthesia/PianoRenderer.cs
@@ -179,6 +179,7 @@ public static void RenderKeyboard()
                 cur_key++;
             }
         }
+
         ImGui.PopFont();
     }
 }
diff --git a/Openthesia/Program.cs b/Openthesia/Program.cs
@@ -5,7 +5,6 @@
 using System.Numerics;
 using ImGuiNET;
 using Vanara.PInvoke;
-using Newtonsoft.Json;
 
 namespace Openthesia;
 
@@ -18,6 +17,7 @@ class Program
     private static ImGuiController _controller;
     private static Vector3 _clearColor = new(0.45f, 0.55f, 0.6f);
 
+    [STAThread]
     static void Main(string[] args)
     {
         User32.SetProcessDPIAware();
diff --git a/Openthesia/ProgramData.cs b/Openthesia/ProgramData.cs
@@ -8,7 +8,7 @@ namespace Openthesia;
 
 public class ProgramData
 {
-    public static readonly string ProgramVersion = "1.2.3";
+    public static readonly string ProgramVersion = "1.3.0";
     public static IntPtr LogoImage;
     public static string SettingsPath = Path.Combine(KnownFolders.RoamingAppData.Path, "Openthesia", "Settings.json");
     public static string HandsDataPath = Path.Combine(KnownFolders.RoamingAppData.Path, "Openthesia\\HandsData");
@@ -32,6 +32,8 @@ private class SettingsJson
         public bool UpDirection;
         public bool ShowTextNotes;
         public ScreenCanvas.TextTypes TextType;
+        public bool SoundFontEngine;
+        public int SoundFontLatency;
     }
 
     public static void Initialize()
@@ -48,6 +50,11 @@ public static void Initialize()
             Settings.SetOutputDevice(0);
         }
         ImGuiTheme.PushTheme();
+
+        if (Settings.SoundFontEngine)
+        {
+            SoundFontPlayer.Initialize();
+        }   
     }
 
     public static void LoadSettings()
@@ -93,6 +100,8 @@ public static void LoadSettings()
                 ScreenCanvas.SetUpDirection(storedSettings.UpDirection);
                 ScreenCanvas.SetTextNotes(storedSettings.ShowTextNotes);
                 ScreenCanvas.TextType = storedSettings.TextType;
+                Settings.SetSoundFontEngine(storedSettings.SoundFontEngine);
+                Settings.SetSoundFontLatency(storedSettings.SoundFontLatency < 15 ? Settings.SoundFontLatency : storedSettings.SoundFontLatency);
             }
             catch (Exception ex)
             {
@@ -130,6 +139,8 @@ public static void SaveSettings()
             UpDirection = ScreenCanvas.UpDirection,
             ShowTextNotes = ScreenCanvas.ShowTextNotes,
             TextType = ScreenCanvas.TextType,
+            SoundFontEngine = Settings.SoundFontEngine,
+            SoundFontLatency = Settings.SoundFontLatency,
         };
 
         string json = JsonConvert.SerializeObject(data, settings);
diff --git a/Openthesia/Resources/SustainPedalOff.png b/Openthesia/Resources/SustainPedalOff.png
diff --git a/Openthesia/Resources/SustainPedalOn.png b/Openthesia/Resources/SustainPedalOn.png
diff --git a/Openthesia/ScreenCanvas.cs b/Openthesia/ScreenCanvas.cs
diff --git a/Openthesia/Settings.cs b/Openthesia/Settings.cs
diff --git a/Openthesia/SoundFontPlayer.cs b/Openthesia/SoundFontPlayer.cs

Original file line number	Diff line number	Diff line change
`@@ -9,6 +9,8 @@ public class Drawings`
`9`	`9`	`public static IntPtr C;`
`10`	`10`	`public static IntPtr CSharp;`
`11`	`11`	`public static IntPtr CSharpWhite;`
	`12`	`+ public static IntPtr SustainPedalOff;`
	`13`	`+ public static IntPtr SustainPedalOn;`
`12`	`14`
`13`	`15`	`public static void RenderMatrixBackground()`
`14`	`16`	`{`
Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@ namespace Openthesia;`
`5`	`5`
`6`	`6`	`public class MidiPlayer`
`7`	`7`	`{`
	`8`	`+ public static SoundFontPlayer SoundFontEngine;`
`8`	`9`	`public static Playback Playback;`
`9`	`10`	`public static MetricTimeSpan Time;`
`10`	`11`	`public static float Seconds;`