Support applying pitch from wave part (stakira#2020)

* Implement graceful interruption for transcribe audio

* Use `CancellationTokenSource` to address thread race

* Do UI cleanup in all paths

* Track a pending-terminate flag before `runOptions` is initialized

* Support applying pitch from wave part

* Handle slicing boundaries correctly, add string resources

Author: yqzhishen

OpenUtau.Core/Analysis/Rmvpe.cs

@@ -7,7 +7,6 @@
 using NAudio.Wave;
 using NAudio.Wave.SampleProviders;
 using OpenUtau.Core.Ustx;
-using OpenUtau.Core.Util;
 using Serilog;
 
 namespace OpenUtau.Core.Analysis;
@@ -124,11 +123,11 @@ static List<NoteSegment> BuildSegments(UProject project, UVoicePart part) {
             .ToList();
     }
 
-    public void ApplyToPart(UProject project, UVoicePart part) {
-        ApplyToPart(project, part, BuildSegments(project, part));
+    public void ApplyToPart(UProject project, UVoicePart part, double offsetMs = 0) {
+        ApplyToPart(project, part, BuildSegments(project, part), offsetMs);
     }
 
-    void ApplyToPart(UProject project, UVoicePart part, IReadOnlyList<NoteSegment> notes) {
+    void ApplyToPart(UProject project, UVoicePart part, IReadOnlyList<NoteSegment> notes, double offsetMs) {
         if (MidiPitch.Length == 0 || notes.Count == 0 || !project.expressions.TryGetValue(Format.Ustx.PITD, out var descriptor)) {
             Log.Information(
                 "RMVPE apply skipped. pitch={PitchCount} notes={NoteCount} hasPITD={HasPitd}",
@@ -146,7 +145,7 @@ void ApplyToPart(UProject project, UVoicePart part, IReadOnlyList<NoteSegment> n
         for (int i = 0; i < MidiPitch.Length; ++i) {
             var midiPitch = MidiPitch[i];
             var localTimeMs = i * frameMs;
-            var absoluteTimeMs = partStartMs + localTimeMs;
+            var absoluteTimeMs = partStartMs + localTimeMs + offsetMs;
             while (noteIndex + 1 < notes.Count && notes[noteIndex].onsetMs + notes[noteIndex].durationMs <= absoluteTimeMs) {
                 noteIndex++;
             }
@@ -183,8 +182,17 @@ void ApplyToPart(UProject project, UVoicePart part, IReadOnlyList<NoteSegment> n
         AppendSmoothedPoints(curve, pendingPoints);
         curve.Simplify();
         if (curve.xs.Count > 0) {
-            part.curves.RemoveAll(c => c.abbr == Format.Ustx.PITD);
-            part.curves.Add(curve);
+            var oldCurve = part.curves.FirstOrDefault(c => c.abbr == Format.Ustx.PITD);
+            var oldXs = oldCurve?.xs.ToArray();
+            var oldYs = oldCurve?.ys.ToArray();
+            DocManager.Inst.ExecuteCmd(new MergedSetCurveCommand(
+                project,
+                part,
+                Format.Ustx.PITD,
+                oldXs,
+                oldYs,
+                curve.xs.ToArray(),
+                curve.ys.ToArray()));
         }
         Log.Information("RMVPE applied pitch curve. points={PointCount}", curve.xs.Count);
     }
@@ -246,8 +254,24 @@ static string ResolveModelPath() {
             ?? Path.Combine(PathManager.Inst.DependencyPath, "rmvpe", "rmvpe.onnx");
     }
 
-    public RmvpeResult? Infer(UWavePart wavePart) {
-        var mono = ToMono(wavePart);
+    public RmvpeResult? Infer(UWavePart wavePart, double startMs = 0, double endMs = 0) {
+        int startSample = 0;
+        int endSample = wavePart.Samples.Length / wavePart.channels;
+        int totalSamples = endSample;
+        if (startMs > 0 || endMs > 0) {
+            if (startMs > 0) {
+                startSample = (int)(startMs * wavePart.sampleRate / 1000);
+            }
+            if (endMs > 0) {
+                endSample = (int)(endMs * wavePart.sampleRate / 1000);
+            }
+            startSample = Math.Clamp(startSample, 0, totalSamples);
+            endSample = Math.Clamp(endSample, startSample, totalSamples);
+        }
+        if (endSample <= startSample) {
+            return null;
+        }
+        var mono = ToMono(wavePart.Samples, startSample, endSample, wavePart.channels);
         var resampled = ResampleTo16k(mono, wavePart.sampleRate);
         var waveform = new DenseTensor<float>(new[] { 1, resampled.Length });
         for (int i = 0; i < resampled.Length; ++i) {
@@ -360,18 +384,18 @@ static string ResolveUvOutputName(InferenceSession session) {
             ?? throw new InvalidDataException("RMVPE model must expose a uv output.");
     }
 
-    static float[] ToMono(UWavePart wavePart) {
-        if (wavePart.channels == 1) {
-            return wavePart.Samples;
+    static float[] ToMono(float[] samples, int startSample, int endSample, int channels) {
+        if (channels == 1 && startSample == 0 && endSample == samples.Length) {
+            return samples;
         }
-        var mono = new float[wavePart.Samples.Length / wavePart.channels];
+        var mono = new float[endSample - startSample];
         for (int i = 0; i < mono.Length; ++i) {
             float sum = 0;
-            var offset = i * wavePart.channels;
-            for (int ch = 0; ch < wavePart.channels; ++ch) {
-                sum += wavePart.Samples[offset + ch];
+            var offset = (startSample + i) * channels;
+            for (int ch = 0; ch < channels; ++ch) {
+                sum += samples[offset + ch];
             }
-            mono[i] = sum / wavePart.channels;
+            mono[i] = sum / channels;
         }
         return mono;
     }

OpenUtau/Strings/Strings.axaml

@@ -63,6 +63,11 @@
   <system:String x:Key="context.part.split">Split part at playhead</system:String>
   <system:String x:Key="context.part.transcribe">Transcribe audio to create a note part</system:String>
   <system:String x:Key="context.part.transcribing">Transcribing</system:String>
+  <system:String x:Key="context.part.applypitch">Apply pitch from...</system:String>
+  <system:String x:Key="context.part.nopitchsource">No wave part available</system:String>
+  <system:String x:Key="context.part.extractingpitch">Extracting pitch</system:String>
+  <system:String x:Key="context.part.nopitchregion">There is no overlapping region between the source wave part and the target note part.</system:String>
+  <system:String x:Key="context.part.nopitchdetected">No pitch detected from the source wave part.</system:String>
   <system:String x:Key="context.pitch.easein">Ease in</system:String>
   <system:String x:Key="context.pitch.easeinout">Ease in/out</system:String>
   <system:String x:Key="context.pitch.easeout">Ease out</system:String>
@@ -126,7 +131,7 @@ Do you want to continue by splitting at the nearest position after current playh
   <system:String x:Key="dialogs.transcribe.game.notfound">GAME is not installed.</system:String>
   <system:String x:Key="dialogs.transcribe.rmvpe.notfound">RMVPE is not installed. Install it from the Package Manager.</system:String>
   <system:String x:Key="dialogs.transcribe.allnotfound">No transcription models are installed. Please install GAME via the Package Manager or download manually from {0}.</system:String>
-  <system:String x:Key="dialogs.transcribe.rmvpe">Predict f0 (PITD) using RMVPE</system:String>
+  <system:String x:Key="dialogs.transcribe.rmvpe">Extract pitch using RMVPE</system:String>
   <system:String x:Key="dialogs.transcribe.game.options">GAME Options</system:String>
   <system:String x:Key="dialogs.transcribe.game.language">Language</system:String>
   <system:String x:Key="dialogs.transcribe.game.language.universal">[universal]</system:String>

OpenUtau/Strings/Strings.zh-CN.axaml

@@ -14,6 +14,11 @@
   <system:String x:Key="context.part.replaceaudio">替换音频</system:String>
   <system:String x:Key="context.part.transcribe">将音频识别为音符区段</system:String>
   <system:String x:Key="context.part.transcribing">正在识别音符</system:String>
+  <system:String x:Key="context.part.applypitch">从音频区段应用音高</system:String>
+  <system:String x:Key="context.part.nopitchsource">无可用的音频区段</system:String>
+  <system:String x:Key="context.part.extractingpitch">正在提取音高</system:String>
+  <system:String x:Key="context.part.nopitchregion">源音频区段与目标音符区段无重叠区域。</system:String>
+  <system:String x:Key="context.part.nopitchdetected">未从源音频区段中提取到音高。</system:String>
   <system:String x:Key="context.pitch.easein">缓入</system:String>
   <system:String x:Key="context.pitch.easeinout">缓入缓出</system:String>
   <system:String x:Key="context.pitch.easeout">缓出</system:String>
@@ -69,6 +74,7 @@
   <system:String x:Key="dialogs.transcribe.some.notfound">SOME 当前未安装。</system:String>
   <system:String x:Key="dialogs.transcribe.game.notfound">GAME 当前未安装。</system:String>
   <system:String x:Key="dialogs.transcribe.allnotfound">当前未安装任何音频转写模型。请通过包管理器安装 GAME，或从此处手动下载：{0}</system:String>
+  <system:String x:Key="dialogs.transcribe.rmvpe">使用 RMVPE 提取音高</system:String>
   <system:String x:Key="dialogs.transcribe.game.options">GAME 选项</system:String>
   <system:String x:Key="dialogs.transcribe.game.language">语言</system:String>
   <system:String x:Key="dialogs.transcribe.game.language.universal">[通用]</system:String>

OpenUtau/ViewModels/MainWindowViewModel.cs

@@ -25,6 +25,7 @@ public class PartsContextMenuArgs {
         public ReactiveCommand<UPart, Unit>? PartTranscribeCommand { get; set; }
         public ReactiveCommand<UPart, Unit>? PartMergeCommand { get; set; }
         public ReactiveCommand<UPart, Unit>? PartSplitCommand { get; set; }
+        public IEnumerable<MenuItemViewModel> PartApplyPitchMenuItems { get; set; } = new List<MenuItemViewModel>();
     }
 
     public class RecentFileInfo {

OpenUtau/Views/MainWindow.axaml

@@ -368,14 +368,14 @@
                           IsVisible="{Binding IsWavePart}"
                           Command="{Binding PartReplaceAudioCommand}"
                           CommandParameter="{Binding Part}"/>
-                <MenuItem Header="{DynamicResource context.part.rename}"
-                          IsVisible="{Binding IsVoicePart}"
-                          Command="{Binding PartRenameCommand}"
-                          CommandParameter="{Binding Part}"/>
                 <MenuItem Header="{DynamicResource context.part.transcribe}"
                           IsVisible="{Binding IsWavePart}"
                           Command="{Binding PartTranscribeCommand}"
                           CommandParameter="{Binding Part}"/>
+                <MenuItem Header="{DynamicResource context.part.rename}"
+                          IsVisible="{Binding IsVoicePart}"
+                          Command="{Binding PartRenameCommand}"
+                          CommandParameter="{Binding Part}"/>
                 <MenuItem Header="{DynamicResource context.part.merge}"
                           IsVisible="{Binding IsVoicePart}"
                           Command="{Binding PartMergeCommand}"
@@ -384,6 +384,10 @@
                           IsVisible="{Binding IsVoicePart}"
                           Command="{Binding PartSplitCommand}"
                           CommandParameter="{Binding Part}"/>
+                <MenuItem Header="{DynamicResource context.part.applypitch}"
+                          IsVisible="{Binding IsVoicePart}"
+                          ItemsSource="{Binding PartApplyPitchMenuItems}"
+                          Classes="MenuItemWithSubMenu"/>
               </ContextMenu>
             </c:PartsCanvas.ContextMenu>
           </c:PartsCanvas>