Skip to content

Commit b8c49d0

Browse files
dbogdanovdbogdanov
authored
Merge pull request #1437 from xaviliz/add-new-algo-audio2midi
Add new algo audio2midi
2 parents eaf8ddf + 5b9499d commit b8c49d0

File tree

9 files changed

+550
-14
lines changed

9 files changed

+550
-14
lines changed

‎src/algorithms/tonal/audio2midi.cpp‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#include "audio2midi.h"
2+
3+
using namespace std;
4+
using namespace essentia;
5+
using namespace standard;
6+
7+
const char *Audio2Midi::name = "Audio2Midi";
8+
const char *Audio2Midi::category = "Pitch";
9+
const char *Audio2Midi::description = DOC("Wrapper around Audio2Pitch and Pitch2Midi for real time application. This algorithm has a state that is used to estimate note on/off events based on consequent compute() calls.");
10+
11+
void Audio2Midi::configure()
12+
{
13+
_sampleRate = parameter("sampleRate").toReal();
14+
_hopSize = parameter("hopSize").toInt();
15+
_minFrequency = parameter("minFrequency").toReal();
16+
_maxFrequency = parameter("maxFrequency").toReal();
17+
_tuningFrequency = parameter("tuningFrequency").toInt();
18+
_pitchConfidenceThreshold = parameter("pitchConfidenceThreshold").toReal();
19+
_loudnessThreshold = parameter("loudnessThreshold").toReal();
20+
_transposition = parameter("transpositionAmount").toInt();
21+
_minOccurrenceRate = parameter("minOccurrenceRate").toReal();
22+
_midiBufferDuration = parameter("midiBufferDuration").toReal();
23+
_minNoteChangePeriod = parameter("minNoteChangePeriod").toReal();
24+
_minOnsetCheckPeriod = parameter("minOnsetCheckPeriod").toReal();
25+
_minOffsetCheckPeriod = parameter("minOffsetCheckPeriod").toReal();
26+
27+
// define frameSize depending on sampleRate
28+
if (static_cast<int>(_sampleRate) <= 16000){
29+
_frameSize = 2048;
30+
}
31+
else if (static_cast<int>(_sampleRate) <= 24000){
32+
_frameSize = 4096;
33+
}
34+
else {
35+
_frameSize = 8192;
36+
}
37+
38+
_applyTimeCompensation = parameter("applyTimeCompensation").toBool();
39+
40+
_lowpass->configure(INHERIT("sampleRate"),
41+
"cutoffFrequency", 1000);
42+
_framebuffer->configure("bufferSize", _frameSize);
43+
_audio2pitch->configure(INHERIT("sampleRate"),
44+
"frameSize", _frameSize,
45+
"pitchAlgorithm", _pitchAlgorithm,
46+
"minFrequency", _minFrequency,
47+
"maxFrequency", _maxFrequency,
48+
INHERIT("pitchConfidenceThreshold"),
49+
INHERIT("loudnessThreshold"));
50+
51+
_pitch2midi->configure(INHERIT("sampleRate"),
52+
INHERIT("hopSize"),
53+
INHERIT("minOccurrenceRate"),
54+
INHERIT("applyTimeCompensation"),
55+
"minOnsetCheckPeriod", _minOnsetCheckPeriod,
56+
"minOffsetCheckPeriod", _minOffsetCheckPeriod,
57+
"minNoteChangePeriod", _minNoteChangePeriod,
58+
"midiBufferDuration", _midiBufferDuration,
59+
"minFrequency", _minFrequency,
60+
"tuningFrequency", _tuningFrequency,
61+
"transpositionAmount", _transposition);
62+
}
63+
64+
void Audio2Midi::compute()
65+
{
66+
// get ref to input
67+
const std::vector<Real> &frame = _frame.get();
68+
Real& pitch = _pitch.get();
69+
Real& loudness = _loudness.get();
70+
vector<string>& messageType = _messageType.get();
71+
vector<Real>& midiNoteNumber = _midiNoteNumber.get();
72+
vector<Real>& timeCompensation = _timeCompensation.get();
73+
74+
_lowpass->input("signal").set(frame);
75+
_lowpass->output("signal").set(lpFrame);
76+
77+
_framebuffer->input("frame").set(lpFrame);
78+
_framebuffer->output("frame").set(analysisFrame);
79+
80+
_audio2pitch->input("frame").set(analysisFrame);
81+
_audio2pitch->output("pitch").set(pitch);
82+
_audio2pitch->output("pitchConfidence").set(pitchConfidence);
83+
_audio2pitch->output("loudness").set(loudness);
84+
_audio2pitch->output("voiced").set(voiced);
85+
86+
_pitch2midi->input("pitch").set(pitch);
87+
_pitch2midi->input("voiced").set(voiced);
88+
_pitch2midi->output("midiNoteNumber").set(midiNoteNumber);
89+
_pitch2midi->output("timeCompensation").set(timeCompensation);
90+
_pitch2midi->output("messageType").set(messageType);
91+
92+
_lowpass->compute();
93+
_framebuffer->compute();
94+
_audio2pitch->compute();
95+
_pitch2midi->compute();
96+
97+
}

‎src/algorithms/tonal/audio2midi.h‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
#ifndef ESSENTIA_AUDIO2MIDI_H
2+
#define ESSENTIA_AUDIO2MIDI_H
3+
4+
#include "algorithmfactory.h"
5+
6+
namespace essentia {
7+
namespace standard {
8+
9+
class Audio2Midi : public Algorithm {
10+
protected:
11+
Input<std::vector<Real>> _frame;
12+
Output<Real> _pitch;
13+
Output<Real> _loudness;
14+
Output<std::vector<std::string> > _messageType;
15+
Output<std::vector<Real> > _midiNoteNumber;
16+
Output<std::vector<Real> > _timeCompensation;
17+
18+
Algorithm* _lowpass;
19+
Algorithm* _framebuffer;
20+
Algorithm* _audio2pitch;
21+
Algorithm* _pitch2midi;
22+
23+
Real _sampleRate;
24+
int _frameSize;
25+
int _hopSize;
26+
std::string _pitchAlgorithm = "pitchyinfft";
27+
std::string _loudnessAlgorithm = "rms";
28+
Real _minFrequency;
29+
Real _maxFrequency;
30+
int _tuningFrequency;
31+
Real _pitchConfidenceThreshold, _loudnessThreshold, _minOccurrenceRate;
32+
Real _midiBufferDuration;
33+
Real _minNoteChangePeriod;
34+
Real _minOnsetCheckPeriod;
35+
Real _minOffsetCheckPeriod;
36+
37+
bool _applyTimeCompensation;
38+
int _transposition;
39+
40+
// Containers
41+
std::vector<Real> lpFrame, analysisFrame;
42+
Real pitch, pitchConfidence, loudness;
43+
std::vector<Real> midiNoteNumber, timeCompensation;
44+
std::vector<std::string> messageType;
45+
Real onsetTimeCompensation, offsetTimeCompensation;
46+
47+
int voiced;
48+
49+
public:
50+
Audio2Midi() {
51+
declareInput(_frame, "frame", "the input frame to analyse");
52+
declareOutput(_pitch, "pitch", "pitch given in Hz");
53+
declareOutput(_loudness, "loudness", "detected loudness in decibels");
54+
declareOutput(_messageType, "messageType", "the output of MIDI message type, as string, {noteoff, noteon, noteoff-noteon}");
55+
declareOutput(_midiNoteNumber, "midiNoteNumber", "the output of detected MIDI note number, as integer, in range [0,127]");
56+
declareOutput(_timeCompensation, "timeCompensation", "time to be compensated in the messages");
57+
58+
_lowpass = AlgorithmFactory::create("LowPass");
59+
_framebuffer = AlgorithmFactory::create("FrameBuffer");
60+
_audio2pitch = AlgorithmFactory::create("Audio2Pitch");
61+
_pitch2midi = AlgorithmFactory::create("Pitch2Midi");
62+
}
63+
64+
~Audio2Midi() {
65+
delete _lowpass;
66+
delete _framebuffer;
67+
delete _audio2pitch;
68+
delete _pitch2midi;
69+
}
70+
71+
void declareParameters() {
72+
declareParameter("sampleRate", "sample rate of incoming audio frames", "[8000,inf)", 44100);
73+
declareParameter("hopSize", "equivalent to I/O buffer size", "[1,inf)", 32);
74+
declareParameter("minFrequency", "minimum frequency to detect in Hz", "[10,20000]", 60.0);
75+
declareParameter("maxFrequency", "maximum frequency to detect in Hz", "[10,20000]", 2300.0);
76+
declareParameter("tuningFrequency", "tuning frequency for semitone index calculation, corresponding to A3 [Hz]", "{432,440}", 440);
77+
declareParameter("pitchConfidenceThreshold", "level of pitch confidence above which note ON/OFF start to be considered", "[0,1]", 0.25);
78+
declareParameter("loudnessThreshold", "loudness level above/below which note ON/OFF start to be considered, in decibels", "[-inf,0]", -51.0);
79+
declareParameter("transpositionAmount", "Apply transposition (in semitones) to the detected MIDI notes.", "(-69,50)", 0);
80+
declareParameter("minOccurrenceRate", "rate of predominant pitch occurrence in MidiPool buffer to consider note ON event", "[0,1]", 0.5);
81+
declareParameter("midiBufferDuration", "duration in seconds of buffer used for voting in MidiPool algorithm", "[0.005,0.5]", 0.05); // 15ms
82+
declareParameter("minNoteChangePeriod", "minimum time to wait until a note change is detected (testing only)", "(0,1]", 0.030);
83+
declareParameter("minOnsetCheckPeriod", "minimum time to wait until an onset is detected (testing only)", "(0,1]", 0.075);
84+
declareParameter("minOffsetCheckPeriod", "minimum time to wait until an offset is detected (testing only)", "(0,1]", 0.2);
85+
declareParameter("applyTimeCompensation", "whether to apply time compensation correction to MIDI note detection", "{true,false}", true);
86+
}
87+
88+
void configure();
89+
void compute();
90+
91+
static const char* name;
92+
static const char* category;
93+
static const char* description;
94+
};
95+
96+
97+
} // namespace standard
98+
} // namespace essentia
99+
100+
#endif

‎src/algorithms/tonal/pitch2midi.cpp‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ void Pitch2Midi::configure()
1414
_sampleRate = parameter("sampleRate").toReal();
1515
_hopSize = parameter("hopSize").toInt();
1616
_minFrequency = parameter("minFrequency").toReal();
17-
_minOcurrenceRate = parameter("minOcurrenceRate").toReal();
17+
_minOccurrenceRate = parameter("minOccurrenceRate").toReal();
1818
_bufferDuration = parameter("midiBufferDuration").toReal();
1919
_minOnsetCheckPeriod = parameter("minOnsetCheckPeriod").toReal();
2020
_minOffsetCheckPeriod = parameter("minOffsetCheckPeriod").toReal();
@@ -33,8 +33,8 @@ void Pitch2Midi::configure()
3333
_offsetCheckCounter = 0;
3434
_onsetCheckCounter = 0;
3535

36-
_minOcurrenceRatePeriod = _minOcurrenceRate * _bufferDuration;
37-
_minOcurrenceRateThreshold = _minOcurrenceRatePeriod / _frameTime;
36+
_minOccurrenceRatePeriod = _minOccurrenceRate * _bufferDuration;
37+
_minOccurrenceRateThreshold = _minOccurrenceRatePeriod / _frameTime;
3838

3939
// estimate buffer capacity
4040
int c = static_cast<int>( round( _sampleRate / float(_hopSize) * _bufferDuration ) );
@@ -151,7 +151,6 @@ void Pitch2Midi::compute()
151151
_noteOff = true;
152152
updateDnote();
153153
setOutputs(dnote, 0.0, _minNoteChangePeriod);
154-
//E_INFO("offset(unvoiced frame)");
155154
_unvoicedFrameCounter = 0;
156155
_offsetCheckCounter = 0;
157156
_onsetCheckCounter = 0;
@@ -220,27 +219,27 @@ void Pitch2Midi::compute()
220219
if (!hasCoherence() && _NOTED_ON) {
221220
if (_maxVoted[0] != 0.0) {
222221
_onsetCheckCounter++;
223-
// combines checker with minOcurrenceRate
224-
if ((_onsetCheckCounter > _minOcurrenceRateThreshold)){
222+
// combines checker with minOccurrenceRate
223+
if ((_onsetCheckCounter > _minOccurrenceRateThreshold)){
225224
_NOTED_ON = true;
226225
if (note != _maxVoted[0]){ // avoid note slicing effect
227226
_noteOff = true;
228227
_noteOn = true;
229228
updateDnote();
230229
note = _maxVoted[0];
231230
}
232-
//E_INFO("off-onset(" << _maxVoted[0] << ", uncoherent & NOTED): " << _onsetCheckCounter << " - " << _minOcurrenceRateThreshold);
231+
//E_INFO("off-onset(" << _maxVoted[0] << ", uncoherent & NOTED): " << _onsetCheckCounter << " - " << _minOccurrenceRateThreshold);
233232
_offsetCheckCounter = 0;
234233
_onsetCheckCounter = 0;
235234
}
236235
}
237236
// output the max-voted midi note to avoid unestable midi note numbers
238-
setOutputs(_maxVoted[0], _minOcurrenceRatePeriod, _minOcurrenceRatePeriod);
237+
setOutputs(_maxVoted[0], _minOccurrenceRatePeriod, _minOccurrenceRatePeriod);
239238
return;
240239
}
241240

242241
if (!hasCoherence() && !_NOTED_ON) {
243-
if (_maxVoted[1] > _minOcurrenceRate) {
242+
if (_maxVoted[1] > _minOccurrenceRate) {
244243
_onsetCheckCounter++;
245244

246245
if (_onsetCheckCounter > _minOnsetCheckThreshold) {

‎src/algorithms/tonal/pitch2midi.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ namespace standard {
2626
Real _sampleRate;
2727
int _hopSize;
2828
Real _minFrequency;
29-
Real _minOcurrenceRate;
29+
Real _minOccurrenceRate;
3030
Real _minOnsetCheckPeriod;
3131
Real _minOffsetCheckPeriod;
3232
Real _minNoteChangePeriod;
@@ -66,8 +66,8 @@ namespace standard {
6666
int _onsetCheckCounter;
6767

6868
Real _frameTime;
69-
Real _minOcurrenceRateThreshold;
70-
Real _minOcurrenceRatePeriod;
69+
Real _minOccurrenceRateThreshold;
70+
Real _minOccurrenceRatePeriod;
7171

7272
// former Pitch2Midi outputs, now interal vars
7373
Real _midiNoteNumberTransposed;
@@ -89,7 +89,7 @@ namespace standard {
8989
declareParameter("sampleRate", "Audio sample rate", "[8000,inf)", 44100);
9090
declareParameter("hopSize", "Pitch Detection analysis hop size in samples, equivalent to I/O buffer size", "[1,inf)", 128);
9191
declareParameter("minFrequency", "minimum detectable frequency", "[20,20000]", 60.0);
92-
declareParameter("minOcurrenceRate", "minimum number of times a midi note has to ocur compared to total capacity", "[0,1]", 0.5);
92+
declareParameter("minOccurrenceRate", "minimum number of times a midi note has to ocur compared to total capacity", "[0,1]", 0.5);
9393
declareParameter("midiBufferDuration", "duration in seconds of buffer used for voting in the note toggle detection algorithm", "[0.005,0.5]", 0.015); // 15ms
9494
declareParameter("minNoteChangePeriod", "minimum time to wait until a note change is detected (s)", "(0,1]", 0.030);
9595
declareParameter("minOnsetCheckPeriod", "minimum time to wait until an onset is detected (s)", "(0,1]", 0.075);

‎test/src/unittests/tonal/audio2midi/359500__mtg__sax-tenor-e-major.npy‎

608 Bytes
Binary file not shown.

‎test/src/unittests/tonal/audio2midi/359628__mtg__sax-tenor-d-minor.npy‎

672 Bytes
Binary file not shown.

‎test/src/unittests/tonal/audio2midi/387517__deleted_user_7267864__saxophone-going-up.npy‎

448 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)