-
-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Expand file tree
/
Copy pathaudio_reactive.cpp
More file actions
2360 lines (2110 loc) · 114 KB
/
audio_reactive.cpp
File metadata and controls
2360 lines (2110 loc) · 114 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#include "wled.h"
#ifdef ARDUINO_ARCH_ESP32
#include <driver/i2s.h>
#include <driver/adc.h>
#endif
#if defined(ARDUINO_ARCH_ESP32) && (defined(WLED_DEBUG) || defined(SR_DEBUG))
#include <esp_timer.h>
#endif
/*
* Usermods allow you to add own functionality to WLED more easily
* See: https://github.com/wled-dev/WLED/wiki/Add-own-functionality
*
* This is an audioreactive v2 usermod.
* ....
*/
#define FFT_PREFER_EXACT_PEAKS // use Blackman-Harris FFT windowing instead of Flat Top -> results in "sharper" peaks and less "leaking" into other frequencies (credits to @softhack)
/*
* Note on FFT variants:
* - ArduinoFFT: uses floating point calculations, very slow on S2 and C3 (no FPU)
* - ESP-IDF DSP library:
- faster but uses ~13k of extra flash on ESP32 and S3
* - uses integer math on S2 and C3: slightly less accurate but over 10x faster than ArduinoFFT and uses less flash
- not available in IDF < 4.4
* - ArduinoFFT is used by default on ESP32 and S3
* - ESP-IDF DSP FFT with integer math is used by default on S2 and C3
* - defines:
* - UM_AUDIOREACTIVE_USE_ARDUINO_FFT: use ArduinoFFT library for FFT
* - UM_AUDIOREACTIVE_USE_ESPDSP_FFT: use ESP-IDF DSP for FFT
*/
//#define UM_AUDIOREACTIVE_USE_ESPDSP_FFT // default on S2 and C3
//#define UM_AUDIOREACTIVE_USE_INTEGER_FFT // use integer FFT if using ESP-IDF DSP library, always used on S2 and C3 (UM_AUDIOREACTIVE_USE_ARDUINO_FFT takes priority)
//#define UM_AUDIOREACTIVE_USE_ARDUINO_FFT // default on ESP32 and S3
#if !defined(FFTTASK_PRIORITY)
#define FFTTASK_PRIORITY 1 // standard: looptask prio
//#define FFTTASK_PRIORITY 2 // above looptask, below asyc_tcp
//#define FFTTASK_PRIORITY 4 // above asyc_tcp
#endif
// Comment/Uncomment to toggle usb serial debugging
// #define MIC_LOGGER // MIC sampling & sound input debugging (serial plotter)
// #define FFT_SAMPLING_LOG // FFT result debugging
// #define SR_DEBUG // generic SR DEBUG messages
#ifdef SR_DEBUG
#define DEBUGSR_PRINT(x) DEBUGOUT.print(x)
#define DEBUGSR_PRINTLN(x) DEBUGOUT.println(x)
#define DEBUGSR_PRINTF(x...) DEBUGOUT.printf(x)
#else
#define DEBUGSR_PRINT(x)
#define DEBUGSR_PRINTLN(x)
#define DEBUGSR_PRINTF(x...)
#endif
#if defined(MIC_LOGGER) || defined(FFT_SAMPLING_LOG)
#define PLOT_PRINT(x) DEBUGOUT.print(x)
#define PLOT_PRINTLN(x) DEBUGOUT.println(x)
#define PLOT_PRINTF(x...) DEBUGOUT.printf(x)
#else
#define PLOT_PRINT(x)
#define PLOT_PRINTLN(x)
#define PLOT_PRINTF(x...)
#endif
#define MAX_PALETTES 5
static volatile bool disableSoundProcessing = false; // if true, sound processing (FFT, filters, AGC) will be suspended. "volatile" as its shared between tasks.
static uint8_t audioSyncEnabled = 0; // bit field: bit 0 - send, bit 1 - receive (config value)
static bool udpSyncConnected = false; // UDP connection status -> true if connected to multicast group
#define NUM_GEQ_CHANNELS 16 // number of frequency channels. Don't change !!
// audioreactive variables
#ifdef ARDUINO_ARCH_ESP32
#ifndef SR_AGC // Automatic gain control mode
#define SR_AGC 0 // default mode = off
#endif
static float micDataReal = 0.0f; // MicIn data with full 24bit resolution - lowest 8bit after decimal point
static float multAgc = 1.0f; // sample * multAgc = sampleAgc. Our AGC multiplier
static float sampleAvg = 0.0f; // Smoothed Average sample - sampleAvg < 1 means "quiet" (simple noise gate)
static float sampleAgc = 0.0f; // Smoothed AGC sample
static uint8_t soundAgc = SR_AGC; // Automatic gain control: 0 - off, 1 - normal, 2 - vivid, 3 - lazy (config value)
#endif
//static float volumeSmth = 0.0f; // either sampleAvg or sampleAgc depending on soundAgc; smoothed sample
static float FFT_MajorPeak = 1.0f; // FFT: strongest (peak) frequency
static float FFT_Magnitude = 0.0f; // FFT: volume (magnitude) of peak frequency
static bool samplePeak = false; // Boolean flag for peak - used in effects. Responding routine may reset this flag. Auto-reset after strip.getFrameTime()
static bool udpSamplePeak = false; // Boolean flag for peak. Set at the same time as samplePeak, but reset by transmitAudioData
static unsigned long timeOfPeak = 0; // time of last sample peak detection.
static uint8_t fftResult[NUM_GEQ_CHANNELS]= {0};// Our calculated freq. channel result table to be used by effects
// TODO: probably best not used by receive nodes
//static float agcSensitivity = 128; // AGC sensitivity estimation, based on agc gain (multAgc). calculated by getSensitivity(). range 0..255
// user settable parameters for limitSoundDynamics()
#ifdef UM_AUDIOREACTIVE_DYNAMICS_LIMITER_OFF
static bool limiterOn = false; // bool: enable / disable dynamics limiter
#else
static bool limiterOn = true;
#endif
static uint16_t attackTime = 80; // int: attack time in milliseconds. Default 0.08sec
static uint16_t decayTime = 1400; // int: decay time in milliseconds. Default 1.40sec
// peak detection
#ifdef ARDUINO_ARCH_ESP32
static void detectSamplePeak(void); // peak detection function (needs scaled FFT results in vReal[]) - no used for 8266 receive-only mode
#endif
static void autoResetPeak(void); // peak auto-reset function
static uint8_t maxVol = 31; // (was 10) Reasonable value for constant volume for 'peak detector', as it won't always trigger (deprecated)
static uint8_t binNum = 8; // Used to select the bin for FFT based beat detection (deprecated)
#ifdef ARDUINO_ARCH_ESP32
#if !defined(UM_AUDIOREACTIVE_USE_ESPDSP_FFT) && (defined(CONFIG_IDF_TARGET_ESP32S3) || defined(CONFIG_IDF_TARGET_ESP32))
#define UM_AUDIOREACTIVE_USE_ARDUINO_FFT // use ArduinoFFT library for FFT instead of ESP-IDF DSP library by default on ESP32 and S3
#endif
#if ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(4, 4, 0)
#define UM_AUDIOREACTIVE_USE_ARDUINO_FFT // DSP FFT library is not available in ESP-IDF < 4.4
#endif
#ifdef UM_AUDIOREACTIVE_USE_ARDUINO_FFT
#include <arduinoFFT.h> // ArduinoFFT library for FFT and window functions
#undef UM_AUDIOREACTIVE_USE_INTEGER_FFT // arduinoFFT has not integer support
#else
#include "dsps_fft2r.h" // ESP-IDF DSP library for FFT and window functions
#ifdef FFT_PREFER_EXACT_PEAKS
#include "dsps_wind_blackman_harris.h"
#else
#include "dsps_wind_flat_top.h"
#endif
#if defined(CONFIG_IDF_TARGET_ESP32S2) || defined(CONFIG_IDF_TARGET_ESP32C3)
#define UM_AUDIOREACTIVE_USE_INTEGER_FFT // always use integer FFT on ESP32-S2 and ESP32-C3
#endif
#endif
#if !defined(UM_AUDIOREACTIVE_USE_INTEGER_FFT)
using FFTsampleType = float;
using FFTmathType = float;
#define FFTabs fabsf
#else
using FFTsampleType = int16_t;
using FFTmathType = int32_t;
#define FFTabs abs
#endif
// These are the input and output vectors. Input vectors receive computed results from FFT.
static FFTsampleType* valFFT = nullptr;
#ifdef UM_AUDIOREACTIVE_USE_ARDUINO_FFT
static float* vImag = nullptr; // imaginary part of FFT results
#endif
// pre-computed window function
static FFTsampleType* windowFFT = nullptr;
// use audio source class (ESP32 specific)
#include "audio_source.h"
constexpr i2s_port_t I2S_PORT = I2S_NUM_0; // I2S port to use (do not change !)
constexpr int BLOCK_SIZE = 128; // I2S buffer size (samples)
// globals
static uint8_t inputLevel = 128; // UI slider value
#ifndef SR_SQUELCH
static uint8_t soundSquelch = 10; // squelch value for volume reactive routines (config value)
#else
static uint8_t soundSquelch = SR_SQUELCH; // squelch value for volume reactive routines (config value)
#endif
#ifndef SR_GAIN
static uint8_t sampleGain = 60; // sample gain (config value)
#else
static uint8_t sampleGain = SR_GAIN; // sample gain (config value)
#endif
// user settable options for FFTResult scaling
static uint8_t FFTScalingMode = 3; // 0 none; 1 optimized logarithmic; 2 optimized linear; 3 optimized square root
//
// AGC presets
// Note: in C++, "const" implies "static" - no need to explicitly declare everything as "static const"
//
#define AGC_NUM_PRESETS 3 // AGC presets: normal, vivid, lazy
const double agcSampleDecay[AGC_NUM_PRESETS] = { 0.9994f, 0.9985f, 0.9997f}; // decay factor for sampleMax, in case the current sample is below sampleMax
const float agcZoneLow[AGC_NUM_PRESETS] = { 32, 28, 36}; // low volume emergency zone
const float agcZoneHigh[AGC_NUM_PRESETS] = { 240, 240, 248}; // high volume emergency zone
const float agcZoneStop[AGC_NUM_PRESETS] = { 336, 448, 304}; // disable AGC integrator if we get above this level
const float agcTarget0[AGC_NUM_PRESETS] = { 112, 144, 164}; // first AGC setPoint -> between 40% and 65%
const float agcTarget0Up[AGC_NUM_PRESETS] = { 88, 64, 116}; // setpoint switching value (a poor man's bang-bang)
const float agcTarget1[AGC_NUM_PRESETS] = { 220, 224, 216}; // second AGC setPoint -> around 85%
const double agcFollowFast[AGC_NUM_PRESETS] = { 1/192.f, 1/128.f, 1/256.f}; // quickly follow setpoint - ~0.15 sec
const double agcFollowSlow[AGC_NUM_PRESETS] = {1/6144.f,1/4096.f,1/8192.f}; // slowly follow setpoint - ~2-15 secs
const double agcControlKp[AGC_NUM_PRESETS] = { 0.6f, 1.5f, 0.65f}; // AGC - PI control, proportional gain parameter
const double agcControlKi[AGC_NUM_PRESETS] = { 1.7f, 1.85f, 1.2f}; // AGC - PI control, integral gain parameter
const float agcSampleSmooth[AGC_NUM_PRESETS] = { 1/12.f, 1/6.f, 1/16.f}; // smoothing factor for sampleAgc (use rawSampleAgc if you want the non-smoothed value)
// AGC presets end
static AudioSource *audioSource = nullptr;
static bool useBandPassFilter = false; // if true, enables a hard cutoff bandpass filter. Applies after FFT.
static bool useMicFilter = false; // if true, enables a IIR bandpass filter 80Hz-20Khz to remove noise. Applies before FFT.
////////////////////
// Begin FFT Code //
////////////////////
// some prototypes, to ensure consistent interfaces
static float fftAddAvg(int from, int to); // average of several FFT result bins
void FFTcode(void * parameter); // audio processing task: read samples, run FFT, fill GEQ channels from FFT results
static void runMicFilter(uint16_t numSamples, FFTsampleType *sampleBuffer);
static void postProcessFFTResults(bool noiseGateOpen, int numberOfChannels); // post-processing and post-amp of GEQ channels
static TaskHandle_t FFT_Task = nullptr;
// Table of multiplication factors so that we can even out the frequency response.
static float fftResultPink[NUM_GEQ_CHANNELS] = { 1.70f, 1.71f, 1.73f, 1.78f, 1.68f, 1.56f, 1.55f, 1.63f, 1.79f, 1.62f, 1.80f, 2.06f, 2.47f, 3.35f, 6.83f, 9.55f };
// globals and FFT Output variables shared with animations
#if defined(WLED_DEBUG) || defined(SR_DEBUG)
static uint64_t fftTime = 0;
static uint64_t sampleTime = 0;
#endif
// FFT Task variables (filtering and post-processing)
static float fftCalc[NUM_GEQ_CHANNELS] = {0.0f}; // Try and normalize fftBin values to a max of 4096, so that 4096/16 = 256.
static float fftAvg[NUM_GEQ_CHANNELS] = {0.0f}; // Calculated frequency channel results, with smoothing (used if dynamics limiter is ON)
static float paletteBandAvg[NUM_GEQ_CHANNELS] = {0.0f}; // Slowly smoothed band averages used only by audio palettes 3 & 4 (EMA, alpha=0.05 → ~400ms time constant at 20ms cycle)
static constexpr float PALETTE_SMOOTHING = 0.05f; // EMA smoothing factor for paletteBandAvg: 0.05 gives ~400ms time constant; increase for faster response, decrease for slower
#ifdef SR_DEBUG
static float fftResultMax[NUM_GEQ_CHANNELS] = {0.0f}; // A table used for testing to determine how our post-processing is working.
#endif
// audio source parameters and constant
constexpr SRate_t SAMPLE_RATE = 22050; // Base sample rate in Hz - 22Khz is a standard rate. Physical sample time -> 23ms
//constexpr SRate_t SAMPLE_RATE = 16000; // 16kHz - use if FFTtask takes more than 20ms. Physical sample time -> 32ms
//constexpr SRate_t SAMPLE_RATE = 20480; // Base sample rate in Hz - 20Khz is experimental. Physical sample time -> 25ms
//constexpr SRate_t SAMPLE_RATE = 10240; // Base sample rate in Hz - previous default. Physical sample time -> 50ms
#define FFT_MIN_CYCLE 21 // minimum time before FFT task is repeated. Use with 22Khz sampling
//#define FFT_MIN_CYCLE 30 // Use with 16Khz sampling
//#define FFT_MIN_CYCLE 23 // minimum time before FFT task is repeated. Use with 20Khz sampling
//#define FFT_MIN_CYCLE 46 // minimum time before FFT task is repeated. Use with 10Khz sampling
// FFT Constants
constexpr uint16_t samplesFFT = 512; // Samples in an FFT batch - This value MUST ALWAYS be a power of 2
constexpr uint16_t samplesFFT_2 = 256; // meaningfull part of FFT results - only the "lower half" contains useful information.
// the following are observed values, supported by a bit of "educated guessing"
//#define FFT_DOWNSCALE 0.65f // 20kHz - downscaling factor for FFT results - "Flat-Top" window @20Khz, old freq channels
#ifdef FFT_PREFER_EXACT_PEAKS
#define FFT_DOWNSCALE 0.40f // downscaling factor for FFT results, RMS averaging for "Blackman-Harris" Window @22kHz (credit to MM)
#else
#define FFT_DOWNSCALE 0.46f // downscaling factor for FFT results - for "Flat-Top" window @22Khz, new freq channels
#endif
#define LOG_256 5.54517744f // log(256)
// Create FFT object
// lib_deps += https://github.com/kosme/arduinoFFT#develop @ 1.9.2
// these options actually cause slow-downs on all esp32 processors, don't use them.
// #define FFT_SPEED_OVER_PRECISION // enables use of reciprocals (1/x etc) - not faster on ESP32
// #define FFT_SQRT_APPROXIMATION // enables "quake3" style inverse sqrt - slower on ESP32
// Below options are forcing ArduinoFFT to use sqrtf() instead of sqrt()
// #define sqrt_internal sqrtf // see https://github.com/kosme/arduinoFFT/pull/83 - since v2.0.0 this must be done in build_flags
// Helper functions
// compute average of several FFT result bins
static float fftAddAvg(int from, int to) {
FFTmathType result = 0;
for (int i = from; i <= to; i++) {
result += valFFT[i];
}
#if !defined(UM_AUDIOREACTIVE_USE_INTEGER_FFT)
result = result * 0.0625; // divide by 16 to reduce magnitude. Want end result to be scaled linear and ~4096 max.
#else
result *= 32; // scale result to match float values. note: raw scaling value between float and int is 512, float version is scaled down by 16
#endif
return float(result) / float(to - from + 1); // return average as float
}
//
// FFT main task
//
void FFTcode(void * parameter)
{
DEBUGSR_PRINT("FFT started on core: "); DEBUGSR_PRINTLN(xPortGetCoreID());
#ifdef UM_AUDIOREACTIVE_USE_ARDUINO_FFT
// allocate FFT buffers on first call
if (valFFT == nullptr) valFFT = (float*) calloc(samplesFFT, sizeof(float));
if (vImag == nullptr) vImag = (float*) calloc(samplesFFT, sizeof(float));
if ((valFFT == nullptr) || (vImag == nullptr)) {
// something went wrong
if (valFFT) free(valFFT); valFFT = nullptr;
if (vImag) free(vImag); vImag = nullptr;
return;
}
// Create FFT object with weighing factor storage
ArduinoFFT<float> FFT = ArduinoFFT<float>(valFFT, vImag, samplesFFT, SAMPLE_RATE, true);
#elif !defined(UM_AUDIOREACTIVE_USE_INTEGER_FFT)
// allocate and initialize FFT buffers on first call
// note: free() is never used on these pointers. If it ever is implemented, this implementation can cause memory leaks (need to free raw pointers)
if (valFFT == nullptr) {
float* raw_buffer = (float*)heap_caps_malloc((2 * samplesFFT * sizeof(float)) + 16, MALLOC_CAP_8BIT);
if ((raw_buffer == nullptr)) return; // something went wrong
valFFT = (float*)(((uintptr_t)raw_buffer + 15) & ~15); // SIMD requires aligned memory to 16-byte boundary. note in IDF5 there is MALLOC_CAP_SIMD available
}
// create window
if (windowFFT == nullptr) {
float* raw_buffer = (float*)heap_caps_malloc((samplesFFT * sizeof(float)) + 16, MALLOC_CAP_8BIT);
if ((raw_buffer == nullptr)) return; // something went wrong
windowFFT = (float*)(((uintptr_t)raw_buffer + 15) & ~15); // SIMD requires aligned memory to 16-byte boundary
}
if (dsps_fft2r_init_fc32(NULL, samplesFFT) != ESP_OK) return; // initialize FFT tables
// create window function for FFT
#ifdef FFT_PREFER_EXACT_PEAKS
dsps_wind_blackman_harris_f32(windowFFT, samplesFFT);
#else
dsps_wind_flat_top_f32(windowFFT, samplesFFT);
#endif
#else
// allocate and initialize integer FFT buffers on first call
if (valFFT == nullptr) valFFT = (int16_t*) calloc(sizeof(int16_t), samplesFFT * 2);
if ((valFFT == nullptr)) return; // something went wrong
// create window
if (windowFFT == nullptr) windowFFT = (int16_t*) calloc(sizeof(int16_t), samplesFFT);
if ((windowFFT == nullptr)) return; // something went wrong
if (dsps_fft2r_init_sc16(NULL, samplesFFT) != ESP_OK) return; // initialize FFT tables
// create window function for FFT
float *windowFloat = (float*) calloc(sizeof(float), samplesFFT); // temporary buffer for window function
if ((windowFloat == nullptr)) return; // something went wrong
#ifdef FFT_PREFER_EXACT_PEAKS
dsps_wind_blackman_harris_f32(windowFloat, samplesFFT);
#else
dsps_wind_flat_top_f32(windowFloat, samplesFFT);
#endif
// convert float window to 16-bit int
for (int i = 0; i < samplesFFT; i++) {
windowFFT[i] = (int16_t)(windowFloat[i] * 32767.0f);
}
free(windowFloat); // free temporary buffer
#endif
// see https://www.freertos.org/vtaskdelayuntil.html
const TickType_t xFrequency = FFT_MIN_CYCLE * portTICK_PERIOD_MS;
TickType_t xLastWakeTime = xTaskGetTickCount();
for(;;) {
delay(1); // DO NOT DELETE THIS LINE! It is needed to give the IDLE(0) task enough time and to keep the watchdog happy.
// taskYIELD(), yield(), vTaskDelay() and esp_task_wdt_feed() didn't seem to work.
// Don't run FFT computing code if we're in Receive mode or in realtime mode
if (disableSoundProcessing || (audioSyncEnabled & 0x02)) {
vTaskDelayUntil( &xLastWakeTime, xFrequency); // release CPU, and let I2S fill its buffers
continue;
}
#if defined(WLED_DEBUG) || defined(SR_DEBUG)
uint64_t start = esp_timer_get_time();
bool haveDoneFFT = false; // indicates if second measurement (FFT time) is valid
#endif
// get a fresh batch of samples from I2S
if (audioSource) audioSource->getSamples(valFFT, samplesFFT); // note: valFFT is used as a int16_t buffer on C3 and S2, could optimize RAM use by only allocating half the size (but makes code harder to read)
#if defined(WLED_DEBUG) || defined(SR_DEBUG)
if (start < esp_timer_get_time()) { // filter out overflows
uint64_t sampleTimeInMillis = (esp_timer_get_time() - start +5ULL) / 10ULL; // "+5" to ensure proper rounding
sampleTime = (sampleTimeInMillis*3 + sampleTime*7)/10; // smooth
}
start = esp_timer_get_time(); // start measuring FFT time
#endif
xLastWakeTime = xTaskGetTickCount(); // update "last unblocked time" for vTaskDelay
// band pass filter - can reduce noise floor by a factor of 50 and avoid aliasing effects to base & high frequency bands
// downside: frequencies below 100Hz will be ignored
if (useMicFilter) runMicFilter(samplesFFT, valFFT);
// find highest sample in the batch
FFTsampleType maxSample = 0; // max sample from FFT batch
for (int i=0; i < samplesFFT; i++) {
// pick our our current mic sample - we take the max value from all samples that go into FFT
if ((valFFT[i] <= (INT16_MAX - 1024)) && (valFFT[i] >= (INT16_MIN + 1024))) //skip extreme values - normally these are artefacts
if (FFTabs(valFFT[i]) > maxSample) maxSample = FFTabs(valFFT[i]);
}
// release highest sample to volume reactive effects early - not strictly necessary here - could also be done at the end of the function
// early release allows the filters (getSample() and agcAvg()) to work with fresh values - we will have matching gain and noise gate values when we want to process the FFT results.
micDataReal = maxSample;
#ifdef SR_DEBUG
if (true) { // this allows measure FFT runtimes, as it disables the "only when needed" optimization
#else
if (sampleAvg > 0.25f) { // noise gate open means that FFT results will be used. Don't run FFT if results are not needed.
#endif
#ifdef UM_AUDIOREACTIVE_USE_ARDUINO_FFT
// run Arduino FFT (takes 3-5ms on ESP32, ~12ms on ESP32-S2, ~20ms on ESP32-C3)
memset(vImag, 0, samplesFFT * sizeof(float)); // set imaginary parts to 0
FFT.dcRemoval(); // remove DC offset
#ifdef FFT_PREFER_EXACT_PEAKS
FFT.windowing(FFTWindow::Blackman_Harris, FFTDirection::Forward); // Weigh data using "Blackman- Harris" window - sharp peaks due to excellent sideband rejection
#else
FFT.windowing( FFTWindow::Flat_top, FFTDirection::Forward); // Weigh data using "Flat Top" function - better amplitude accuracy
#endif
FFT.compute( FFTDirection::Forward ); // Compute FFT
FFT.complexToMagnitude(); // Compute magnitudes
valFFT[0] = 0; // The remaining DC offset on the signal produces a strong spike on position 0 that should be eliminated to avoid issues.
FFT.majorPeak(&FFT_MajorPeak, &FFT_Magnitude); // let the effects know which freq was most dominant
// note: scaling is done in fftAddAvg(), so we don't scale here
#else
// run run float DSP FFT (takes ~x ms on ESP32, ~x ms on ESP32-S2, , ~x ms on ESP32-C3) TODO: test and fill in these values
// remove DC offset
FFTmathType sum = 0;
for (int i = 0; i < samplesFFT; i++) sum += valFFT[i];
FFTmathType mean = sum / (FFTmathType)samplesFFT;
for (int i = 0; i < samplesFFT; i++) valFFT[i] -= mean;
#if !defined(UM_AUDIOREACTIVE_USE_INTEGER_FFT)
//apply window function to samples and fill buffer with interleaved complex values [Re,Im,Re,Im,...]
for (int i = samplesFFT - 1; i >= 0 ; i--) {
// fill the buffer back to front to avoid overwriting samples
float windowed_sample = valFFT[i] * windowFFT[i];
valFFT[i * 2] = windowed_sample;
valFFT[i * 2 + 1] = 0.0; // set imaginary part to zero
}
#ifdef CONFIG_IDF_TARGET_ESP32S3
dsps_fft2r_fc32_aes3(valFFT, samplesFFT); // ESP32 S3 optimized version of FFT
#elif defined(CONFIG_IDF_TARGET_ESP32)
dsps_fft2r_fc32_ae32(valFFT, samplesFFT); // ESP32 optimized version of FFT
#else
dsps_fft2r_fc32_ansi(valFFT, samplesFFT); // perform FFT using ANSI C implementation
#endif
dsps_bit_rev_fc32(valFFT, samplesFFT); // bit reverse
valFFT[0] = 0; // set DC bin to 0, as it is not needed and can cause issues
// convert to magnitude & find FFT_MajorPeak and FFT_Magnitude
FFT_MajorPeak = 0;
FFT_Magnitude = 0;
for (int i = 1; i < samplesFFT_2; i++) { // skip [0] as it is DC offset
float real_part = valFFT[i * 2];
float imag_part = valFFT[i * 2 + 1];
valFFT[i] = sqrtf(real_part * real_part + imag_part * imag_part);
if (valFFT[i] > FFT_Magnitude) {
FFT_Magnitude = valFFT[i];
FFT_MajorPeak = i*(SAMPLE_RATE/samplesFFT);
}
// note: scaling is done in fftAddAvg(), so we don't scale here
}
#else
// run integer DSP FFT (takes ~x ms on ESP32, ~x ms on ESP32-S2, , ~1.5 ms on ESP32-C3) TODO: test and fill in these values
//apply window function to samples and fill buffer with interleaved complex values [Re,Im,Re,Im,...]
for (int i = samplesFFT - 1; i >= 0 ; i--) {
// fill the buffer back to front to avoid overwriting samples
int16_t windowed_sample = ((int32_t)valFFT[i] * (int32_t)windowFFT[i]) >> 15; // both values are ±15bit
valFFT[i * 2] = windowed_sample;
valFFT[i * 2 + 1] = 0; // set imaginary part to zero
}
dsps_fft2r_sc16_ansi(valFFT, samplesFFT); // perform FFT on complex value pairs (Re,Im)
dsps_bit_rev_sc16_ansi(valFFT, samplesFFT); // bit reverse i.e. "unshuffle" the results
valFFT[0] = 0; // set DC bin to 0, as it is not needed and can cause issues
// convert to magnitude, FFT returns interleaved complex values [Re,Im,Re,Im,...]
int FFT_MajorPeak_int = 0;
int FFT_Magnitude_int = 0;
for (int i = 1; i < samplesFFT_2; i++) { // skip [0], it is DC offset
int32_t real_part = valFFT[i * 2];
int32_t imag_part = valFFT[i * 2 + 1];
valFFT[i] = sqrt32_bw(real_part * real_part + imag_part * imag_part); // note: this should never overflow as Re and Im form a vector of maximum length 32767
if (valFFT[i] > FFT_Magnitude_int) {
FFT_Magnitude_int = valFFT[i];
FFT_MajorPeak_int = ((i * SAMPLE_RATE)/samplesFFT);
}
// note: scaling is done in fftAddAvg(), so we don't scale here
}
FFT_Magnitude = FFT_Magnitude_int * 512; // scale to match raw float value
FFT_MajorPeak = FFT_MajorPeak_int;
FFT_Magnitude = FFT_Magnitude_int;
#endif
#endif
FFT_MajorPeak = constrain(FFT_MajorPeak, 1.0f, 11025.0f); // restrict value to range expected by effects
#if defined(WLED_DEBUG) || defined(SR_DEBUG)
haveDoneFFT = true;
#endif
} else { // noise gate closed - only clear results as FFT was skipped. MIC samples are still valid when we do this -> set all samples to 0
memset(valFFT, 0, samplesFFT * sizeof(FFTsampleType));
FFT_MajorPeak = 1;
FFT_Magnitude = 0.001;
}
// mapping of FFT result bins to frequency channels
if (fabsf(sampleAvg) > 0.5f) { // noise gate open
#if 0
/* This FFT post processing is a DIY endeavour. What we really need is someone with sound engineering expertise to do a great job here AND most importantly, that the animations look GREAT as a result.
*
* Andrew's updated mapping of 256 bins down to the 16 result bins with Sample Freq = 10240, samplesFFT = 512 and some overlap.
* Based on testing, the lowest/Start frequency is 60 Hz (with bin 3) and a highest/End frequency of 5120 Hz in bin 255.
* Now, Take the 60Hz and multiply by 1.320367784 to get the next frequency and so on until the end. Then determine the bins.
* End frequency = Start frequency * multiplier ^ 16
* Multiplier = (End frequency/ Start frequency) ^ 1/16
* Multiplier = 1.320367784
*/ // Range
fftCalc[ 0] = fftAddAvg(2,4); // 60 - 100
fftCalc[ 1] = fftAddAvg(4,5); // 80 - 120
fftCalc[ 2] = fftAddAvg(5,7); // 100 - 160
fftCalc[ 3] = fftAddAvg(7,9); // 140 - 200
fftCalc[ 4] = fftAddAvg(9,12); // 180 - 260
fftCalc[ 5] = fftAddAvg(12,16); // 240 - 340
fftCalc[ 6] = fftAddAvg(16,21); // 320 - 440
fftCalc[ 7] = fftAddAvg(21,29); // 420 - 600
fftCalc[ 8] = fftAddAvg(29,37); // 580 - 760
fftCalc[ 9] = fftAddAvg(37,48); // 740 - 980
fftCalc[10] = fftAddAvg(48,64); // 960 - 1300
fftCalc[11] = fftAddAvg(64,84); // 1280 - 1700
fftCalc[12] = fftAddAvg(84,111); // 1680 - 2240
fftCalc[13] = fftAddAvg(111,147); // 2220 - 2960
fftCalc[14] = fftAddAvg(147,194); // 2940 - 3900
fftCalc[15] = fftAddAvg(194,250); // 3880 - 5000 // avoid the last 5 bins, which are usually inaccurate
#else
/* new mapping, optimized for 22050 Hz by softhack007 */
// bins frequency range
if (useBandPassFilter) {
// skip frequencies below 100hz
fftCalc[ 0] = 0.8f * fftAddAvg(3,4);
fftCalc[ 1] = 0.9f * fftAddAvg(4,5);
fftCalc[ 2] = fftAddAvg(5,6);
fftCalc[ 3] = fftAddAvg(6,7);
// don't use the last bins from 206 to 255.
fftCalc[15] = fftAddAvg(165,205) * 0.75f; // 40 7106 - 8828 high -- with some damping
} else {
fftCalc[ 0] = fftAddAvg(1,2); // 1 43 - 86 sub-bass
fftCalc[ 1] = fftAddAvg(2,3); // 1 86 - 129 bass
fftCalc[ 2] = fftAddAvg(3,5); // 2 129 - 216 bass
fftCalc[ 3] = fftAddAvg(5,7); // 2 216 - 301 bass + midrange
// don't use the last bins from 216 to 255. They are usually contaminated by aliasing (aka noise)
fftCalc[15] = fftAddAvg(165,215) * 0.70f; // 50 7106 - 9259 high -- with some damping
}
fftCalc[ 4] = fftAddAvg(7,10); // 3 301 - 430 midrange
fftCalc[ 5] = fftAddAvg(10,13); // 3 430 - 560 midrange
fftCalc[ 6] = fftAddAvg(13,19); // 5 560 - 818 midrange
fftCalc[ 7] = fftAddAvg(19,26); // 7 818 - 1120 midrange -- 1Khz should always be the center !
fftCalc[ 8] = fftAddAvg(26,33); // 7 1120 - 1421 midrange
fftCalc[ 9] = fftAddAvg(33,44); // 9 1421 - 1895 midrange
fftCalc[10] = fftAddAvg(44,56); // 12 1895 - 2412 midrange + high mid
fftCalc[11] = fftAddAvg(56,70); // 14 2412 - 3015 high mid
fftCalc[12] = fftAddAvg(70,86); // 16 3015 - 3704 high mid
fftCalc[13] = fftAddAvg(86,104); // 18 3704 - 4479 high mid
fftCalc[14] = fftAddAvg(104,165) * 0.88f; // 61 4479 - 7106 high mid + high -- with slight damping
#endif
} else { // noise gate closed - just decay old values
for (int i=0; i < NUM_GEQ_CHANNELS; i++) {
fftCalc[i] *= 0.85f; // decay to zero
if (fftCalc[i] < 4.0f) fftCalc[i] = 0.0f;
}
}
// post-processing of frequency channels (pink noise adjustment, AGC, smoothing, scaling)
postProcessFFTResults((fabsf(sampleAvg) > 0.25f)? true : false , NUM_GEQ_CHANNELS);
#if defined(WLED_DEBUG) || defined(SR_DEBUG)
if (haveDoneFFT && (start < esp_timer_get_time())) { // filter out overflows
uint64_t fftTimeInMillis = ((esp_timer_get_time() - start) +5ULL) / 10ULL; // "+5" to ensure proper rounding
fftTime = (fftTimeInMillis*3 + fftTime*7)/10; // smooth
}
#endif
// run peak detection
autoResetPeak();
detectSamplePeak();
#if !defined(I2S_GRAB_ADC1_COMPLETELY)
if ((audioSource == nullptr) || (audioSource->getType() != AudioSource::Type_I2SAdc)) // the "delay trick" does not help for analog ADC
#endif
vTaskDelayUntil( &xLastWakeTime, xFrequency); // release CPU, and let I2S fill its buffers
} // for(;;)ever
} // FFTcode() task end
///////////////////////////
// Pre / Postprocessing //
///////////////////////////
static void runMicFilter(uint16_t numSamples, FFTsampleType *sampleBuffer) // pre-filtering of raw samples (band-pass)
{
#if !defined(UM_AUDIOREACTIVE_USE_INTEGER_FFT)
// low frequency cutoff parameter - see https://dsp.stackexchange.com/questions/40462/exponential-moving-average-cut-off-frequency (alpha = 2π × fc / fs)
//constexpr float alpha = 0.04f; // 150Hz
//constexpr float alpha = 0.03f; // 110Hz
//constexpr float alpha = 0.0285f; //100Hz
constexpr float alpha = 0.0256f; //90Hz
//constexpr float alpha = 0.0225f; // 80hz
//constexpr float alpha = 0.01693f;// 60hz
// high frequency cutoff parameter
//constexpr float beta1 = 0.75f; // 11Khz
//constexpr float beta1 = 0.82f; // 15Khz
//constexpr float beta1 = 0.8285f; // 18Khz
constexpr float beta1 = 0.85f; // 20Khz
constexpr float beta2 = (1.0f - beta1) / 2.0f;
static float last_vals[2] = { 0.0f }; // FIR high freq cutoff filter
static float lowfilt = 0.0f; // IIR low frequency cutoff filter
for (int i=0; i < numSamples; i++) {
// FIR lowpass, to remove high frequency noise
float highFilteredSample;
if (i < (numSamples-1)) highFilteredSample = beta1*sampleBuffer[i] + beta2*last_vals[0] + beta2*sampleBuffer[i+1]; // smooth out spikes
else highFilteredSample = beta1*sampleBuffer[i] + beta2*last_vals[0] + beta2*last_vals[1]; // special handling for last sample in array
last_vals[1] = last_vals[0];
last_vals[0] = sampleBuffer[i];
sampleBuffer[i] = highFilteredSample;
// IIR highpass, to remove low frequency noise
lowfilt += alpha * (sampleBuffer[i] - lowfilt);
sampleBuffer[i] = sampleBuffer[i] - lowfilt;
}
#else
// low frequency cutoff parameter 17.15 fixed point format
//constexpr int32_t ALPHA_FP = 1311; // 0.04f * (1<<15) (150Hz)
//constexpr int32_t ALPHA_FP = 983; // 0.03f * (1<<15) (110Hz)
//constexpr int32_t ALPHA_FP = 934; // 0.0285f * (1<<15) (100Hz)
constexpr int32_t ALPHA_FP = 840; // 0.0256f * (1<<15) (90Hz)
//constexpr int32_t ALPHA_FP = 737; // 0.0225f * (1<<15) (80Hz)
//constexpr int32_t ALPHA_FP = 555; // 0.01693f * (1<<15) (60Hz)
// high frequency cutoff parameters 16.16 fixed point format
//constexpr int32_t BETA1_FP = 49152; // 0.75f * (1<<16) (11KHz)
//constexpr int32_t BETA1_FP = 53740; // 0.82f * (1<<16) (15KHz)
//constexpr int32_t BETA1_FP = 54297; // 0.8285f * (1<<16) (18KHz)
constexpr int32_t BETA1_FP = 55706; // 0.85f * (1<<16) (20KHz)
constexpr int32_t BETA2_FP = (65536 - BETA1_FP) / 2; // ((1.0f - beta1) / 2.0f) * (1<<16)
static int32_t last_vals[2] = { 0 }; // FIR high freq cutoff filter (scaled by sample range)
static int32_t lowfilt_fp = 0; // IIR low frequency cutoff filter (16.16 fixed point)
for (int i = 0; i < numSamples; i++) {
// FIR lowpass filter to remove high frequency noise
int32_t highFilteredSample_fp;
if (i < (numSamples - 1))
highFilteredSample_fp = (BETA1_FP * (int32_t)sampleBuffer[i] + BETA2_FP * last_vals[0] + BETA2_FP * (int32_t)sampleBuffer[i + 1]) >> 16; // smooth out spikes
else
highFilteredSample_fp = (BETA1_FP * (int32_t)sampleBuffer[i] + BETA2_FP * last_vals[0] + BETA2_FP * last_vals[1]) >> 16; // special handling for last sample in array
last_vals[1] = last_vals[0];
last_vals[0] = (int32_t)sampleBuffer[i];
lowfilt_fp += ALPHA_FP * (highFilteredSample_fp - (lowfilt_fp >> 15)); // low pass filter in 17.15 fixed point format
sampleBuffer[i] = highFilteredSample_fp - (lowfilt_fp >> 15);
}
#endif
}
static void postProcessFFTResults(bool noiseGateOpen, int numberOfChannels) // post-processing and post-amp of GEQ channels
{
for (int i=0; i < numberOfChannels; i++) {
if (noiseGateOpen) { // noise gate open
// Adjustment for frequency curves.
fftCalc[i] *= fftResultPink[i];
if (FFTScalingMode > 0) fftCalc[i] *= FFT_DOWNSCALE; // adjustment related to FFT windowing function
// Manual linear adjustment of gain using sampleGain adjustment for different input types.
fftCalc[i] *= soundAgc ? multAgc : ((float)sampleGain/40.0f * (float)inputLevel/128.0f + 1.0f/16.0f); //apply gain, with inputLevel adjustment
if(fftCalc[i] < 0) fftCalc[i] = 0;
}
// smooth results - rise fast, fall slower
if(fftCalc[i] > fftAvg[i]) // rise fast
fftAvg[i] = fftCalc[i] *0.75f + 0.25f*fftAvg[i]; // will need approx 2 cycles (50ms) for converging against fftCalc[i]
else { // fall slow
if (decayTime < 1000) fftAvg[i] = fftCalc[i]*0.22f + 0.78f*fftAvg[i]; // approx 5 cycles (225ms) for falling to zero
else if (decayTime < 2000) fftAvg[i] = fftCalc[i]*0.17f + 0.83f*fftAvg[i]; // default - approx 9 cycles (225ms) for falling to zero
else if (decayTime < 3000) fftAvg[i] = fftCalc[i]*0.14f + 0.86f*fftAvg[i]; // approx 14 cycles (350ms) for falling to zero
else fftAvg[i] = fftCalc[i]*0.1f + 0.9f*fftAvg[i]; // approx 20 cycles (500ms) for falling to zero
}
// constrain internal vars - just to be sure
fftCalc[i] = constrain(fftCalc[i], 0.0f, 1023.0f);
fftAvg[i] = constrain(fftAvg[i], 0.0f, 1023.0f);
float currentResult;
if(limiterOn == true)
currentResult = fftAvg[i];
else
currentResult = fftCalc[i];
switch (FFTScalingMode) {
case 1:
// Logarithmic scaling
currentResult *= 0.42f; // 42 is the answer ;-)
currentResult -= 8.0f; // this skips the lowest row, giving some room for peaks
if (currentResult > 1.0f) currentResult = logf(currentResult); // log to base "e", which is the fastest log() function
else currentResult = 0.0f; // special handling, because log(1) = 0; log(0) = undefined
currentResult *= 0.85f + (float(i)/18.0f); // extra up-scaling for high frequencies
currentResult = mapf(currentResult, 0, LOG_256, 0, 255); // map [log(1) ... log(255)] to [0 ... 255]
break;
case 2:
// Linear scaling
currentResult *= 0.30f; // needs a bit more damping, get stay below 255
currentResult -= 4.0f; // giving a bit more room for peaks
if (currentResult < 1.0f) currentResult = 0.0f;
currentResult *= 0.85f + (float(i)/1.8f); // extra up-scaling for high frequencies
break;
case 3:
// square root scaling
currentResult *= 0.38f;
currentResult -= 6.0f;
if (currentResult > 1.0f) currentResult = sqrtf(currentResult);
else currentResult = 0.0f; // special handling, because sqrt(0) = undefined
currentResult *= 0.85f + (float(i)/4.5f); // extra up-scaling for high frequencies
currentResult = mapf(currentResult, 0.0, 16.0, 0.0, 255.0); // map [sqrt(1) ... sqrt(256)] to [0 ... 255]
break;
case 0:
default:
// no scaling - leave freq bins as-is
currentResult -= 4; // just a bit more room for peaks
break;
}
// Now, let's dump it all into fftResult. Need to do this, otherwise other routines might grab fftResult values prematurely.
if (soundAgc > 0) { // apply extra "GEQ Gain" if set by user
float post_gain = (float)inputLevel/128.0f;
if (post_gain < 1.0f) post_gain = ((post_gain -1.0f) * 0.8f) +1.0f;
currentResult *= post_gain;
}
fftResult[i] = constrain((int)currentResult, 0, 255);
}
}
////////////////////
// Peak detection //
////////////////////
// peak detection is called from FFT task when vReal[] contains valid FFT results
static void detectSamplePeak(void) {
bool havePeak = false;
// softhack007: this code continuously triggers while amplitude in the selected bin is above a certain threshold. So it does not detect peaks - it detects high activity in a frequency bin.
// Poor man's beat detection by seeing if sample > Average + some value.
// This goes through ALL of the 255 bins - but ignores stupid settings
// Then we got a peak, else we don't. The peak has to time out on its own in order to support UDP sound sync.
if ((sampleAvg > 1) && (maxVol > 0) && (binNum > 4) && (valFFT[binNum] > maxVol) && ((millis() - timeOfPeak) > 100)) {
havePeak = true;
}
if (havePeak) {
samplePeak = true;
timeOfPeak = millis();
udpSamplePeak = true;
}
}
#endif
static void autoResetPeak(void) {
uint16_t peakDelay = max(uint16_t(50), strip.getFrameTime());
if (millis() - timeOfPeak > peakDelay) { // Auto-reset of samplePeak after at least one complete frame has passed.
samplePeak = false;
if (audioSyncEnabled == 0) udpSamplePeak = false; // this is normally reset by transmitAudioData
}
}
////////////////////
// usermod class //
////////////////////
//class name. Use something descriptive and leave the ": public Usermod" part :)
class AudioReactive : public Usermod {
private:
#ifdef ARDUINO_ARCH_ESP32
#ifndef AUDIOPIN
int8_t audioPin = -1;
#else
int8_t audioPin = AUDIOPIN;
#endif
#ifndef SR_DMTYPE // I2S mic type
uint8_t dmType = 1; // 0=none/disabled/analog; 1=generic I2S
#define SR_DMTYPE 1 // default type = I2S
#else
uint8_t dmType = SR_DMTYPE;
#endif
#ifndef I2S_SDPIN // aka DOUT
int8_t i2ssdPin = 32;
#else
int8_t i2ssdPin = I2S_SDPIN;
#endif
#ifndef I2S_WSPIN // aka LRCL
int8_t i2swsPin = 15;
#else
int8_t i2swsPin = I2S_WSPIN;
#endif
#ifndef I2S_CKPIN // aka BCLK
int8_t i2sckPin = 14; /*PDM: set to I2S_PIN_NO_CHANGE*/
#else
int8_t i2sckPin = I2S_CKPIN;
#endif
#ifndef MCLK_PIN
int8_t mclkPin = I2S_PIN_NO_CHANGE; /* ESP32: only -1, 0, 1, 3 allowed*/
#else
int8_t mclkPin = MCLK_PIN;
#endif
#endif
// new "V2" audiosync struct - 44 Bytes
struct __attribute__ ((packed)) audioSyncPacket { // "packed" ensures that there are no additional gaps
char header[6]; // 06 Bytes offset 0
uint8_t reserved1[2]; // 02 Bytes, offset 6 - gap required by the compiler - not used yet
float sampleRaw; // 04 Bytes offset 8 - either "sampleRaw" or "rawSampleAgc" depending on soundAgc setting
float sampleSmth; // 04 Bytes offset 12 - either "sampleAvg" or "sampleAgc" depending on soundAgc setting
uint8_t samplePeak; // 01 Bytes offset 16 - 0 no peak; >=1 peak detected. In future, this will also provide peak Magnitude
uint8_t reserved2; // 01 Bytes offset 17 - for future extensions - not used yet
uint8_t fftResult[16]; // 16 Bytes offset 18
uint16_t reserved3; // 02 Bytes, offset 34 - gap required by the compiler - not used yet
float FFT_Magnitude; // 04 Bytes offset 36
float FFT_MajorPeak; // 04 Bytes offset 40
};
// old "V1" audiosync struct - 83 Bytes payload, 88 bytes total (with padding added by compiler) - for backwards compatibility
struct audioSyncPacket_v1 {
char header[6]; // 06 Bytes
uint8_t myVals[32]; // 32 Bytes
int sampleAgc; // 04 Bytes
int sampleRaw; // 04 Bytes
float sampleAvg; // 04 Bytes
bool samplePeak; // 01 Bytes
uint8_t fftResult[16]; // 16 Bytes
double FFT_Magnitude; // 08 Bytes
double FFT_MajorPeak; // 08 Bytes
};
#define UDPSOUND_MAX_PACKET 88 // max packet size for audiosync
// set your config variables to their boot default value (this can also be done in readFromConfig() or a constructor if you prefer)
#ifdef UM_AUDIOREACTIVE_ENABLE
bool enabled = true;
#else
bool enabled = false;
#endif
bool initDone = false;
bool addPalettes = false;
int8_t palettes = 0;
// variables for UDP sound sync
WiFiUDP fftUdp; // UDP object for sound sync (from WiFi UDP, not Async UDP!)
unsigned long lastTime = 0; // last time of running UDP Microphone Sync
const uint16_t delayMs = 10; // I don't want to sample too often and overload WLED
uint16_t audioSyncPort= 11988;// default port for UDP sound sync
bool updateIsRunning = false; // true during OTA.
#ifdef ARDUINO_ARCH_ESP32
// used for AGC
int last_soundAgc = -1; // used to detect AGC mode change (for resetting AGC internal error buffers)
double control_integrated = 0.0; // persistent across calls to agcAvg(); "integrator control" = accumulated error
// variables used by getSample() and agcAvg()
int16_t micIn = 0; // Current sample starts with negative values and large values, which is why it's 16 bit signed
double sampleMax = 0.0; // Max sample over a few seconds. Needed for AGC controller.
double micLev = 0.0; // Used to convert returned value to have '0' as minimum. A leveller
float expAdjF = 0.0f; // Used for exponential filter.
float sampleReal = 0.0f; // "sampleRaw" as float, to provide bits that are lost otherwise (before amplification by sampleGain or inputLevel). Needed for AGC.
int16_t sampleRaw = 0; // Current sample. Must only be updated ONCE!!! (amplified mic value by sampleGain and inputLevel)
int16_t rawSampleAgc = 0; // not smoothed AGC sample
#endif
// variables used in effects
float volumeSmth = 0.0f; // either sampleAvg or sampleAgc depending on soundAgc; smoothed sample
int16_t volumeRaw = 0; // either sampleRaw or rawSampleAgc depending on soundAgc
float my_magnitude =0.0f; // FFT_Magnitude, scaled by multAgc
// used to feed "Info" Page
unsigned long last_UDPTime = 0; // time of last valid UDP sound sync datapacket
int receivedFormat = 0; // last received UDP sound sync format - 0=none, 1=v1 (0.13.x), 2=v2 (0.14.x)
float maxSample5sec = 0.0f; // max sample (after AGC) in last 5 seconds
unsigned long sampleMaxTimer = 0; // last time maxSample5sec was reset
#define CYCLE_SAMPLEMAX 3500 // time window for merasuring
// strings to reduce flash memory usage (used more than twice)
static const char _name[];
static const char _enabled[];
static const char _config[];
static const char _dynamics[];
static const char _frequency[];
static const char _inputLvl[];
#if defined(ARDUINO_ARCH_ESP32) && !defined(CONFIG_IDF_TARGET_ESP32S2) && !defined(CONFIG_IDF_TARGET_ESP32C3) && !defined(CONFIG_IDF_TARGET_ESP32S3)
static const char _analogmic[];
#endif
static const char _digitalmic[];
static const char _addPalettes[];
static const char UDP_SYNC_HEADER[];
static const char UDP_SYNC_HEADER_v1[];
// private methods
void removeAudioPalettes(void);
void createAudioPalettes(void);
CRGB getCRGBForBand(int x, int pal);
void fillAudioPalettes(void);
////////////////////
// Debug support //
////////////////////
void logAudio()
{
if (disableSoundProcessing && (!udpSyncConnected || ((audioSyncEnabled & 0x02) == 0))) return; // no audio availeable
#ifdef MIC_LOGGER
// Debugging functions for audio input and sound processing. Comment out the values you want to see
PLOT_PRINT("micReal:"); PLOT_PRINT(micDataReal); PLOT_PRINT("\t");
PLOT_PRINT("volumeSmth:"); PLOT_PRINT(volumeSmth); PLOT_PRINT("\t");
//PLOT_PRINT("volumeRaw:"); PLOT_PRINT(volumeRaw); PLOT_PRINT("\t");
PLOT_PRINT("DC_Level:"); PLOT_PRINT(micLev); PLOT_PRINT("\t");
//PLOT_PRINT("sampleAgc:"); PLOT_PRINT(sampleAgc); PLOT_PRINT("\t");
//PLOT_PRINT("sampleAvg:"); PLOT_PRINT(sampleAvg); PLOT_PRINT("\t");
//PLOT_PRINT("sampleReal:"); PLOT_PRINT(sampleReal); PLOT_PRINT("\t");
#ifdef ARDUINO_ARCH_ESP32
//PLOT_PRINT("micIn:"); PLOT_PRINT(micIn); PLOT_PRINT("\t");
//PLOT_PRINT("sample:"); PLOT_PRINT(sample); PLOT_PRINT("\t");
//PLOT_PRINT("sampleMax:"); PLOT_PRINT(sampleMax); PLOT_PRINT("\t");
//PLOT_PRINT("samplePeak:"); PLOT_PRINT((samplePeak!=0) ? 128:0); PLOT_PRINT("\t");
//PLOT_PRINT("multAgc:"); PLOT_PRINT(multAgc, 4); PLOT_PRINT("\t");
#endif
PLOT_PRINTLN();
#endif
#ifdef FFT_SAMPLING_LOG
#if 0
for(int i=0; i<NUM_GEQ_CHANNELS; i++) {
PLOT_PRINT(fftResult[i]);
PLOT_PRINT("\t");
}
PLOT_PRINTLN();
#endif
// OPTIONS are in the following format: Description \n Option
//
// Set true if wanting to see all the bands in their own vertical space on the Serial Plotter, false if wanting to see values in Serial Monitor
const bool mapValuesToPlotterSpace = false;
// Set true to apply an auto-gain like setting to to the data (this hasn't been tested recently)
const bool scaleValuesFromCurrentMaxVal = false;
// prints the max value seen in the current data
const bool printMaxVal = false;
// prints the min value seen in the current data
const bool printMinVal = false;
// if !scaleValuesFromCurrentMaxVal, we scale values from [0..defaultScalingFromHighValue] to [0..scalingToHighValue], lower this if you want to see smaller values easier
const int defaultScalingFromHighValue = 256;
// Print values to terminal in range of [0..scalingToHighValue] if !mapValuesToPlotterSpace, or [(i)*scalingToHighValue..(i+1)*scalingToHighValue] if mapValuesToPlotterSpace
const int scalingToHighValue = 256;
// set higher if using scaleValuesFromCurrentMaxVal and you want a small value that's also the current maxVal to look small on the plotter (can't be 0 to avoid divide by zero error)
const int minimumMaxVal = 1;
int maxVal = minimumMaxVal;
int minVal = 0;
for(int i = 0; i < NUM_GEQ_CHANNELS; i++) {
if(fftResult[i] > maxVal) maxVal = fftResult[i];
if(fftResult[i] < minVal) minVal = fftResult[i];
}
for(int i = 0; i < NUM_GEQ_CHANNELS; i++) {
PLOT_PRINT(i); PLOT_PRINT(":");
PLOT_PRINTF("%04ld ", map(fftResult[i], 0, (scaleValuesFromCurrentMaxVal ? maxVal : defaultScalingFromHighValue), (mapValuesToPlotterSpace*i*scalingToHighValue)+0, (mapValuesToPlotterSpace*i*scalingToHighValue)+scalingToHighValue-1));
}
if(printMaxVal) {
PLOT_PRINTF("maxVal:%04d ", maxVal + (mapValuesToPlotterSpace ? 16*256 : 0));
}
if(printMinVal) {
PLOT_PRINTF("%04d:minVal ", minVal); // printed with value first, then label, so negative values can be seen in Serial Monitor but don't throw off y axis in Serial Plotter
}
if(mapValuesToPlotterSpace)
PLOT_PRINTF("max:%04d ", (printMaxVal ? 17 : 16)*256); // print line above the maximum value we expect to see on the plotter to avoid autoscaling y axis
else {
PLOT_PRINTF("max:%04d ", 256);
}
PLOT_PRINTLN();
#endif // FFT_SAMPLING_LOG
} // logAudio()
#ifdef ARDUINO_ARCH_ESP32
//////////////////////
// Audio Processing //
//////////////////////
/*
* A "PI controller" multiplier to automatically adjust sound sensitivity.
*
* A few tricks are implemented so that sampleAgc does't only utilize 0% and 100%:
* 0. don't amplify anything below squelch (but keep previous gain)
* 1. gain input = maximum signal observed in the last 5-10 seconds
* 2. we use two setpoints, one at ~60%, and one at ~80% of the maximum signal
* 3. the amplification depends on signal level:
* a) normal zone - very slow adjustment
* b) emergency zone (<10% or >90%) - very fast adjustment
*/
void agcAvg(unsigned long the_time)
{
const int AGC_preset = (soundAgc > 0)? (soundAgc-1): 0; // make sure the _compiler_ knows this value will not change while we are inside the function
float lastMultAgc = multAgc; // last multiplier used
float multAgcTemp = multAgc; // new multiplier
float tmpAgc = sampleReal * multAgc; // what-if amplified signal
float control_error; // "control error" input for PI control
if (last_soundAgc != soundAgc)
control_integrated = 0.0; // new preset - reset integrator
// For PI controller, we need to have a constant "frequency"
// so let's make sure that the control loop is not running at insane speed