Skip to content

Add autoleveling feature (Probalby not ready for commit) #224

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cnlohr wants to merge 3 commits into from
Closed

Add autoleveling feature (Probalby not ready for commit) #224

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
02038de
Add autoleveling feature.
cnlohr Jan 27, 2023
adb2015
Adjust filter to make it time invariant.
cnlohr Jan 27, 2023
5c71dff
Further tweaks, to pull levels, etc. from self texture.
cnlohr Jan 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Packages/com.llealloo.audiolink/Runtime/Shaders/AudioLink.cginc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
#define ALPASS_FILTEREDVU uint2(24,28) //Size: 4, 4
#define ALPASS_FILTEREDVU_INTENSITY uint2(24,28) //Size: 4, 1
#define ALPASS_FILTEREDVU_MARKER uint2(24,29) //Size: 4, 1
// There is an undocumented, probably changing area having to do with beat detection at 28,28
#define ALPASS_FILTEREDVU_LEVELTIMING uint2(32,28) //Size: 4, 4
#define ALPASS_GLOBAL_STRINGS uint2(40,28) //Size: 8, 4

#define ALPASS_VUHISTORY uint2(0,32) //Size: 128, 5
Expand Down
134 changes: 107 additions & 27 deletions Packages/com.llealloo.audiolink/Runtime/Shaders/AudioLink.shader
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ Shader "AudioLink/Internal/AudioLink"
_Threshold2("Threshold 2", Range(0.0, 1.0)) = 0.45
_Threshold3("Threshold 3", Range(0.0, 1.0)) = 0.45
[ToggleUI] _EnableAutogain("Enable Autogain", float) = 1

[ToggleUI] _EnableAutoLevel("Enable Autolevel", float) = 0

_AutogainDerate ("Autogain Derate", Range(.001, .5)) = 0.1
_SourceVolume("Audio Source Volume", float) = 1
_SourceDistance("Distance to Source", float) = 1
Expand Down Expand Up @@ -103,6 +106,7 @@ Shader "AudioLink/Internal/AudioLink"

// Extra Properties
uniform float _EnableAutogain;
uniform float _EnableAutoLevel;
uniform float _AutogainDerate;

// Set by Udon
Expand Down Expand Up @@ -142,6 +146,34 @@ Shader "AudioLink/Internal/AudioLink"
(float)((val >> 30) & 0x3ff)
);
}

inline float Compute4BandRaw( int band )
{
const float audioThresholds[4] = {_Threshold0, _Threshold1, _Threshold2, _Threshold3};
const float audioBands[4] = {_X0, _X1, _X2, _X3};
float threshold = AudioLinkGetSelfPixelData( ALPASS_FILTEREDVU_LEVELTIMING + uint2( 0, band ) ).z;

// Get average of samples in the band
float total = 0.;
uint totalBins = AUDIOLINK_EXPBINS * AUDIOLINK_EXPOCT;
uint binStart = AudioLinkRemap(audioBands[band], 0., 1., AUDIOLINK_4BAND_FREQFLOOR * totalBins, AUDIOLINK_4BAND_FREQCEILING * totalBins);
uint binEnd = (band != 3) ? AudioLinkRemap(audioBands[band + 1], 0., 1., AUDIOLINK_4BAND_FREQFLOOR * totalBins, AUDIOLINK_4BAND_FREQCEILING * totalBins) : AUDIOLINK_4BAND_FREQCEILING * totalBins;
for (uint i=binStart; i<binEnd; i++)
{
int2 spectrumCoord = int2(i % AUDIOLINK_WIDTH, i / AUDIOLINK_WIDTH);
float rawMagnitude = AudioLinkGetSelfPixelData(ALPASS_DFT + spectrumCoord).y;
total += rawMagnitude;
}
float magnitude = total / (binEnd - binStart);

// Log attenuation
magnitude = (magnitude * (log(1.1) / (log(1.1 + pow(_LogAttenuation, 4) * (1.0 - magnitude))))) / pow(threshold, 2);

// Contrast
magnitude = (magnitude * tan(1.57 * _ContrastSlope) + magnitude + _ContrastOffset * tan(1.57 * _ContrastSlope) - tan(1.57 * _ContrastSlope));

return magnitude;
}
ENDCG

Name "Pass1AudioDFT"
Expand Down Expand Up @@ -302,44 +334,22 @@ Shader "AudioLink/Internal/AudioLink"
{
AUDIO_LINK_ALPHA_START(ALPASS_AUDIOLINK)

float audioBands[4] = {_X0, _X1, _X2, _X3};
float audioThresholds[4] = {_Threshold0, _Threshold1, _Threshold2, _Threshold3};

int band = min(coordinateLocal.y, 3);
int delay = coordinateLocal.x;
if (delay == 0)
{
// Get average of samples in the band
float total = 0.;
uint totalBins = AUDIOLINK_EXPBINS * AUDIOLINK_EXPOCT;
uint binStart = AudioLinkRemap(audioBands[band], 0., 1., AUDIOLINK_4BAND_FREQFLOOR * totalBins, AUDIOLINK_4BAND_FREQCEILING * totalBins);
uint binEnd = (band != 3) ? AudioLinkRemap(audioBands[band + 1], 0., 1., AUDIOLINK_4BAND_FREQFLOOR * totalBins, AUDIOLINK_4BAND_FREQCEILING * totalBins) : AUDIOLINK_4BAND_FREQCEILING * totalBins;
float threshold = audioThresholds[band];
for (uint i=binStart; i<binEnd; i++)
{
int2 spectrumCoord = int2(i % AUDIOLINK_WIDTH, i / AUDIOLINK_WIDTH);
float rawMagnitude = AudioLinkGetSelfPixelData(ALPASS_DFT + spectrumCoord).y;
total += rawMagnitude;
}
float magnitude = total / (binEnd - binStart);

// Log attenuation
magnitude = saturate(magnitude * (log(1.1) / (log(1.1 + pow(_LogAttenuation, 4) * (1.0 - magnitude))))) / pow(threshold, 2);

// Contrast
magnitude = saturate(magnitude * tan(1.57 * _ContrastSlope) + magnitude + _ContrastOffset * tan(1.57 * _ContrastSlope) - tan(1.57 * _ContrastSlope));
// Is left-most pixel.
float magnitude = saturate( Compute4BandRaw( band ) );

// Fade
float lastMagnitude = AudioLinkGetSelfPixelData(ALPASS_AUDIOLINK + int2(0, band)).y;
lastMagnitude -= -1.0 * pow(_FadeLength-1.0, 3); // Inverse cubic remap
lastMagnitude = saturate(lastMagnitude * (1.0 + (pow(lastMagnitude - 1.0, 4.0) * _FadeExpFalloff) - _FadeExpFalloff)); // Exp falloff

magnitude = max(lastMagnitude, magnitude);

return float4(magnitude, magnitude, magnitude, 1.);

// If part of the delay
} else {
// If part of the delay
// Slide pixels (coordinateLocal.x > 0)
float4 lastvalTiming = AudioLinkGetSelfPixelData(ALPASS_GENERALVU + int2(4, 1)); // Timing for 4-band, move at 90 Hz.
lastvalTiming.x += unity_DeltaTime.x * AUDIOLINK_4BAND_TARGET_RATE;
Expand Down Expand Up @@ -1216,9 +1226,9 @@ Shader "AudioLink/Internal/AudioLink"
{
// PEMA
}
else
else if (coordinateLocal.x < 8 )
{
// BEAT DETECTION STILL IN EARLY DEVELOPMENT - DO NOT USE
// BEAT DETECTION STILL IN EARLY DEVELOPMENT - DO NOT USE
float4 prev = AudioLinkGetSelfPixelData(coordinateGlobal.xy);
if( coordinateLocal.x == 5 )
{
Expand Down Expand Up @@ -1249,6 +1259,76 @@ Shader "AudioLink/Internal/AudioLink"
//Assume beats in the range of 80..160 BPM only.
}
}
else if( coordinateLocal.x < 12 )
{
// 4x4 area for ALPASS_FILTEREDVU_LEVELTIMING

// Split into 4 columns. Each row is a band.
int band = coordinateLocal.y;
const float4 audioBands = {_X0, _X1, _X2, _X3};
const float4 audioThresholds = {_Threshold0, _Threshold1, _Threshold2, _Threshold3};

switch( coordinateLocal.x )
{
case 8:
{
// A basic recomputation of the 4-band. Except raw.
// This first column of 4 rows is:
// R: Current raw band magnitude.
// G: Parameters
// Row 0: "Is Autoleveling Enabled"
// B: Threshold amount from auto-level.
// A: "Confidence" in this. If confidence is low, threshold will fall.

float magnitude = Compute4BandRaw(band);
float4 self = AudioLinkGetSelfPixelData( ALPASS_FILTEREDVU + coordinateLocal.xy );

float para = 0.0;
if( coordinateLocal.x == 8 )
para = _EnableAutoLevel;
else
para = 0.0; // There are 3 more floats that can be used.

// invamt should be between 0 for no boost to 0.3ish
float amt = self.z;
float confidence = self.w;

if( _EnableAutoLevel )
{
// Compare the current value to "Magnitude" if it's very low, need to boost it.
if( magnitude < 0.5 )
{
confidence = ( confidence - 0.05 * unity_DeltaTime.x );
amt = lerp( amt - 0.1 * unity_DeltaTime.x, amt, saturate( confidence ) );
}
else
{
// When we are above the threshold.
confidence = 1.1; // Make sure it can't slop down for a few seconds.
amt += 0.5 * unity_DeltaTime.x;
}
amt = clamp( amt, 0.1, audioThresholds[band] );
}
else
{
amt = audioThresholds[band];
}

return float4( magnitude, para, amt, confidence );
}
case 9:
{
// Include configured bands and threshold.
if( band == 0 ) return audioBands;
if( band == 1 ) return audioThresholds;
return 0.0;
}
case 10:
case 11:
// Reserved
return 0.0;
}
}
return 1;
}
ENDCG
Expand Down
20 changes: 6 additions & 14 deletions Packages/com.llealloo.audiolink/Runtime/Shaders/AudioLinkSpectrumUI.shader
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,6 @@ Shader "AudioLink/Internal/AudioLinkSpectrumUI"
_X1("X1", Range(0.242, 0.387)) = 0.25
_X2("X2", Range(0.461, 0.628)) = 0.5
_X3("X3", Range(0.704, 0.953)) = 0.75
_Threshold0("Threshold 0", Range(0.0, 1.0)) = 0.45
_Threshold1("Threshold 1", Range(0.0, 1.0)) = 0.45
_Threshold2("Threshold 2", Range(0.0, 1.0)) = 0.45
_Threshold3("Threshold 3", Range(0.0, 1.0)) = 0.45

}
SubShader
Expand Down Expand Up @@ -67,10 +63,6 @@ Shader "AudioLink/Internal/AudioLinkSpectrumUI"
uniform float _X1;
uniform float _X2;
uniform float _X3;
uniform float _Threshold0;
uniform float _Threshold1;
uniform float _Threshold2;
uniform float _Threshold3;

float _SpectrumGain;
float _SpectrumColorMix;
Expand Down Expand Up @@ -121,13 +113,14 @@ Shader "AudioLink/Internal/AudioLinkSpectrumUI"
fixed4 frag(v2f IN) : SV_Target
{
float2 iuv = IN.uv;
float audioBands[4] = {_X0, _X1, _X2, _X3};
float audioThresholds[4] = {_Threshold0, _Threshold1, _Threshold2, _Threshold3};
float4 intensity = 0;
uint totalBins = AUDIOLINK_EXPBINS * AUDIOLINK_EXPOCT;
uint noteno = AudioLinkRemap(iuv.x, 0., 1., AUDIOLINK_4BAND_FREQFLOOR * totalBins, AUDIOLINK_4BAND_FREQCEILING * totalBins);
float notenof = AudioLinkRemap(iuv.x, 0., 1., AUDIOLINK_4BAND_FREQFLOOR * totalBins, AUDIOLINK_4BAND_FREQCEILING * totalBins);

float4 audioBands = AudioLinkData( ALPASS_FILTEREDVU_LEVELTIMING + uint2( 1, 0 ) );
float4 audioSetLevels = AudioLinkData( ALPASS_FILTEREDVU_LEVELTIMING + uint2( 1, 1 ) );

{
float4 spectrum_value_lower = AudioLinkData(float2(fmod(noteno, 128), (noteno/128)+4.0));
float4 spectrum_value_higher = AudioLinkData(float2(fmod(noteno+1, 128), ((noteno+1)/128)+4.0));
Expand All @@ -152,10 +145,9 @@ Shader "AudioLink/Internal/AudioLinkSpectrumUI"
{
band += (iuv.x > audioBands[j]);
}
for (int k=0; k<4; k++)
{
threshold += (band == k) * saturate(_ThresholdThickness - abs(iuv.y - lerp(minHeight, maxHeight, audioThresholds[k]))) * 1000.;
}

float tHoldLevel = AudioLinkData( ALPASS_FILTEREDVU_LEVELTIMING + uint2( 0, band ) ).z;
threshold += saturate(_ThresholdThickness - abs(iuv.y - lerp(minHeight, maxHeight, tHoldLevel))) * 1000.;
threshold = saturate(threshold) * (1. - round((iuv.x % _ThresholdDottedLine) / _ThresholdDottedLine));
threshold *= (iuv.x > _X0);

Expand Down