Skip to content
This repository was archived by the owner on Apr 18, 2024. It is now read-only.

Commit 61e6d9f

Browse files
bmartelyyassi-heartex
bmartel
and
yyassi-heartex
authored
fix: LSDV-4532: Audio v3 visual data is offset to sound playback (#1166)
* Revert "Revert feat: DEV-4034: Rendering performance improvements for large-duration audio (#1138) (#1160)" This reverts commit d84c38e. * fix: LSDV-4532: Correct decoded audio data offset * update the removal grace period for audio cached decoders * only pause playback if it was playing in disconnect, remove redundant methods * force buffer audio playback by playing then pausing, and ensure media has ability to playback through buffer * adding comment about the duration offset start for decoding audio * fix: LSDV-4532: Audio sometimes goes missing during playback * fix: LSDV-4532: Waveform sometimes blank when toggling visibility off then on * fix: LSDV-4532: Sound sometimes delayed to playback * when layer visibility changes we now fire renderAvailableChannels * add crossorigin anonymous to audio element * testing cors issue * changing back to proper src * fix: LSDV-4532: Adding more tests fixing playbackRate setting * adding the audioelement selector to the waitForAudio test helper * e2e tests are being run on production builds so testid needs to exist always on waveform * moving out audio v3 specific check from waitForAudio as there are other tests which cannot see this element --------- Co-authored-by: Yousif Yassi <[email protected]>
1 parent d56dd55 commit 61e6d9f

24 files changed

+1316
-569
lines changed

Diff for: ‎e2e/fragments/AtAudioView.js

+99
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,28 @@
11
/* global inject */
22
const { I } = inject();
3+
const assert = require('assert');
34

45
const Helpers = require('../tests/helpers');
56

67
module.exports = {
78
_stageSelector: '#waveform-layer-main',
89
_progressBarSelector: 'loading-progress-bar',
10+
_controlMenuSelector: '.lsf-audio-control',
11+
_settingsMenuSelector: '.lsf-audio-config',
12+
_volumeSliderSelector: '.lsf-audio-slider__range',
13+
_volumeInputSelector: '.lsf-audio-slider__input',
14+
_muteButtonSelector: '.lsf-audio-control__mute-button',
15+
_playbackSpeedSliderSelector: '.lsf-audio-config__modal > .lsf-audio-slider:nth-child(1) .lsf-audio-slider__range',
16+
_playbackSpeedInputSelector: '.lsf-audio-config__modal > .lsf-audio-slider:nth-child(1) .lsf-audio-slider__input',
17+
_amplitudeSliderSelector: '.lsf-audio-config__modal > .lsf-audio-slider:nth-child(2) .lsf-audio-slider__range',
18+
_amplitudeInputSelector: '.lsf-audio-config__modal > .lsf-audio-slider:nth-child(2) .lsf-audio-slider__input',
19+
_hideTimelineButtonSelector: '.lsf-audio-config__buttons > .lsf-audio-config__menu-button:nth-child(1)',
20+
_hideWaveformButtonSelector: '.lsf-audio-config__buttons > .lsf-audio-config__menu-button:nth-child(2)',
21+
_audioElementSelector: '[data-testid="waveform-audio"]',
22+
_seekBackwardButtonSelector: '.lsf-audio-tag .lsf-timeline-controls__main-controls > .lsf-timeline-controls__group:nth-child(2) > button:nth-child(1)',
23+
_playButtonSelector: '.lsf-audio-tag .lsf-timeline-controls__main-controls > .lsf-timeline-controls__group:nth-child(2) > button:nth-child(2)',
24+
_seekForwardButtonSelector: '.lsf-audio-tag .lsf-timeline-controls__main-controls > .lsf-timeline-controls__group:nth-child(2) > button:nth-child(3)',
25+
926
_stageBbox: { x: 0, y: 0, width: 0, height: 0 },
1027

1128
async lookForStage() {
@@ -43,4 +60,86 @@ module.exports = {
4360
I.clickAt(this._stageBbox.x + x, this._stageBbox.y + this._stageBbox.height / 2);
4461
I.wait(1); // We gotta wait here because clicks on the canvas are not processed immediately
4562
},
63+
64+
toggleControlsMenu() {
65+
I.click(this._controlMenuSelector);
66+
},
67+
68+
toggleSettingsMenu() {
69+
I.click(this._settingsMenuSelector);
70+
},
71+
72+
async seeVolume(value) {
73+
this.toggleControlsMenu();
74+
I.seeInField(this._volumeInputSelector, value);
75+
I.seeInField(this._volumeSliderSelector, value);
76+
const volume = await I.grabAttributeFrom(this._audioElementSelector, 'volume');
77+
const muted = await I.grabAttributeFrom(this._audioElementSelector, 'muted');
78+
79+
if (muted) {
80+
assert.equal(volume, null, 'Volume doesn\'t match in audio element');
81+
} else {
82+
assert.equal(volume, value / 100, 'Volume doesn\'t match in audio element');
83+
}
84+
this.toggleControlsMenu();
85+
},
86+
87+
setVolumeInput(value) {
88+
this.toggleControlsMenu();
89+
I.fillField(this._volumeInputSelector, value);
90+
this.toggleControlsMenu();
91+
},
92+
93+
async seePlaybackSpeed(value) {
94+
this.toggleSettingsMenu();
95+
96+
I.seeInField(this._playbackSpeedInputSelector, value);
97+
I.seeInField(this._playbackSpeedSliderSelector, value);
98+
const playbackSpeed = await I.grabAttributeFrom(this._audioElementSelector, 'playbackRate');
99+
100+
assert.equal(playbackSpeed, value, 'Playback speed doesn\'t match in audio element');
101+
102+
this.toggleSettingsMenu();
103+
},
104+
105+
setPlaybackSpeedInput(value) {
106+
this.toggleSettingsMenu();
107+
I.fillField(this._playbackSpeedInputSelector, value);
108+
this.toggleSettingsMenu();
109+
},
110+
111+
async seeAmplitude(value) {
112+
this.toggleSettingsMenu();
113+
114+
I.seeInField(this._amplitudeInputSelector, value);
115+
I.seeInField(this._amplitudeSliderSelector, value);
116+
117+
this.toggleSettingsMenu();
118+
},
119+
120+
setAmplitudeInput(value) {
121+
this.toggleSettingsMenu();
122+
I.fillField(this._amplitudeInputSelector, value);
123+
this.toggleSettingsMenu();
124+
},
125+
126+
clickMuteButton() {
127+
this.toggleControlsMenu();
128+
I.click(this._muteButtonSelector);
129+
this.toggleControlsMenu();
130+
},
131+
132+
clickPlayButton() {
133+
I.click(this._playButtonSelector);
134+
},
135+
136+
clickPauseButton() {
137+
I.click(this._playButtonSelector);
138+
},
139+
140+
async seeIsPlaying(playing) {
141+
const isPaused = await I.grabAttributeFrom(this._audioElementSelector, 'paused');
142+
143+
assert.equal(!isPaused, playing, 'Audio is not playing');
144+
},
46145
};

Diff for: ‎e2e/tests/audio/audio-controls.test.js

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
2+
/* global Feature, Scenario */
3+
4+
Feature('Audio Controls');
5+
6+
const config = `
7+
<View>
8+
<Header value="Select regions:"></Header>
9+
<Labels name="label" toName="audio" choice="multiple">
10+
<Label value="Beat" background="yellow"></Label>
11+
<Label value="Voice" background="red"></Label>
12+
<Label value="Guitar" background="blue"></Label>
13+
<Label value="Other"></Label>
14+
</Labels>
15+
<Header value="Select genre:"></Header>
16+
<Choices name="choice" toName="audio" choice="multiple">
17+
<Choice value="Lo-Fi" />
18+
<Choice value="Rock" />
19+
<Choice value="Pop" />
20+
</Choices>
21+
<Header value="Listen the audio:"></Header>
22+
<AudioPlus name="audio" value="$url"></AudioPlus>
23+
</View>
24+
`;
25+
26+
const data = {
27+
url: 'https://htx-misc.s3.amazonaws.com/opensource/label-studio/examples/audio/barradeen-emotional.mp3',
28+
};
29+
30+
const annotations = [
31+
{
32+
from_name: 'choice',
33+
id: 'hIj6zg57SY',
34+
to_name: 'audio',
35+
type: 'choices',
36+
origin: 'manual',
37+
value: {
38+
choices: ['Lo-Fi'],
39+
},
40+
},
41+
{
42+
from_name: 'label',
43+
id: 'JhxupEJWlW',
44+
to_name: 'audio',
45+
original_length: 98.719925,
46+
type: 'labels',
47+
origin: 'manual',
48+
value: {
49+
channel: 1,
50+
end: 59.39854733358493,
51+
labels: ['Other'],
52+
start: 55.747572792986325,
53+
},
54+
},
55+
];
56+
57+
const params = { annotations: [{ id: 'test', result: annotations }], config, data };
58+
59+
Scenario('Check the audio controls work', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
60+
LabelStudio.setFeatureFlags({
61+
ff_front_dev_2715_audio_3_280722_short: true,
62+
});
63+
I.amOnPage('/');
64+
65+
LabelStudio.init(params);
66+
67+
await AtAudioView.waitForAudio();
68+
69+
I.waitForDetached('loading-progress-bar', 10);
70+
71+
await AtAudioView.lookForStage();
72+
73+
AtSidebar.seeRegions(1);
74+
75+
I.say('Check the volume updates');
76+
77+
await AtAudioView.seeVolume(100);
78+
79+
AtAudioView.setVolumeInput(50);
80+
81+
await AtAudioView.seeVolume(50);
82+
83+
I.say('Check can be muted');
84+
85+
AtAudioView.clickMuteButton();
86+
87+
await AtAudioView.seeVolume(0);
88+
89+
I.say('Check the playback speed updates');
90+
91+
await AtAudioView.seePlaybackSpeed(1);
92+
93+
AtAudioView.setPlaybackSpeedInput(2);
94+
95+
await AtAudioView.seePlaybackSpeed(2);
96+
97+
I.say('Check the amplitude updates');
98+
99+
await AtAudioView.seeAmplitude(1);
100+
101+
AtAudioView.setAmplitudeInput(2);
102+
103+
await AtAudioView.seeAmplitude(2);
104+
105+
I.say('Check can be played');
106+
107+
await AtAudioView.seeIsPlaying(false);
108+
109+
AtAudioView.clickPlayButton();
110+
111+
await AtAudioView.seeIsPlaying(true);
112+
113+
I.say('Check can be paused');
114+
115+
AtAudioView.clickPauseButton();
116+
117+
await AtAudioView.seeIsPlaying(false);
118+
});

Diff for: ‎e2e/tests/audio-paragraphs.test.js renamed to ‎e2e/tests/audio/audio-paragraphs.test.js

+13-18
Original file line numberDiff line numberDiff line change
@@ -93,29 +93,28 @@ const annotations = [
9393

9494
const params = { annotations: [{ id: 'test', result: annotations }], config, data };
9595

96-
Scenario('Check audio clip is played when using the new sync option', async function({ I, LabelStudio, AtAudioView }) {
96+
Scenario('Check audio clip is played when using the new sync option', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
97+
LabelStudio.setFeatureFlags({
98+
fflag_feat_front_dev_2461_audio_paragraphs_seek_chunk_position_short: true,
99+
ff_front_dev_2715_audio_3_280722_short: true,
100+
});
97101

98102
I.amOnPage('/');
99103

100-
const hasFFDev1713 = await LabelStudio.hasFF('ff_front_DEV_1713_audio_ui_150222_short');
101-
const hasFFDev2461 = await LabelStudio.hasFF('fflag_feat_front_dev_2461_audio_paragraphs_seek_chunk_position_short');
102-
103-
if (!hasFFDev1713) {
104-
return;
105-
}
106-
107104
LabelStudio.init(params);
108105

109106
await AtAudioView.waitForAudio();
110107

111-
I.wait(1);
108+
I.waitForDetached('loading-progress-bar', 10);
109+
110+
await AtAudioView.lookForStage();
111+
112+
AtSidebar.seeRegions(2);
112113

113114
const [startingAudioPlusTime, startingParagraphAudioTime] = await AtAudioView.getCurrentAudioTime();
114115

115-
if (hasFFDev2461) {
116-
assert.equal(startingAudioPlusTime, startingParagraphAudioTime);
117-
assert.equal(startingParagraphAudioTime, 0);
118-
}
116+
assert.equal(startingAudioPlusTime, startingParagraphAudioTime);
117+
assert.equal(startingParagraphAudioTime, 0);
119118

120119
I.click('[aria-label="play-circle"]');
121120

@@ -125,10 +124,6 @@ Scenario('Check audio clip is played when using the new sync option', async func
125124

126125
const expectedSeekTime = Math.round(seekAudioPlusTime);
127126

128-
if (hasFFDev2461) {
129-
assert.equal(expectedSeekTime, Math.round(seekParagraphAudioTime), `Expected seek time to be ${expectedSeekTime} but was ${Math.round(seekParagraphAudioTime)}`);
130-
} else {
131-
assert(expectedSeekTime > 0, 'Expected seek time to be greater than 0');
132-
}
127+
assert.equal(expectedSeekTime, Math.round(seekParagraphAudioTime), `Expected seek time to be ${expectedSeekTime} but was ${Math.round(seekParagraphAudioTime)}`);
133128
});
134129

Diff for: ‎e2e/tests/audio.test.js renamed to ‎e2e/tests/audio/audio-regions.test.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ const annotations = [
5555

5656
const params = { annotations: [{ id: 'test', result: annotations }], config, data };
5757

58-
Scenario('Check if regions is selected', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
58+
Scenario('Check if regions are selected', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
5959
LabelStudio.setFeatureFlags({
6060
ff_front_dev_2715_audio_3_280722_short: true,
6161
});
@@ -69,7 +69,6 @@ Scenario('Check if regions is selected', async function({ I, LabelStudio, AtAudi
6969

7070
await AtAudioView.lookForStage();
7171

72-
7372
AtSidebar.seeRegions(1);
7473

7574
// creating a new region

Diff for: ‎package.json

+5-2
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@
4444
"start": "yarn run copy-examples && node dev-server.js",
4545
"test:coverage": "yarn test -- --coverage",
4646
"test:watch": "react-scripts test",
47-
"test": "yarn jest src"
47+
"test": "yarn jest src",
48+
"postinstall": "sh scripts/postinstall.sh"
4849
},
4950
"husky": {
5051
"hooks": {
@@ -85,6 +86,7 @@
8586
],
8687
"dependencies": {
8788
"@thi.ng/rle-pack": "^2.1.6",
89+
"audio-file-decoder": "^2.3.0",
8890
"babel-preset-react-app": "^9.1.1",
8991
"d3": "^5.16.0",
9092
"magic-wand-js": "^1.0.0",
@@ -114,8 +116,8 @@
114116
"@types/jest": "^29.2.3",
115117
"@types/keymaster": "^1.6.30",
116118
"@types/lodash.ismatch": "^4.4.6",
117-
"@types/offscreencanvas": "^2019.6.4",
118119
"@types/nanoid": "^3.0.0",
120+
"@types/offscreencanvas": "^2019.6.4",
119121
"@types/react-dom": "^17.0.11",
120122
"@types/react-window": "^1.8.5",
121123
"@types/strman": "^2.0.0",
@@ -140,6 +142,7 @@
140142
"enzyme-to-json": "^3.5.0",
141143
"eslint": "^8.28.0",
142144
"eslint-webpack-plugin": "^3.0.1",
145+
"file-loader": "^6.2.0",
143146
"html-webpack-plugin": "^5.3.1",
144147
"husky": "^3.1.0",
145148
"identity-obj-proxy": "^3.0.0",

Diff for: ‎scripts/postinstall.sh

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
cp ./node_modules/audio-file-decoder/decode-audio.wasm ./node_modules/audio-file-decoder/dist/decode-audio.wasm

Diff for: ‎src/components/Timeline/Controls/AudioControl.tsx

+4-4
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { IconSoundConfig, IconSoundMutedConfig } from '../../../assets/icons/tim
66
import { ControlButton } from '../Controls';
77
import { Slider } from './Slider';
88

9-
const MAX_VOL = 200;
9+
const MAX_VOL = 100;
1010

1111
export interface AudioControlProps {
1212
volume: number;
@@ -38,15 +38,15 @@ export const AudioControl: FC<AudioControlProps> = ({
3838
onVolumeChange?.(0);
3939
return;
4040
}
41-
if (_volumeValue > (MAX_VOL)) {
41+
if (_volumeValue > MAX_VOL) {
4242
onVolumeChange?.(MAX_VOL / 100);
4343
return;
4444
} else if (_volumeValue < 0) {
4545
onVolumeChange?.(0);
4646
return;
4747
}
4848

49-
onVolumeChange?.(_volumeValue / MAX_VOL * 2);
49+
onVolumeChange?.(_volumeValue / MAX_VOL);
5050
};
5151

5252
const handleSetMute = () => {
@@ -60,7 +60,7 @@ export const AudioControl: FC<AudioControlProps> = ({
6060
<Slider
6161
min={0}
6262
max={MAX_VOL}
63-
value={Math.round(volume * MAX_VOL / 2)}
63+
value={Math.round(volume * MAX_VOL)}
6464
onChange={handleSetVolume}
6565
description={'Volume'}
6666
info={'Increase or decrease the volume of the audio'}

0 commit comments

Comments
 (0)