|
78 | 78 | "## Part 1: Recording and Preparation\n", |
79 | 79 | "\n", |
80 | 80 | "For a speech/hearing course, everyone should record the same **standard sentence**\n", |
81 | | - "so that results can be compared across students. We also load a second recording\n", |
82 | | - "from a different speaker for voice conversion later.\n", |
| 81 | + "so that results can be compared across students.\n", |
83 | 82 | "\n", |
84 | 83 | "The standard sentence below is designed to exercise a wide variety of English\n", |
85 | 84 | "phonemes and articulatory gestures:\n", |
86 | 85 | "\n", |
87 | 86 | "> *\"The beige ant carried a muffin over the head of Liberty!\"*\n", |
88 | 87 | "\n", |
89 | 88 | "If you are running in Google Colab with a microphone, you can record your own\n", |
90 | | - "voice. Otherwise, sample files are downloaded automatically." |
| 89 | + "voice. Otherwise, a sample file is downloaded automatically.\n" |
91 | 90 | ] |
92 | 91 | }, |
93 | 92 | { |
|
158 | 157 | "source": [ |
159 | 158 | "from pathlib import Path\n", |
160 | 159 | "\n", |
161 | | - "# Load recorded audio if available, otherwise download sample files\n", |
| 160 | + "# Load recorded audio if available, otherwise download a sample file\n", |
162 | 161 | "import urllib.request\n", |
163 | 162 | "\n", |
164 | 163 | "base_url = \"https://github.com/sensein/senselab/raw/main/src/tests/data_for_testing\"\n", |
| 164 | + "dest = Path(\"tutorial_audio_files/audio_48khz_mono_16bits.wav\")\n", |
| 165 | + "if not dest.exists():\n", |
| 166 | + " Path(\"tutorial_audio_files\").mkdir(exist_ok=True)\n", |
| 167 | + " urllib.request.urlretrieve(f\"{base_url}/audio_48khz_mono_16bits.wav\", str(dest))\n", |
165 | 168 | "\n", |
166 | | - "# Download sample files as fallback\n", |
167 | | - "for fname in [\"audio_48khz_mono_16bits.wav\", \"audio_48khz_stereo_16bits.wav\"]:\n", |
168 | | - " dest = Path(\"tutorial_audio_files\") / fname\n", |
169 | | - " if not dest.exists():\n", |
170 | | - " urllib.request.urlretrieve(f\"{base_url}/{fname}\", str(dest))\n", |
171 | | - "\n", |
172 | | - "# Use recording if available, otherwise use sample\n", |
| 169 | + "# Use recording if available\n", |
173 | 170 | "rec_path = Path(\"tutorial_audio_files/my_recording.wav\")\n", |
174 | 171 | "if rec_path.exists() and rec_path.stat().st_size > 1000:\n", |
175 | | - " sentence1 = Audio(filepath=str(rec_path.resolve()))\n", |
176 | | - " print(\"Sentence 1: using your recording\")\n", |
| 172 | + " audio = Audio(filepath=str(rec_path.resolve()))\n", |
| 173 | + " print(\"Using your recording\")\n", |
177 | 174 | "else:\n", |
178 | | - " sentence1 = Audio(filepath=os.path.abspath(\"tutorial_audio_files/audio_48khz_mono_16bits.wav\"))\n", |
179 | | - " print(\"Sentence 1: using sample audio\")\n", |
180 | | - "\n", |
181 | | - "# Second audio for speaker comparison (always sample)\n", |
182 | | - "sentence2_raw = Audio(filepath=os.path.abspath(\"tutorial_audio_files/audio_48khz_stereo_16bits.wav\"))\n", |
| 175 | + " audio = Audio(filepath=os.path.abspath(\"tutorial_audio_files/audio_48khz_mono_16bits.wav\"))\n", |
| 176 | + " print(\"Using sample audio\")\n", |
183 | 177 | "\n", |
184 | 178 | "# Preprocess: mono + 16kHz\n", |
185 | | - "sentence1_16k = resample_audios(downmix_audios_to_mono([sentence1]), resample_rate=16000)[0]\n", |
186 | | - "sentence2_16k = resample_audios(downmix_audios_to_mono([sentence2_raw]), resample_rate=16000)[0]\n", |
187 | | - "\n", |
188 | | - "print(f\"Sentence 1: {sentence1_16k.waveform.shape[1] / 16000:.2f}s at 16kHz\")\n", |
189 | | - "print(f\"Sentence 2: {sentence2_16k.waveform.shape[1] / 16000:.2f}s at 16kHz (sample, for comparison)\")" |
| 179 | + "audio_16k = resample_audios(downmix_audios_to_mono([audio]), resample_rate=16000)[0]\n", |
| 180 | + "print(f\"Audio: {audio_16k.waveform.shape[1] / 16000:.2f}s at 16kHz\")" |
190 | 181 | ] |
191 | 182 | }, |
192 | 183 | { |
|
195 | 186 | "metadata": {}, |
196 | 187 | "outputs": [], |
197 | 188 | "source": [ |
198 | | - "print(\"Sentence 1:\")\n", |
199 | | - "play_audio(sentence1_16k)" |
| 189 | + "play_audio(audio_16k)" |
200 | 190 | ] |
201 | 191 | }, |
202 | 192 | { |
|
205 | 195 | "metadata": {}, |
206 | 196 | "outputs": [], |
207 | 197 | "source": [ |
208 | | - "print(\"Sentence 2:\")\n", |
209 | | - "play_audio(sentence2_16k)" |
210 | | - ] |
211 | | - }, |
212 | | - { |
213 | | - "cell_type": "code", |
214 | | - "execution_count": null, |
215 | | - "metadata": {}, |
216 | | - "outputs": [], |
217 | | - "source": [ |
218 | | - "# Waveforms side by side\n", |
219 | | - "fig, axes = plt.subplots(2, 1, figsize=(12, 4), sharex=False)\n", |
220 | | - "\n", |
221 | | - "wf1 = sentence1_16k.waveform.squeeze().numpy()\n", |
222 | | - "wf2 = sentence2_16k.waveform.squeeze().numpy()\n", |
223 | | - "t1 = np.linspace(0, len(wf1) / 16000, len(wf1))\n", |
224 | | - "t2 = np.linspace(0, len(wf2) / 16000, len(wf2))\n", |
225 | | - "\n", |
226 | | - "axes[0].plot(t1, wf1, linewidth=0.3, color=\"steelblue\")\n", |
227 | | - "axes[0].set_ylabel(\"Amplitude\")\n", |
228 | | - "axes[0].set_title(\"Sentence 1\")\n", |
229 | | - "axes[0].grid(True, alpha=0.2)\n", |
230 | | - "\n", |
231 | | - "axes[1].plot(t2, wf2, linewidth=0.3, color=\"coral\")\n", |
232 | | - "axes[1].set_ylabel(\"Amplitude\")\n", |
233 | | - "axes[1].set_title(\"Sentence 2\")\n", |
234 | | - "axes[1].set_xlabel(\"Time (seconds)\")\n", |
235 | | - "axes[1].grid(True, alpha=0.2)\n", |
| 198 | + "# Waveform visualization\n", |
| 199 | + "wf = audio_16k.waveform.squeeze().cpu().numpy()\n", |
| 200 | + "duration = len(wf) / audio_16k.sampling_rate\n", |
| 201 | + "time_axis = np.linspace(0, duration, len(wf), endpoint=False)\n", |
236 | 202 | "\n", |
| 203 | + "fig, ax = plt.subplots(figsize=(12, 3))\n", |
| 204 | + "ax.plot(time_axis, wf, linewidth=0.3, color=\"steelblue\")\n", |
| 205 | + "ax.set_xlabel(\"Time (seconds)\")\n", |
| 206 | + "ax.set_ylabel(\"Amplitude\")\n", |
| 207 | + "ax.set_title(\"Audio Waveform\")\n", |
| 208 | + "ax.grid(True, alpha=0.2)\n", |
237 | 209 | "plt.tight_layout()\n", |
238 | 210 | "plt.show()" |
239 | 211 | ] |
|
271 | 243 | "outputs": [], |
272 | 244 | "source": [ |
273 | 245 | "# Extract SPARC articulatory features\n", |
274 | | - "sparc_features = SparcFeatureExtractor.extract_sparc_features(audios=[sentence1_16k], device=device, resample=True)\n", |
| 246 | + "sparc_features = SparcFeatureExtractor.extract_sparc_features(audios=[audio_16k], device=device, resample=True)\n", |
275 | 247 | "features1 = sparc_features[0]\n", |
276 | 248 | "print(f\"EMA shape: {features1['ema'].shape} (channels x frames)\")\n", |
277 | 249 | "print(f\"Pitch shape: {features1['pitch'].shape}\")\n", |
|
414 | 386 | "print(f\"Resynthesized: {resynthesized.waveform.shape[1] / resynthesized.sampling_rate:.2f}s\")\n", |
415 | 387 | "\n", |
416 | 388 | "print(\"\\nOriginal:\")\n", |
417 | | - "play_audio(sentence1_16k)" |
| 389 | + "play_audio(audio_16k)" |
418 | 390 | ] |
419 | 391 | }, |
420 | 392 | { |
|
445 | 417 | "outputs": [], |
446 | 418 | "source": [ |
447 | 419 | "# Extract acoustic pitch contour\n", |
448 | | - "pitch_result = extract_pitch_from_audios([sentence1_16k], freq_low=80, freq_high=500)\n", |
| 420 | + "pitch_result = extract_pitch_from_audios([audio_16k], freq_low=80, freq_high=500)\n", |
449 | 421 | "pitch_contour = pitch_result[0][\"pitch\"].squeeze()\n", |
450 | 422 | "\n", |
451 | 423 | "# Build time axis (default hop length for torchaudio pitch detection)\n", |
|
488 | 460 | "outputs": [], |
489 | 461 | "source": [ |
490 | 462 | "# Extract phonetic posteriorgrams\n", |
491 | | - "ppgs = extract_ppgs_from_audios([sentence1_16k], device=device)\n", |
| 463 | + "ppgs = extract_ppgs_from_audios([audio_16k], device=device)\n", |
492 | 464 | "ppg1 = ppgs[0]\n", |
493 | 465 | "print(f\"PPG shape: {ppg1.shape}\")\n", |
494 | 466 | "print(\"Each column is a time frame; each row is a phoneme probability.\")" |
|
501 | 473 | "outputs": [], |
502 | 474 | "source": [ |
503 | 475 | "# Phoneme duration analysis from PPG\n", |
504 | | - "durations = extract_mean_phoneme_durations(sentence1_16k, ppg1)\n", |
| 476 | + "durations = extract_mean_phoneme_durations(audio_16k, ppg1)\n", |
505 | 477 | "print(f\"Analysis: {durations['frame_count']} frames over {durations['analysis_duration_seconds']:.2f}s\")\n", |
506 | 478 | "print(\"\\n=== Top 10 Phonemes by Total Duration ===\")\n", |
507 | 479 | "for p in sorted(durations[\"phoneme_durations\"], key=lambda x: x[\"total_duration_seconds\"], reverse=True)[:10]:\n", |
|
531 | 503 | "\n", |
532 | 504 | "# Get PPG segments for plotting\n", |
533 | 505 | "frame_major = to_frame_major_posteriorgram(ppg1)\n", |
534 | | - "segments = extract_ppg_segments(sentence1_16k, frame_major)\n", |
| 506 | + "segments = extract_ppg_segments(audio_16k, frame_major)\n", |
535 | 507 | "\n", |
536 | 508 | "# Convert PPG segments to plot_aligned_panels format\n", |
537 | 509 | "ppg_segments = [\n", |
|
544 | 516 | "pitch_data = [(time_pitch[voiced_mask], pitch_contour[voiced_mask], \"F0\", \"steelblue\")]\n", |
545 | 517 | "\n", |
546 | 518 | "plot_aligned_panels(\n", |
547 | | - " sentence1_16k,\n", |
| 519 | + " audio_16k,\n", |
548 | 520 | " [\n", |
549 | 521 | " {\"type\": \"waveform\"},\n", |
550 | 522 | " {\"type\": \"spectrogram\", \"mel\": False},\n", |
|
588 | 560 | "source": [ |
589 | 561 | "# Get PPG segments for the comparison\n", |
590 | 562 | "frame_major = to_frame_major_posteriorgram(ppg1)\n", |
591 | | - "segments = extract_ppg_segments(sentence1_16k, frame_major)\n", |
| 563 | + "segments = extract_ppg_segments(audio_16k, frame_major)\n", |
592 | 564 | "ppg_segments = [\n", |
593 | 565 | " {\"label\": seg[\"phoneme\"], \"start\": seg[\"start_seconds\"], \"end\": seg[\"start_seconds\"] + seg[\"duration_seconds\"]}\n", |
594 | 566 | " for seg in segments\n", |
|
620 | 592 | " ema_data.append((time_ema, signal, name, color_map.get(artic, \"gray\")))\n", |
621 | 593 | "\n", |
622 | 594 | "plot_aligned_panels(\n", |
623 | | - " sentence1_16k,\n", |
| 595 | + " audio_16k,\n", |
624 | 596 | " [\n", |
625 | 597 | " {\"type\": \"waveform\"},\n", |
626 | 598 | " {\"type\": \"features\", \"data\": pitch_comparison_data, \"style\": \"line\"},\n", |
|
0 commit comments