Adding sound to dialogues, Undertale style

src/createGame.ts

@@ -21,7 +21,10 @@ export const createGame = <T extends string>(
 	const soundPlayer = initSoundPlayer(config)
 	const camera = initCamera(config)
 	const renderer = initRenderer(config)
-	const dialog = initDialog(config)
+	const dialog = initDialog({
+		...config,
+		soundPlayer,
+	})
 	const prompt = initPrompt(config)
 	const messageBox = initMessageBox(config)
 	const gameFilter = initFilter(renderer.canvas, config.filter)

src/dialog.ts

@@ -1,12 +1,14 @@
 import { Char, getColorFrompalette, TextFx } from './lib'
 import { RendererParams } from './renderer'
+import { type Sound } from 'pfxr'
+import { SoundPlayer } from './sound.js'
 
 export type DialogParams = {
 	dialogBackground: string | number
 	dialogColor: string | number
 	dialogBorder: string | number
-	dialogInternvalMs?: number
 	colors: RendererParams['colors']
+	soundPlayer?: SoundPlayer
 }
 
 const CANVAS_SIZE = 384
@@ -36,6 +38,9 @@ export class Dialog {
 	#lastFrameTime = 0
 
 	#textFx: TextFx
+	#soundPlayer?: SoundPlayer
+	#currentVoiceOverride?: { template?: 'BLIP' | 'HIT' | 'PICKUP' | 'JUMP' | 'FALL' | 'POWERUP' | 'LASER' | 'BLIP_RANDOM', seed?: number | null }
+	#dialogSeed?: number
 
 	isOpen = false
 
@@ -44,22 +49,17 @@ export class Dialog {
 	#contentColor: string
 	#borderColor: string
 
-	#animationIntervalMs?: number
-
 	#boxHeight: number
 	#boxWidth: number
 	#boxX: number
 	#boxY: number
 
 	constructor(params: DialogParams) {
 		this.#configColors = params.colors
-		this.#backgroundColor = getColorFrompalette(
-			params.dialogBackground,
-			params.colors,
-		)
-		this.#contentColor = getColorFrompalette(params.dialogColor, params.colors)
-		this.#borderColor = getColorFrompalette(params.dialogBorder, params.colors)
-		this.#animationIntervalMs = params.dialogInternvalMs
+		this.#backgroundColor = this.#getColor(params.dialogBackground)
+		this.#contentColor = this.#getColor(params.dialogColor)
+		this.#borderColor = this.#getColor(params.dialogBorder)
+		this.#soundPlayer = params.soundPlayer
 
 		this.#canvas = document.createElement('canvas')
 		this.#canvas.style.setProperty('position', 'absolute')
@@ -85,10 +85,11 @@ export class Dialog {
 		this.#textFx = new TextFx('|', this.#contentColor, params.colors)
 	}
 
-	async open(text: string) {
-		if (text.length <= 0) return
+	async open(text: string, defaultVoice?: { template?: 'BLIP' | 'HIT' | 'PICKUP' | 'JUMP' | 'FALL' | 'POWERUP' | 'LASER' | 'BLIP_RANDOM', seed?: number | null } | null, voiceOverride?: { template?: 'BLIP' | 'HIT' | 'PICKUP' | 'JUMP' | 'FALL' | 'POWERUP' | 'LASER' | 'BLIP_RANDOM', seed?: number | null }) {
 		this.isOpen = true
 		this.#canvas.style.setProperty('display', 'block')
+		this.#currentVoiceOverride = voiceOverride || defaultVoice || undefined
+		this.#dialogSeed = undefined // Reset dialog seed for new dialog
 
 		this.#remainingLines = this.#textFx.parseText(text, MAX_CHARS_PER_LINE)
 		this.#currentLineQueue = this.#remainingLines.shift()
@@ -124,11 +125,7 @@ export class Dialog {
 
 	#update = (time: number) => {
 		this.#animationId = requestAnimationFrame(this.#update)
-		if (
-			time - this.#lastFrameTime <
-			(this.#animationIntervalMs || ANIMATION_INTERVAL_MS)
-		)
-			return
+		if (time - this.#lastFrameTime < ANIMATION_INTERVAL_MS) return
 		this.#lastFrameTime = time
 		if (
 			this.#currentLineQueue?.length === 0 &&
@@ -140,7 +137,42 @@ export class Dialog {
 
 		let newChar = this.#currentLineQueue?.shift()
 
-		if (newChar) this.#displayedLines[this.#lineCursor]?.push(newChar)
+		if (newChar) {
+			this.#displayedLines[this.#lineCursor]?.push(newChar)
+			// Play MIDI-style voice sound for non-space characters - only if voice is explicitly defined
+			if (this.#soundPlayer && this.#currentVoiceOverride && this.#currentVoiceOverride.template && newChar.value !== ' ') {
+				const template = this.#currentVoiceOverride.template
+				const seed = this.#currentVoiceOverride.seed
+
+				let finalTemplate = template
+				let characterSeed = seed ?? 0
+
+				if (template === 'BLIP_RANDOM') {
+					// BLIP_RANDOM: new random seed for each character
+					finalTemplate = 'BLIP'
+					characterSeed = Math.floor(Math.random() * 1000)
+				} else if (seed === null) {
+					// seed=null: random seed per dialog (not per character)
+					characterSeed = this.#dialogSeed ?? (this.#dialogSeed = Math.floor(Math.random() * 1000))
+				}
+
+				// Play voice sample with appropriate variation
+				const velocity = 0.8 + Math.random() * 0.4 // 0.8-1.2 volume variation
+
+				// Determine if we should use variation
+				let useVariation = false
+				if (template === 'BLIP_RANDOM') {
+					// BLIP_RANDOM: each character gets random variation
+					useVariation = true
+				} else if (seed === null) {
+					// seed=null: use random variation within template
+					useVariation = true
+				}
+
+				// Play voice sample
+				this.#soundPlayer.playVoiceSample(finalTemplate, velocity, useVariation)
+			}
+		}
 		this.#render(time)
 	}
 
@@ -150,6 +182,8 @@ export class Dialog {
 		this.#lineCursor = 0
 		this.#resolvePromise?.()
 		this.#animationId && cancelAnimationFrame(this.#animationId)
+
+		// No cleanup needed for individual sounds
 	}
 
 	#resizeCanvas = () => {
@@ -185,6 +219,11 @@ export class Dialog {
 		this.#ctx.fillStyle = this.#backgroundColor
 		this.#ctx.fillRect(this.#boxX, this.#boxY, this.#boxWidth, this.#boxHeight)
 	}
+
+	#getColor(color: string | number) {
+		if (typeof color === 'string') return color
+		return this.#configColors[color] ?? 'black'
+	}
 }
 
 export const initDialog = (params: DialogParams) => new Dialog(params)

src/gameApi.ts

@@ -31,7 +31,7 @@ export const initGameApi = <T extends string>(
 			symbol: T,
 			params: Unwrap<Partial<Omit<ActorState<T>, 'symbol'>>>,
 		) => gameState.actors.setAll(symbol, params),
-		openDialog: (text: string) => dialog.open(text),
+		openDialog: (text: string, voiceOverride?: { template?: 'BLIP' | 'HIT' | 'PICKUP' | 'JUMP' | 'FALL' | 'POWERUP' | 'LASER' | 'BLIP_RANDOM', seed?: number | null }) => dialog.open(text, undefined, voiceOverride),
 		prompt: (...options: string[]) => prompt.open(...options),
 		openMenu: (options: MenuOption) => prompt.openMenu(options),
 		openMessage: (...args: string[]) => messageBox.open(args),

src/gameLoop.ts

@@ -62,7 +62,7 @@ class GameLoop<T extends string> {
 		}
 	}
 	async #openDialog(actor: ActorFacade<T>) {
-		if (actor.dialog) await this.#dialog.open(actor.dialog)
+		if (actor.dialog) await this.#dialog.open(actor.dialog, actor.voice)
 	}
 
 	async #end(actor: ActorFacade<T>) {

src/gameState/actorFacade.ts

@@ -48,6 +48,14 @@ export class ActorFacade<T extends string> {
 		this.#setActor('dialog', value)
 	}
 
+	get voice() {
+		return this.#getActor()?.voice ?? null
+	}
+
+	set voice(value: ActorState<T>['voice']) {
+		this.#setActor('voice', value)
+	}
+
 	get visible() {
 		return this.#getActor()?.visible ?? false
 	}

src/gameState/actors.ts

@@ -166,6 +166,7 @@ export class Actors<T extends string> {
 			sprite: template.sprite ?? null,
 			position: [x, y],
 			dialog: template.dialog ?? null,
+			voice: template.voice ?? null,
 			isOnScreen: false,
 			end: template.end ?? null,
 			sound: template.sound ?? null,

src/gameState/types.ts

@@ -24,6 +24,7 @@ export type ActorState<T extends string> = {
 	sprite: Tile | null
 	sound: UnTuplify<PlaySoundArgs> | null
 	dialog: string | null
+	voice: { template?: 'BLIP' | 'HIT' | 'PICKUP' | 'JUMP' | 'FALL' | 'POWERUP' | 'LASER' | 'BLIP_RANDOM', seed?: number | null } | null
 	solid: boolean
 	visible: boolean
 	isOnScreen: boolean