Streaming audio display

.changeset/yummy-experts-push.md

@@ -0,0 +1,6 @@
+---
+"@gradio/audio": patch
+"gradio": patch
+---
+
+fix:Streaming audio display

js/audio/player/AudioPlayer.svelte

@@ -5,6 +5,8 @@
 	import WaveSurfer from "wavesurfer.js";
 	import { skip_audio, process_audio } from "../shared/utils";
 	import WaveformControls from "../shared/WaveformControls.svelte";
+	import StreamingAudio from "./StreamingAudio.svelte";
+	import WaveformWrapper from "../shared/waveform_wrapper";
 	import { Empty } from "@gradio/atoms";
 	import { resolve_wasm_src } from "@gradio/wasm/svelte";
 	import type { FileData } from "@gradio/client";
@@ -31,7 +33,7 @@
 	export let handle_reset_value: () => void = () => {};
 
 	let container: HTMLDivElement;
-	let waveform: WaveSurfer | undefined;
+	let waveform: WaveSurfer | WaveformWrapper | undefined;
 	let playing = false;
 
 	let timeRef: HTMLTimeElement;
@@ -64,6 +66,7 @@
 				return waveform.load(resolved_src);
 			}
 		});
+		attach_waveform_events(waveform);
 	};
 
 	$: if (!value?.is_stream && container !== undefined && container !== null) {
@@ -73,52 +76,12 @@
 		playing = false;
 	}
 
-	$: waveform?.on("decode", (duration: any) => {
-		audio_duration = duration;
-		durationRef && (durationRef.textContent = format_time(duration));
-	});
-
-	$: waveform?.on(
-		"timeupdate",
-		(currentTime: any) =>
-			timeRef && (timeRef.textContent = format_time(currentTime))
-	);
-
-	$: waveform?.on("ready", () => {
-		if (!waveform_settings.autoplay) {
-			waveform?.stop();
-		} else {
-			waveform?.play();
-		}
-	});
-
-	$: waveform?.on("finish", () => {
-		if (loop) {
-			waveform?.play();
-		} else {
-			playing = false;
-			dispatch("stop");
-		}
-	});
-	$: waveform?.on("pause", () => {
-		playing = false;
-		dispatch("pause");
-	});
-	$: waveform?.on("play", () => {
-		playing = true;
-		dispatch("play");
-	});
-
-	$: waveform?.on("load", () => {
-		dispatch("load");
-	});
-
 	const handle_trim_audio = async (
 		start: number,
 		end: number
 	): Promise<void> => {
 		mode = "";
-		const decodedData = waveform?.getDecodedData();
+		const decodedData = (waveform as undefined | WaveSurfer)?.getDecodedData();
 		if (decodedData)
 			await process_audio(
 				decodedData,
@@ -156,7 +119,9 @@
 			hls.loadSource(value.url);
 			hls.attachMedia(audio_player);
 			hls.on(Hls.Events.MANIFEST_PARSED, function () {
-				if (waveform_settings.autoplay) audio_player.play();
+				if (waveform_settings.autoplay) {
+					audio_player.play();
+				}
 			});
 			hls.on(Hls.Events.ERROR, function (event, data) {
 				console.error("HLS error:", event, data);
@@ -199,18 +164,72 @@
 			}
 		});
 	});
+
+	function attach_waveform_events(
+		waveform: WaveformWrapper | WaveSurfer
+	): void {
+		waveform.on("decode", (duration: any) => {
+			audio_duration = duration;
+			durationRef && (durationRef.textContent = format_time(duration));
+		});
+
+		waveform.on(
+			"timeupdate",
+			(currentTime: any) =>
+				timeRef && (timeRef.textContent = format_time(currentTime))
+		);
+
+		waveform.on("ready", () => {
+			if (!waveform_settings.autoplay) {
+				waveform?.stop();
+			} else {
+				waveform?.play();
+			}
+		});
+
+		waveform.on("finish", () => {
+			if (loop) {
+				waveform?.play();
+			} else {
+				playing = false;
+				dispatch("stop");
+			}
+		});
+		waveform.on("pause", () => {
+			playing = false;
+			dispatch("pause");
+		});
+
+		waveform.on("play", () => {
+			playing = true;
+			dispatch("play");
+		});
+
+		waveform.on("load", () => {
+			dispatch("load");
+		});
+	}
+
+	function make_waveform(stream_active: boolean): void {
+		if (stream_active) {
+			waveform = new WaveformWrapper(audio_player);
+			attach_waveform_events(waveform);
+		}
+	}
+
+	$: make_waveform(stream_active);
 </script>
 
 <audio
 	class="standard-player"
-	class:hidden={!(value && value.is_stream)}
-	controls
+	class:hidden={true}
 	autoplay={waveform_settings.autoplay}
 	on:load
 	bind:this={audio_player}
 	on:ended={() => dispatch("stop")}
 	on:play={() => dispatch("play")}
 />
+
 {#if value === null}
 	<Empty size="small">
 		<Music />
@@ -258,6 +277,25 @@
 		/>
 		<!-- {/if} -->
 	</div>
+{:else if stream_active}
+	<StreamingAudio audio_source={audio_player} />
+	<WaveformControls
+		{container}
+		{waveform}
+		{playing}
+		{audio_duration}
+		{i18n}
+		{interactive}
+		{handle_trim_audio}
+		bind:mode
+		bind:trimDuration
+		bind:show_volume_slider
+		show_redo={interactive}
+		{handle_reset_value}
+		{waveform_options}
+		{trim_region_settings}
+		{editable}
+	/>
 {/if}
 
 <style>

js/audio/player/StreamingAudio.svelte

@@ -0,0 +1,85 @@
+<script lang="ts">
+	import { onMount, onDestroy } from "svelte";
+
+	export let numBars = 16;
+	export let audio_source: HTMLAudioElement;
+
+	let audioContext: AudioContext;
+	let analyser: AnalyserNode;
+	let dataArray: Uint8Array;
+	let animationId: number;
+
+	$: containerWidth = `calc((var(--boxSize) + var(--gutter)) * ${numBars})`;
+
+	onMount(() => {
+		setupAudioContext();
+	});
+
+	onDestroy(() => {
+		if (animationId) {
+			cancelAnimationFrame(animationId);
+		}
+		if (audioContext) {
+			audioContext.close();
+		}
+	});
+
+	function setupAudioContext(): void {
+		audioContext = new (window.AudioContext || window.webkitAudioContext)();
+		analyser = audioContext.createAnalyser();
+		const source = audioContext.createMediaElementSource(audio_source);
+		source.connect(analyser);
+		analyser.connect(audioContext.destination);
+
+		analyser.fftSize = 64;
+		dataArray = new Uint8Array(analyser.frequencyBinCount);
+
+		updateBars();
+	}
+
+	function updateBars(): void {
+		analyser.getByteFrequencyData(dataArray);
+
+		const bars = document.querySelectorAll(".box");
+		for (let i = 0; i < bars.length; i++) {
+			const barHeight = (dataArray[i] / 255) * 2; // Amplify the effect
+			// @ts-ignore
+			bars[i].style.transform = `scaleY(${Math.max(0.1, barHeight)})`;
+		}
+
+		animationId = requestAnimationFrame(updateBars);
+	}
+</script>
+
+<div class="waveContainer">
+	<div class="boxContainer" style:width={containerWidth}>
+		{#each Array(numBars) as _}
+			<div class="box"></div>
+		{/each}
+	</div>
+</div>
+
+<style>
+	.waveContainer {
+		position: relative;
+		display: flex;
+		flex-direction: column;
+		align-items: center;
+	}
+
+	.boxContainer {
+		display: flex;
+		justify-content: space-between;
+		height: 32px;
+		--boxSize: 8px;
+		--gutter: 4px;
+	}
+
+	.box {
+		height: 100%;
+		width: var(--boxSize);
+		background: var(--color-accent);
+		border-radius: 8px;
+		transition: transform 0.05s ease;
+	}
+</style>

js/audio/shared/utils.ts

@@ -1,5 +1,6 @@
 import type WaveSurfer from "wavesurfer.js";
 import { audioBufferToWav } from "./audioBufferToWav";
+import WaveformWrapper from "./waveform_wrapper";
 
 export interface LoadedParams {
 	autoplay?: boolean;
@@ -63,7 +64,10 @@ export function loaded(
 	}
 }
 
-export const skip_audio = (waveform: WaveSurfer, amount: number): void => {
+export const skip_audio = (
+	waveform: WaveSurfer | WaveformWrapper,
+	amount: number
+): void => {
 	if (!waveform) return;
 	waveform.skip(amount);
 };