Send audio over WebTransport streams.

Author: jianjunz

src/samples/conference/public/scripts/media-worker.js

@@ -3,21 +3,30 @@
 // SPDX-License-Identifier: Apache-2.0
 
 /* eslint-disable require-jsdoc */
-/* global VideoEncoder, VideoDecoder, EncodedVideoChunk */
+/* global AudioEncoder, EncodedAudioChunk, VideoEncoder, VideoDecoder,
+ * EncodedVideoChunk */
 
-let bidirectionalStreamWritable, videoEncoder, frameBuffer, sendStreamWriter,
-    mediaSession, datagramReceiver, videoDecoder;
+let videoBidiStreamWritable, audioEncoder, videoEncoder, frameBuffer,
+    audioSendStreamWriter, videoSendStreamWriter, mediaSession,
+    datagramReceiver, videoDecoder;
 // 4 bytes for frame size before each frame. The 1st byte is reserved, always 0.
 const sizePrefix = 4;
 
 onmessage = (e) => {
-  if (e.data[0] === 'video-source') {
-    readVideoData(e.data[1]);
-  } else if (e.data[0] === 'send-stream') {
-    bidirectionalStreamWritable = e.data[1];
-    sendStreamWriter = bidirectionalStreamWritable.getWriter();
-    writeTrackId();
-    // initVideoEncoder();
+  if (e.data[0] === 'audio-source') {
+    readMediaData(e.data[1], 'audio');
+  } else if (e.data[0] === 'video-source') {
+    readMediaData(e.data[1], 'video');
+  } else if (e.data[0] === 'send-stream-audio') {
+    const audioBidiStreamWritable = e.data[1];
+    audioSendStreamWriter = audioBidiStreamWritable.getWriter();
+    writeTrackId('audio', audioSendStreamWriter);
+    initAudioEncoder();
+  } else if (e.data[0] === 'send-stream-video') {
+    videoBidiStreamWritable = e.data[1];
+    videoSendStreamWriter = videoBidiStreamWritable.getWriter();
+    writeTrackId('video', videoSendStreamWriter);
+    initVideoEncoder();
   } else if (e.data[0] === 'datagram-receiver') {
     datagramReceiver = e.data[1];
   } else if (e.data[0] === 'encoded-video-frame') {
@@ -30,7 +39,8 @@ onmessage = (e) => {
 };
 
 async function videoOutput(chunk, metadata) {
-  if (bidirectionalStreamWritable) {
+  return;
+  if (videoBidiStreamWritable) {
     if (!frameBuffer ||
         frameBuffer.byteLength < chunk.byteLength + sizePrefix) {
       frameBuffer = new ArrayBuffer(chunk.byteLength + sizePrefix);
@@ -40,21 +50,48 @@ async function videoOutput(chunk, metadata) {
     const dataView =
         new DataView(frameBuffer, 0, chunk.byteLength + sizePrefix);
     dataView.setUint32(0, chunk.byteLength);
-    await sendStreamWriter.ready;
-    await sendStreamWriter.write(dataView);
-    console.log('Write a frame.');
+    await videoSendStreamWriter.ready;
+    await videoSendStreamWriter.write(dataView);
   }
 }
 
 function videoError(error) {
-  console.log('Encode error, ' + error);
+  console.log('Video encode error, ' + error.message);
 }
 
-async function writeTrackId() {
+async function audioOutput(chunk, metadata) {
+  if (audioSendStreamWriter) {
+    if (!frameBuffer ||
+        frameBuffer.byteLength < chunk.byteLength + sizePrefix) {
+      frameBuffer = new ArrayBuffer(chunk.byteLength + sizePrefix);
+    }
+    const bufferView = new Uint8Array(frameBuffer, sizePrefix);
+    chunk.copyTo(bufferView);
+    const dataView =
+        new DataView(frameBuffer, 0, chunk.byteLength + sizePrefix);
+    dataView.setUint32(0, chunk.byteLength);
+    await audioSendStreamWriter.ready;
+    await audioSendStreamWriter.write(dataView);
+    console.log('Wrote an audio frame. '+chunk.byteLength);
+  }
+}
+
+function audioError(error) {
+  console.log(`Audio encode error: ${error.message}`);
+}
+
+async function writeTrackId(kind, writer) {
   const id = new Uint8Array(16);
-  id[16] = 2;
-  await sendStreamWriter.ready;
-  sendStreamWriter.write(id);
+  id[15] = (kind === 'audio' ? 1 : 2);
+  await writer.ready;
+  writer.write(id);
+  console.log('Wrote track ID for '+kind);
+}
+
+function initAudioEncoder() {
+  audioEncoder = new AudioEncoder({output: audioOutput, error: audioError});
+  audioEncoder.configure(
+      {codec: 'opus', numberOfChannels: 1, sampleRate: 48000});
 }
 
 function initVideoEncoder() {
@@ -85,16 +122,20 @@ function webCodecsErrorCallback(error) {
   console.log('error: ' + error.message);
 }
 
-// Read data from video track.
-async function readVideoData(readable) {
+// Read data from media track.
+async function readMediaData(readable, kind) {
   const reader = readable.getReader();
   while (true) {
     const {value, done} = await reader.read();
     if (done) {
       console.log('MediaStream ends.');
       break;
     }
-    videoEncoder.encode(value);
+    if (kind === 'audio') {
+      audioEncoder.encode(value);
+    } else if (kind === 'video') {
+      videoEncoder.encode(value);
+    }
     value.close();
   }
 }

src/samples/conference/public/scripts/quic.js

@@ -8,6 +8,7 @@
 
 let quicChannel = null;
 let bidirectionalStream = null;
+let bidiAudioStream = null;
 let writeTask, mediaStream, mediaWorker, conferenceId, myId, mixedStream, generatorWriter;
 
 window.Module={};
@@ -114,18 +115,22 @@ async function attachReader(stream) {
 }
 
 async function createSendChannel() {
-  bidirectionalStream = await conference.createSendStream();
+  //bidirectionalStream = await conference.createSendStream();
+  bidiAudioStream = await conference.createSendStream();
+  const localStream = new Owt.Base.LocalStream(
+      bidiAudioStream,
+      new Owt.Base.StreamSourceInfo('mic', undefined, undefined));
+  attachReader(bidiAudioStream);
+  const publication = await conference.publish(localStream, {
+    audio: {codec: 'opus', numberOfChannels: 2, sampleRate: 48000},
+    video: false,
+    transport: {type: 'quic'},
+  });
   // const localStream = new Owt.Base.LocalStream(
   //     bidirectionalStream,
-  //     new Owt.Base.StreamSourceInfo(undefined, 'camera', undefined));
-  // attachReader(bidirectionalStream);
+  //     new Owt.Base.StreamSourceInfo(undefined, undefined, true));
   // const publication = await conference.publish(
-  //     localStream, {video: {codec: 'h264'}, transport: {type: 'quic'}});
-  const localStream = new Owt.Base.LocalStream(
-      bidirectionalStream,
-      new Owt.Base.StreamSourceInfo(undefined, undefined, true));
-  const publication = await conference.publish(
-      localStream, {transport: {type: 'quic'}});
+  //     localStream, {transport: {type: 'quic'}});
   console.log(publication);
   updateConferenceStatus('Created send channel.');
 }
@@ -146,13 +151,23 @@ async function writeUuid() {
   return;
 }
 
-async function writeVideoData() {
-  mediaStream = await navigator.mediaDevices.getUserMedia({video: true});
-  const track = new MediaStreamTrackProcessor(mediaStream.getVideoTracks()[0]);
-  mediaWorker.postMessage(['video-source', track.readable], [track.readable]);
+async function writeMediaData() {
+  mediaStream =
+      await navigator.mediaDevices.getUserMedia({audio: true, video: true});
+  const audioTrack =
+      new MediaStreamTrackProcessor(mediaStream.getAudioTracks()[0]);
+  mediaWorker.postMessage(
+      ['audio-source', audioTrack.readable], [audioTrack.readable]);
   mediaWorker.postMessage(
-      ['send-stream', bidirectionalStream.writable],
-      [bidirectionalStream.writable]);
+      ['send-stream-audio', bidiAudioStream.writable],
+      [bidiAudioStream.writable]);
+  // const videoTrack =
+  //     new MediaStreamTrackProcessor(mediaStream.getVideoTracks()[0]);
+  // mediaWorker.postMessage(
+  //     ['video-source', videoTrack.readable], [videoTrack.readable]);
+  // mediaWorker.postMessage(
+  //     ['send-stream-video', bidirectionalStream.writable],
+  //     [bidirectionalStream.writable]);
 }
 
 async function writeData() {
@@ -173,12 +188,12 @@ window.addEventListener('load', () => {
 });
 
 document.getElementById('start-sending').addEventListener('click', async () => {
-  if (!bidirectionalStream) {
+  if (!bidiAudioStream) {
     updateConferenceStatus('Stream is not created.');
     return;
   }
-  //writeVideoData();
-  writeTask = setInterval(writeData, 200);
+  writeMediaData();
+  //writeTask = setInterval(writeData, 200);
   updateConferenceStatus('Started sending.');
 });