From 9962c0bfd35bc7018784bd14e68a01fdaddc0a56 Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 5 Jul 2021 17:11:06 +0100 Subject: [PATCH 1/2] Added an audio recorder --- lib/machines/audio-recorder.machine.ts | 213 +++++++++++++++++++++++++ lib/machines/audio-recorder.mdx | 19 +++ lib/metadata.ts | 5 + package.json | 1 + yarn.lock | 5 + 5 files changed, 243 insertions(+) create mode 100644 lib/machines/audio-recorder.machine.ts create mode 100644 lib/machines/audio-recorder.mdx diff --git a/lib/machines/audio-recorder.machine.ts b/lib/machines/audio-recorder.machine.ts new file mode 100644 index 0000000..a7e61f6 --- /dev/null +++ b/lib/machines/audio-recorder.machine.ts @@ -0,0 +1,213 @@ +import { + assign, + createMachine, + DoneInvokeEvent, + forwardTo, + Sender, +} from 'xstate'; + +export interface AudioRecorderMachineContext { + stream?: MediaStream; + mediaChunks: Blob[]; +} + +export type AudioRecorderMachineEvent = + | { + type: 'RETRY'; + } + | { + type: 'RECORD'; + } + | { + type: 'AUDIO_CHUNK_RECEIVED'; + blob: Blob; + } + | { + type: 'PAUSE'; + } + | { + type: 'RESUME'; + } + | { + type: 'STOP'; + } + | { + type: 'DOWNLOAD'; + }; + +const audioRecorderMachine = createMachine< + AudioRecorderMachineContext, + AudioRecorderMachineEvent +>( + { + id: 'audioRecorder', + initial: 'idle', + context: { + mediaChunks: [], + }, + exit: ['removeMediaStream'], + states: { + idle: { + on: { + RECORD: { + target: 'requestingAudioOptions', + }, + }, + }, + requestingAudioOptions: { + invoke: { + src: 'requestAudioOptions', + onError: { + target: 'couldNotRetrieveAudioOptions', + }, + onDone: { + target: 'recording', + actions: 'assignStreamToContext', + }, + }, + }, + recordingFailed: { + on: { + RETRY: 'recording', + }, + }, + recording: { + on: { + AUDIO_CHUNK_RECEIVED: { + actions: 'appendBlob', + }, + STOP: { + target: 'complete', + }, + }, + invoke: { + id: 'recording', + src: 'recordFromStream', + onError: { + target: 'recordingFailed', + actions: (context, error) => { + console.error(error); + }, + }, + }, + initial: 'running', + states: { + running: { + on: { + PAUSE: { + target: 'paused', + actions: forwardTo('recording'), + }, + }, + }, + paused: { + on: { + RESUME: { + target: 'running', + actions: forwardTo('recording'), + }, + }, + }, + }, + }, + complete: { + on: { + RETRY: { + target: 'recording', + actions: 'clearBlobData', + }, + DOWNLOAD: { + actions: 'downloadBlob', + }, + }, + }, + couldNotRetrieveAudioOptions: { + on: { + RETRY: 'requestingAudioOptions', + }, + }, + }, + }, + { + actions: { + downloadBlob: (context) => { + const blob = new Blob(context.mediaChunks, { + type: 'audio/ogg; codecs=opus', + }); + const url = URL.createObjectURL(blob); + const downloadLink = document.createElement('a'); + + downloadLink.href = url; + downloadLink.download = `file.ogg`; + document.body.appendChild(downloadLink); // Required for FF + downloadLink.click(); + }, + removeMediaStream: (context) => { + if (context.stream) { + context.stream.getTracks().forEach((track) => { + track.stop(); + }); + } + }, + assignStreamToContext: assign((context, event) => { + return { + stream: (event as DoneInvokeEvent).data + .stream, + }; + }), + clearBlobData: assign((context) => { + return { + mediaChunks: [], + }; + }), + appendBlob: assign((context, event) => { + if (event.type !== 'AUDIO_CHUNK_RECEIVED') return {}; + return { + mediaChunks: [...context.mediaChunks, event.blob], + }; + }), + }, + services: { + recordFromStream: (context) => (send, onReceive) => { + const mediaRecorder = new MediaRecorder(context.stream); + + mediaRecorder.ondataavailable = (e) => { + send({ + type: 'AUDIO_CHUNK_RECEIVED', + blob: e.data, + }); + }; + mediaRecorder.start(200); + + onReceive((event) => { + if (event.type === 'PAUSE') { + mediaRecorder.pause(); + } else if (event.type === 'RESUME') { + mediaRecorder.resume(); + } + }); + + return () => { + if (mediaRecorder.state !== 'inactive') { + mediaRecorder.stop(); + } + }; + }, + requestAudioOptions: async () => { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: true, + }); + + return { + stream, + }; + }, + }, + }, +); + +export default audioRecorderMachine; + +export interface RequestAudioOptionsOutput { + stream: MediaStream; +} diff --git a/lib/machines/audio-recorder.mdx b/lib/machines/audio-recorder.mdx new file mode 100644 index 0000000..7e0d404 --- /dev/null +++ b/lib/machines/audio-recorder.mdx @@ -0,0 +1,19 @@ +# Audio Recorder + +This machine handles audio recording, including pausing/resuming, and downloading. It's fully implemented in the browser, so feel free to try recording yourself! + +## Getting access to audio + +In the browser, you need to request access to the user's audio in order to record it. When the user presses RECORD, we enter requestingAudioOptions, where the requestAudioOptions service is invoked. + +If that check fails, via error.platform.requestAudioOptions, you'll be sent to the couldNotRetrieveAudioOptions state, where you can RETRY to try again. + +## Recording + +Once the device check completes, it saves a [MediaStream](https://developer.mozilla.org/en-US/docs/Web/API/MediaStream) to stream and we enter recording.running. The recordFromStream service manages the [MediaRecorder](https://developer.mozilla.org/en-US/docs/Web/API/MediaRecorder) and fires back the data to context via AUDIO_CHUNK_RECEIVED. + +Pressing PAUSE will pause the recording and head to recording.paused. Pressing RESUME starts the recording again. + +## Stopping and downloading + +Once you're done with the recording, you can STOP it. This heads to the complete state, where you can RETRY to go again or DOWNLOAD to grab the recording. diff --git a/lib/metadata.ts b/lib/metadata.ts index 29e7e1c..2825814 100644 --- a/lib/metadata.ts +++ b/lib/metadata.ts @@ -7,6 +7,11 @@ export interface MetadataItem { } export const metadata: Record = { + "audio-recorder": { + title: "Audio Recorder", + icon: "PlaylistAddCheckOutlined", + version: "0.1.0", + }, authentication: { title: 'Authentication', icon: 'LockOpenOutlined', diff --git a/package.json b/package.json index f460cb8..99b1548 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "xstate": "^4.19.1" }, "devDependencies": { + "@types/dom-mediacapture-record": "^1.0.9", "husky": "^6.0.0", "lint-staged": "^10.5.4", "prettier": "^2.2.1" diff --git a/yarn.lock b/yarn.lock index 90b93cc..a46598f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -752,6 +752,11 @@ lodash.merge "^4.6.2" lodash.uniq "^4.5.0" +"@types/dom-mediacapture-record@^1.0.9": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@types/dom-mediacapture-record/-/dom-mediacapture-record-1.0.9.tgz#42eca652b9da7f8e1dfe5b264865c675f3fc344c" + integrity sha512-RKxDs2fTvWfvYGCwp2RgER3HkLCysOA/1FmLIIO++9wKT+y3XQHWi0pfx1pg4rMOy6E/NUy4uaS/xJiHj6NOuA== + "@types/fined@*": version "1.1.2" resolved "https://registry.yarnpkg.com/@types/fined/-/fined-1.1.2.tgz#05d2b9f93d144855c97c18c9675f424ed01400c4" From 1ce4a39945cdd77be5bd34012656dddacdf3bae9 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 16 Jul 2021 10:51:50 +0100 Subject: [PATCH 2/2] Added device selection machine --- lib/Icons.tsx | 1 + .../audio-video-device-selection.machine.ts | 158 ++++++++++++++++++ lib/machines/audio-video-device-selection.mdx | 3 + lib/metadata.ts | 11 +- package.json | 2 +- pages/index.mdx | 7 +- 6 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 lib/machines/audio-video-device-selection.machine.ts create mode 100644 lib/machines/audio-video-device-selection.mdx diff --git a/lib/Icons.tsx b/lib/Icons.tsx index 6177dd2..4fe11f6 100644 --- a/lib/Icons.tsx +++ b/lib/Icons.tsx @@ -21,3 +21,4 @@ export { default as Twitter } from '@material-ui/icons/Twitter'; export { default as ImportContactsOutlined } from '@material-ui/icons/ImportContactsOutlined'; export { default as TimerOutlined } from '@material-ui/icons/TimerOutlined'; export { default as TabOutlined } from '@material-ui/icons/TabOutlined'; +export { default as MicOutlined } from '@material-ui/icons/MicOutlined'; diff --git a/lib/machines/audio-video-device-selection.machine.ts b/lib/machines/audio-video-device-selection.machine.ts new file mode 100644 index 0000000..ccf7608 --- /dev/null +++ b/lib/machines/audio-video-device-selection.machine.ts @@ -0,0 +1,158 @@ +import { + assign, + createMachine, + DoneInvokeEvent, + EventObject, + MachineOptions, +} from 'xstate'; + +export interface AudioVideoDeviceSelectionMachineContext { + audioInputDevices: MediaDeviceInfo[]; + audioOutputDevices: MediaDeviceInfo[]; + videoInputDevices: MediaDeviceInfo[]; + selectedAudioInputDevice?: MediaDeviceInfo; + selectedAudioOutputDevice?: MediaDeviceInfo; + selectedVideoInputDevice?: MediaDeviceInfo; + formValues: { username: string; password: string }; +} + +export type AudioVideoDeviceSelectionMachineEvent = + | { + type: 'CHOOSE_AUDIO_INPUT_DEVICE'; + index: number; + } + | { + type: 'CHOOSE_AUDIO_OUTPUT_DEVICE'; + index: number; + } + | { + type: 'CHOOSE_VIDEO_DEVICE'; + index: number; + }; + +export type DevicesDoneEvent = DoneInvokeEvent<{ + devices: MediaDeviceInfo[]; +}>; + +const createAudioVideoDeviceSelectionMachineOptions = < + TContext, + TEvent extends EventObject +>( + options: Partial>, +): Partial> => { + return options; +}; + +const audioVideoDeviceSelectionMachine = createMachine< + AudioVideoDeviceSelectionMachineContext, + AudioVideoDeviceSelectionMachineEvent +>( + { + id: 'audioVideoDeviceSelection', + initial: 'requesting devices', + context: { + audioInputDevices: [], + audioOutputDevices: [], + videoInputDevices: [], + formValues: { + username: '', + password: '', + }, + }, + states: { + 'requesting devices': { + invoke: { + src: 'requestAudioOptions', + onError: { + target: 'could not retrieve devices', + }, + onDone: { + actions: [ + 'assignDevicesToContext', + 'assignDefaultDevicesToContext', + ], + target: 'got devices', + }, + }, + }, + 'could not retrieve devices': {}, + 'got devices': { + on: { + CHOOSE_AUDIO_INPUT_DEVICE: { + cond: (context, event) => + Boolean(context.audioInputDevices[event.index]), + actions: assign((context, event) => { + return { + selectedAudioInputDevice: + context.audioInputDevices[event.index], + }; + }), + }, + CHOOSE_AUDIO_OUTPUT_DEVICE: { + cond: (context, event) => + Boolean(context.audioOutputDevices[event.index]), + actions: assign((context, event) => { + return { + selectedAudioOutputDevice: + context.audioOutputDevices[event.index], + }; + }), + }, + CHOOSE_VIDEO_DEVICE: { + cond: (context, event) => + Boolean(context.videoInputDevices[event.index]), + actions: assign((context, event) => { + return { + selectedVideoInputDevice: + context.videoInputDevices[event.index], + }; + }), + }, + }, + }, + }, + }, + { + actions: { + assignDevicesToContext: assign((context, event: unknown) => { + return { + audioInputDevices: (event as DevicesDoneEvent).data.devices.filter( + (device) => device.deviceId && device.kind === 'audioinput', + ), + audioOutputDevices: (event as DevicesDoneEvent).data.devices.filter( + (device) => device.deviceId && device.kind === 'audiooutput', + ), + videoInputDevices: (event as DevicesDoneEvent).data.devices.filter( + (device) => device.deviceId && device.kind === 'videoinput', + ), + }; + }), + assignDefaultDevicesToContext: assign((context) => { + return { + selectedAudioInputDevice: context.audioInputDevices[0], + selectedAudioOutputDevice: context.audioOutputDevices[0], + selectedVideoInputDevice: context.videoInputDevices[0], + }; + }), + }, + services: { + requestAudioOptions: async () => { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: true, + video: true, + }); + const devices = await navigator.mediaDevices.enumerateDevices(); + + stream.getTracks().forEach((track) => { + track.stop(); + }); + + return { + devices, + }; + }, + }, + }, +); + +export default audioVideoDeviceSelectionMachine; diff --git a/lib/machines/audio-video-device-selection.mdx b/lib/machines/audio-video-device-selection.mdx new file mode 100644 index 0000000..7b1c947 --- /dev/null +++ b/lib/machines/audio-video-device-selection.mdx @@ -0,0 +1,3 @@ +# Audio/Video Device selection + + diff --git a/lib/metadata.ts b/lib/metadata.ts index 2825814..bc78eff 100644 --- a/lib/metadata.ts +++ b/lib/metadata.ts @@ -7,10 +7,15 @@ export interface MetadataItem { } export const metadata: Record = { - "audio-recorder": { - title: "Audio Recorder", + "audio-video-device-selection": { + title: "Audio/Video Device selection", icon: "PlaylistAddCheckOutlined", - version: "0.1.0", + version: "0.1.1", + }, + 'audio-recorder': { + title: 'Audio Recorder', + icon: 'MicOutlined', + version: '0.1.1', }, authentication: { title: 'Authentication', diff --git a/package.json b/package.json index 99b1548..b4d240d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "xstate-catalogue", - "version": "0.1.0", + "version": "0.1.1", "author": "Matt Pocock", "license": "MIT", "scripts": { diff --git a/pages/index.mdx b/pages/index.mdx index 20dc80e..f99e380 100644 --- a/pages/index.mdx +++ b/pages/index.mdx @@ -1,6 +1,6 @@ -import { CategoryList, CategorySection } from "../lib/CategoryList"; -import { FrontPageHero } from "../lib/FrontPageHero"; -import * as Icons from "../lib/Icons"; +import { CategoryList, CategorySection } from '../lib/CategoryList'; +import { FrontPageHero } from '../lib/FrontPageHero'; +import * as Icons from '../lib/Icons'; @@ -54,6 +54,7 @@ import * as Icons from "../lib/Icons";
+