From b4d952ca760ddb2295437415958139fbfa523357 Mon Sep 17 00:00:00 2001 From: terry feng Date: Fri, 26 Apr 2024 01:42:10 -0700 Subject: [PATCH] add audio recorder feature --- .eslintrc.json | 3 +- index.html | 3 + package-lock.json | 32 +++++- package.json | 4 +- public/img/armed-button.svg | 5 + public/img/record-button.svg | 4 + public/img/stop-button.svg | 4 + public/js/wave-worker.js | 102 +++++++++++++++++++ src/components/chuckBar.ts | 9 ++ src/components/console.ts | 11 +++ src/components/recorder.ts | 184 +++++++++++++++++++++++++++++++++++ src/components/vmMonitor.ts | 8 ++ src/host.ts | 10 ++ 13 files changed, 376 insertions(+), 3 deletions(-) create mode 100644 public/img/armed-button.svg create mode 100644 public/img/record-button.svg create mode 100644 public/img/stop-button.svg create mode 100644 public/js/wave-worker.js create mode 100644 src/components/recorder.ts diff --git a/.eslintrc.json b/.eslintrc.json index 9002cd2..f301ef5 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -18,7 +18,8 @@ "rules": { "indent": [ "error", - 4 + 4, + { "SwitchCase": 1 } ], "linebreak-style": [ "error", diff --git a/index.html b/index.html index 9aeca9d..e36a18a 100644 --- a/index.html +++ b/index.html @@ -253,6 +253,9 @@ + diff --git a/package-lock.json b/package-lock.json index 54dd921..431a5d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "webchuck-ide", "version": "2.1.1", "dependencies": { + "@xterm/addon-web-links": "^0.11.0", "js-search": "^2.0.1", "jszip": "^3.10.1", "monaco-editor": "^0.37.1", @@ -15,7 +16,8 @@ "pako": "^2.1.0", "webchuck": "github:ccrma/webchuck", "xterm": "^5.3.0", - "xterm-addon-fit": "^0.8.0" + "xterm-addon-fit": "^0.8.0", + "xterm-link-provider": "^1.3.1" }, "devDependencies": { "@types/js-search": "^1.4.4", @@ -820,6 +822,20 @@ "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", "dev": true }, + "node_modules/@xterm/addon-web-links": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@xterm/addon-web-links/-/addon-web-links-0.11.0.tgz", + "integrity": "sha512-nIHQ38pQI+a5kXnRaTgwqSHnX7KE6+4SVoceompgHL26unAxdfP6IPqUTSYPQgSwM56hsElfoNrrW5V7BUED/Q==", + "peerDependencies": { + "@xterm/xterm": "^5.0.0" + } + }, + "node_modules/@xterm/xterm": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-5.5.0.tgz", + "integrity": "sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A==", + "peer": true + }, "node_modules/acorn": { "version": "8.11.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", @@ -2969,6 +2985,20 @@ "xterm": "^5.0.0" } }, + "node_modules/xterm-link-provider": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/xterm-link-provider/-/xterm-link-provider-1.3.1.tgz", + "integrity": "sha512-uOlaIeUED6kJeL2nIIf5YwreO0obMhsC0RWypEUmWkz7SAQewzgwdWFjQ2He7NGcT93c4KUf8bRgAu8cV9bAYA==", + "dependencies": { + "xterm": "^4.6.0" + } + }, + "node_modules/xterm-link-provider/node_modules/xterm": { + "version": "4.19.0", + "resolved": "https://registry.npmjs.org/xterm/-/xterm-4.19.0.tgz", + "integrity": "sha512-c3Cp4eOVsYY5Q839dR5IejghRPpxciGmLWWaP9g+ppfMeBChMeLa1DCA+pmX/jyDZ+zxFOmlJL/82qVdayVoGQ==", + "deprecated": "This package is now deprecated. Move to @xterm/xterm instead." + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/package.json b/package.json index b4e1360..75fc5a2 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "vite": "^4.2.0" }, "dependencies": { + "@xterm/addon-web-links": "^0.11.0", "js-search": "^2.0.1", "jszip": "^3.10.1", "monaco-editor": "^0.37.1", @@ -31,6 +32,7 @@ "pako": "^2.1.0", "webchuck": "github:ccrma/webchuck", "xterm": "^5.3.0", - "xterm-addon-fit": "^0.8.0" + "xterm-addon-fit": "^0.8.0", + "xterm-link-provider": "^1.3.1" } } diff --git a/public/img/armed-button.svg b/public/img/armed-button.svg new file mode 100644 index 0000000..ad9290a --- /dev/null +++ b/public/img/armed-button.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/public/img/record-button.svg b/public/img/record-button.svg new file mode 100644 index 0000000..a9be114 --- /dev/null +++ b/public/img/record-button.svg @@ -0,0 +1,4 @@ + + + + diff --git a/public/img/stop-button.svg b/public/img/stop-button.svg new file mode 100644 index 0000000..dede648 --- /dev/null +++ b/public/img/stop-button.svg @@ -0,0 +1,4 @@ + + + + diff --git a/public/js/wave-worker.js b/public/js/wave-worker.js new file mode 100644 index 0000000..6086879 --- /dev/null +++ b/public/js/wave-worker.js @@ -0,0 +1,102 @@ +self.onmessage = function (e) { + var wavPCM = new WavePCM(e["data"]["config"]); + wavPCM.record(e["data"]["pcmArrays"]); + wavPCM.requestData(); +}; + +var WavePCM = function (config) { + this.sampleRate = config["sampleRate"] || 48000; + this.bitDepth = config["bitDepth"] || 16; + this.recordedBuffers = []; + this.bytesPerSample = this.bitDepth / 8; +}; + +WavePCM.prototype.record = function (buffers) { + this.numberOfChannels = this.numberOfChannels || buffers.length; + var bufferLength = buffers[0].length; + var reducedData = new Uint8Array( + bufferLength * this.numberOfChannels * this.bytesPerSample + ); + + // Interleave + for (var i = 0; i < bufferLength; i++) { + for (var channel = 0; channel < this.numberOfChannels; channel++) { + var outputIndex = + (i * this.numberOfChannels + channel) * this.bytesPerSample; + var sample = buffers[channel][i]; + + // Check for clipping + if (sample > 1) { + sample = 1; + } else if (sample < -1) { + sample = -1; + } + + // bit reduce and convert to uInt + switch (this.bytesPerSample) { + case 4: + sample = sample * 2147483648; + reducedData[outputIndex] = sample; + reducedData[outputIndex + 1] = sample >> 8; + reducedData[outputIndex + 2] = sample >> 16; + reducedData[outputIndex + 3] = sample >> 24; + break; + + case 3: + sample = sample * 8388608; + reducedData[outputIndex] = sample; + reducedData[outputIndex + 1] = sample >> 8; + reducedData[outputIndex + 2] = sample >> 16; + break; + + case 2: + sample = sample * 32768; + reducedData[outputIndex] = sample; + reducedData[outputIndex + 1] = sample >> 8; + break; + + case 1: + reducedData[outputIndex] = (sample + 1) * 128; + break; + + default: + throw "Only 8, 16, 24 and 32 bits per sample are supported"; + } + } + } + + this.recordedBuffers.push(reducedData); +}; + +WavePCM.prototype.requestData = function () { + var bufferLength = this.recordedBuffers[0].length; + var dataLength = this.recordedBuffers.length * bufferLength; + var headerLength = 44; + var wav = new Uint8Array(headerLength + dataLength); + var view = new DataView(wav.buffer); + + view.setUint32(0, 1380533830, false); // RIFF identifier 'RIFF' + view.setUint32(4, 36 + dataLength, true); // file length minus RIFF identifier length and file description length + view.setUint32(8, 1463899717, false); // RIFF type 'WAVE' + view.setUint32(12, 1718449184, false); // format chunk identifier 'fmt ' + view.setUint32(16, 16, true); // format chunk length + view.setUint16(20, 1, true); // sample format (raw) + view.setUint16(22, this.numberOfChannels, true); // channel count + view.setUint32(24, this.sampleRate, true); // sample rate + view.setUint32( + 28, + this.sampleRate * this.bytesPerSample * this.numberOfChannels, + true + ); // byte rate (sample rate * block align) + view.setUint16(32, this.bytesPerSample * this.numberOfChannels, true); // block align (channel count * bytes per sample) + view.setUint16(34, this.bitDepth, true); // bits per sample + view.setUint32(36, 1684108385, false); // data chunk identifier 'data' + view.setUint32(40, dataLength, true); // data chunk length + + for (var i = 0; i < this.recordedBuffers.length; i++) { + wav.set(this.recordedBuffers[i], i * bufferLength + headerLength); + } + + self.postMessage(wav, [wav.buffer]); + self.close(); +}; diff --git a/src/components/chuckBar.ts b/src/components/chuckBar.ts index 01e61a6..fb59b2c 100644 --- a/src/components/chuckBar.ts +++ b/src/components/chuckBar.ts @@ -17,6 +17,7 @@ import { theChuck, startChuck, connectMic } from "@/host"; import Editor from "@/components/monaco/editor"; import VmMonitor from "@/components/vmMonitor"; +import Recorder from "@/components/recorder"; // detect operating system const isWindows = navigator.userAgent.includes("Windows"); @@ -28,6 +29,7 @@ export default class ChuckBar { public static playButton: HTMLButtonElement; public static replaceButton: HTMLButtonElement; public static removeButton: HTMLButtonElement; + public static recordButton: HTMLButtonElement; public static running: boolean = false; @@ -43,6 +45,8 @@ export default class ChuckBar { document.querySelector("#replaceButton")!; ChuckBar.removeButton = document.querySelector("#removeButton")!; + ChuckBar.recordButton = + document.querySelector("#recordButton")!; // Add tooltips ChuckBar.webchuckButton.title = `Start ChucK VM [${metaKey} + .]`; @@ -50,6 +54,7 @@ export default class ChuckBar { ChuckBar.playButton.title = `Run [${metaKey} + Enter]`; ChuckBar.replaceButton.title = `Replace [${metaKey} + \\]`; ChuckBar.removeButton.title = `Remove [${metaKey} + ⌫]`; + ChuckBar.recordButton.title = `Record`; // Add button event listeners ChuckBar.webchuckButton.addEventListener("click", async () => { @@ -68,6 +73,9 @@ export default class ChuckBar { ChuckBar.removeButton.addEventListener("click", async () => { ChuckBar.removeCode(); }); + + // Configure the recorder button + new Recorder(ChuckBar.recordButton); } static runEditorCode() { @@ -112,6 +120,7 @@ export default class ChuckBar { ChuckBar.playButton.disabled = false; ChuckBar.replaceButton.disabled = false; ChuckBar.removeButton.disabled = false; + ChuckBar.recordButton.disabled = false; ChuckBar.running = true; } diff --git a/src/components/console.ts b/src/components/console.ts index f92dd77..91ee746 100644 --- a/src/components/console.ts +++ b/src/components/console.ts @@ -9,9 +9,14 @@ import { Terminal } from "xterm"; import { FitAddon } from "xterm-addon-fit"; +// import { WebLinksAddon } from "@xterm/addon-web-links"; import "@styles/xterm.css"; import { theChuck } from "@/host"; +import { LinkProvider } from "xterm-link-provider"; + +// Define a custom regular expression that matches blob URIs +const blobRegex = /(blob:https?:\/\/\S+)/; export default class Console { public static terminal: Terminal; @@ -44,6 +49,12 @@ export default class Console { Console.terminal.open(Console.terminalElement); Console.fit(); + // Blob Links + Console.terminal.registerLinkProvider(new LinkProvider(Console.terminal, blobRegex, (e, uri) => { + window.open(uri, "_blank"); + })); + // Console.terminal.loadAddon(new WebLinksAddon()); + // Resize listener window.addEventListener("resize", () => { Console.resizeConsole(); diff --git a/src/components/recorder.ts b/src/components/recorder.ts new file mode 100644 index 0000000..6abd6bd --- /dev/null +++ b/src/components/recorder.ts @@ -0,0 +1,184 @@ +import Console from "./console"; +import ProjectSystem from "./fileExplorer/projectSystem"; +import VmMonitor from "./vmMonitor"; + +enum RecordState { + stopped = 0, + recording = 1, + armed = 2, +} + +enum RecordButtonImage { + record = "img/record-button.svg", + stop = "img/stop-button.svg", + armed = "img/armed-button.svg", +} + +export default class Recorder { + private static state: RecordState = RecordState.stopped; + public static recordButton: HTMLButtonElement; + public static recordImage: HTMLImageElement; + public static playButton: HTMLButtonElement; + public static removeButton: HTMLButtonElement; + + private static stream: MediaStreamAudioDestinationNode; + private static recorder: MediaRecorder; + private static buffer: Blob[]; + + + constructor(recordButton: HTMLButtonElement) { + Recorder.recordButton = recordButton; + Recorder.recordButton.title = `Record`; + Recorder.recordButton.addEventListener("click", async () => { + Recorder.recordPressed(); + }); + Recorder.recordImage = document.getElementById( + "recordImage" + )! as HTMLImageElement; + + // Get references to Chuck Buttons + Recorder.playButton = document.getElementById( + "playButton" + )! as HTMLButtonElement; + Recorder.removeButton = document.getElementById( + "removeButton" + )! as HTMLButtonElement; + // Add special event listener to automatically stop recording when only 1 shred is left + Recorder.removeButton.addEventListener("click", () => { + if (Recorder.state === RecordState.recording && VmMonitor.getNumShreds() === 1) { + Recorder.stopRecording(); + } + }); + + } + + static configureRecorder(audioContext: AudioContext, gainNode: GainNode) { + // Record functionality + Recorder.stream = audioContext.createMediaStreamDestination(); + Recorder.recorder = new MediaRecorder(Recorder.stream.stream); + gainNode.connect(Recorder.stream); + Recorder.buffer = []; + + Recorder.recorder.ondataavailable = (e) => { + Recorder.buffer.push(e.data); + }; + + Recorder.recorder.onstop = async () => { + const blob = new Blob(Recorder.buffer, { + type: Recorder.recorder.mimeType, + }); + const arrayBuffer = await blob.arrayBuffer(); + const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + const wavBlob = await convertAudioBufferToWavBlob(audioBuffer); + + // Get current file and date:time (local) + const filename = ProjectSystem.activeFile.getFilename().slice(0, -3); + const now = new Date(); + const date = now.toLocaleDateString().replace(/\//g, "-"); + const time = now.toLocaleTimeString().replace(/:/g, "-").replace(/\s/g, ""); + const dateTime = `${date}_${time}`; + + const blobLink = getBlobLink(wavBlob, filename + "_" + dateTime + ".wav"); + Console.print(`Preview recording at \x1b[38;2;34;178;254m${blobLink}\x1b[0m`); + + Recorder.buffer = []; + }; + } + + static recordPressed() { + const numActiveShreds = VmMonitor.getNumShreds(); + + // Record FSM + switch (Recorder.state) { + case RecordState.stopped: + if (numActiveShreds === 0) { + Recorder.armRecorder(); + } else { + Recorder.startRecording(); + } + break; + case RecordState.recording: + Recorder.stopRecording(); + break; + case RecordState.armed: + Recorder.disarmRecorder(); + break; + } + } + + static startRecording() { + Recorder.state = RecordState.recording; + Console.print("\x1b[31mrecording...\x1b[0m"); // Print in red + Recorder.recordImage.src = RecordButtonImage.stop; + + Recorder.playButton.removeEventListener("click", Recorder.startRecording); + Recorder.recorder.start(); + } + + static stopRecording() { + Recorder.state = RecordState.stopped; + Console.print("recording stopped..."); + Recorder.recordImage.src = RecordButtonImage.record; + + Recorder.recorder.stop(); + } + + static armRecorder() { + Recorder.state = RecordState.armed; + Console.print("armed for recording..."); + Recorder.recordImage.src = RecordButtonImage.armed; + + Recorder.playButton.addEventListener("click", Recorder.startRecording); + } + + static disarmRecorder() { + Recorder.state = RecordState.stopped; + Recorder.recordImage.src = RecordButtonImage.record; + Recorder.playButton.removeEventListener("click", Recorder.startRecording); + } +} + +//-------------------------------------- +// Recorder Utility Functions +//-------------------------------------- +/** + * Convert an AudioBuffer to a Wav Blob + * @param audioBuffer audio buffer to convert + * @returns Promise that resolves to a Blob + */ +export async function convertAudioBufferToWavBlob(audioBuffer: AudioBuffer): Promise { + return new Promise((resolve) => { + // This file is in `public/js` + const worker = new Worker("./js/wave-worker.js"); + + worker.onmessage = (e: MessageEvent) => { + const blob = new Blob([e.data.buffer], { type: "audio/wav" }); + resolve(blob); + }; + + const pcmArrays: Float32Array[] = []; + for (let i = 0; i < audioBuffer.numberOfChannels; i++) { + pcmArrays.push(audioBuffer.getChannelData(i)); + } + + worker.postMessage({ + pcmArrays, + config: { sampleRate: audioBuffer.sampleRate }, + }); + }); +} + +/** + * Get the link to download a blob + * @param blob blob to download + * @param name name of blob + * @returns url to the blob + */ +export function getBlobLink(blob: Blob, name: string): string { + const blobUrl = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = blobUrl; + link.download = name; + // link.click(); + return link.href; +} \ No newline at end of file diff --git a/src/components/vmMonitor.ts b/src/components/vmMonitor.ts index 51836eb..52bca89 100644 --- a/src/components/vmMonitor.ts +++ b/src/components/vmMonitor.ts @@ -17,6 +17,7 @@ import Editor from "@components/monaco/editor"; export default class VmMonitor { public static vmContainer: HTMLDivElement; public static shredTableBody: HTMLTableSectionElement; + private static numShreds: number = 0; private static shredsToRows: { [key: number]: HTMLTableRowElement } = {}; @@ -113,6 +114,8 @@ export default class VmMonitor { () => {} ); }); + + this.numShreds++; } /** @@ -125,8 +128,13 @@ export default class VmMonitor { VmMonitor.shredsToRows[theShred] ); delete VmMonitor.shredsToRows[theShred]; + this.numShreds--; } } + + static getNumShreds(): number { + return this.numShreds; + } } /** diff --git a/src/host.ts b/src/host.ts index 0e32572..5399842 100644 --- a/src/host.ts +++ b/src/host.ts @@ -18,6 +18,7 @@ import Visualizer from "@/components/visualizer"; import HidPanel from "@/components/hidPanel"; import ChuckBar from "@/components/chuckBar"; import ProjectSystem from "@/components/fileExplorer/projectSystem"; +import Recorder from "@/components/recorder"; // WebChucK source const DEV_CHUCK_SRC = "https://chuck.stanford.edu/webchuck/dev/"; // dev webchuck src @@ -35,6 +36,9 @@ let sampleRate: number = 0; let analyser: AnalyserNode; let visual: Visualizer; +// Recorder +let recordGain: GainNode; + // HID let hid: HID; @@ -110,6 +114,12 @@ export async function startChuck() { // Start audio visualizer startVisualizer(); + // Configure Recorder + recordGain = audioContext.createGain(); + recordGain.gain.value = .96; // so it doesn't clip + theChuck.connect(recordGain); + Recorder.configureRecorder(audioContext, recordGain); + // Start HID, mouse and keyboard on hid = await HID.init(theChuck); new HidPanel(hid);