first commit

2023-08-01 14:20:25 +02:00
commit e142078e83
17 changed files with 639 additions and 0 deletions
--- a/public/cool-background.png
+++ b/public/cool-background.png
--- a/public/favicon.ico
+++ b/public/favicon.ico
--- a/public/index.html
+++ b/public/index.html
@@ -0,0 +1,22 @@
+
+<!doctype html>
+<html>
+<head>
+	<meta name="viewport" content="width=device-width,initial-scale=1">
+	<title>Deepspeech Demo</title>
+	<link rel="stylesheet" type="text/css" href="styles.css"/>
+</head>
+<body>
+	<main>
+		<canvas id="analyser" width="512" height="250"></canvas>
+		<canvas id="wavedisplay" width="512" height="250"></canvas>
+		<button onclick="toggleRecording(this)">Listen</button>
+		<div id="result"></div>
+	</main>
+	<script src="adapter.js"></script>
+	<script src="socket.io.js"></script>
+	<script src="socket.io-stream.js"></script>
+	<script src="recorder.js"></script>
+	<script src="main.js"></script>	
+</body>
+</html>
--- a/public/main.js
+++ b/public/main.js
@@ -0,0 +1,128 @@
+let audioContext = new window.AudioContext()
+let audioInput = null
+let realAudioInput = null
+let inputPoint = null
+let audioRecorder = null
+let socket = null
+let analyserContext = null
+let canvasWidth = null
+let canvasHeight = null
+let analyserNode = null
+
+const drawBuffer = (width, height, context, data) => {
+  const step = Math.ceil(data.length / width)
+  const amp = height / 2
+  context.fillStyle = 'silver'
+  context.clearRect(0, 0, width, height)
+  for (let i = 0; i < width; i++) {
+    let min = 1.0
+    let max = -1.0
+    for (let j = 0; j < step; j++) {
+      let datum = data[(i * step) + j]
+      if (datum < min) min = datum
+      if (datum > max) max = datum
+    }
+    context.fillRect(i, (1 + min) * amp, 1, Math.max(1, (max - min) * amp))
+  }
+}
+
+const gotBuffers = buffers => {
+  let canvas = document.getElementById('wavedisplay')
+  drawBuffer(canvas.width, canvas.height, canvas.getContext('2d'), buffers[0])
+  audioRecorder.exportWAV(doneEncoding)
+}
+
+const doneEncoding = blob => {
+  const stream = window.ss.createStream()
+  document.getElementById('result').textContent = 'Analysing...'
+  window.ss(socket).emit('audio', stream)
+  window.ss.createBlobReadStream(blob).pipe(stream)
+}
+
+const toggleRecording = element => {
+  if (element.classList.contains('recording')) {
+    element.textContent = 'Listen'
+    audioRecorder.stop()
+    element.classList.remove('recording')
+    audioRecorder.getBuffers(gotBuffers)
+    return
+  }
+  element.textContent = 'Listening...'
+  if (!audioRecorder) return
+  element.classList.add('recording')
+  audioRecorder.clear()
+  audioRecorder.record()
+}
+
+const updateAnalysers = time => {
+  if (!analyserContext) {
+    const canvas = document.getElementById('analyser')
+    canvasWidth = canvas.width
+    canvasHeight = canvas.height
+    analyserContext = canvas.getContext('2d')
+  }
+
+  // analyzer draw code here
+  const SPACING = 3
+  const BAR_WIDTH = 1
+  const numBars = Math.round(canvasWidth / SPACING)
+  const freqByteData = new Uint8Array(analyserNode.frequencyBinCount)
+  analyserNode.getByteFrequencyData(freqByteData)
+  analyserContext.clearRect(0, 0, canvasWidth, canvasHeight)
+  analyserContext.fillStyle = '#F6D565'
+  analyserContext.lineCap = 'round'
+  const multiplier = analyserNode.frequencyBinCount / numBars
+
+  // Draw rectangle for each frequency bin.
+  for (let i = 0; i < numBars; ++i) {
+    let magnitude = 0
+    const offset = Math.floor(i * multiplier)
+    // gotta sum/average the block, or we miss narrow-bandwidth spikes
+    for (var j = 0; j < multiplier; j++) magnitude += freqByteData[offset + j]
+    magnitude = magnitude / multiplier
+    analyserContext.fillStyle = `hsl( ${Math.round((i * 360) / numBars)}, 100%, 50%)`
+    analyserContext.fillRect(i * SPACING, canvasHeight, BAR_WIDTH, -magnitude)
+  }
+  window.requestAnimationFrame(updateAnalysers)
+}
+
+const gotStream = stream => {
+  inputPoint = audioContext.createGain()
+  // Create an AudioNode from the stream.
+  realAudioInput = audioContext.createMediaStreamSource(stream)
+  audioInput = realAudioInput
+  audioInput.connect(inputPoint)
+  analyserNode = audioContext.createAnalyser()
+  analyserNode.fftSize = 2048
+  inputPoint.connect(analyserNode)
+  audioRecorder = new window.Recorder(inputPoint)
+  let zeroGain = audioContext.createGain()
+  zeroGain.gain.value = 0.0
+  inputPoint.connect(zeroGain)
+  zeroGain.connect(audioContext.destination)
+  updateAnalysers()
+}
+
+const initAudio = () => {
+  navigator.mediaDevices.getUserMedia({
+    audio: {
+      mandatory: {
+        googEchoCancellation: true,
+        googAutoGainControl: true,
+        googNoiseSuppression: true,
+        googHighpassFilter: true
+      },
+      optional: []
+    }
+  }).then(gotStream).catch(console.error)
+}
+
+const connect = () => {
+  socket = window.io.connect(window.location.origin)
+  window.ss(socket).on('news', (stream, data) => {
+    document.getElementById('result').textContent = data.text
+  })
+}
+
+connect()
+initAudio()
--- a/public/recorder.js
+++ b/public/recorder.js
@@ -0,0 +1,48 @@
+let Recorder = function (source) {
+  const bufferLen = 4096
+  let recording = false
+  let currCallback = null
+
+  this.context = source.context
+  if (!this.context.createScriptProcessor) {
+    this.node = this.context.createJavaScriptNode(bufferLen, 2, 2)
+  } else {
+    this.node = this.context.createScriptProcessor(bufferLen, 2, 2)
+  }
+
+  const worker = new Worker('./recorderWorker.js')
+  worker.postMessage({
+    command: 'init',
+    config: {sampleRate: this.context.sampleRate}
+  })
+
+  this.record = () => { recording = true }
+  this.stop = () => { recording = false }
+  this.clear = () => worker.postMessage({command: 'clear'})
+  this.getBuffers = cb => {
+    currCallback = cb
+    worker.postMessage({command: 'getBuffers'})
+  }
+
+  this.exportWAV = cb => {
+    currCallback = cb
+    worker.postMessage({command: 'exportWAV', type: 'audio/wav'})
+  }
+
+  this.node.onaudioprocess = e => {
+    if (!recording) return
+    worker.postMessage({
+      command: 'record',
+      buffer: [
+        e.inputBuffer.getChannelData(0),
+        e.inputBuffer.getChannelData(1)
+      ]
+    })
+  }
+
+  worker.onmessage = e => currCallback(e.data)
+  source.connect(this.node)
+  this.node.connect(this.context.destination)
+}
+
+window.Recorder = Recorder
--- a/public/recorderWorker.js
+++ b/public/recorderWorker.js
@@ -0,0 +1,111 @@
+let recLength = 0
+let recBuffersL = []
+let recBuffersR = []
+let sampleRate
+
+this.onmessage = e => {
+  if (e.data.command === 'init') init(e.data.config)
+  if (e.data.command === 'record') record(e.data.buffer)
+  if (e.data.command === 'exportWAV') exportWAV(e.data.type)
+  if (e.data.command === 'getBuffers') getBuffers()
+  if (e.data.command === 'clear') clear()
+}
+
+const mergeBuffers = (recBuffers, recLength) => {
+  let result = new Float32Array(recLength)
+  let offset = 0
+  for (let i = 0; i < recBuffers.length; i++) {
+    result.set(recBuffers[i], offset)
+    offset += recBuffers[i].length
+  }
+  return result
+}
+
+const init = config => { sampleRate = config.sampleRate }
+
+const record = inputBuffer => {
+  recBuffersL.push(inputBuffer[0])
+  recBuffersR.push(inputBuffer[1])
+  recLength += inputBuffer[0].length
+}
+
+const exportWAV = type => {
+  const bufferL = mergeBuffers(recBuffersL, recLength)
+  const bufferR = mergeBuffers(recBuffersR, recLength)
+  const interleaved = interleave(bufferL, bufferR)
+  const dataview = encodeWAV(interleaved)
+  const audioBlob = new Blob([dataview], {type})
+  this.postMessage(audioBlob)
+}
+
+const getBuffers = () => {
+  let buffers = []
+  buffers.push(mergeBuffers(recBuffersL, recLength))
+  buffers.push(mergeBuffers(recBuffersR, recLength))
+  this.postMessage(buffers)
+}
+
+const clear = () => {
+  recLength = 0
+  recBuffersL = []
+  recBuffersR = []
+}
+
+const interleave = (inputL, inputR) => {
+  const length = inputL.length + inputR.length
+  let result = new Float32Array(length)
+  let index = 0
+  let inputIndex = 0
+  while (index < length) {
+    result[index++] = inputL[inputIndex]
+    result[index++] = inputR[inputIndex]
+    inputIndex++
+  }
+  return result
+}
+
+const floatTo16BitPCM = (output, offset, input) => {
+  for (let i = 0; i < input.length; i++, offset += 2) {
+    let s = Math.max(-1, Math.min(1, input[i]))
+    output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true)
+  }
+}
+
+const writeString = (view, offset, string) => {
+  for (let i = 0; i < string.length; i++) {
+    view.setUint8(offset + i, string.charCodeAt(i))
+  }
+}
+
+const encodeWAV = samples => {
+  let buffer = new ArrayBuffer(44 + samples.length * 2)
+  let view = new DataView(buffer)
+  /* RIFF identifier */
+  writeString(view, 0, 'RIFF')
+  /* file length */
+  view.setUint32(4, 32 + samples.length * 2, true)
+  /* RIFF type */
+  writeString(view, 8, 'WAVE')
+  /* format chunk identifier */
+  writeString(view, 12, 'fmt ')
+  /* format chunk length */
+  view.setUint32(16, 16, true)
+  /* sample format (raw) */
+  view.setUint16(20, 1, true)
+  /* channel count */
+  view.setUint16(22, 2, true)
+  /* sample rate */
+  view.setUint32(24, sampleRate, true)
+  /* byte rate (sample rate * block align) */
+  view.setUint32(28, sampleRate * 4, true)
+  /* block align (channel count * bytes per sample) */
+  view.setUint16(32, 4, true)
+  /* bits per sample */
+  view.setUint16(34, 16, true)
+  /* data chunk identifier */
+  writeString(view, 36, 'data')
+  /* data chunk length */
+  view.setUint32(40, samples.length * 2, true)
+  floatTo16BitPCM(view, 44, samples)
+  return view
+}
--- a/public/screenshot.png
+++ b/public/screenshot.png
--- a/public/styles.css
+++ b/public/styles.css
@@ -0,0 +1,60 @@
+html {overflow: hidden}
+
+body {
+  font: 14pt Arial, sans-serif;
+  background: url('cool-background.png');
+  display: flex;
+  flex-direction: row;
+  height: 100vh;
+  width: 100%;
+  margin: 0}
+
+canvas {
+  display: flex;
+  align-self: top;
+  background: #202020;
+  width: 50%;
+  height: 25%;
+  margin: 2rem;
+  box-shadow: 0px 0px 10px blue}
+
+main {
+  height: 80%;
+  width: 100%;
+  display: flex;
+  flex-direction: column;
+  justify-content: flex-start;
+  align-items: center}
+
+button {
+  display: inline-block;
+  border-radius: 4px;
+  background-color: #f4511e;
+  border: none;
+  color: #FFFFFF;
+  text-align: center;
+  font-size: 28px;
+  padding: 20px;
+  width: 200px;
+  transition: all 0.5s;
+  cursor: pointer;
+  margin: 5px}
+
+#result {
+  color: #fff;
+  background-color: #333;
+  width: 50%;
+  padding: 2rem;
+  margin-top: 5%;
+  border-radius: 20px}
+
+@media (orientation: landscape) {
+  body {flex-direction: row}
+  #controls {
+    flex-direction: column;
+    height: 100%;
+    width: 10%}
+  main {
+    height: 100%;
+    width: 90%}
+}