Real Time sound synthesis with Jupyter

%%bash 
#Install support for websockets
pip install autobahn
pip install ipywidgets
pip install plotly

Requirement already satisfied: autobahn in /usr/local/lib/python3.5/dist-packages
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.5/dist-packages (from autobahn)
....

import asyncio
from autobahn.asyncio.websocket import WebSocketServerProtocol, WebSocketServerFactory
#import websockets
import numpy
import threading
import time
import plotly
from plotly.graph_objs import Scatter, Layout
plotly.offline.init_notebook_mode(connected=True)

from ipywidgets import interact, interactive, fixed, interact_manual
import ipywidgets as widgets

server = None
loop_thread = None

buffer_size = 4096
sample_rate = 44100.0

#note float32 is essential, the same format is used by javascript
#could be that when endianness is not equal between client and server
#additional processing is needed
buffer = numpy.zeros(buffer_size,dtype=numpy.float32)

#create an array of 5 frequencies
component_count = 5
sine_frequencies = numpy.linspace(start = 200, stop = 400, num = component_count)
#reshape from (5,) to (1,5)
sine_frequencies = numpy.reshape( sine_frequencies, (1, component_count))

#create a (1,5) array with the phase change per sample
sine_frequencies_angle_per_s = ( sine_frequencies * 2.0 * numpy.pi ) 

#store the start phases for each buffer fill
#by using a start phase you won't get glitches in the signal
#when chaning the frequency
sine_start_phases = numpy.zeros(sine_frequencies.shape)

#create buffers for each sine component -> (buffer_size,components)
sine_components = numpy.zeros(
    (buffer_size,sine_frequencies.shape[0]),
    dtype=numpy.float32)

#calculate the time of each sample in seconds (buffer_size,1)
sample_times = numpy.reshape(numpy.linspace(start = 0,stop = buffer_size / sample_rate, num = buffer_size),(buffer_size,1))

def setSineFrequency(component, frequency):
    """This function is called when a slider is moved"""
    global sine_frequencies, sine_frequencies_angle_per_s
    sine_frequencies[0,component] = frequency 
    sine_frequencies_angle_per_s = ( sine_frequencies * 2.0 * numpy.pi )

# Create a bunch of sliders for each frequency component
for component in range(sine_frequencies.shape[1]):
    interact(setSineFrequency, 
             component = fixed(component),
             frequency = widgets.FloatSlider(
                 min=20,max=20000,
                 step=1,
                 value=sine_frequencies[0,component]))

def fillBuffer():
    """Fill a single buffer with the sum of the sine components"""
    global sine_start_phases, sine_components    
    sine_phases = sine_start_phases + sample_times * sine_frequencies_angle_per_s
    sine_components = numpy.sin(sine_phases) * 1 / component_count
    numpy.sum(sine_components, axis = 1, out = buffer)
    
    sine_start_phases += (buffer_size + 1) * sine_frequencies_angle_per_s / sample_rate
    
# To test
#setSineFrequency(0,220)
glued = numpy.zeros(2*4096)
fillBuffer()
glued[0:4096] = buffer[0:4096]
#setSineFrequency(0,440)
fillBuffer()
glued[4096:8192] = buffer[0:4096]

x = plotly.offline.iplot({
    "data": [Scatter(y=glued)],
    "layout": Layout(title="Two buffers")
})

#Create a thread to run the asyncio loop
#It calls loop_run_forever

loop = asyncio.get_event_loop()

class WebIOThread(threading.Thread):
    def __init__(self):
        super().__init__()
        self.loop = asyncio.get_event_loop()
        self.should_quit = False
    def quit(self):
        self.should_quit = True
        if (self.loop.is_running):
            self.loop.call_soon_threadsafe(self.loop.stop)
    
    def run(self):
        self.running = True
        print("thread start")    
        while (not self.should_quit):
        #logging.debug('running')
            print("start loop")
            self.loop.run_forever()
            print("loop exited")
            if (not self.should_quit):
                time.sleep(1)
        print("thread exit")    
        self.running = False

#Define a simple websocket protocol that handles requests
#for a sample buffers
class SignalGeneratorProtocol(WebSocketServerProtocol):
    def onConnect(self, request):
        print("Client connecting: {}".format(request.peer))

    def onOpen(self):
        print("WebSocket connection open.")

    def onMessage(self, payload, isBinary):
        # if the client sends (any) message then
        # assume it wants a new buffer of samples
        #print(".")
        fillBuffer()
        self.sendMessage(buffer.tobytes(), True)
           
    def onClose(self, wasClean, code, reason):
        print("WebSocket connection closed: {}".format(reason))

factory = WebSocketServerFactory()
factory.protocol = SignalGeneratorProtocol

#Ever time this cell is executed, cleanup
#previous servers

if server is not None:
    print("Close existing server")
    server.close()
    server = None

if (loop_thread is not None):
    print("Close existing loop thread")
    loop_thread.quit()
    loop_thread.join()
    loop_thread = None
    
loop = asyncio.get_event_loop()
coro = loop.create_server(factory, '0.0.0.0', 8889)
server = loop.run_until_complete(coro)

loop_thread = WebIOThread()
loop_thread.start()

Close existing server
Close existing loop thread
loop exited
thread exit
thread start
start loop

%%javascript 

if (!window.audioContext) {
    window.audioContext = new AudioContext();
}

// Connect a web socket to a localhost server on port 8889
var ws = new WebSocket("ws://127.0.0.1:8889");
ws.binaryType = 'arraybuffer';
window.ws = ws;

// bufferSize should be the same on client and server
var bufferSize = 4096;
var player = (function() {
    var sampleNumber = 0;
    var node = window.audioContext.createScriptProcessor(bufferSize, 1, 1);
    node.onaudioprocess = function(e) {
        var output = e.outputBuffer.getChannelData(0);
        
        // does the websocket receive any buffers with samples
        if (ws.received_buffers.length>=1) {
            // playback the first received buffer
            output.set(ws.received_buffers.shift());
        }
        if (ws.received_buffers.length<10) {
            // The number of sample buffers in the queue is getting low
            // request more buffers
            if (ws.readyState==1) {
                // Web Socket is connected, send data using send()
                ws.send("Get block");
                ws.send("Get block");
            }
        }
    }
    return node;
})();


// called when the  websocket opened
ws.onopen = function()
{
  console.log("Websocket connected");
  ws.received_buffers = [] ;
  ws.received_frames = 0
  // request a buffer of samples
  ws.send("Get block");
  ws.is_connected = false;
};

ws.onmessage = function (evt) 
{ 
  // a new buffer of samples was received
  ws.received_buffers.push(new Float32Array(evt.data)); //.set(evt.data);
  //ws.last_received_buffer.set(evt.data);
  ws.received_frames += 1;
  if (!ws.is_connected) {
      player.connect(window.audioContext.destination);  
      ws.is_connected = true;
  }
};

ws.onclose = function()
{ 
  player.disconnect();
  ws.is_connected = false;
 
  // websocket is closed.
  console.log("Connection is closed..."); 
};
            

Client connecting: tcp:172.17.0.1:50694
WebSocket connection open.

%%javascript 
console.log(window.ws.received_buffers.length)
window.ws.close()

WebSocket connection closed: None