2
\$\begingroup\$

I normally program in a linux vm and decided to do some machine learning with my webcam when I realized the vm didn't have access to my camera hardware.

That kicked-off a mini project to create a plug-and-play module to access the webcam in a browser and redirect through a Go backend so I could work around the limitations of my vm but also reuse the code in future webapps. After a bit of research WebRTC seemed like the best way to handle a full video stream (as opposed to grabbing and posting frames from canvas)

Right now the code uses websockets to establish a webrtc Peer Connection between the browser and server and write the webcam video stream to disk. Next steps are replacing saveToDisk with a gRPC endpoint exposing the stream for processing via Python/Tensorflow but haven't gotten to that yet

Since I've never written in Javascript or used Websockets or WebRTC before any and all feedback is appreciated! I tried to keep the Go and JS APIs as similar as possible for ease of use. I also didn't include the basic server in main.go or index.html since they just call the functions below (hence plug-and-play; simply serve webRTCHandle and load webRTCClient however you want to automatically get access to the webcam stream) but can if anyone wants to see

JS:

export function webRTCClient() { 
    var conn = new connHandler()

    conn.startUserMedia()
    conn.sendIceCandidates()
    conn.startListener()
}

// Default ICEServers - Google provides public testing server
const defaultIceServers = {
    iceServers: [
        {
            urls: 'stun:stun.l.google.com:19302'
        }
    ]
}

const userMediaSettings = { video: true }

class connHandler {
    constructor() {
        // Establish websocket signalling channel and new webRTCPeerConnection
        this.socket = new WebSocket("ws://" + window.location.hostname + ":8080/ws")
        this.peerConnection = new RTCPeerConnection(defaultIceServers)
        this.listener = null

        // Server (offer) and client (answer) can't switch once established
        // Client signals server when renegotiation is necessary
        this.peerConnection.onnegotiationneeded = e => {
            this.send("renegotiate")
        }
    }

    send(payload) {
        this.socket.send(JSON.stringify(payload))
    }

    startUserMedia() {
        // Acces getUserMedia API and stream video track
        // Add audio with "audio: true" and getAudioTracks()
        let conn = this
        const f = async function startStream() {
            let source = await navigator.mediaDevices.getUserMedia(userMediaSettings)
            for (const track of source.getVideoTracks()) {
                conn.peerConnection.addTrack(track, source)
            }
        }
        f()
    }

    sendIceCandidates() {
        // Whenever there is a new ICECandidate send to server
        this.peerConnection.onicecandidate = event => {
            if (event.candidate != null) {
             this.send(event.candidate)
            }
        }
    }

    startListener() {
        // Client receives ICECandidates or Descriptions
        this.socket.onmessage = event => {
            var message = JSON.parse(event.data)
            if (message.sdp) {
                // Once local and remote descriptions are synced connection will be formed
                this.peerConnection.setRemoteDescription(new RTCSessionDescription(message))
                this.peerConnection.createAnswer()
                    .then(answer => {
                        this.peerConnection.setLocalDescription(answer)
                        this.send(answer)
                    })
            } else if (message.candidate) {
                this.peerConnection.addIceCandidate(message)
            }
        }
    }
}

Go:

package main

import (
    "encoding/json"
    "fmt"
    "log"
    "net/http"

    "time"

    "github.com/gorilla/websocket"

    "github.com/pion/rtcp"
    "github.com/pion/webrtc"
    "github.com/pion/webrtc/pkg/media"
    "github.com/pion/webrtc/pkg/media/h264writer"
)

// Implements http.Handler interface so works with any server (httprouter, gorillca/mux, etc)
func webRTCHandle(w http.ResponseWriter, r *http.Request) {
    conn, err := newConnHandler(w, r)
    if err != nil {
        fmt.Fprint(w, "Unable to connect: ", err)
    }

    conn.startUserMedia()
    conn.sendIceCandidates()
    conn.sendOffer()
    conn.startListener()
}

var upgrader = websocket.Upgrader{
    ReadBufferSize:  512,
    WriteBufferSize: 512,
}

var defaultIceServers = webrtc.Configuration{
    ICEServers: []webrtc.ICEServer{
        {
            URLs: []string{"stun:stun.l.google.com:19302"},
        },
    },
}

type connHandler struct {
    socket *websocket.Conn
    peerConnection *webrtc.PeerConnection
}

func newConnHandler(w http.ResponseWriter, r *http.Request) (connHandler, error) {
    conn := connHandler{}

    // Upgrade http to websocket
    ws, err := upgrader.Upgrade(w, r, nil)
    if err != nil {
        return conn, err
    }

    // Create new webrtc peer connection obj
    // Note: connection to client won't be formed until local and remote descriptions are set
    pc, err := newPeerConnection()
    if err != nil {
        return conn, err
    }

    conn.socket = ws
    conn.peerConnection = pc

    return conn, nil
}

func newPeerConnection() (*webrtc.PeerConnection, error) {
    m := webrtc.MediaEngine{}
    m.RegisterCodec(webrtc.NewRTPH264Codec(webrtc.DefaultPayloadTypeH264, 90000))
    api := webrtc.NewAPI(webrtc.WithMediaEngine(m))

    peerConnection, err := api.NewPeerConnection(defaultIceServers)
    if err != nil {
        return nil, err
    }

    if _, err = peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo); err != nil {
        return nil, err
    }

    return peerConnection, nil
}

func (conn *connHandler) send(message interface{}) {
    payload, err := json.Marshal(message)
    if err != nil {
        log.Println(err)
    }

    err = conn.socket.WriteMessage(websocket.TextMessage, payload)
    if err != nil {
        log.Println("message not sent")
    }
}

func (conn *connHandler) startUserMedia() {
    h264File, err := h264writer.New("output.264")
    if err != nil {
        log.Println(err)
    }

    conn.peerConnection.OnTrack(func(track *webrtc.Track, receiver *webrtc.RTPReceiver) {
        // Send heartbeat - copied from Pion example code
        go func() {
            ticker := time.NewTicker(time.Second * 3)
            for range ticker.C {
                errSend := conn.peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: track.SSRC()}})
                if errSend != nil {
                    log.Println(errSend)
                }
            }
        }()

        codec := track.Codec()

        if codec.Name == webrtc.H264 {
            saveToDisk(h264File, track)
        }
    })
}

func (conn *connHandler) sendIceCandidates() {
    conn.peerConnection.OnICECandidate(func(c *webrtc.ICECandidate) {
        if c == nil {
            return
        }
        conn.send(c.ToJSON())
    })
}

func (conn *connHandler) sendOffer() {
    offer, err := conn.peerConnection.CreateOffer(nil)
    if err != nil {
        log.Println(err)
    }

    if err = conn.peerConnection.SetLocalDescription(offer); err != nil {
        log.Println(err)
    }

    conn.send(offer)
}

func (conn *connHandler) startListener() {
    var renegotiate string
    var candidate webrtc.ICECandidateInit
    var sessDesc webrtc.SessionDescription

    for {
        _, p, err := conn.socket.ReadMessage()
        if err != nil {
            conn.peerConnection.Close()
            return
        }

        if err = json.Unmarshal(p, &renegotiate); renegotiate == "renegotiate" {
            conn.sendOffer()
            renegotiate = ""

        } else if err = json.Unmarshal(p, &sessDesc); sessDesc.Type == webrtc.SDPTypeAnswer {
            conn.peerConnection.SetRemoteDescription(sessDesc)
            sessDesc = webrtc.SessionDescription{}

        } else if err = json.Unmarshal(p, &candidate); err != nil {
            conn.peerConnection.AddICECandidate(candidate)
            candidate = webrtc.ICECandidateInit{}
            
        }
    }
}

// Copied from Pion example code
// Replace with gRPC endpoint
func saveToDisk(i media.Writer, track *webrtc.Track) {
    defer func() {
        if err := i.Close(); err != nil {
            log.Println(err)
        }
    }()

    for {
        rtpPacket, err := track.ReadRTP()
        if err != nil {
            log.Println(err)
        }

        if err := i.WriteRTP(rtpPacket); err != nil {
            log.Println(err)
        }
    }
}
\$\endgroup\$
1
  • \$\begingroup\$ Your JavaScript WebRTC code looks great to me. I don't know golang but from inspection it looks great too. \$\endgroup\$ – SamFast Aug 26 '20 at 13:15

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.