Story

Background

I created a exploration vehicle (https://goo.gl/T6YwAz), that is in its first throw sending camera data packed into events of the IoT Hub. It's a very expensive solution and i need a different. WebSockets seem to be the way to go. I can host a web application inside Azure and keep it that way perfectly scalable! After all - to keep the IoT spirit alive - we assume someday everyone in the world to use our device.

Remote control application with camera display

How it works

Device

The Device connects to a WebSocket enabled server with its DeviceId and sends captured data as a binary message.

Retrospective

I decided to capture single frames in first version and see what I can learn out of it. Looking back, it seems to be the best choice. Next time I would capture a stream and convert them into single images to hopefully get a better frame rate. Yet, the biggest bottleneck is still the upstream connection to server. Getting something like 20FPS seems right now rather unrealistic.

Code

WebSocketCamera.csC#

Device class responsible for capturing camera data and sending it to webserver.

usingSystem;usingSystem.Collections.Generic;usingSystem.Net.Http;usingSystem.Runtime.InteropServices.WindowsRuntime;usingSystem.Threading;usingSystem.Threading.Tasks;usingWindows.Media.Capture;usingWindows.Media.MediaProperties;usingWindows.Networking.Sockets;usingWindows.Storage.Streams;namespaceSimpleController.Domain{publicclassWebSocketCamera{publicclassPutRequest{publicstringImage{get;set;}}privateCancellationTokenSourcemCancelaationToken;privateLowLagPhotoCapturemLowLagCapture;privateMediaCapturemMediaCapture;privateDateTimemStartTime;privateTimeSpanmAutoStopAfter=newTimeSpan(0,5,0);privateStreamWebSocketmStreamWebSocket;privateasyncTaskinit(){try{mMediaCapture=newMediaCapture();awaitmMediaCapture.InitializeAsync();mLowLagCapture=awaitmMediaCapture.PrepareLowLagPhotoCaptureAsync(ImageEncodingProperties.CreateJpeg());mStreamWebSocket=newStreamWebSocket();mStreamWebSocket.Closed+=MStreamWebSocket_Closed;}catch(Exceptionex){AppInsights.Client.TrackException(ex);}}privatevoidMStreamWebSocket_Closed(IWebSocketsender,WebSocketClosedEventArgsargs){closeSocket(sender);}publicasyncTaskStartCapture(){mCancelaationToken?.Cancel();if(mStreamWebSocket!=null){closeSocket(mStreamWebSocket);}awaitinit();mCancelaationToken=newCancellationTokenSource();mStartTime=DateTime.UtcNow;try{awaitmStreamWebSocket.ConnectAsync(newUri($"{Globals.WEBSOCKET_ENDPOINT}?device={MainPage.GetUniqueDeviceId()}"));vartask=Task.Run(async()=>{varsocket=mStreamWebSocket;while(!mCancelaationToken.IsCancellationRequested){try{varcapturedPhoto=awaitmLowLagCapture.CaptureAsync();using(varrac=capturedPhoto.Frame.CloneStream()){vardr=newDataReader(rac.GetInputStreamAt(0));varbytes=newbyte[rac.Size];awaitdr.LoadAsync((uint)rac.Size);dr.ReadBytes(bytes);awaitsocket.OutputStream.WriteAsync(bytes.AsBuffer());}}catch(Exceptionex){AppInsights.Client.TrackException(ex);}if((DateTime.UtcNow-mStartTime)>mAutoStopAfter){AppInsights.Client.TrackEvent("CameraAutoTurnOff");mCancelaationToken.Cancel();}}},mCancelaationToken.Token);}catch(Exceptionex){mStreamWebSocket.Dispose();mStreamWebSocket=null;AppInsights.Client.TrackException(ex);}}privatevoidcloseSocket(IWebSocketwebSocket){try{webSocket.Close(1000,"Closed due to user request.");}catch(Exceptionex){AppInsights.Client.TrackException(ex);}}privatestaticasyncTasksendData(byte[]bytes){try{HttpClienthttp=newHttpClient();varresponse=awaithttp.PutAsJsonAsync<PutRequest>(newUri("http://sgnexus.azurewebsites.net/api/imagedata/"+"f56bccc1-4075-d40f-7d0d-34c16a1411e0"),newPutRequest{Image=Convert.ToBase64String(bytes)});varcode=response.StatusCode;AppInsights.Client.TrackEvent("CameraDataSent",newDictionary<string,string>{{"responsecode",code.ToString()}});}catch(Exceptionex){AppInsights.Client.TrackException(ex);}}publicasyncTaskStopCapture(){mCancelaationToken.Cancel();mCancelaationToken=null;mStreamWebSocket=null;}}}

ImageSenderWebSocketMiddleware.csC#

Server-side class for receiving data from sender (device) and delegating it further to receiver (PC).

usingMicrosoft.AspNetCore.Http;usingSystem;usingSystem.Collections.Generic;usingSystem.Linq;usingSystem.Net.WebSockets;usingSystem.Threading;usingSystem.Threading.Tasks;namespaceSGNexus{publicclassImageSenderWebSocketMiddleware{readonlyRequestDelegatemNext;publicImageSenderWebSocketMiddleware(RequestDelegatenext){mNext=next;}publicasyncTaskInvoke(HttpContexthttp){if(http.WebSockets.IsWebSocketRequest&&http.Request.Query.ContainsKey("device")){vardeviceid=http.Request.Query["device"].ToString();varwebSocket=awaithttp.WebSockets.AcceptWebSocketAsync();if(webSocket.State==WebSocketState.Open){while(webSocket.State==WebSocketState.Open){varbuffer=newArraySegment<Byte>(newByte[4096]);varreceived=awaitwebSocket.ReceiveAsync(buffer,CancellationToken.None);switch(received.MessageType){caseWebSocketMessageType.Close:awaitwebSocket.CloseAsync(WebSocketCloseStatus.NormalClosure,"Closed in server by the client",CancellationToken.None);continue;caseWebSocketMessageType.Binary:List<byte>data=newList<byte>(buffer.Take(received.Count));while(received.EndOfMessage==false){received=awaitwebSocket.ReceiveAsync(buffer,CancellationToken.None);data.AddRange(buffer.Take(received.Count));}varsocketconnectionList=ImageReceiverWebSocketMiddleware.Connections.Where(x=>x.DeviceId.Equals(deviceid,StringComparison.Ordinal)).ToArray();foreach(varsocketconnectioninsocketconnectionList){vardestsocket=socketconnection.SocketConnection;if(destsocket.State==System.Net.WebSockets.WebSocketState.Open){vartype=WebSocketMessageType.Binary;try{awaitdestsocket.SendAsync(newArraySegment<byte>(data.ToArray()),type,true,CancellationToken.None);}catch(Exceptionex){AppInsights.Client.TrackException(ex);}}else{AppInsights.Client.TrackTrace("Removing closed connection");ImageReceiverWebSocketMiddleware.Connections.Remove(socketconnection);}}break;}}}}else{awaitmNext.Invoke(http);}}publicclassSocketConnections{publicstringDeviceId{get;set;}publicWebSocketSocketConnection{get;set;}}}}

ImageReceiverWebSocketMiddleware.csC#

Server-side class responsible for accepting PC connections. Stores them in a public list.

usingMicrosoft.AspNetCore.Http;usingSystem;usingSystem.Collections.Generic;usingSystem.Linq;usingSystem.Net.WebSockets;usingSystem.Threading;usingSystem.Threading.Tasks;namespaceSGNexus{publicclassImageReceiverWebSocketMiddleware{publicstaticList<SocketConnections>Connections{get;set;}readonlyRequestDelegatemNext;staticImageReceiverWebSocketMiddleware(){Connections=newList<SocketConnections>();}publicImageReceiverWebSocketMiddleware(RequestDelegatenext){mNext=next;}publicasyncTaskInvoke(HttpContexthttp){if(http.WebSockets.IsWebSocketRequest&&http.Request.Query.ContainsKey("device")){vardeviceid=http.Request.Query["device"].ToString();varwebSocket=awaithttp.WebSockets.AcceptWebSocketAsync();if(webSocket.State==WebSocketState.Open){varexistigsocketconnection=ImageReceiverWebSocketMiddleware.Connections.Where(x=>x.DeviceId.Equals(deviceid)).FirstOrDefault();if(existigsocketconnection!=null){ImageReceiverWebSocketMiddleware.Connections.Remove(existigsocketconnection);}Connections.Add(newSocketConnections{DeviceId=deviceid,SocketConnection=webSocket});while(webSocket.State==WebSocketState.Open){varbuffer=newArraySegment<Byte>(newByte[4096]);varreceived=awaitwebSocket.ReceiveAsync(buffer,CancellationToken.None);switch(received.MessageType){caseWebSocketMessageType.Close:varsocket=Connections.Where(x=>x.SocketConnection==webSocket).First();Connections.Remove(socket);awaitwebSocket.CloseAsync(WebSocketCloseStatus.NormalClosure,"Closed in server by the client",CancellationToken.None);continue;}}}}else{awaitmNext.Invoke(http);}}publicclassSocketConnections{publicstringDeviceId{get;set;}publicWebSocketSocketConnection{get;set;}}}}

EventsReaderViewModel.csC#

PC-Application class that connects to and receives data from webserver.