Janus/WebRTC frontend (#943)

* Make `<remove-screen>` content injectable

* Add explanation how to use the slot

* Optionally provide WebRTC video stream

* Migrate Janus Gateway to 1.0.0

* Refactor, cleanups

* Read config from settings file

* Docs

* Clarify comments / log messages

Co-authored-by: Jan Heuermann <jan@jotaen.net>

app/static/js/webrtc-video.js

@@ -0,0 +1,131 @@
+/**
+ * The initialization of the WebRTC Video stream from the Janus plugin.
+ *
+ * Import this as `type="module"`, to avoid polluting the global namespace.
+ *
+ * This file implicitly depends on the following libraries to be loaded into the
+ * global namespace, in this order:
+ * - WebRTC Adapter 8.1.1
+ * - Janus Gateway 1.0.0
+ *
+ * See here for the Janus Gateway API reference:
+ * https://janus.conf.meetecho.com/docs/JS.html
+ */
+
+// Parameters for the setup.
+const config = {
+  // Set to `true` to turn on all internal Janus logging. Make sure to set the
+  // log level in your browser to debug/verbose.
+  isDebug: false,
+
+  // The hostname of the device. For development, you can replace this with the
+  // hostname of your testing device.
+  deviceHostname: location.host,
+
+  // Whether the connection should be established with SSL.
+  useSSL: location.protocol === "https:",
+};
+
+// Initialize library.
+Janus.init({
+  debug: config.isDebug ? "all" : false,
+});
+
+// Establish connection to the server.
+const janus = new Janus({
+  server: `${config.useSSL ? "wss" : "ws"}://${config.deviceHostname}/janus/ws`,
+  success: attachToJanusPlugin,
+  error: function (error) {
+    console.error("Failed to connect to Janus: " + error);
+  },
+});
+
+function attachToJanusPlugin() {
+  let janusPluginHandle = null;
+
+  janus.attach({
+    plugin: "janus.plugin.ustreamer",
+    opaqueId: "tinypilot-" + Janus.randomString(8),
+
+    /**
+     * This callback is triggered when the ICE state for the PeerConnection
+     * associated to the handle changes.
+     * ICE = Interactive Connectivity Establishment
+     * See https://developer.mozilla.org/en-US/docs/Glossary/ICE
+     * @param {string} state E.g., "connected" or "failed".
+     */
+    iceState: function (state) {
+      console.debug("ICE Connection State changed to: " + state);
+    },
+
+    /**
+     * The plugin handle was successfully created and is ready to be used.
+     * @param {object} pluginHandle The Janus plugin handle.
+     */
+    success: function (pluginHandle) {
+      janusPluginHandle = pluginHandle;
+      console.debug("Successfully created Janus plugin handle.");
+
+      // This makes the uStreamer plugin generate a webrtc offer that will be
+      // received in the onmessage handler.
+      janusPluginHandle.send({ message: { request: "watch" } });
+    },
+
+    /**
+     * The plugin handle was NOT successfully created
+     * @param error
+     */
+    error: function (error) {
+      console.error("Failed to create Janus plugin handle: " + error);
+    },
+
+    /**
+     * A message/event has been received from the plugin.
+     * @param {object} msg
+     * @param {object|null} jsep JSEP = JavaScript Session Establishment Protocol
+     */
+    onmessage: function (msg, jsep) {
+      if (!jsep) {
+        return;
+      }
+      janusPluginHandle.createAnswer({
+        jsep: jsep,
+        // Client only receives media and does not interact with data channels.
+        media: { audioSend: false, videoSend: false, data: false },
+        success: function (jsep) {
+          // Send back the generated webrtc response.
+          janusPluginHandle.send({
+            message: { request: "start" },
+            jsep: jsep,
+          });
+        },
+        error: function (error) {
+          console.error("failed to create answer SDP: " + error);
+        },
+      });
+    },
+
+    /**
+     * A remote media track is available and ready to be consumed.
+     * @param {MediaStreamTrack} track https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack
+     * @param {string} mid The Media-ID.
+     * @param {boolean} added Whether a track was added or removed.
+     */
+    onremotetrack: function (track, mid, added) {
+      console.debug(`Remote track changed. mid:"${mid}" added:"${added}`);
+      const videoElement = document.getElementById("webrtc-output");
+
+      if (!added) {
+        videoElement.srcObject = null;
+        return;
+      }
+
+      // According to the examples/tests in the Janus repository, the track
+      // object should be cloned.
+      // https://github.com/meetecho/janus-gateway/blob/4110eea4568926dc18642a544718c87118629253/html/streamingtest.js#L249-L250
+      const stream = new MediaStream();
+      stream.addTrack(track.clone());
+      videoElement.srcObject = stream;
+    },
+  });
+}

app/templates/index.html

@@ -58,7 +58,11 @@
           id="remote-screen"
           milliseconds-between-mouse-events="600"
         >
+          {% if use_webrtc_remote_screen %}
+          <video id="webrtc-output" autoplay playsinline muted></video>
+          {% else %}
           <img src="/stream?advance_headers=1" />
+          {% endif %}
         </remote-screen>
 
         <on-screen-keyboard id="on-screen-keyboard"></on-screen-keyboard>
@@ -69,6 +73,18 @@
     <script src="/js/paste.js"></script>
     <script type="module" src="/js/app.js"></script>
 
+    {% if use_webrtc_remote_screen %}
+    <script
+      type="text/javascript"
+      src="/third-party/webrtc-adapter/8.1.1/adapter.min.js"
+    ></script>
+    <script
+      type="text/javascript"
+      src="/third-party/janus-gateway/1.0.0/janus.js"
+    ></script>
+    <script type="module" src="/js/webrtc-video.js"></script>
+    {% endif %}
+
     <div style="opacity: 0;">
       <!-- Force browser to preload font variants in order to
            prevent potential display flickering -->

app/views.py

@@ -13,6 +13,8 @@ _DEFAULT_HOSTNAME = 'tinypilot'
 def index_get():
     return flask.render_template(
         'index.html',
+        use_webrtc_remote_screen=flask.current_app.config.get(
+            'USE_WEBRTC_REMOTE_SCREEN', False),
         page_title_prefix=_page_title_prefix(),
         custom_elements_files=find_files.custom_elements_files())
 

dev_app_settings.cfg

@@ -3,3 +3,5 @@
 
 KEYBOARD_PATH = '/dev/null'
 MOUSE_PATH = '/dev/null'
+
+USE_WEBRTC_REMOTE_SCREEN = True