webrtc calls in webview to typescript (#592)

* mobile: webrtc calls in webview (typescript WIP)

* typsecript compiles

* fix error messages

* TS works in chrome

* include ICE candidates into offer/answer, report connection state changes to host, end call on disconnection

* refactor, readme for .js file
This commit is contained in:
Evgeny Poberezkin
2022-05-02 15:48:39 +01:00
committed by GitHub
parent 18d1a0605e
commit f78ec3584f
12 changed files with 1107 additions and 276 deletions
@@ -0,0 +1 @@
Do NOT edit call.js here, it is compiled from call.ts in packages/simplex-chat-webrtc
@@ -4,8 +4,8 @@
<link href="./style.css" rel="stylesheet" />
</head>
<body>
<video id="incoming-video-stream" autoplay></video>
<video id="outgoing-video-stream" muted autoplay></video>
<video id="remote-video-stream" autoplay></video>
<video id="local-video-stream" muted autoplay></video>
</body>
<footer>
<script src="./call.js"></script>
+427 -271
View File
@@ -1,299 +1,455 @@
"use strict";
// Inspired by
// https://github.com/webrtc/samples/blob/gh-pages/src/content/insertable-streams/endtoend-encryption
let incomingVideo = document.getElementById("incoming-video-stream")
let outgoingVideo = document.getElementById("outgoing-video-stream")
incomingVideo.style.opacity = 0
outgoingVideo.style.opacity = 0
incomingVideo.onplaying = () => {
incomingVideo.style.opacity = 1
}
outgoingVideo.onplaying = () => {
outgoingVideo.style.opacity = 1
}
var CallMediaType;
(function (CallMediaType) {
CallMediaType["Audio"] = "audio";
CallMediaType["Video"] = "video";
})(CallMediaType || (CallMediaType = {}));
// STUN servers
const peerConnectionConfig = {
iceServers: [{urls: ["stun:stun.l.google.com:19302"]}],
iceCandidatePoolSize: 10,
encodedInsertableStreams: true,
}
let keyGenConfig = {
name: "AES-GCM",
length: 256,
tagLength: 128,
}
let keyUsages = ["encrypt", "decrypt"]
// Hardcode a key for development
let keyData = {alg: "A256GCM", ext: true, k: "JCMDWkhxLmPDhua0BUdhgv6Ac6hOtB9frSxJlnkTAK8", key_ops: keyUsages, kty: "oct"}
let pc
let key
let IV_LENGTH = 12
iceServers: [{ urls: ["stun:stun.l.google.com:19302"] }],
iceCandidatePoolSize: 10,
encodedInsertableStreams: true,
};
const keyAlgorithm = {
name: "AES-GCM",
length: 256
};
const keyUsages = ["encrypt", "decrypt"];
let pc;
const IV_LENGTH = 12;
const initialPlainTextRequired = {
key: 10,
delta: 3,
undefined: 1,
}
// let encryptKeyRepresentation
let candidates = []
run()
async function run() {
pc = new RTCPeerConnection(peerConnectionConfig)
pc.onicecandidate = (event) => {
// add candidate to maintained list to be sent all at once
if (event.candidate) {
candidates.push(event.candidate)
key: 10,
delta: 3,
undefined: 1,
};
const defaultCallConfig = {
iceCandidates: {
delay: 2000,
extrasInterval: 2000,
extrasTimeout: 8000
}
}
pc.onicegatheringstatechange = (_) => {
if (pc.iceGatheringState == "complete") {
// Give command for other caller to use
console.log(JSON.stringify({action: "processIceCandidates", content: candidates}))
};
async function initializeCall(config, mediaType, aesKey) {
const conn = new RTCPeerConnection(peerConnectionConfig);
const remoteStream = new MediaStream();
const localStream = await navigator.mediaDevices.getUserMedia(callMediaContraints(mediaType));
await setUpMediaStreams(conn, localStream, remoteStream, aesKey);
conn.addEventListener("connectionstatechange", connectionStateChange);
const iceCandidates = new Promise((resolve, _) => {
let candidates = [];
let resolved = false;
let extrasInterval;
let extrasTimeout;
const delay = setTimeout(() => {
if (!resolved) {
resolveIceCandidates();
extrasInterval = setInterval(() => {
sendIceCandidates();
}, config.iceCandidates.extrasInterval);
extrasTimeout = setTimeout(() => {
clearInterval(extrasInterval);
sendIceCandidates();
}, config.iceCandidates.extrasTimeout);
}
}, config.iceCandidates.delay);
conn.onicecandidate = ({ candidate: c }) => c && candidates.push(c);
conn.onicegatheringstatechange = () => {
if (conn.iceGatheringState == "complete") {
if (resolved) {
if (extrasInterval)
clearInterval(extrasInterval);
if (extrasTimeout)
clearTimeout(extrasTimeout);
sendIceCandidates();
}
else {
resolveIceCandidates();
}
}
};
function resolveIceCandidates() {
if (delay)
clearTimeout(delay);
resolved = true;
const iceCandidates = candidates.slice();
candidates = [];
resolve(iceCandidates);
}
function sendIceCandidates() {
if (candidates.length === 0)
return;
const iceCandidates = candidates.slice();
candidates = [];
sendMessageToNative({ type: "ice", iceCandidates });
}
});
return { connection: conn, iceCandidates };
function connectionStateChange() {
sendMessageToNative({
type: "connection",
state: {
connectionState: conn.connectionState,
iceConnectionState: conn.iceConnectionState,
iceGatheringState: conn.iceGatheringState,
signalingState: conn.signalingState,
}
});
if (conn.connectionState == "disconnected" || conn.connectionState == "failed") {
conn.removeEventListener("connectionstatechange", connectionStateChange);
sendMessageToNative({ type: "ended" });
conn.close();
pc = undefined;
resetVideoElements();
}
}
}
let remoteStream = new MediaStream()
key = await crypto.subtle.importKey("jwk", keyData, keyGenConfig, true, keyUsages)
let localStream = await getLocalVideoStream()
setUpVideos(pc, localStream, remoteStream)
}
async function processCommand(data) {
switch (data.action) {
case "initiateCall":
console.log("initiating call")
let result = await makeOffer(pc)
// Give command for callee to use
console.log(
JSON.stringify({
action: "processAndAnswerOffer",
content: result,
})
)
return result
case "processAndAnswerOffer":
await processOffer(data.content)
let answer = await answerOffer(pc)
// Give command for callee to use
console.log(
JSON.stringify({
action: "processOffer",
content: answer,
})
)
return answer
case "processOffer":
await processOffer(data.content)
break
case "processIceCandidates":
processIceCandidates(data.content)
break
default:
console.log("JS: Unknown Command")
}
// TODO remove WCallCommand from parameter type
function sendMessageToNative(msg) {
console.log(JSON.stringify(msg));
}
async function makeOffer(pc) {
// For initiating a call. Send offer to callee
let offerDescription = await pc.createOffer()
await pc.setLocalDescription(offerDescription)
let offer = {
sdp: offerDescription.sdp,
type: offerDescription.type,
}
return offer
// TODO remove WCallCommand from result type
async function processCommand(command) {
let resp;
switch (command.type) {
case "capabilities":
const encryption = supportsInsertableStreams();
resp = { type: "capabilities", capabilities: { encryption } };
break;
case "start":
console.log("starting call");
if (pc) {
resp = { type: "error", message: "start: call already started" };
}
else if (!supportsInsertableStreams() && command.aesKey) {
resp = { type: "error", message: "start: encryption is not supported" };
}
else {
try {
const { media, aesKey } = command;
const call = await initializeCall(defaultCallConfig, media, aesKey);
const { connection, iceCandidates } = call;
pc = connection;
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
// for debugging, returning the command for callee to use
resp = { type: "accept", offer, iceCandidates: await iceCandidates, media, aesKey };
// resp = {type: "offer", offer, iceCandidates: await iceCandidates}
}
catch (e) {
resp = { type: "error", message: e.message };
}
}
break;
case "accept":
if (pc) {
resp = { type: "error", message: "accept: call already started" };
}
else if (!supportsInsertableStreams() && command.aesKey) {
resp = { type: "error", message: "accept: encryption is not supported" };
}
else {
try {
const call = await initializeCall(defaultCallConfig, command.media, command.aesKey);
const { connection, iceCandidates } = call;
pc = connection;
await pc.setRemoteDescription(new RTCSessionDescription(command.offer));
const answer = await pc.createAnswer();
await pc.setLocalDescription(answer);
addIceCandidates(pc, command.iceCandidates);
// same as command for caller to use
resp = { type: "answer", answer, iceCandidates: await iceCandidates };
}
catch (e) {
resp = { type: "error", message: e.message };
}
}
break;
case "answer":
if (!pc) {
resp = { type: "error", message: "answer: call not started" };
}
else if (!pc.localDescription) {
resp = { type: "error", message: "answer: local description is not set" };
}
else if (pc.currentRemoteDescription) {
resp = { type: "error", message: "answer: remote description already set" };
}
else {
await pc.setRemoteDescription(new RTCSessionDescription(command.answer));
addIceCandidates(pc, command.iceCandidates);
resp = { type: "ok" };
}
break;
case "ice":
if (pc) {
addIceCandidates(pc, command.iceCandidates);
resp = { type: "ok" };
}
else {
resp = { type: "error", message: "ice: call not started" };
}
break;
case "end":
if (pc) {
pc.close();
pc = undefined;
resetVideoElements();
resp = { type: "ok" };
}
else {
resp = { type: "error", message: "end: call not started" };
}
break;
default:
resp = { type: "error", message: "unknown command" };
break;
}
sendMessageToNative(resp);
return resp;
}
async function answerOffer(pc) {
let answerDescription = await pc.createAnswer()
await pc.setLocalDescription(answerDescription)
let answer = {
sdp: answerDescription.sdp,
type: answerDescription.type,
}
return answer
function addIceCandidates(conn, iceCandidates) {
for (const c of iceCandidates) {
conn.addIceCandidate(new RTCIceCandidate(c));
}
}
function processIceCandidates(iceCandidates) {
iceCandidates.forEach((candidate) => processIceCandidate(candidate))
async function setUpMediaStreams(pc, localStream, remoteStream, aesKey) {
var _a;
const videos = getVideoElements();
if (!videos)
throw Error("no video elements");
let key;
if (aesKey) {
const keyData = decodeBase64(encodeAscii(aesKey));
if (keyData)
key = await crypto.subtle.importKey("raw", keyData, keyAlgorithm, false, keyUsages);
}
for (const track of localStream.getTracks()) {
pc.addTrack(track, localStream);
}
if (key) {
console.log("set up encryption for sending");
for (const sender of pc.getSenders()) {
setupPeerTransform(sender, encodeFunction(key));
}
}
// Pull tracks from remote stream as they arrive add them to remoteStream video
pc.ontrack = (event) => {
if (key) {
console.log("set up decryption for receiving");
setupPeerTransform(event.receiver, decodeFunction(key));
}
for (const track of event.streams[0].getTracks()) {
remoteStream.addTrack(track);
}
};
// We assume VP8 encoding in the decode/encode stages to get the initial
// bytes to pass as plaintext so we enforce that here.
// VP8 is supported by all supports of webrtc.
// Use of VP8 by default may also reduce depacketisation issues.
// We do not encrypt the first couple of bytes of the payload so that the
// video elements can work by determining video keyframes and the opus mode
// being used. This appears to be necessary for any video feed at all.
// For VP8 this is the content described in
// https://tools.ietf.org/html/rfc6386#section-9.1
// which is 10 bytes for key frames and 3 bytes for delta frames.
// For opus (where encodedFrame.type is not set) this is the TOC byte from
// https://tools.ietf.org/html/rfc6716#section-3.1
const capabilities = RTCRtpSender.getCapabilities("video");
if (capabilities) {
const { codecs } = capabilities;
const selectedCodecIndex = codecs.findIndex((c) => c.mimeType === "video/VP8");
const selectedCodec = codecs[selectedCodecIndex];
codecs.splice(selectedCodecIndex, 1);
codecs.unshift(selectedCodec);
for (const t of pc.getTransceivers()) {
if (((_a = t.sender.track) === null || _a === void 0 ? void 0 : _a.kind) === "video") {
t.setCodecPreferences(codecs);
}
}
}
// setupVideoElement(videos.local)
// setupVideoElement(videos.remote)
videos.local.srcObject = localStream;
videos.remote.srcObject = remoteStream;
}
function processIceCandidate(iceCandidate) {
let candidate = new RTCIceCandidate(iceCandidate)
pc.addIceCandidate(candidate)
function callMediaContraints(mediaType) {
switch (mediaType) {
case CallMediaType.Audio:
return { audio: true, video: false };
case CallMediaType.Video:
return {
audio: true,
video: {
frameRate: 24,
width: {
min: 480,
ideal: 720,
max: 1280,
},
aspectRatio: 1.33,
},
};
}
}
async function processOffer(offer) {
// Negotiating initial connection
if (!pc.currentRemoteDescription) {
let remoteSessionDescription = new RTCSessionDescription(offer)
await pc.setRemoteDescription(remoteSessionDescription)
}
function supportsInsertableStreams() {
return ("createEncodedStreams" in RTCRtpSender.prototype)
&& ("createEncodedStreams" in RTCRtpReceiver.prototype);
}
function setUpVideos(pc, localStream, remoteStream) {
localStream.getTracks().forEach((track) => {
pc.addTrack(track, localStream)
})
pc.getSenders().forEach(setupSenderTransform)
// Pull tracks from remote stream as they arrive add them to remoteStream video
pc.ontrack = (event) => {
setupReceiverTransform(event.receiver)
event.streams[0].getTracks().forEach((track) => {
remoteStream.addTrack(track)
})
}
// We assume VP8 encoding in the decode/encode stages to get the initial
// bytes to pass as plaintext so we enforce that here.
// VP8 is supported by all supports of webrtc.
// Use of VP8 by default may also reduce depacketisation issues.
// We do not encrypt the first couple of bytes of the payload so that the
// video elements can work by determining video keyframes and the opus mode
// being used. This appears to be necessary for any video feed at all.
// For VP8 this is the content described in
// https://tools.ietf.org/html/rfc6386#section-9.1
// which is 10 bytes for key frames and 3 bytes for delta frames.
// For opus (where encodedFrame.type is not set) this is the TOC byte from
// https://tools.ietf.org/html/rfc6716#section-3.1
const {codecs} = RTCRtpSender.getCapabilities("video")
const selectedCodecIndex = codecs.findIndex((c) => c.mimeType === "video/VP8")
const selectedCodec = codecs[selectedCodecIndex]
codecs.splice(selectedCodecIndex, 1)
codecs.unshift(selectedCodec)
const transceiver = pc.getTransceivers().find((t) => t.sender && t.sender.track.kind === "video")
transceiver.setCodecPreferences(codecs)
outgoingVideo.srcObject = localStream
incomingVideo.srcObject = remoteStream
function resetVideoElements() {
const videos = getVideoElements();
if (!videos)
return;
videos.local.srcObject = null;
videos.remote.srcObject = null;
}
async function getLocalVideoStream() {
return await navigator.mediaDevices.getUserMedia({
audio: true,
video: {
frameRate: 24,
width: {
min: 480,
ideal: 720,
max: 1280,
},
aspectRatio: 1.33,
},
})
function getVideoElements() {
const local = document.getElementById("local-video-stream");
const remote = document.getElementById("remote-video-stream");
if (!(local && remote && local instanceof HTMLMediaElement && remote instanceof HTMLMediaElement))
return;
return { local, remote };
}
function endCall() {
pc.close()
}
function toggleVideo(b) {
if (b == "true") {
localStream.getVideoTracks()[0].enabled = true
} else {
localStream.getVideoTracks()[0].enabled = false
}
}
// function setupVideoElement(video: HTMLElement) {
// // TODO use display: none
// video.style.opacity = "0"
// video.onplaying = () => {
// video.style.opacity = "1"
// }
// }
// what does it do?
// function toggleVideo(b) {
// if (b == "true") {
// localStream.getVideoTracks()[0].enabled = true
// } else {
// localStream.getVideoTracks()[0].enabled = false
// }
// }
function f() {
console.log("Debug Function")
return "Debugging Return"
console.log("Debug Function");
return "Debugging Return";
}
/* Stream Transforms */
function setupSenderTransform(sender) {
const senderStreams = sender.createEncodedStreams()
const transformStream = new TransformStream({
transform: encodeFunction,
})
senderStreams.readable.pipeThrough(transformStream).pipeTo(senderStreams.writable)
function setupPeerTransform(peer, transform) {
const streams = peer.createEncodedStreams();
streams.readable.pipeThrough(new TransformStream({ transform })).pipeTo(streams.writable);
}
function setupReceiverTransform(receiver) {
const receiverStreams = receiver.createEncodedStreams()
const transformStream = new TransformStream({
transform: decodeFunction,
})
receiverStreams.readable.pipeThrough(transformStream).pipeTo(receiverStreams.writable)
}
/* Cryptography */
function encodeFunction(frame, controller) {
// frame is an RTCEncodedAudioFrame
// frame.data is ArrayBuffer
let data = new Uint8Array(frame.data)
let n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0
let iv = randomIV()
let initial = data.subarray(0, n)
let plaintext = data.subarray(n, data.byteLength)
crypto.subtle
.encrypt({name: "AES-GCM", iv: iv.buffer}, key, plaintext)
.then((c) => {
frame.data = concatN(initial, new Uint8Array(c), iv).buffer
controller.enqueue(frame)
})
.catch((e) => {
console.log("encrypt error")
endCall()
throw e
})
function encodeFunction(key) {
return async (frame, controller) => {
const data = new Uint8Array(frame.data);
const n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0;
const iv = randomIV();
const initial = data.subarray(0, n);
const plaintext = data.subarray(n, data.byteLength);
try {
const ciphertext = await crypto.subtle.encrypt({ name: "AES-GCM", iv: iv.buffer }, key, plaintext);
frame.data = concatN(initial, new Uint8Array(ciphertext), iv).buffer;
controller.enqueue(frame);
}
catch (e) {
console.log(`encryption error ${e}`);
throw e;
}
};
}
function decodeFunction(frame, controller) {
let data = new Uint8Array(frame.data)
let n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0
let initial = data.subarray(0, n)
let ciphertext = data.subarray(n, data.byteLength - IV_LENGTH)
let iv = data.subarray(data.byteLength - IV_LENGTH, data.byteLength)
crypto.subtle
.decrypt({name: "AES-GCM", iv: iv}, key, ciphertext)
.then((p) => {
frame.data = concatN(initial, new Uint8Array(p)).buffer
controller.enqueue(frame)
})
.catch((e) => {
console.log("decrypt error")
endCall()
throw e
})
function decodeFunction(key) {
return async (frame, controller) => {
const data = new Uint8Array(frame.data);
const n = frame instanceof RTCEncodedVideoFrame ? initialPlainTextRequired[frame.type] : 0;
const initial = data.subarray(0, n);
const ciphertext = data.subarray(n, data.byteLength - IV_LENGTH);
const iv = data.subarray(data.byteLength - IV_LENGTH, data.byteLength);
try {
const plaintext = await crypto.subtle.decrypt({ name: "AES-GCM", iv }, key, ciphertext);
frame.data = concatN(initial, new Uint8Array(plaintext)).buffer;
controller.enqueue(frame);
}
catch (e) {
console.log(`decryption error ${e}`);
throw e;
}
};
}
class RTCEncodedVideoFrame {
constructor(type, data) {
this.type = type;
this.data = data;
}
}
function randomIV() {
return crypto.getRandomValues(new Uint8Array(IV_LENGTH))
return crypto.getRandomValues(new Uint8Array(IV_LENGTH));
}
async function loadKey(keyData) {
key = await crypto.subtle.importKey("jwk", keyData, keyGenConfig, false, keyUsages)
}
const char_equal = "=".charCodeAt(0);
function concatN(...bs) {
const a = new Uint8Array(bs.reduce((size, b) => size + b.byteLength, 0))
bs.reduce((offset, b) => {
a.set(b, offset)
return offset + b.byteLength
}, 0)
return a
const a = new Uint8Array(bs.reduce((size, b) => size + b.byteLength, 0));
bs.reduce((offset, b) => {
a.set(b, offset);
return offset + b.byteLength;
}, 0);
return a;
}
async function generateKey() {
crypto.subtle
.generateKey(keyGenConfig, true, keyUsages)
.then((k) => {
encryptKey = k
return crypto.subtle.exportKey("jwk", encryptKey)
})
.then((r) => {
encryptKeyRepresentation = r
console.log(
JSON.stringify({
action: "processDecryptionKey",
content: {
key: encryptKeyRepresentation,
iv: encryptIv,
},
})
)
})
function encodeAscii(s) {
const a = new Uint8Array(s.length);
let i = s.length;
while (i--)
a[i] = s.charCodeAt(i);
return a;
}
function decodeAscii(a) {
let s = "";
for (let i = 0; i < a.length; i++)
s += String.fromCharCode(a[i]);
return s;
}
const base64chars = new Uint8Array("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".split("").map((c) => c.charCodeAt(0)));
const base64lookup = new Array(256);
base64chars.forEach((c, i) => (base64lookup[c] = i));
function encodeBase64(a) {
const len = a.length;
const b64len = Math.ceil(len / 3) * 4;
const b64 = new Uint8Array(b64len);
let j = 0;
for (let i = 0; i < len; i += 3) {
b64[j++] = base64chars[a[i] >> 2];
b64[j++] = base64chars[((a[i] & 3) << 4) | (a[i + 1] >> 4)];
b64[j++] = base64chars[((a[i + 1] & 15) << 2) | (a[i + 2] >> 6)];
b64[j++] = base64chars[a[i + 2] & 63];
}
if (len % 3)
b64[b64len - 1] = char_equal;
if (len % 3 === 1)
b64[b64len - 2] = char_equal;
return b64;
}
function decodeBase64(b64) {
let len = b64.length;
if (len % 4)
return;
let bLen = (len * 3) / 4;
if (b64[len - 1] === char_equal) {
len--;
bLen--;
if (b64[len - 1] === char_equal) {
len--;
bLen--;
}
}
const bytes = new Uint8Array(bLen);
let i = 0;
let pos = 0;
while (i < len) {
const enc1 = base64lookup[b64[i++]];
const enc2 = i < len ? base64lookup[b64[i++]] : 0;
const enc3 = i < len ? base64lookup[b64[i++]] : 0;
const enc4 = i < len ? base64lookup[b64[i++]] : 0;
if (enc1 === undefined || enc2 === undefined || enc3 === undefined || enc4 === undefined)
return;
bytes[pos++] = (enc1 << 2) | (enc2 >> 4);
bytes[pos++] = ((enc2 & 15) << 4) | (enc3 >> 2);
bytes[pos++] = ((enc3 & 3) << 6) | (enc4 & 63);
}
return bytes;
}
//# sourceMappingURL=call.js.map
@@ -5,14 +5,14 @@ html, body {
padding: 0;
margin: 0;
}
#incoming-video-stream {
#remote-video-stream {
position: absolute;
width: 100%;
height: 100%;
object-fit: cover;
}
#outgoing-video-stream {
#local-video-stream {
position: absolute;
width: 30%;
max-width: 30%;
@@ -58,7 +58,7 @@ fun VideoCallView(close: () -> Unit) {
}
val localContext = LocalContext.current
val iceCandidateCommand = remember { mutableStateOf("") }
val commandToShow = remember { mutableStateOf("processCommand({action: \"initiateCall\"})") }
val commandToShow = remember { mutableStateOf("processCommand({type: 'start', media: 'video', aesKey: 'FwW+t6UbnwHoapYOfN4mUBUuqR7UtvYWxW16iBqM29U='})") }
val assetLoader = WebViewAssetLoader.Builder()
.addPathHandler("/assets/www/", WebViewAssetLoader.AssetsPathHandler(localContext))
.build()