From 85bd49f5b87957644264a393515baa4152fd7561 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Sun, 16 Jan 2022 02:37:38 +1100 Subject: [PATCH 01/10] boilerplate stuff --- bundle/src/Server.ts | 28 ++++++++++----------- util/src/util/Constants.ts | 3 +++ webrtc/.vscode/launch.json | 23 +++++++++++++++++ webrtc/package-lock.json | Bin 25101 -> 31537 bytes webrtc/package.json | 4 ++- webrtc/src/Server.ts | 36 +++++++++++---------------- webrtc/src/opcodes/Connect.ts | 5 ++++ webrtc/src/opcodes/Heartbeat.ts | 7 ++++++ webrtc/src/opcodes/Identify.ts | 20 +++++++++++++++ webrtc/src/opcodes/Resume.ts | 5 ++++ webrtc/src/opcodes/SelectProtocol.ts | 16 ++++++++++++ webrtc/src/opcodes/Speaking.ts | 6 +++++ webrtc/src/opcodes/index.ts | 35 ++++++++++++++++++++++++++ webrtc/src/start.ts | 1 + webrtc/src/util/Heartbeat.ts | 18 ++++++++++++++ webrtc/src/util/index.ts | 1 + webrtc/tsconfig.json | 19 +++++++++++--- 17 files changed, 187 insertions(+), 40 deletions(-) create mode 100644 webrtc/.vscode/launch.json create mode 100644 webrtc/src/opcodes/Connect.ts create mode 100644 webrtc/src/opcodes/Heartbeat.ts create mode 100644 webrtc/src/opcodes/Identify.ts create mode 100644 webrtc/src/opcodes/Resume.ts create mode 100644 webrtc/src/opcodes/SelectProtocol.ts create mode 100644 webrtc/src/opcodes/Speaking.ts create mode 100644 webrtc/src/opcodes/index.ts create mode 100644 webrtc/src/util/Heartbeat.ts create mode 100644 webrtc/src/util/index.ts diff --git a/bundle/src/Server.ts b/bundle/src/Server.ts index 71a60d499..bc1d7cbce 100644 --- a/bundle/src/Server.ts +++ b/bundle/src/Server.ts @@ -50,20 +50,20 @@ async function main() { endpointPublic: `ws://localhost:${port}`, }), }, - // regions: { - // default: "fosscord", - // useDefaultAsOptimal: true, - // available: [ - // { - // id: "fosscord", - // name: "Fosscord", - // endpoint: "127.0.0.1:3001", - // vip: false, - // custom: false, - // deprecated: false, - // }, - // ], - // }, + regions: { + default: "fosscord", + useDefaultAsOptimal: true, + available: [ + { + id: "fosscord", + name: "Fosscord", + endpoint: "127.0.0.1:3004", + vip: false, + custom: false, + deprecated: false, + }, + ], + }, } as any); //Sentry diff --git a/util/src/util/Constants.ts b/util/src/util/Constants.ts index 5fdf5bc09..a1892105d 100644 --- a/util/src/util/Constants.ts +++ b/util/src/util/Constants.ts @@ -73,7 +73,10 @@ export const VoiceOPCodes = { HEARTBEAT: 3, SESSION_DESCRIPTION: 4, SPEAKING: 5, + HEARTBEAT_ACK: 6, + RESUME: 7, HELLO: 8, + RESUMED: 9, CLIENT_CONNECT: 12, CLIENT_DISCONNECT: 13, }; diff --git a/webrtc/.vscode/launch.json b/webrtc/.vscode/launch.json new file mode 100644 index 000000000..92403164c --- /dev/null +++ b/webrtc/.vscode/launch.json @@ -0,0 +1,23 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "ts-node", + "type": "node", + "request": "launch", + "args": [ + "src/start.ts" + ], + "runtimeArgs": [ + "-r", + "ts-node/register" + ], + "cwd": "${workspaceRoot}", + "protocol": "inspector", + "internalConsoleOptions": "openOnSessionStart" + } + ] +} \ No newline at end of file diff --git a/webrtc/package-lock.json b/webrtc/package-lock.json index a5db2de13632936e54cc30ed1e8df396443d47a0..6c3726dc425b0a2cc59c41eb82b859538beb60c6 100644 GIT binary patch literal 31537 zcmeHQ*OKa5dcNN0DNgm}U>YQpaQTdfBt#G(M0R<*6hT4>CBk^-*}awmh(ulT*U()_T4~%ryN1r3hU_9pMUxG?G3yI|K3~&oXNWqIBjT>VtLU$b@&Fn zJUA&Uj$+A*C242U;H;_0n&8=i1Ml{G<-Pij$Daq6vt^~|2X3^G?_eG8MEaKRNw%eG z>!Kt08+h;Vs|Xq|6<%x~pCsQg@FjTpzoa)2HgoUePyW$Syb`=-L=M57^V<#F5N1|_^rtA|2gZ17&ganSY7-RYCry3`|v)KRi$6};Qo*G{$d zZ@(p%l$8)%EMZXqu41h%7=4^>lsr6W)pW%0>g=X=7vp@1eafb9FL>j}6qLPzB) zIY8$!;_-{Je?S7vHdusE`-6?oteo*ikyl-3GwANjRUW6`Q4`lUkKV(i`Zr;MrkJ1+ zCe)Qp|!Xh8up|VhSFDExQ^*c@EjF>Q1m(){nL31a*&&D(LKLHNfT^ajLi1 z^`3)Q20e4vsSta5*%T{BfgJ^IXR^=X%uiv0-NPhN=$}@W*HB|ACRl_C@&psSTy@dH zu{NevhG~%#ViW|Zu5az5r22NaS&Vn{3QCQdi^;6r(>8{PjHp%BWLHcXQ^cd7tK%gp zE3(=?T82wA-6_}j-F!ma!vy`C_=HF?AtFrBCzvR!gCTaP_R5olHcxK1QptCj$pk;F zU7e|v5DV=#_kq`_YQkpJo3rC^tatg^05=_pmN`EKIEiMOaipQn25k zn~>)rTsbK>hAQi`e4fS6eowNK#hA>4dxz$pY;|0r-a3w%mO?g|0&OCTaJ`!n!G1NM zI8t|wS%t&J><7QELV4qo2K$RWGI z0eXFjfA%5_dk=uH_lgmPCkV(Ta*kHIwZ+Jmxq&!b4Hlt+_6p*z#<>l4Z){3dzzr)_(T+r}=u50|LHupwINx3_fV zZ{Tcv%h{OVfp-%7^FSEC{Uu)_#vJ)K;pVRM0K?vq_sWZ$2!a~Ghwi!coj^IX-@r@Kh*AsZSF}9y{WWll{dNBgg%h)w}#ws>GwvU8A zXk&c8L=U%(+JG3JbWPZmUBfdLp*_j^CV9%rb6gNPszp5t&i@J*{4i1z1y3Qc_~SWn;c0ySS=l<%18|*R z1H1sCCu~#eIl>b79h8spGjYf?fFi;Rz1}(;W1;IcsZb1EQRBLfN27 zvODeSjq>&+4wO-Ue>&_a?daGbSXSm2wowC1)+{Iy{{eN)w^(h(83b_hZLRs6K#W-f z#J+)Rz_V$TAHVx9*=7kO5G>Djy^^Z9w&z7!E=hgd5Cy3dR{>d~h~aLj?)UrmYZqU9Q3k)6;xW>CKF~fK5sE zxG1BHJk!fPvQe1}^@<*l^u==hbMvFkkM&-D%Hb@mknO(%6OnHFw>M2u6cUF4AAAG+ zR35oNk}KsgQswnL8lVKY2>dQaUw*fCd7M_1r5?Sg=vJ9^I>JKNJbN4}hfa6a z>Yaogc|zS)@1Pb~-mOlyvdNg<6f;pn>f5@jmuSYI%q>1Uu;vn2Fn_!ql`3{Gi(Hyk z%vU}#L`Ah-Q^-vXR)7h6(PU>R0z@)LfRSL-_!ti5IFX#*P!ddeGw&{RoM<{6Ya>U4 z=Nzf+w|fg|TAqkive3hcQiH|?v$osq270;B3u|Rq<}#2_-L{AkCcXL2)-Yc ztZv+U8125uoC!66dGe|l5qZL*F1~Lc$fYve59R}-Q7i8`ia7O-%|6Syh0?fA9gZZ% zVw;JxQBB>@^a;-|+5tnkR@hrv>ZGa_Nc?2VlrHsTc8m=P9byjf$w}d@!(7~Hx>nAWg6?G3&0l=jwLnQMgG-bOl6zOicKOr2B5 z%5rtYh~d63Eek76D4V{_ctg8-ENw=G#(^D_C(|l-u!nvddlvt#T|rXRz$mJ5br!p% zN4~wmyeX_tLuUI9b<4Sq8dw^vc>-osqk;KK^Dd< zq}*T;vm#DvGrHB3WV{yCFoBrk%WZpwXoCfFTAio~F{$_ZND6T)9C(ob9+a z+<>B?fHGlGgO%hPARuSGs0wpw14weu$JvV~;4~{^!z$^$f!=G*@zta|=~XO^w}K&!W=>N_h(l%5I~5A2S*VQ%V{;MUk~1hgv-YWj z1DO-JtuZq_Wj|1#Pi=la@|XA}2W#M4yk?&hY5?8j6~GfZD36taEFE{7X1G8a7-1|{ z(rnjodIj2N$J|WrumhshRB_!yPj)c%c5Sjlc=SQ@JBH2rL)2gRQ*t}L>e2_~p4WlpCvPmS%!ppag5+7-bdM2T0@M zDCpu6OOl~|0?fp2-*9lojDOZ_rfxt2X=IQhG7mTyDQ+t59mbgAswTU#$&HTd`N*m_ z^^xbL4Mb_wlnuC2sr&YpvCk1iP-K$Ev3wpf>@^nUv2PYlfJo!ToIp$rEIEErj7U6K zFf72b5~U9Up6-|&sF?Xmo!-ksPBI$=hnvE*Je!4b1053!RwRhh%$gL)Igd3<<(4vF zR{^pdHqEYr4>}gQQ2oZKWlPibGd9YkDDq=7C4J#{P*BayD1Zk6Hf)EBKAFF1rm$6titAp&>1>z> z^NKGzc^86U^DYNK4**h6HqfJ{8)Kl5;izYu+@^u-XvJXdty|)yN^Lz5J-cF&m^mBO z@cv}A-`1-GW1(V;kzn=@-4?YeuQz1HWvkdB+>a+{W7e^<8~8Z0m2JGd;o(dl=4S=> zOb;TFHgy8)0~$J?06`fY-WuSN=<}hxW0c@{*fb{egb8GOR|3g`zSOLj<+3xu$8Cf% zR|Z4l^?j$#Z`+4@$(kRPi8^VIoOWFs_&qxi#qr|eeryDec$FCaKMl%@|{D(A)Z=Ct6Oe(UyBZbg+Pr9K$Erkg=D1^z&CQS^Se}_tqP;*VcVQb>&~JNmC>D5!vw(1 zdgD}T%%Y!6NSd%=$Mis^2ZHrD%<LP?>;fh28b;UYR-bOHRv3>hc1nA= zXoLjiPYSwN3Ad&3QRvM_<focm^jL1 zwZT^PKIjoq>I}ZeaY8guk*2pNMr(BG&Z0cTtkY`zYSOKop|&D7Rlk|&LJ;2R(? zIjhvOKlS$4*nxUr#@!DJG#A|$6~+}!fbi9Egz_`Rp*ii!cH9`Zw9=AuG_;0uZ7FP? zke*`N8#|nv46$pgqCGnv+wvIoOXP_=l|%aAss~48xbbUz^}xMc7r@?4%J4`(P?j(noUp+=IWxu6R?yV#<`zF9a7ItnMVY)~gP zNhifEanewlYPV|#Z7o?%+UYXl;5d-h$8J*_6)0=yE^%qqcJHT5ui8d()(cYmuLa2$ zf?(B`gD84?RB1wr^`XFv=+ZVOGq72;S*N)?1+?U-+WwxZ`{2*s+<)xezNGZBB;g7& z9y3scz*~Nz2#0O5x-%=aK(DdYj%n}K^VSkMv@vE;=K-(>6tK|8^gi1^?H8aq$l%1GJQ{MdaY#0w=FhGwsnD&5N!%nx$U0Ooxt5$xQ`(Ua z6=%mIwGgBA`{RHtV%N^db%2wzS?8PntVPaH1bdY^ifq<1Q4^P3(C#brVY599hI9pX zWofHJ*r?ZbLHo2a340Ej23F-YI$@?KEfmI^kP3zy*+oG8%(g{-*ta{igEH-*V_9UW z4n4W3?gtP;-TXiY0|+&zDB5P8AMjDaKxZP**Psjn_lUK!< z$`i=fW!e~pVPnB}hDf(b6Fgc$N4P#(?&{ldzq&Jy6tY^^hWwOt_normJDbK@Mz?*f z;ZCbfool#VcHY`a177F)Wz-{N>LQ093)%TDRsQ5v+DHd7DbWmBsyzR6J-h;Er(mij zlV<(!2{=jg_!DWHRi-%ifd-s1iD@Lo+f4H{2w2bg?BsMY>70haY)0>c%y2{pX5r2@ z;<}qCxlZPN*p!fbD-&#Ak}~ISoXyKInk)r=D6lvgyK>q!NBGL0sL~rf|59Vk*`L{o$?07v5`^Vo< zknVqQE7!d++1rWl@J<^{;y7gQ@Zsm9I{?3OD)p)OoM()0^bg=1m`IPkkxvG!*Hj}l zlpSS;RV@8y z8B4Pe$vbDfK%Oh(LBR3tCxy~0{<+zTc)0qtoq&T+kwn5>I3IxWJV>iO*?`ju+wdF9 z1hr19#yB%(?o6bE9Fp>UM3i}y_Kq^wpHVZ=EzehIO$DZw9%~FgcbZ*c1B?YV8Xsr4 zuT1#9y~C6?Y4-ARxABTTXrw4>4$c7+77hG^+Sr@#d5R0#Bp}Z5Tgh=o#vT~Oo@dgoEU~i zRa6$c+5uafoGD!%^OcpTm+5wA*r-^NNtyQBvXp<3ba0bL&0Qp>gW+sSq@qX(c>xuiM`LQr|Cxe-h26FDZ83oKW>nC7BW+Z5g9 zBcJ;Iz}}15ub2@6Um`HnH6RO(ddlxP0Q z8;i>J&w8%!;AE-!qcvkJ>trHjieE8f*F&Kmwfo82a*suSvPVJ zJ6$iy!O;RzaI~O>1V4`rMsUd4X;U! z3a5(gfy0t&P+n4E>4Xr@qHocM)iRuK}hQ+Ucmzn$P|%(iM))w z=b*ghd-nf)^atcC09cAPIE)>&e7r7^&<($3BBReniVv>=STXaAmgK^~YW;N2@W?#h zCzjynyq0}_#|81!aZR3-0hG`U$S}D4;!ooE5bnZ|ITwZ(s}CQ^0``LC9#D>im_DmK z{No?K@!@`9ULY5S#DaZV69KYQR&ObWdTK%#}V2OegMU-#9!+CfW_Z( zA3l*CGtWWiD?D%F%<^$UAWz!A##0q;86dm6Fi8#a2x)ENGK2%)06!leBJf3@8;}Fi z<7Wok)ZlW&1Xj~tO6*({1)Ajg1kfk&=dn8Z$3G?@)5(!1MZcp3B0~{?kU4tV1_Aq4 zf7p`&&MG5q(z)65XhQ$BJ&$O?RsJj^{LJh1kdG(V_qG$T)ZkG9VhR<(B`F6T1 zQ-@9`&Y)ZP`N8uu=Umgif%OYcj^7(|xmkM@P`qXra%lrO$~_-P8c%e`4;~i3Wwz8z zrekn`Q!-Cxxwx7+aGj@oJ{x}hWb@uee2-RCZ_&n`0|2lya)1&e^NIqb37 z_`y@nw_Bfvz>z9}-2ZH%IRn_8Cldb-8T#o)l5c^^=?c0QlZ;Zn;h?zr;-wHHB^Pl} z{G5@<@pzP__|#PK?b5F8L97HaMPPW_Z|7l`o3(w7;o)0UGSN&V9GURwA2B$5mb0sN z&>Qr>`~>i2PBW)c^_7NxZ#nR~V*G{XW}`I3%@V)FsBHSroo-h5m8N5F zHPPR9GWKdvH7`p8!F;hD<%%ZI16RjC$uR7fo`1b6R^5mPj0t|Sk=N`*WgZiLy(!mQ zx&G+m{_PX4`NseHIuUnV8~^NuKW+IlRDr~2jy}$*-m(5aY*G~lh994Ao1GoG5syr4 z@maU$Ob;ym{Q7Cx<;nEvIq0|^@yrvCuQo_}UixKx?q-|5?)2uZYIxX73ncU1UTy~r;Pu4){@SOestbRy0B%TEXO5*wiOh>{p(c^UF z51!h~;kr1Oi}Hbx$$tFAUUb29%JoL#zv__QnfbCKY> mEcy$Mx}6tyze)E0?z~!7QC$9;!`GhEf*T6vuYUQ-v14G{uAWI&nIXn?%w+1I2e&Cy!q$>xQ-kLRIh_i?X&6ugLzDS_n-qYurgbb zTHR8&1o+gEal+|>n-Acoowuk1z1!l@K5GphPn$D|{D9%y8EHK&^#koXMOC^?B-Qq< z5#0*kr+i^wJW%cXQt~nh^S!m4QwpstCy2vi4b#Cu`pQ5xyk`0>1jr6kq9pX z<6VHO7K{4zgVjm(qVW`>t{#(h5S*VvP`EU68V(sy_+axeTv?pY6vcpC?A6J2e<@6c zT?r;e6yW{X#=$yL6E7t=kO0%ui$j z{pwl@W$>_&+9*iPCb#P4%E>q;g)`k;THv{88>g$SRJR_jxxM9{E3lfZmC|%8mmY|l z%fXa)CAI7)V{tKr=9IK6(5pq)nISfa2VOOq)Q>F6xcbbAYh(A-M2-|XdkxLbSPdL_ z%dQ8yJUy=Q4VV%JPnC_M`qugROlg_UtO+H-Hr%Z;k9K#dK12)?qqF=cS&kJGjF|co2XuHT8EE1pn2gKDBQt^8YQVCwJQy7)P`w%);8wOjC-{0Kjao* zzukmj(Dckf(Ejp-HiXvnfWhO2Bf82?wlxdp|yIdyUDsjeRe&c3Mrd7Ej3bn zjbM|>%Ag&Tc!sVobt9piZ?h8Oxd25KghVbCq^d%$mu)Ns;_LBfO(+U-zFbq_Y4oUu zrV*ZDrKFofBRJ9yC*>~KmJLkn^ywUVw9 zLaWura+0!o)(TfAEaRBdJ)-MN=yy8sP&4q{jA<_qMo~w#gMANA&{j=2TNwi0qvuC| zdTY^=i8j-L7!enXQhK2D!c2`O3h5x7V5mT4r4#BUJDCtxVV4S2BEJ!e5dKxtPY0Uo zNl!ZETW!`FwLz{D>t-moYzuZ{O5Y{*iIIC8-u8qM$4DpU#ID#+j%d|Ze|qfo$w`C$ zvfTiTWeTyv&-&k0S8M@Mv4rV=`*C)6{P^rytLjhM#-YFQtIFnrlklWP2j7UsiBi7H z0m4sDlA45YF!R{ZmJu8T-g16-j9FV|D0Z;KrxrEE*E>Ua2`BC(tRoN5Umo2wWJiur z?+o{Y&z$$eB7YkGa2i44jbKC1!H@i*|LE!2-SPAxzHeMy7*oH>hsWScaqeEn^)R62 zhyN}L(z_A`&|*z+iaWGBX>e}a^3DA6cB#?3x-0H4Yh3sXe~b6J#PmCv@oy4k?jI9H peOq`$xBrf^?|Ee4wbmXc64E<%*X`b6XVn*aH^y~Z1Q{+H{six0 { - socket.on("message", (message) => { - socket.emit( - JSON.stringify({ - op: 2, - d: { - ssrc: 1, - ip: "127.0.0.1", - port: 3004, - modes: [ - "xsalsa20_poly1305", - "xsalsa20_poly1305_suffix", - "xsalsa20_poly1305_lite", - ], - heartbeat_interval: 1, - }, - }) - ); + this.ws.on("connection", async (socket: WebSocket) => { + await setHeartbeat(socket); + + socket.on("message", async (message: string) => { + const payload: Payload = JSON.parse(message); + + if (OPCodeHandlers[payload.op]) + await OPCodeHandlers[payload.op](socket, payload); + else + console.error(`Unimplemented`, payload) }); }); } async listen(): Promise { // @ts-ignore - await (db as Promise); + await initDatabase(); await Config.init(); console.log("[DB] connected"); console.log(`[WebRTC] online on 0.0.0.0:${port}`); diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts new file mode 100644 index 000000000..5cc66506b --- /dev/null +++ b/webrtc/src/opcodes/Connect.ts @@ -0,0 +1,5 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; + +export async function onConnect(socket: WebSocket, data: Payload) { +} \ No newline at end of file diff --git a/webrtc/src/opcodes/Heartbeat.ts b/webrtc/src/opcodes/Heartbeat.ts new file mode 100644 index 000000000..04150e36b --- /dev/null +++ b/webrtc/src/opcodes/Heartbeat.ts @@ -0,0 +1,7 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; +import { setHeartbeat } from "./../util"; + +export async function onHeartbeat(socket: WebSocket, data: Payload) { + await setHeartbeat(socket); +} \ No newline at end of file diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts new file mode 100644 index 000000000..2026d7c9d --- /dev/null +++ b/webrtc/src/opcodes/Identify.ts @@ -0,0 +1,20 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index" +import { VoiceOPCodes } from "@fosscord/util"; + +export async function onIdentify(socket: WebSocket, data: Payload) { + socket.send(JSON.stringify({ + op: VoiceOPCodes.READY, + d: { + ssrc: 1, + ip: "127.0.0.1", + port: 3005, + modes: [ + "xsalsa20_poly1305", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305_lite", + ], + heartbeat_interval: 1, + }, + })); +} \ No newline at end of file diff --git a/webrtc/src/opcodes/Resume.ts b/webrtc/src/opcodes/Resume.ts new file mode 100644 index 000000000..de21eba6f --- /dev/null +++ b/webrtc/src/opcodes/Resume.ts @@ -0,0 +1,5 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; + +export async function onResume(socket: WebSocket, data: Payload) { +} \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts new file mode 100644 index 000000000..f1732dd98 --- /dev/null +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -0,0 +1,16 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; +import { VoiceOPCodes } from "@fosscord/util"; + +export async function onSelectProtocol(socket: WebSocket, data: Payload) { + socket.send(JSON.stringify({ + op: VoiceOPCodes.SESSION_DESCRIPTION, + d: { + video_codec: "H264", + secret_key: new Array(32).fill(null).map(x => Math.random() * 256), + mode: "aead_aes256_gcm_rtpsize", + media_session_id: "d8eb5c84d987c6642ec4ce72ffa97f00", + audio_codec: "opus", + } + })); +} \ No newline at end of file diff --git a/webrtc/src/opcodes/Speaking.ts b/webrtc/src/opcodes/Speaking.ts new file mode 100644 index 000000000..14f86b3c2 --- /dev/null +++ b/webrtc/src/opcodes/Speaking.ts @@ -0,0 +1,6 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index" +import { VoiceOPCodes } from "@fosscord/util"; + +export async function onSpeaking(socket: WebSocket, data: Payload) { +} \ No newline at end of file diff --git a/webrtc/src/opcodes/index.ts b/webrtc/src/opcodes/index.ts new file mode 100644 index 000000000..2fe69c386 --- /dev/null +++ b/webrtc/src/opcodes/index.ts @@ -0,0 +1,35 @@ +import { WebSocket } from "@fosscord/gateway"; +import { VoiceOPCodes } from "@fosscord/util"; + +export interface Payload { + op: number; + d?: any; + s?: number; + t?: string; +} + +import { onIdentify } from "./Identify"; +import { onSelectProtocol } from "./SelectProtocol"; +import { onHeartbeat } from "./Heartbeat"; +import { onSpeaking } from "./Speaking"; +import { onResume } from "./Resume"; +import { onConnect } from "./Connect"; + +export type OPCodeHandler = (this: WebSocket, data: Payload) => any; + +export default { + [VoiceOPCodes.IDENTIFY]: onIdentify, //op 0 + [VoiceOPCodes.SELECT_PROTOCOL]: onSelectProtocol, //op 1 + //op 2 voice_ready + [VoiceOPCodes.HEARTBEAT]: onHeartbeat, //op 3 + //op 4 session_description + [VoiceOPCodes.SPEAKING]: onSpeaking, //op 5 + //op 6 heartbeat_ack + [VoiceOPCodes.RESUME]: onResume, //op 7 + //op 8 hello + //op 9 resumed + //op 10? + //op 11? + [VoiceOPCodes.CLIENT_CONNECT]: onConnect, //op 12 + //op 13? +}; \ No newline at end of file diff --git a/webrtc/src/start.ts b/webrtc/src/start.ts index 68867a2cf..5614982dc 100644 --- a/webrtc/src/start.ts +++ b/webrtc/src/start.ts @@ -1,3 +1,4 @@ import { Server } from "./Server"; const server = new Server(); +server.listen(); \ No newline at end of file diff --git a/webrtc/src/util/Heartbeat.ts b/webrtc/src/util/Heartbeat.ts new file mode 100644 index 000000000..7b5ed9cd4 --- /dev/null +++ b/webrtc/src/util/Heartbeat.ts @@ -0,0 +1,18 @@ +import { WebSocket, CLOSECODES } from "@fosscord/gateway"; +import { VoiceOPCodes } from "@fosscord/util"; + +export async function setHeartbeat(socket: WebSocket) { + if (socket.heartbeatTimeout) clearTimeout(socket.heartbeatTimeout); + + socket.heartbeatTimeout = setTimeout(() => { + return socket.close(CLOSECODES.Session_timed_out); + }, 1000 * 45); + + socket.send(JSON.stringify({ + op: VoiceOPCodes.HEARTBEAT_ACK, + d: { + v: 6, + heartbeat_interval: 13750, + } + })); +} \ No newline at end of file diff --git a/webrtc/src/util/index.ts b/webrtc/src/util/index.ts new file mode 100644 index 000000000..e8557452b --- /dev/null +++ b/webrtc/src/util/index.ts @@ -0,0 +1 @@ +export * from "./Heartbeat" \ No newline at end of file diff --git a/webrtc/tsconfig.json b/webrtc/tsconfig.json index 77353db0d..fb93b0bdc 100644 --- a/webrtc/tsconfig.json +++ b/webrtc/tsconfig.json @@ -1,5 +1,8 @@ { "include": ["src/**/*.ts"], + "ts-node": { + "require": ["tsconfig-paths/register"], + }, "compilerOptions": { /* Visit https://aka.ms/tsconfig.json to read more about this file */ @@ -18,7 +21,7 @@ "sourceMap": true /* Generates corresponding '.map' file. */, // "outFile": "./", /* Concatenate and emit output to single file. */ "outDir": "./dist/" /* Redirect output structure to the directory. */, - "rootDir": "./src/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */, + "rootDir": "../" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */, // "composite": true, /* Enable project compilation */ // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ // "removeComments": true, /* Do not emit comments to output. */ @@ -62,11 +65,19 @@ // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ /* Experimental Options */ - // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ - // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ + "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ /* Advanced Options */ "skipLibCheck": true /* Skip type checking of declaration files. */, - "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ + "forceConsistentCasingInFileNames": true, /* Disallow inconsistently-cased references to the same file. */ + + "baseUrl": "../", + "paths": { + "@fosscord/api": ["api/src/index"], + "@fosscord/gateway": ["gateway/src/index"], + "@fosscord/cdn": ["cdn/src/index"], + "@fosscord/util": ["util/src/index"] + }, } } From b1dc6b34ddb0c27b99f4203043d4f2ba0fcee375 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Sun, 16 Jan 2022 03:35:30 +1100 Subject: [PATCH 02/10] messing around with things I don't understand --- webrtc/src/Server.ts | 70 ++++++++++++++++++++++++++-- webrtc/src/opcodes/Connect.ts | 3 +- webrtc/src/opcodes/Heartbeat.ts | 3 +- webrtc/src/opcodes/Identify.ts | 7 ++- webrtc/src/opcodes/Resume.ts | 3 +- webrtc/src/opcodes/SelectProtocol.ts | 5 +- webrtc/src/opcodes/Speaking.ts | 3 +- 7 files changed, 81 insertions(+), 13 deletions(-) diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 06a36df95..cdda10ec6 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -1,15 +1,19 @@ import { Server as WebSocketServer } from "ws"; -import { WebSocket, CLOSECODES, Payload, OPCODES } from "@fosscord/gateway"; +import { WebSocket, Payload, } from "@fosscord/gateway"; import { Config, initDatabase } from "@fosscord/util"; import OPCodeHandlers from "./opcodes"; -import { setHeartbeat } from "./util" -import mediasoup from "mediasoup"; +import { setHeartbeat } from "./util"; +import * as mediasoup from "mediasoup"; +import { types as MediasoupTypes } from "mediasoup"; var port = Number(process.env.PORT); if (isNaN(port)) port = 3004; export class Server { public ws: WebSocketServer; + public mediasoupWorkers: MediasoupTypes.Worker[] = []; + public mediasoupRouters: MediasoupTypes.Router[] = []; + public mediasoupTransports: MediasoupTypes.Transport[] = []; constructor() { this.ws = new WebSocketServer({ @@ -23,9 +27,9 @@ export class Server { const payload: Payload = JSON.parse(message); if (OPCodeHandlers[payload.op]) - await OPCodeHandlers[payload.op](socket, payload); + await OPCodeHandlers[payload.op].call(this, socket, payload); else - console.error(`Unimplemented`, payload) + console.error(`Unimplemented`, payload); }); }); } @@ -34,7 +38,63 @@ export class Server { // @ts-ignore await initDatabase(); await Config.init(); + await this.createWorkers(); console.log("[DB] connected"); console.log(`[WebRTC] online on 0.0.0.0:${port}`); } + + async createWorkers(): Promise { + const numWorkers = 1; + for (let i = 0; i < numWorkers; i++) { + const worker = await mediasoup.createWorker(); + if (!worker) return; + + worker.on("died", () => { + console.error("mediasoup worker died"); + }); + + worker.observer.on("newrouter", async (router: MediasoupTypes.Router) => { + console.log("new router"); + + this.mediasoupRouters.push(router); + + router.observer.on("newtransport", (transport: MediasoupTypes.Transport) => { + console.log("new transport"); + + this.mediasoupTransports.push(transport); + }) + + await router.createWebRtcTransport({ + listenIps: [{ ip: "127.0.0.1" }], + enableUdp: true, + enableTcp: true, + preferUdp: true + }); + }); + + await worker.createRouter({ + mediaCodecs: [ + { + kind: "audio", + mimeType: "audio/opus", + clockRate: 48000, + channels: 2 + }, + { + kind: "video", + mimeType: "video/H264", + clockRate: 90000, + parameters: + { + "packetization-mode": 1, + "profile-level-id": "42e01f", + "level-asymmetry-allowed": 1 + } + } + ] + }); + + this.mediasoupWorkers.push(worker); + } + } } diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts index 5cc66506b..5db116388 100644 --- a/webrtc/src/opcodes/Connect.ts +++ b/webrtc/src/opcodes/Connect.ts @@ -1,5 +1,6 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; +import { Server } from "../Server" -export async function onConnect(socket: WebSocket, data: Payload) { +export async function onConnect(this: Server, socket: WebSocket, data: Payload) { } \ No newline at end of file diff --git a/webrtc/src/opcodes/Heartbeat.ts b/webrtc/src/opcodes/Heartbeat.ts index 04150e36b..06d6bcb1a 100644 --- a/webrtc/src/opcodes/Heartbeat.ts +++ b/webrtc/src/opcodes/Heartbeat.ts @@ -1,7 +1,8 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { setHeartbeat } from "./../util"; +import { Server } from "../Server" -export async function onHeartbeat(socket: WebSocket, data: Payload) { +export async function onHeartbeat(this: Server, socket: WebSocket, data: Payload) { await setHeartbeat(socket); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 2026d7c9d..6043a4602 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -1,14 +1,17 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index" import { VoiceOPCodes } from "@fosscord/util"; +import { Server } from "../Server" -export async function onIdentify(socket: WebSocket, data: Payload) { +export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { ssrc: 1, ip: "127.0.0.1", - port: 3005, + + //@ts-ignore + port: this.mediasoupTransports[0].iceCandidates.port, modes: [ "xsalsa20_poly1305", "xsalsa20_poly1305_suffix", diff --git a/webrtc/src/opcodes/Resume.ts b/webrtc/src/opcodes/Resume.ts index de21eba6f..dcd4f4cda 100644 --- a/webrtc/src/opcodes/Resume.ts +++ b/webrtc/src/opcodes/Resume.ts @@ -1,5 +1,6 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; +import { Server } from "../Server" -export async function onResume(socket: WebSocket, data: Payload) { +export async function onResume(this: Server, socket: WebSocket, data: Payload) { } \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index f1732dd98..fcc45855e 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -1,15 +1,16 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; +import { Server } from "../Server" -export async function onSelectProtocol(socket: WebSocket, data: Payload) { +export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { video_codec: "H264", secret_key: new Array(32).fill(null).map(x => Math.random() * 256), mode: "aead_aes256_gcm_rtpsize", - media_session_id: "d8eb5c84d987c6642ec4ce72ffa97f00", + media_session_id: this.mediasoupTransports[0].id, audio_codec: "opus", } })); diff --git a/webrtc/src/opcodes/Speaking.ts b/webrtc/src/opcodes/Speaking.ts index 14f86b3c2..861a7c3db 100644 --- a/webrtc/src/opcodes/Speaking.ts +++ b/webrtc/src/opcodes/Speaking.ts @@ -1,6 +1,7 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index" import { VoiceOPCodes } from "@fosscord/util"; +import { Server } from "../Server" -export async function onSpeaking(socket: WebSocket, data: Payload) { +export async function onSpeaking(this: Server, socket: WebSocket, data: Payload) { } \ No newline at end of file From 2e573cc3056da60a45556179687fb4c720af1227 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Mon, 17 Jan 2022 02:59:26 +1100 Subject: [PATCH 03/10] more fuckery --- webrtc/package-lock.json | Bin 31537 -> 32074 bytes webrtc/package.json | 1 + webrtc/src/Server.ts | 31 ++++++++---- webrtc/src/opcodes/Connect.ts | 4 ++ webrtc/src/opcodes/Identify.ts | 68 ++++++++++++++++++++++++-- webrtc/src/opcodes/SelectProtocol.ts | 70 +++++++++++++++++++++++++-- webrtc/src/start.ts | 6 +++ 7 files changed, 161 insertions(+), 19 deletions(-) diff --git a/webrtc/package-lock.json b/webrtc/package-lock.json index 6c3726dc425b0a2cc59c41eb82b859538beb60c6..43bb8cd83ccd4da91b13dac0631ab7a60e07109f 100644 GIT binary patch delta 321 zcmdn^jq%hk#t8?6Q}Rnv^U9R06qMo&jr0ukOq6ss?&_CfgNRRd^c7_V3s0UHsV@pv ztFNmMk<^7~n0!!7J}f^xr7$VDvOF!V%EK_nBs{`9z$4tzAS67pps>K)v(hK6Fe|el z#H6$!-_tGFz$3u7u*5IFq%u3Oz{MofGu^-m$j?nl4{**f3imDZEitbQ$t<57=%_OJ b0E^saPiYBz5}g!D8zD-t3gxTq`4Czqu;rf&u`Gr3(iD diff --git a/webrtc/package.json b/webrtc/package.json index 8c66245dc..d5a994a18 100644 --- a/webrtc/package.json +++ b/webrtc/package.json @@ -18,6 +18,7 @@ "typescript": "^4.3.2" }, "dependencies": { + "dotenv": "^12.0.4", "mediasoup": "^3.9.5", "node-turn": "^0.0.6", "tsconfig-paths": "^3.12.0", diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index cdda10ec6..1d2e73e77 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -54,21 +54,32 @@ export class Server { }); worker.observer.on("newrouter", async (router: MediasoupTypes.Router) => { - console.log("new router"); + console.log("new router created [id:%s]", router.id); this.mediasoupRouters.push(router); - router.observer.on("newtransport", (transport: MediasoupTypes.Transport) => { - console.log("new transport"); + router.observer.on("newtransport", async (transport: MediasoupTypes.Transport) => { + console.log("new transport created [id:%s]", transport.id); + + await transport.enableTraceEvent(); + + transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => { + console.log("new producer created [id:%s]", producer.id); + }); + + transport.observer.on("newconsumer", (consumer: MediasoupTypes.Consumer) => { + console.log("new consumer created [id:%s]", consumer.id); + }); + + transport.observer.on("newdataproducer", (dataProducer) => { + console.log("new data producer created [id:%s]", dataProducer.id); + }); + + transport.on("trace", (trace) => { + console.log(trace); + }); this.mediasoupTransports.push(transport); - }) - - await router.createWebRtcTransport({ - listenIps: [{ ip: "127.0.0.1" }], - enableUdp: true, - enableTcp: true, - preferUdp: true }); }); diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts index 5db116388..b312d6f2e 100644 --- a/webrtc/src/opcodes/Connect.ts +++ b/webrtc/src/opcodes/Connect.ts @@ -3,4 +3,8 @@ import { Payload } from "./index"; import { Server } from "../Server" export async function onConnect(this: Server, socket: WebSocket, data: Payload) { + socket.send(JSON.stringify({ + op: 15, + d: { any: 100 } + })) } \ No newline at end of file diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 6043a4602..6bbed04c4 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -1,9 +1,67 @@ import { WebSocket } from "@fosscord/gateway"; -import { Payload } from "./index" +import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; -import { Server } from "../Server" +import { Server } from "../Server"; +import * as mediasoup from "mediasoup"; +import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; + +const test = "extmap-allow-mixed\na=ice-ufrag:ilWh\na=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ\na=ice-options:trickle\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\na=rtpmap:111 opus/48000/2\na=extmap:14 urn:ietf:params:rtp-hdrext:toffset\na=extmap:13 urn:3gpp:video-orientation\na=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\na=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\na=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\na=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\na=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\na=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\na=rtpmap:96 VP8/90000\na=rtpmap:97 rtx/90000"; export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { + var transport = await this.mediasoupRouters[0].createWebRtcTransport({ + listenIps: [{ ip: "127.0.0.1" }], + enableUdp: true, + enableTcp: true, + preferUdp: true, + }); + + const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; + const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; + + var producer = await transport.produce( + { + kind: "audio", + rtpParameters: + { + mid: "1", + codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { + return { + mimeType: x.mimeType, + kind: x.kind, + clockRate: x.clockRate, + channels: x.channels, + payloadType: x.preferredPayloadType as number + }; + }), + headerExtensions: test.split("\na=").map((x, i) => ({ + id: i + 1, + uri: x, + })) + } + }); + + const consumer = await transport.consume( + { + producerId: producer.id, + rtpCapabilities: + { + codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { + return { + mimeType: x.mimeType, + kind: x.kind, + clockRate: x.clockRate, + channels: x.channels, + payloadType: x.preferredPayloadType as number + }; + }), + headerExtensions: test.split("\na=").map((x, i) => ({ + kind: "audio", + preferredId: i + 1, + uri: x, + })) + } + }); + socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { @@ -11,11 +69,11 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) ip: "127.0.0.1", //@ts-ignore - port: this.mediasoupTransports[0].iceCandidates.port, + port: transport.iceCandidates[0].port, modes: [ "xsalsa20_poly1305", - "xsalsa20_poly1305_suffix", - "xsalsa20_poly1305_lite", + // "xsalsa20_poly1305_suffix", + // "xsalsa20_poly1305_lite", ], heartbeat_interval: 1, }, diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index fcc45855e..24e8ef5f0 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -1,17 +1,79 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; -import { Server } from "../Server" +import { Server } from "../Server"; + +/* + { + op: 1, + d: { + protocol: "webrtc", + data: " + a=extmap-allow-mixed + a=ice-ufrag:ilWh + a=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ + a=ice-options:trickle + a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level + a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time + a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 + a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid + a=rtpmap:111 opus/48000/2 + a=extmap:14 urn:ietf:params:rtp-hdrext:toffset + a=extmap:13 urn:3gpp:video-orientation + a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay + a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type + a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing + a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space + a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id + a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id + a=rtpmap:96 VP8/90000 + a=rtpmap:97 rtx/90000 + ", + sdp: "same data as in d.data? also not documented by discord", + codecs: [ + { + name: "opus", + type: "audio", + priority: 1000, + payload_type: 111, + rtx_payload_type: null, + }, + { + name: "H264", + type: "video", + priority: 1000, + payload_type: 102, + rtx_payload_type: 121, + }, + { + name: "VP8", + type: "video", + priority: 2000, + payload_type: 96, + rtx_payload_type: 97, + }, + { + name: "VP9", + type: "video", + priority: 3000, + payload_type: 98, + rtx_payload_type: 99, + }, + ], + rtc_connection_id: "b3c8628a-edb5-49ae-b860-ab0d2842b104", + }, + } +*/ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { - video_codec: "H264", + video_codec: data.d.codecs.find((x: any) => x.type === "video").name, secret_key: new Array(32).fill(null).map(x => Math.random() * 256), - mode: "aead_aes256_gcm_rtpsize", + mode: "xsalsa20_poly1305", media_session_id: this.mediasoupTransports[0].id, - audio_codec: "opus", + audio_codec: data.d.codecs.find((x: any) => x.type === "audio").name, } })); } \ No newline at end of file diff --git a/webrtc/src/start.ts b/webrtc/src/start.ts index 5614982dc..299bfce8e 100644 --- a/webrtc/src/start.ts +++ b/webrtc/src/start.ts @@ -1,4 +1,10 @@ +import { config } from "dotenv"; +config(); + import { Server } from "./Server"; +//testing +process.env.DATABASE = "../bundle/database.db"; + const server = new Server(); server.listen(); \ No newline at end of file From 4847351daa226cbd71fc8676b6be7516c6e76253 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Fri, 21 Jan 2022 21:04:45 +1100 Subject: [PATCH 04/10] mmmm --- webrtc/package-lock.json | Bin 32074 -> 33275 bytes webrtc/package.json | 2 + webrtc/src/opcodes/Identify.ts | 79 ++++++++++----------------- webrtc/src/opcodes/SelectProtocol.ts | 37 +++++++++++++ webrtc/src/opcodes/index.ts | 2 + 5 files changed, 69 insertions(+), 51 deletions(-) diff --git a/webrtc/package-lock.json b/webrtc/package-lock.json index 43bb8cd83ccd4da91b13dac0631ab7a60e07109f..afba7e761c8a7595e1dff36c4faee723403a8d91 100644 GIT binary patch delta 723 zcmX^0i}80e(*}Mc!^!N7_2SqhP4rARmoqNl zJ_lxker- z0mYG_e#w=Y8Bwmr?v>7w7L~=mkshUi?rA<5?oovnmVUV=Il)z-KH(u%QTde~CF$B> z{`#INMR^rb9szj)!R~pElOM9`Y!>2X)8s++8OVK;1yzmYQCx!QvdMzUDv?21QI!$K zd12*VmCmIh`KBfA0TtQ-QQ_GxzRv09`Tjly!8sXOE-6uI<)Qh0h38*PNaAzqeRl$llu3hv~L)a2~c nBE78Q$%!&Do5fASr3Z1y%5F~g?^GZ=WW#6^vYQ|0r}F{;8%F!G delta 38 ucmey}%yjA(;|6}l$@+}to6j&#GA=r&*ZvX4T*(h0O~~qIdx?K@QRY diff --git a/webrtc/package.json b/webrtc/package.json index d5a994a18..b9bac3564 100644 --- a/webrtc/package.json +++ b/webrtc/package.json @@ -13,6 +13,7 @@ "license": "ISC", "devDependencies": { "@types/node": "^15.6.1", + "@types/sdp-transform": "^2.4.5", "@types/ws": "^7.4.4", "ts-node": "^10.4.0", "typescript": "^4.3.2" @@ -21,6 +22,7 @@ "dotenv": "^12.0.4", "mediasoup": "^3.9.5", "node-turn": "^0.0.6", + "sdp-transform": "^2.14.1", "tsconfig-paths": "^3.12.0", "ws": "^7.4.6" } diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 6bbed04c4..c31870c8f 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -2,10 +2,6 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; import { Server } from "../Server"; -import * as mediasoup from "mediasoup"; -import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; - -const test = "extmap-allow-mixed\na=ice-ufrag:ilWh\na=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ\na=ice-options:trickle\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\na=rtpmap:111 opus/48000/2\na=extmap:14 urn:ietf:params:rtp-hdrext:toffset\na=extmap:13 urn:3gpp:video-orientation\na=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\na=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\na=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\na=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\na=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\na=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\na=rtpmap:96 VP8/90000\na=rtpmap:97 rtx/90000"; export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { var transport = await this.mediasoupRouters[0].createWebRtcTransport({ @@ -15,52 +11,31 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) preferUdp: true, }); - const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; - const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; - - var producer = await transport.produce( - { - kind: "audio", - rtpParameters: - { - mid: "1", - codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { - return { - mimeType: x.mimeType, - kind: x.kind, - clockRate: x.clockRate, - channels: x.channels, - payloadType: x.preferredPayloadType as number - }; - }), - headerExtensions: test.split("\na=").map((x, i) => ({ - id: i + 1, - uri: x, - })) - } - }); - - const consumer = await transport.consume( - { - producerId: producer.id, - rtpCapabilities: - { - codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { - return { - mimeType: x.mimeType, - kind: x.kind, - clockRate: x.clockRate, - channels: x.channels, - payloadType: x.preferredPayloadType as number - }; - }), - headerExtensions: test.split("\na=").map((x, i) => ({ - kind: "audio", - preferredId: i + 1, - uri: x, - })) - } - }); + /* + //discord proper sends: + { + "streams": [ + { "type": "video", "ssrc": 1311885, "rtx_ssrc": 1311886, "rid": "50", "quality": 50, "active": false }, + { "type": "video", "ssrc": 1311887, "rtx_ssrc": 1311888, "rid": "100", "quality": 100, "active": false } + ], + "ssrc": 1311884, + "port": 50008, + "modes": [ + "aead_aes256_gcm_rtpsize", + "aead_aes256_gcm", + "xsalsa20_poly1305_lite_rtpsize", + "xsalsa20_poly1305_lite", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305" + ], + "ip": "109.200.214.158", + "experiments": [ + "bwe_conservative_link_estimate", + "bwe_remote_locus_client", + "fixed_keyframe_interval" + ] + } + */ socket.send(JSON.stringify({ op: VoiceOPCodes.READY, @@ -71,11 +46,13 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) //@ts-ignore port: transport.iceCandidates[0].port, modes: [ - "xsalsa20_poly1305", + "aead_aes256_gcm_rtpsize", + // "xsalsa20_poly1305", // "xsalsa20_poly1305_suffix", // "xsalsa20_poly1305_lite", ], heartbeat_interval: 1, + experiments: [], }, })); } \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index 24e8ef5f0..08335aded 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -2,6 +2,9 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; import { Server } from "../Server"; +import * as mediasoup from "mediasoup"; +import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; +import * as sdpTransform from 'sdp-transform'; /* { @@ -66,6 +69,40 @@ import { Server } from "../Server"; */ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { + const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; + const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; + + const transport = this.mediasoupTransports[0]; //whatever + + const res = sdpTransform.parse(data.d.sdp); + + /* + res.media.map(x => x.rtp).flat(1).map(x => ({ + codec: x.codec, + payloadType: x.payload, + clockRate: x.rate as number, + mimeType: `audio/${x.codec}`, + })), + */ + + const producer = await transport.produce({ + kind: "audio", + rtpParameters: { + mid: "audio", + codecs: [{ + clockRate: 48000, + payloadType: 111, + mimeType: "audio/opus", + channels: 2, + }], + headerExtensions: res.ext?.map(x => ({ + id: x.value, + uri: x.uri, + })) + }, + paused: false, + }); + socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { diff --git a/webrtc/src/opcodes/index.ts b/webrtc/src/opcodes/index.ts index 2fe69c386..36d30e7d5 100644 --- a/webrtc/src/opcodes/index.ts +++ b/webrtc/src/opcodes/index.ts @@ -32,4 +32,6 @@ export default { //op 11? [VoiceOPCodes.CLIENT_CONNECT]: onConnect, //op 12 //op 13? + //op 15? + //op 16? empty data on client send but server sends {"voice":"0.8.24+bugfix.voice.streams.opt.branch-ffcefaff7","rtc_worker":"0.3.14-crypto-collision-copy"} }; \ No newline at end of file From 958d570574ecd5e3e510fa8203edf27b8760763c Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Fri, 4 Feb 2022 18:46:09 +1100 Subject: [PATCH 05/10] ;jondfgsk --- webrtc/src/Server.ts | 19 +++--------- webrtc/src/opcodes/Identify.ts | 7 ++--- webrtc/src/opcodes/SelectProtocol.ts | 46 ++++++++++++++++++---------- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 1d2e73e77..dcbf216a1 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -1,5 +1,5 @@ import { Server as WebSocketServer } from "ws"; -import { WebSocket, Payload, } from "@fosscord/gateway"; +import { WebSocket, Payload, CLOSECODES } from "@fosscord/gateway"; import { Config, initDatabase } from "@fosscord/util"; import OPCodeHandlers from "./opcodes"; import { setHeartbeat } from "./util"; @@ -28,8 +28,10 @@ export class Server { if (OPCodeHandlers[payload.op]) await OPCodeHandlers[payload.op].call(this, socket, payload); - else + else { console.error(`Unimplemented`, payload); + socket.close(CLOSECODES.Unknown_opcode); + } }); }); } @@ -46,7 +48,7 @@ export class Server { async createWorkers(): Promise { const numWorkers = 1; for (let i = 0; i < numWorkers; i++) { - const worker = await mediasoup.createWorker(); + const worker = await mediasoup.createWorker({ logLevel: "debug" }); if (!worker) return; worker.on("died", () => { @@ -91,17 +93,6 @@ export class Server { clockRate: 48000, channels: 2 }, - { - kind: "video", - mimeType: "video/H264", - clockRate: 90000, - parameters: - { - "packetization-mode": 1, - "profile-level-id": "42e01f", - "level-asymmetry-allowed": 1 - } - } ] }); diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index c31870c8f..82f327be1 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -5,7 +5,7 @@ import { Server } from "../Server"; export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { var transport = await this.mediasoupRouters[0].createWebRtcTransport({ - listenIps: [{ ip: "127.0.0.1" }], + listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }], enableUdp: true, enableTcp: true, preferUdp: true, @@ -40,10 +40,9 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { + streams: [], ssrc: 1, - ip: "127.0.0.1", - - //@ts-ignore + ip: transport.iceCandidates[0].ip, port: transport.iceCandidates[0].port, modes: [ "aead_aes256_gcm_rtpsize", diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index 08335aded..36527a8be 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -68,6 +68,8 @@ import * as sdpTransform from 'sdp-transform'; } */ +var test_hasMadeProducer = false; + export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; @@ -85,23 +87,33 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa })), */ - const producer = await transport.produce({ - kind: "audio", - rtpParameters: { - mid: "audio", - codecs: [{ - clockRate: 48000, - payloadType: 111, - mimeType: "audio/opus", - channels: 2, - }], - headerExtensions: res.ext?.map(x => ({ - id: x.value, - uri: x.uri, - })) - }, - paused: false, - }); + if (!test_hasMadeProducer) { + const producer = await transport.produce({ + kind: "audio", + rtpParameters: { + mid: "audio", + codecs: [{ + clockRate: 48000, + payloadType: 111, + mimeType: "audio/opus", + channels: 2, + }], + headerExtensions: res.ext?.map(x => ({ + id: x.value, + uri: x.uri, + })) + }, + paused: false, + }); + + const consumer = await transport.consume({ + producerId: producer.id, + paused: false, + rtpCapabilities, + }) + + test_hasMadeProducer = true; + } socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, From 407af95d94b5efb43c6a1f0849d92098b1cb8090 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Thu, 17 Feb 2022 18:54:16 +1100 Subject: [PATCH 06/10] Discord.js gateway connection when connecting to voice will close 4002 decode error due to not sending self_video field. temp solution: remove it from server. this commit will probably be reverted later; I'm just trying to see if a proper ( not self signed ) SSL cert on slowcord.maddy.k.vu will fix this SSL error I'm receiving. --- bundle/package-lock.json | Bin 576191 -> 589262 bytes bundle/package.json | 1 + gateway/src/schema/VoiceStateUpdateSchema.ts | 6 +-- webrtc/.vscode/launch.json | 4 +- webrtc/src/Server.ts | 14 ++++-- webrtc/src/opcodes/Identify.ts | 45 ++++++++++++++++--- webrtc/src/opcodes/index.ts | 6 +-- 7 files changed, 59 insertions(+), 17 deletions(-) diff --git a/bundle/package-lock.json b/bundle/package-lock.json index 898c041c76670ef3d9676afe6c64b3b6ac5f60e9..9464d164300482e180a4b3eb3e45b12b157e2b16 100644 GIT binary patch delta 17728 zcmcJ1XLuFW*8eko(i^0Z-smBrh;%~l5PFA1dIxw%n7VDvDZ@@pBghV6nyawwBDPb$igql#&U5JEZgOpW9)@l{TwNm{dI#mic@>U;V zqam+{5L!HMOTZ-O!$DM>SC?Ecu5QTYC_61egYIlw4etr+En z5ynr&8E!+6ncWuEOgzck0}S6i?m~OKrUE>jge?BGNvMKH>jMQlm0%dy7+EhLrKA{- z5BfpG6y=7|VBa$*rN+?he$Ysg0-!8Gh&7(u03z?`tF<*ktj`#Q*bj@L(Zprn1+S8HgUOgd3v|8*F~D z|B(uUoq%K*I$QCCZH)yFs4+)b0>5q&f|*{vdmxz-vcY0^`CMf_ZrKeI=P4`sYyTsf zabs@}q_Mk^dQvyKeUtB`(Y+X&?P_^W^DcTCz~FbLn{^EIJ<_=F3vmup~*9| zy1etXi)UTfVq3Z-AB`w?nG$6@yOM9zy0p!0J^H;k`SM`*DMgo+5M$|$)zCaoDdML< zw`a?(TcpMpAG+4vea`6*l_+?=NY#s%-e~Fmaa{``(s<`@FXPO$Jmc!EkNE+bzMwj! zHWmgzUJJn&=KUkoft+@NXK~gKZe+D%OvL=?3CVAvoE~gZf{ci7qa>cSwLo`kPrt2Tl;215-htm)z+<*RRRNdU0Humxci5&&U4+qc z85H>np3rp{DpOjKQs6?v2KwIgOU|`+cnWKozzDj~g`y=pwo}dP=uuDj3N}%(_p4e9 z=$+%yDb>T^ym^l@gzuDho+;ZEBkj6MN%4X;bcMeHa0M%rIyW!at8B9|h>T!}O(H`p z2oqdSf-$}k9`1LrM0sB4B~&DHLWFTI8M^@vh6zP(+~gcmba;LP1AWp#w|JyM`6R`M zghvQl?W>uh)d?5MAu{;xGBvz>N-c#q~g3AL+`Z1V`%U-@-lm+T-hkNQqRcVi^Psn zf=O}}Ayojst>^-r^8US@Pyw!lp*vbqO|aPsJJYAP6kjN-&RI!-wRaV7SemSRliV6Y z2UjKE06UU|t!~8JY>%;U&?$#t>82P4y3t$*%DjbfkXMSQXL0AGYWZp9LpMduD^p~+ zURN+PHRn)#zs{j)wT)R9x&WF~xFx_ZItYQ_bJp$?E$;t0>(G7_EJ2?*(|$M($4$_2 z`vh3Lo%@&a8Em^G2OFtZ+rX-yludA|t?tj0LP}Wn9VMGw>ngP5#zWK5>QFvJ@h20y z3qH0j2!3hJ2J!3-@_Gx~t*nhc>M)>>@B!bPS_4f53StB=(zLH23bwJS{W0C>`dC?+ z-(OIvM0jQdvHgW_ElVTX03nF)NNNodvIH)Enm*_6mLj2G5>CD{*z^F9^^ceUseg(A z@Z49$MD#aNfg6IFmLI07P%;QzVal&!dp@gZxFEy&p+dg38{|F~6w)nSsGx8TavlYc zv`69~iBGvd8hsbd@7X|j`a~Q9U1VuI|EhYa>)Fv#GNRyumkLlmUC4y~;~n&|_5Cvi zGc_g)CkPj0C-Y7DI{s790gj)IieCPypg_rFVGi9ifpniDOtEq}bF}`t3r$FT4ERS$ z(XjSY!3Xjxsg=O5q8bm;puZr3ebWSFy z6Ff-tTw!`8#wVnKEW^(Ch3}ZG$*NUCsASWeq;=>syk1e4yT5yTt>E@p1Y=g{Bg{8*|}9?NyetBvFancFM+w ze(-=1oiWugiBrk&(0V&E~$2{6qsL8jO~-ws$-q;?OKg&(0X$BDU;gNt6e*-e~qraCTETB zRIkDKdL1Tooltw+u$EaVV|tGt-)TVqcAY1VnAopP-*zpMJ9M3tIW1#Cr(s$B$Mvq4 z-Ev0HiM`u)?%ujhRtxyHr`!a*mZ*`!D^P8T8VJ=y>NLvLwP~{CN8Om;HUhwla zty)~x@DWKF!-r>0NXp0_gJe!{d687FR$YiaBSjiFJIH-rtvk6*!y0vaXLRo0zinpg z$!!NrnAobrgqdkGM%V2!vBR_h-5L&?_DX{p9VWIIFuGgUNs}^Oo78_s{}FY2P3!U=da!&Zc>HN^~^%bL`^LK(LNjonz_Ouz!jZ8Iy z9J($vwp>Lxq>L3CL0*BVLwT4~mCU;-wBpSMa^VM|kLG43aA&I=0p*ue1s+UOXOZjo zg*9w=B0)#d&5QuU=ctt-?6z!ku$Wi3$L~*+eN4KcUNu?U% zPcQ5koFu;BwT>r=(^Y3x8$f#06BpW@iK9>shk~O*3`wdlPV?ffh}OA~w@mT_|7og1 zPPGvCn5S2=L)T#5A+;_kYb{nY<0@0U<_mF7Br*s^7vc;z}E7NnORiZBD0IH*prfHN^emnMLop`OVmxlqmQ_mpWD$s;-|9Ha9Gwrb`GTS zb2Q>2A#JdD*m+JoMjVrFd#X0_XO0n9JJ$&(XmqJ6 zSF?(@v`|=+Ep{T(MDab-GOx>GMY4U8s92PTlT%YfQF5MjIHYAual~u7i0uVu{Xr9q zNEm-d-EK77XXZ3N%@oaU7EO*Y4VIe2!rMY^5q)7O@hbG(ii2XHzhfp(YFS zj*F-HDQBDz?@N~GsdYw@g{Q?rn}>y_-->IjG3b*tcs^fv4F;bT2U!Y5&YTtJ+I%PT zUF)ylcBkj6KCu40XnHuB9QOTA?7`el0y%I@Q+Py(s>z1W5S2{E%J&-`&%G~Z0o z+QHDtX5NbHRW@=zVTrPtOXI80EU5q;0eM%^88~v|ru3Z$Z0&&Ug264M2-tB=%&^HV z{<;{(T_8_SAKeo}NahW(60`V3t~8D6Yjq)Kyv($w3;gphF&r{)qZea@hyNh-;UY9v))<=!89K|gp&rq1UrEy=Zi?*A7Zlm z7lL2?J(T&X?crG?Es*T&qG8js;DQ`Yi;XUDT#yR6baO<>p%#p9TAq-nk~GXDkzNCk zWs3B>`HYmXY6d$YTd)uqQh>SlvOmR<@TxAcO2aIfP}w!|M3>rFT5EDu=qd5u2tRz3 z)>9Wzc7ZG25-)@Kh$307LazHrv39AkngUtv0FNW3R9k)3Btz&yn)pjAM1EeZ`%h~e zpi@ZZAT3Zbk|&tNqy(bD-3^s|T@`M<0FEzKwcwVq+;-KvW6tfWB%*9G2-k#@0TfSgSEEv{)l#bWoF>5W6wX5rYihZaem#Yv5< zlZb@XW3^>a@*B#3aH6!^Mv+T$l?1CQJB042suCat6%n3{*CxQs@eak9Ta|aVUtv5n zn4qt#ws9F_aZy1OVw>L5FXdSEm&IE1awa*%hoCbj!_{w8U57L zBKZ{vOOjSQL#a0*6SXi2(n6(b#@uf!!MNIz2WeDWiWK0%epF;D<2+ag<;0eLdzAbh z%t}O8k*4V~_>9Jcp-DYy3itDL202hq`p6#lBfh}Wd*^gbBYE|uZ~?B(*ZrXDKj=2o za?xpbt$}rM!C@&7qHjtbP}RWXq+tVTkki?RksS@B>K-=P=CzTcNI*+zxMYPFf-kYh zjt7?j9ok9vZC*34BkI`b#ads>p98+r$=B_rS&D67->y;ygl|D-eX6SjwkSbnbeC!= zHVRlMxzR`ZRfji4^xD{kCfJ4kP#}jgBn+SSmvaK0`}&eev!n%H)-!l7mPR=_ZAyw3 zOKE;iihSYGaw!DLi^Uky6{Hnbs?n50tdi!M5gW6uzic@bIanaQVipaoC(>7pfm(&q zMlPJlW^y=rUMTH1XP15?fzCdW=|xf#`^g>KB|Ws5CS|*&1W&+RWdUG9;z8CNkXGri za^j2USAGDS26#ys2N50QBS0U?rEjFD(3g%*`4MxI@js)Mmw0PYR!jRMh)9 z17CYceJ=T|ydrP6w91w;1&VS{u3uz^B#-d4b|t_1$Rl*yb5^i?@`ds#ElS=EkJ{kH zD^_};u>P*1k?3f7m+78px8N!f@me-Tr#%I(Fc=>a&y8&1^LFLL9- z_7zL4U~{~@(!EaUo*=V|DgmC<)x#ksQT~jp8Z&G((@^qPqMTz3y<~YcxxTqw=LUKN z+*p83qr8Dw)m*D8YtXQU{JzTpL=umh@;x&V)9T3O{Hs+myjoYTquI`)><2XT+(z=x zmW=b;UpO;#p>i}C)kl6$=6dfpP`+$SY3cWPsFIG-bCd(Dw8H{>5SBS#R@MXI&}({U z=yg<#hnzujJozX?ZtBMtbNFW!)=kh8A!VX`!KN7zljQTB)RR^Qj|?n7yUoz!ohKQ` zGtV<~+pZPi{#zJ&zFVjx zN`R%?V}B7nVNkvtLDnvlbMdmz5jhCiLxdxhXF~ zNzFoeV-TOm^ytQZ`E^L$CkMk{>-G2e2TUp=N$6gAufY5@{J2kkWT_s(DfMj3(p{j_ z7xEr@cbEX9H|g0

y-Wk0EgRfXuG9Oe6-K^w~k2Kd@QvPnwj- z>8dmJKrXR^Nw9;7cMHyWkn*DtMS2{gM6m#cb#@rQ4&4Vzj>~7^Yub5t_9Nlr6Y^$D zd*SO8sS^|>iotO7q&%4Od%Q@GWlo~wBB?U=4fPq;RXxO&%3tz*{wkGES=t9zPs=Qy zO(3~vNOW(`?va8jz%W|ImdDAtyjgO5N8v5Y92YB0uJ-~|46k}Gd;0yUo=vW0lzw$0rESjp*M5!#(wB5++ur%V%9^BZBb6#h34o?|l$f(gkk*jMrcD#^8#0R}yUe_EjCIKLW2R*)cju)d@mk9B1p; z&&No=SDRkk)nZ^6f?$|wO!L{UJ|ty~BADvHrWBpSJ1L5OA&7p#0Pz3D>5JeOiv!LX zCBfhD-^K1Vt&ctgF)^6DyNZF{=oPxZbHHD0;-a_~Cjcha5+i-jZV=hkm3f#{m@fgs z5WHI4XqSL>809zJ0WqF^)e3Mw6Hn#FS?pPuU>Ms5OxiA}7$_T#kOsUU3@}m;xO30X z1Q9Z+FQcFqWY1KHs~j%b&fKE3Xm1F-K3iF0L>*Ou5$QDtFM6?OQHhc3#K$;D{alHI zp`-BH)PJbblad%|#H^K}+B|-l=op52X{ZHSh+d*>=4a}77m4?(*^qrk34y$&cx;wc;|aH*=#D#1C1`49)mi#QZ77pKw4w1>!d^qF8I>*9bc$ugYvc8d)Ttk{aq~V{79+tU$8gUIoN1>ITXiY z&s@;~#(DV~Hf)aD%u`Oav+e1E(0SR4E`Xhv4S=+(N+jZ_80KbW3dcoQTQf1ng9FiH ztzdrmFPI*;aS>3q1Cb)7SJCmh!+Tf=VXLuG3;|-sLyR10Hie`y#T-cSWKKNK3!Aa8 z4rq_X3k$ylW1lKO^G}syu1a8E*#-DxEnpw=atx433Yx|Y$RivgYG;s9=D_`ME-B^* z5cAWR+D$BUYJ-mL={}C(cwy>;{mKT5Y*O&KQqhXncJdtfpO+K=A{5AyA&bi<|b911f|fi)m%|Q76BX^L=YQAJFYr zhL~FzFb*Yfl#aEu#d(+$5X3p6QGp?X01-hRT*POp;0+=|gI+e@6JA6Ky?mStV2BL~ zv`|BUs3Bjdc|tL-^c+FjBGl6fB6|7qE_fo&%brpNo5K#t*_U!)h^$kze|3Qqy?i$Z zvdCP(9t;`C=*CO8bwG;vEP}IwP5PWch~kNU^n=GgD!VMbrG^|s5Lz@=plBilBIBgMfd7c~!9Faa;H51hlk zWyy9;HU@2rfS>vb{dum}@UgO$;geVnh)=92_Js8Q93K)(c9ttweVt+nBx|25Sot|n zC!Q;>s0_kr5>Z}3j|Bg_*b8<*lNbUy9`FML9`%qQRaHNCTBlp7E?nyoWqai z_+SP-pe@3tbH}=v6y1t^=cTr^9Ru5jUXSzy|L&Scz89jRR`{s4l8Pddf+Q;AU?{ky z7E(g|c>xv{sBW?th4OZoYWAKY^y0Og+u%9Dss``)sy_J@REC?k1i&u`UV>3N?_52I z63X#N1o23kodu?rIU@vY?fiE@lD7yqA85LbdPY#IIUJhC3@FSEfdq&^nhI&7u-jQR zT#dyR1jVGGxC3EeZS{onF$fTlRF$OHQ4Ll*heOBu*x+^mlFUa37KuHsb}EEF41A(^ zlIQhRyu)^E!0KlXMj{ACinAO9tM$Ao_9zy@H(X7idtL4Oo@!^U`G4(DRd zMGR6P`${S`QTLh%2ILB{SK$)7S@PwdJ=>}8vmR4SetYD{0ZU>#F3Qz(BC>2m)zOb!#SG7Q+))qs)=&jhZ2$Uyei)?rZV?Ybc~AawwUZuFf)hK8QG4Lp~U$&aoU8hF6L>augJSQ2CtD z2)sukl+oeSJjnKuYNXlNW9D%sQ~lQJ)R2-1>J;;Sm3}5|C#thmYr2#%X%zU$=$-CQ zQw!X@nhHC7)l_&~fVX;2*W*7iIjRhI9;5QyJb=M}{cN?^X*Yp@5G1#nA%-C6sJFXq zVbSdeh%uTBd)~m?^QJ!{x_oX9NAH=Yh@GeyIAvzB4dol0?!QePV zv(UdtuTAPA-UueQKU6~LKc4;2wqEsV}V*jK0AS7z_ z=jhYC&SIo;$`3v~i$mXT)(S6;1rN^kJR3FS`pzBLHhinrd!T55D@FrV9y_FU5%cLKZW}ELE3W7 zgLHbPeq-J)w){#DBuS#yNCe+?4n5oyi03{29AZC@&%oQe&txrHwuvmI2op8`O=1q2 z>!D@ZcSHctVb0$-e%c&At`CUwsmHC_h}H&ho_~+lzOspo0)8wr3nT@JTBZmIaatu3 zQdO(Y|AbD}wUz88+^en)WD5a;c!t6!HMHmagI7(>+ZDfKCehUFjns=CfJmP4@Hk1^ z$+f7Ys#XCGT@Z)yT&PB}w!wCEtV?|}S-WUcxgo!y5=4m6kSD3&&gZrbUMRKwT9HySR9XopnBSE=N>FAQh?%m0C7F9(j?9o#=L^0IlKrP z5($X%x#LCwX_u}YK_z=fkA~pav~`yLfL~`#fzKOiCX^9%mG4BsJkcUX1iY(fV?Jo2 zSq{q_gVbPw&-(d@Z&6^NiW#q^tP3Ivxi*_h}mV zwAK!|Gea>xG+!RT`SomvN0~AJv%6(&v=?$qC-w&r`x6QOwADUYpqL$|*jdVjftw$Ys`%;Ggt+);aK3l*$mZp>Nh|9?aW z-LPh0X4XwJ@7-CfM4?1mpMz+z6bc9lDu+uww9PM}0n3)2HWs~MZZGX4W@^yW1JRlf zs^TSb!B{lR8=jKAfy(Lw6m7A6JNgnR1_%%X6vpL~-0ua&VDKIae)_M7ptl6O+6+W# zva}S}x3EE9OYVB|e^Jp=sQMtA_xJZ@zCLUAuCf;;osxj1;k1h4*DSo`oW6Yu^&r+ z0|3E3&)t}ZPFdQQ{NO^fQC^N2fzTMRpN|k9tJxI@7$|)xrspek*VLllo{Ab;(i5|W zov&-}T0WU{oTjzp21eFR*G3AE+dxv`QVTTk4FoSpA2*O@k?k|JEiVnlIB3rOV>n@LcU*Jk*dLh87;qr@p>J%l6<79DnUDL$EJ~$j@axAb%wyPtq1k z@tkz}kqp_ZE%suoi4(Hma3RJf3ZDUl&kTiGpJ15(AQvw|&H#F)r7>9Q@A?wu>WuKD z_g7jMTe?P58b3I&2Av(Inx5psG3|*-?8TGXMthEoX{SFlE0h=zlm(JNS`UYlZl|@+ z?BN#yH=(4~w^~2*7f#g1;r)x+KJEe$bQ1!NiHu!{@A1@j?nbGzgs)yDjJ_$Cw3F4i z9A!y=%K!3|b>I@*D(ih1VmA&D^%4&^=ixfde09r<2Lgg%LU>8*>8GEwB61KxORQqk zWKttoZ*4zf#J$vp2tSnC=_p+#heP!DIc9^r7Nf_QY3i5N@OYj##@dHBA{~X%O(BJS5JnNnsOy)P&mod0=9K^KIOKdvza~u6_6Ku_3Ei7pd z?xGJ-L(r1KDco0-hTrX$M0GUqL?>1AFvX)KwbUt&CqQ!Ii~Il~(ZQHvNn zhDCKmp(G)B)J^s{AXkR0FXTGlb4VV-i@Ls>^|MT!u)|{hD6R)Qk3o5$F2uo_Z_uzG zZO2U=lJSK}pXh%%B@sw#x1ViaXYJHCaSwxlk~p4Y4=U0Ja#xIik`hi#uM|XRHRy@? zPg*#pK8s3aGefwkX+iLJsT>DmcI&(hf-m=I_$#<#eTlU%IZ&)`G+i5;B7ZM0B7f}H a&7Y2tj$h~who5y&U&qgY1RvM82>%1K4BHw2 delta 14236 zcmaJ|d3a4%*MIhzZ<3oyMwxC5F%v_Tm@ZYNl&ID$qODQ{ZBcWgv_+K5A&W{R=9yH~ zZevK4qDrKO(h^c?s1hmqwra??_TG00ulM`MlY7qIYp;2&y?#g0!|DY~s^>P9^7>2L zg3YExcOTKVpfDvGpFIQp8 z8zj<B%@XQ+M6oG zW?tRRgNd0nfgR}!J*KO{pwCht=M`;@ATGIlq0xFR%sjt63;b=E zrN(UagJmD0)f{z6EX;AJgO@BvBd6_a3@eVKPoy408SHygnX;Ku+M2Bo`NLs<5(>!^ zRX?+EcOP^6zBO*~Tne7eWHhH2u6CMvlsW2DEvUMrl62WK)Ch>oz$d8Uyy62v1O}`@ z7Lix8?RRUSn<?=xI_ zS(G0MUYTlpu)V`+*TDrLX3^0cRBRAzFC&4_u7=?S$!{n~V|0XB{KFZf83cuucns@rgmV|Ku zxiYaq;w|-6X!sfl23xM`1@?c`NEj4C-Zm3XUt>0Qud14tdM1-^%(I_eOSt{*XB+3W z`gtP(-EFD*gO}@P7TQxdw@#R;aecrt@i8Avmyv-A(4)f7(8@P_05m*<>+mrra?@WAQ z-j`}09#_dFBh19>Te$OZEg<<%rr)27 zTyif2yCGT3#NChExUSHB|!4WA|oy^x0*P>YbN;-=Ajzp|iqp0*orNKmhdL zpjLrgLZacjy&m=#ntml=ux_8a$R$vwuAX>ZYqan{BfVRqYow5+nLP2K96P^_WzwD&Ef(k$GNbJ!MkUbI81-9!2m~D`C_l z%xn$QF_R?UiH58nJTf{Lq=RatK8Ktt#0#Djp_h~&S68{Dbne0DtDku^ z-H&836;Vu7vm}qQkkyrN?6K+(`Bex{H*Q<5gf3l)AKa`>LLfN-OJBxmHJJ&CKF59O zyqWl`#yIhY)=!XvS13|&$1A6t`;)rW#XVN326RC<8SU0LXLl7u{VXIvx>i<;g6Z){ zGVM z3!#1uh`mxW;~Um>Cqd9EhBSdGjWEL9*^Ix;BAPE0$Kxp_i9s;P(0pLf3yKfTZcMrm zbN+yF8X3U8qqpEiHdZFT)KWXkWT3baii z+0Sr?&vO`M)KO!z)z7a1#!f^f9ga%Gay0EvXMXeOXzE?`b*EEOOmLIA-)tERgaJaqw0XiU&$jkTw}zuk?AESO9-Gl?Uhd>W|@{?jmbZJC9xnmkUAg=16DRhdg3ZmRWVTf4&8rZQ@bNr!8{ za&P_K!zL34{4H_}OuJ}AQ+q0DBZ2|>(ZF}O9tB4m%hjOmr+9+nU#SX>m_Y)>5|q6< zb+piO3F~~q!7&K~#&hK7Gmz!QvnsJ%2h2~+CM7`dNNosoZ!QIaZ5z><)-JneiCC5C$LAf+fmX$7Qf4ji9y#NCudX- z<)70uo0}e-KSjRq?1j%B8v)>};C-cZhBh6hpOLK4>kRpd7MvsKxa?f|&v`Oj;*Bag z7kLD?t17{;;u5(cLT}%8C6*p9B`negXfaB;FmuL<-I%D zCn+(I)>7^YHy$X_5TB%B9=xSR!h4Igesu3`6q`i>{k)v?)|oeiP|H80h2rvg4ce{L z?n3buB!eFNx|z5>5MIp3_8~qTucX!rRJ-*FHVe!f9YZAzEeD;#@kP9#&Ke!RXa)_dBdQa=rG$xW7~J@iWUfCyGefZ zMh^)jEMLb9Bjff!$pE>Y9s=2|jA#fRh-PKix-SMw z4+Yb^2T3b9LxbDPR=99S>qv`|q%Hi&5So)LnJ9gd976lOCD|l*60Qb^-a*xl?kWdS zbhDj`ry$XJBbZ9qECgDOk`AD8DrHwXCPiw-O$?=vwP1Q-tmNff%&Hk}_<__*6QR0( zqO@Eb=!%*MIPsTcr8_4{4_K%N!Dl~Wp&Mk;-=gi(q&WmOEyWDkwoGb6>wYX9A>1#@ zPRTK}^(^U%#4Hv8gZj(W=%~*myhE}+1xrN~eRYmh;N*gzl{-$t2CFR=k-Z^GW4b0? z8sTJWuqmmusC?n^ zd$JE4T#NY&CALE0&r%V8{-C>F1k}J zFG~aA;7{mdHSUwDkk&v~XzCRVW1)WDW;kx)ar5J`N3~ydhkO9E?xT-lvnA1s@I{!< zmvfa&zb*|RV4X{L0=@`?ihlYM$m)V&nmtSPg&+FsuRv;hsRer*XZG0kg|nx+C4J@W zsqi^uO(t1_kJ3w04aoi-19|o!J(KIG;~gp7UGh~ZAEt-W73ES@2^kQ!h4f^-DU|k> zJWhx{@V68J+wMuMSqq}0?n`}~uI=))M7CtNNDv1bpp%XKN6JJ+bDAn(t5Xw_8)-El zEl!T+{T3~JAT4pWi>w<$RuTz4@mRXW`D4A7kHUHOSL8GTTQ6c;(5{W9(%q_zjU~JK zGGAPhB!}|e4JyXyZ?ZmX82I0nRQT6|manLW$;#W1#RZnNU5u9+Q066n0%;?#@|M1@ z2h;f$InYf~ETOohFPE~iBKBD2Y?oLLmin&K5E3e(UsZvijw(-)~&f+(efj=;89RK2kl&5 z3l$epTMmMTRtwHC{){jcGu(lk`h;mQKCa|=g zyyNM_l*pb1!-M*Aj@xIBFoGF}v7ZZ{iDmp^Lm6ib?2h7pH-K(+=#mT-11dG1$6&!c zk}24`-K>&cD7N#dV#2~41RiDRpF?Vh{4UID@hl^^UB=|?Z!1V0PF|yU6To=|9?iI+ zS*>N>B?p;>#a0-*!tjO)A0<|_Ca%O+ESj$shcSuY$kaa;?N5&*iVu_>mGGu)mzTrw zqf#(UZZFSQ+(J`!t$*;=Q z2sAj4p|E_NULA6Vpz)`k#E`;gjp1Hbc_2&L;Slp13Vx(J8uD==I#mDl`V@2dgb@1b zYjR&>$vnIy@_HZN2gBo~?fMe)d}0XA>Mie(L?L2tDfE+PW8nRa)$&oE9tr~n$fraV zd}E-TWah<%K(|4%K}!e9Q&drdX1pa2F}p?u!`^&70{clTZS=N0TH-J1fuV9kRX75t z-*TjU-+;24x(4qb(PP9!6(?=9|2TQt|9Y{Oz_wI55VlU1b*McN>*-I^<<0(ZB|{FS z8y3hjz1g1;(0jT3HiSQr+d%v(OfTMR`KvH^GwLMq651 zPkC3g^02*$;sq0$Dr}a=hc6b|wVCpzNDJ&<_QbCH!sfvSyTMuM8_ktOo_E-s#saw1 zhnBpkY;yLaMG1;GeBDx6!ugx|wt=EP#v;8eLD>L(Qxz-4m|H6I_K-%jQ(h3XEW?dR zHc}`Nn&n;BSM8OXqNDmD#lS-SfrRa+t1pI>Uf2zFZHG6>cAb@Rp5_#=ISrw57o~|V zy4~pUNSZrc+627aeEkno)Kt9Pq>VS;7J_mi;AKix>i>cAvLY0MTMl@V29&?AD0IO` z%CCNmf-7fgFgV+&38@!T9SE zAqB1=9~>#zfgujoz_YVy5|d}iX!sV`W;43$h}M=p>#g}({KeQ;5pgKRwj2O zw0nftgqb)U9*$G5VzO1Edx!w=p&1HS9)@ z8C}RegL8p~rxn&<1i^s=O|u60cF=aOqd&k_f%uE-UMJY;C)u?1OqqnCbgnG1wr$3j2Oj)?%fI0>@S5 zTkcA=c5A_qeodjm6}MegbQp78S;{+_dDo0YsF+PO26}mdf#%;(e&jtn-CnL-m7tRq z{Wkxe;&57Y?0=M(txOJDDyuWzhlkw|yKu}_Ifnc#44R;B5YG}V8pnq-Bq;SUnnBD( zJrZ(`8`GdQ%K{a1am$6tD)zpNLg8YZR=AMhTacb0o;AgJ`aXF0k~qU^`mKEpaxI!a z_|3%aIQ9&IxOr+5$e6AU7R&HhgCZi_g}}(Vu5Hi9e<_ZbZA6@f5?E>B$0~7dv){uG zoS{_{}RC$IbvY%Lbnu6*>K-~(T;`uc#3ozI*&jRo4P^Xg&q+Z%)6t~PXuxq z(pz~2PWBnt>pX?=`W(2IBHX*m$v!*F^#AP{Y_}hfIS2%%;yA*I>rShrTU651;4Z{Z zSEJb+KEN#8C*)z4wsI7mB8qOu3zXA%f4Bu`201fue2>7cb4ziCdWIjoDBlYPbilde zB~a4^E3FT$6lN;@S*-?_($qi(?y`fU%)~4O9xp>G-25mJ$9UPhF!sHIH#AtzUON4c z&!Z9OeF#&};)LpAoS{M8IKzv!sbRcCz_CJI>wdEcF4QrGL%&AGdGOCfXFRbJF_x(- zRTFxyRYRf8UuY=?OhRIB5@T*TiN3{-iNB?ecoxHLa~^@*#FtR23msSnOGgy`$z(Os zEI2(_=md8qpw5$_ZF8J%7q7+xVqU_O&)`N5yPNqxIS)XuL*V%<^+PCFr>+y*UC*Kr zFc=GkWw_nGLEQ|kFXPmu_?#MNrjfQ#Eud0Yeh1%rWMz*e;;V!-Rfa~5^%JB&&w zFhvwN799j*m*Tei;!mc-cg@6`?HPME2I>ZyE&p(uk%>xCeB8zp0CP6{cH_8^nk)GU zRW}KtaPcItAH@{{-*vd9u|ctn_OR!x)XlZ$S@ct2?+mxnV&T<2Y7uN{i-tyi@&3D4 zJ?Pe-bN|?V>Y+-{CKce%dE;6cLc5Q_Tn3%6pv919P-qJu`A!~De^{h6(iKdV9@ifJ*}84S;?L8RE@ z9hF6R3NNmd9~nFx&XC*TaH2&0&coXO!i{eTDOnIwgooEv3Q9LEjsiyl*ERycPJ5wZ zngb@rI2Y!NY&gBo!)Z7JeVzQV6v+W!K)}}`RJ6STSpj>3Dn+C7oTu>L!B6|2bBPND z=HvsxHpH-iqcLW!`oF6Oo@;o-f}MTKGkSI*!a@X!oW?)jAP56{lB7pW9L+ny;H>Ae zif>m4?Xc{}l0QrI92vg!T<1V6Skz348$z7CxbrWAg0bu`P_!?(oEdAh`V^tyB+(d9 ze_e~hDH+BD#XUZr?RBBRq5FgXTD1!QHVy-&A?sG+0l(8i$kiWlQiO;b!)kw)1~AMmo&3YRiPGnFqPhVnDRG8a!#KTVSoPwnFo=Sfohhg zq^Y)skqD%S3R{SH_@8E)(~sPodN5$HHP=c69Xz!zGQi?WSU-hCFu-?+B-+WzLFPs#i-P-g*FU^!7wYRp8{SyWr zK$ri#wT{fEEBk6&om`-qp9OL$0=d{xySjOf-tHFq8QWiL4*r9*uRU++<3ZYwUT}Vj z)`G*Y6yewD1on;4b_qqh1gQh%NtzBtBXDZ4;8Uysw!P>hxG$WEDG*nRi0dke4jH3? zL5+!8pzL-Ms~$nSPSJ*{Vhcj4`{||`+CV~Qexgm0sPAm8ngsT_`27Sk4rE;-3Gi~d z2J}|C7Kb-Wrmztv;?xL+eh1C%dJwI*SR2RVDi9FcY)Svk&|cE$YgyX&vd2j%f}7j8 zwgSO#wN}K)ILC5|uX1(@}Cnt005fFNegIyc(Mijpe z-K=nfhS|hb@z|2u?Cl=m2+TrH1CphiG{PASgu}9CrT*Kr3V$IW!zH1>qK80Pfp!am zI%`20`!(P%DFT$BL-P0NVtz-pjr7P-Z9bvz9@D;5q1H)l9W6PjO;_p2bDFO^IAh~* z^iuYcl+H<*-HLo6tE1#YlYY_qTbLIFLifv>AN~D`)*n^Llxw0tGjO zmj-f-5D+7LMeuq5cNCt*Fa7$ahS$!d+ZbM)F5-a^(5_6|B0?kJj)o{I%N{50XuF6- zBq58X8YUa*7!=LHqsLTW{LUVNuntdFtM6)WQsbUhf>C;3+X|1~#5HI`4IX|i`_P~M z(i+J$_<{B+fieGTYhAvU+zdYqNK26%wCJI>1I^0(>F`*z3Qs!-5v~@ZZG58P*Hrst zeFYt*=<5|`=Rl7CQN;i7d*g&Ph9B6jkRZC%OK(Q#pWb>iiN4^gk9NkFI|~7#eOq9G zzrI6|9u}bAQe6x!*pFCeYN*~pI4ivpu1{6PVTkax5P9L-7`?6zzwFk-XizQvuuT7` ztG_AH#0GkbLXS7rLs4pG9z^wGU~eZZt_@r0%jnq_`lo(E;i;W_6 z%$p)6hYDc|qLcdSn1vs=)`OryJIn@8+F}sYuZurZ2VsdR@29U4I^WyHFkrfY95EET3B-eu}Qbroku+pE7uqk8s>| z;e<+4@UR_jfd$Na4ej8%|Xn~ zTl((`)m9c9m)&FKG6f5DZADJ1#OGZ_&V+hxFg+p`7Ygx*vB%={y~pTF1$~zQ4aIj= zh|L+?TlvX#p2pt}yIJBd=6X(|=^yGn+yVc7Ei9Wytn#zfn4vZ}Tpq7KcIG*r_YjE- z#!nX9vd2w!Itu(gS-&l;=#tPX5JZN>LD#AJF`-c`g93}Ah{eru(sl_J%1*^^Kz5*d zxa{3h(d9;#86nWDCf=k!or+FlAFW5j3z2d^*q5gB>@TW2#U|pYQy(6J&!BI%* zF_YQXZ81dNHdhwv?a%b(%vesR!q*-oZo2&c_|1j+`U8e5@8P_a^rf`=&x-fb->{?NL)Y=Lh3Fk-lBY#Dmy(|#RB zA7|*TogE4~a*ohAQmqcgP`ChVzp@lm%3w2cJJv6=xh-Q2vg8YJJ?4T=j<60fMxUqqE*-cH19NNFB zG1yr{5gj@t5+*`Hp(3GQX=9vHEA^%RsfeC2Xb2D*>ME+I>;F4gU`!_Z=iIrHH#j=t zP@7eL8uzl%kw;V@MWpDQ;n4;lG_;r3Oxi8c=&8}Iy^VJ@_;4U9x9mkFgw44e;4K+J zyONA?;u^xnhyo3whz9lIkrhbsnl=|d2zkp`&y@lQ`OJa0H(__x`uUy42x`rI) zhF_*bdI?4bhh8$LViy_H4DV2^Pp8X2G5%5nJ;Yw>^4e`)I~EF)ql{R%)X)fmw&_Mc zp62P^bmNI=Uns&cn`KzK$k<4Gn#RxGLac7t_@}$&7&ip|Off=fpB!VEh23`p;lx5i zh0Ngy3&(FlAc7SbiU><@snV_#Xgm$0ci2*8y6}*POYzHr9@~sB;K4RygiAulG6_Gn z+iv_Vv8ziUUAMzf@&9W@;&Ak0yx|K!<{2RNH~7T}pv!j}{hb#@yle$Z(E4j*gKz+^ zZ;U%$)N;_MaJm74DSf@2=a@wp3<-spU-c788q*a1uL!(#P()HN&>q!6zrK3P*e`1A zvD3!aE(WuQ7`5QiS)&Q0oK})VFd&rDfI#k9;~P5ZoN-?D&=a>91WI0qm{;)HW^bZZ z1KTBI66X;LE*bv{9lLN7oRc%0)Nusi@aSKSb)pjA{MB$ahPjuGPvLhzRC?blMj_Px z&Dbo|;lfUQA?&J=S;^b|zCiT;kE=#1_dkj-$|}(f(Zt(EF`@ByjGC2l1t4CsXxClC U#|!p9GD5*yQGp(PWGLkS02YkeZU6uP diff --git a/bundle/package.json b/bundle/package.json index 8915665d5..4dec60ee3 100644 --- a/bundle/package.json +++ b/bundle/package.json @@ -64,6 +64,7 @@ "@fosscord/gateway": "file:../gateway", "@sentry/node": "^6.16.1", "@sentry/tracing": "^6.16.1", + "@yukikaze-bot/erlpack": "^1.0.1", "ajv": "8.6.2", "ajv-formats": "^2.1.1", "amqplib": "^0.8.0", diff --git a/gateway/src/schema/VoiceStateUpdateSchema.ts b/gateway/src/schema/VoiceStateUpdateSchema.ts index 9efa191e7..c046600d9 100644 --- a/gateway/src/schema/VoiceStateUpdateSchema.ts +++ b/gateway/src/schema/VoiceStateUpdateSchema.ts @@ -3,7 +3,7 @@ export const VoiceStateUpdateSchema = { $channel_id: String, self_mute: Boolean, self_deaf: Boolean, - self_video: Boolean, + $self_video: Boolean, //required in docs but bots don't always send it }; export interface VoiceStateUpdateSchema { @@ -11,5 +11,5 @@ export interface VoiceStateUpdateSchema { channel_id?: string; self_mute: boolean; self_deaf: boolean; - self_video: boolean; -} + self_video?: boolean; +} \ No newline at end of file diff --git a/webrtc/.vscode/launch.json b/webrtc/.vscode/launch.json index 92403164c..495841729 100644 --- a/webrtc/.vscode/launch.json +++ b/webrtc/.vscode/launch.json @@ -17,7 +17,9 @@ ], "cwd": "${workspaceRoot}", "protocol": "inspector", - "internalConsoleOptions": "openOnSessionStart" + "internalConsoleOptions": "openOnSessionStart", + "sourceMaps": true, + "resolveSourceMapLocations": null, } ] } \ No newline at end of file diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index dcbf216a1..0145a2219 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -1,7 +1,7 @@ import { Server as WebSocketServer } from "ws"; -import { WebSocket, Payload, CLOSECODES } from "@fosscord/gateway"; +import { WebSocket, CLOSECODES } from "@fosscord/gateway"; import { Config, initDatabase } from "@fosscord/util"; -import OPCodeHandlers from "./opcodes"; +import OPCodeHandlers, { Payload } from "./opcodes"; import { setHeartbeat } from "./util"; import * as mediasoup from "mediasoup"; import { types as MediasoupTypes } from "mediasoup"; @@ -26,8 +26,16 @@ export class Server { socket.on("message", async (message: string) => { const payload: Payload = JSON.parse(message); + console.log(payload); + if (OPCodeHandlers[payload.op]) - await OPCodeHandlers[payload.op].call(this, socket, payload); + try { + await OPCodeHandlers[payload.op].call(this, socket, payload); + } + catch (e) { + console.error(e); + socket.close(CLOSECODES.Unknown_error); + } else { console.error(`Unimplemented`, payload); socket.close(CLOSECODES.Unknown_opcode); diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 82f327be1..e965e3de2 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -1,9 +1,38 @@ -import { WebSocket } from "@fosscord/gateway"; +import { WebSocket, CLOSECODES } from "@fosscord/gateway"; import { Payload } from "./index"; -import { VoiceOPCodes } from "@fosscord/util"; +import { VoiceOPCodes, Session, User, Guild } from "@fosscord/util"; import { Server } from "../Server"; -export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { +export interface IdentifyPayload extends Payload { + d: { + server_id: string, //guild id + session_id: string, //gateway session + streams: Array<{ + type: string, + rid: string, //number + quality: number, + }>, + token: string, //voice_states token + user_id: string, + video: boolean, + }; +} + +export async function onIdentify(this: Server, socket: WebSocket, data: IdentifyPayload) { + + const session = await Session.findOneOrFail( + { session_id: data.d.session_id, }, + { + where: { user_id: data.d.user_id }, + relations: ["user"] + } + ); + const user = session.user; + const guild = await Guild.findOneOrFail({ id: data.d.server_id }); + + if (!guild.members.find(x => x.id === user.id)) + return socket.close(CLOSECODES.Invalid_intent); + var transport = await this.mediasoupRouters[0].createWebRtcTransport({ listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }], enableUdp: true, @@ -40,15 +69,17 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { - streams: [], + streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: 1311886, ssrc: 1311885, active: false, }))], ssrc: 1, ip: transport.iceCandidates[0].ip, port: transport.iceCandidates[0].port, modes: [ "aead_aes256_gcm_rtpsize", - // "xsalsa20_poly1305", - // "xsalsa20_poly1305_suffix", - // "xsalsa20_poly1305_lite", + "aead_aes256_gcm", + "xsalsa20_poly1305_lite_rtpsize", + "xsalsa20_poly1305_lite", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305" ], heartbeat_interval: 1, experiments: [], diff --git a/webrtc/src/opcodes/index.ts b/webrtc/src/opcodes/index.ts index 36d30e7d5..9b1eb270e 100644 --- a/webrtc/src/opcodes/index.ts +++ b/webrtc/src/opcodes/index.ts @@ -3,9 +3,9 @@ import { VoiceOPCodes } from "@fosscord/util"; export interface Payload { op: number; - d?: any; - s?: number; - t?: string; + d: any; + s: number; + t: string; } import { onIdentify } from "./Identify"; From aa8a9eea6b5475d949ee9124a3f47d8166d019fc Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Mon, 7 Mar 2022 19:15:33 +1100 Subject: [PATCH 07/10] augh --- bundle/package.json | 2 +- util/src/util/Constants.ts | 5 +- webrtc/src/Server.ts | 12 +++-- webrtc/src/opcodes/Heartbeat.ts | 4 +- webrtc/src/opcodes/Identify.ts | 37 ++++++++++++--- webrtc/src/opcodes/SelectProtocol.ts | 68 ++++++++++++++++++++++------ webrtc/src/opcodes/Version.ts | 14 ++++++ webrtc/src/opcodes/index.ts | 3 ++ webrtc/src/start.ts | 6 +-- webrtc/src/util/Heartbeat.ts | 21 +++++---- 10 files changed, 133 insertions(+), 39 deletions(-) create mode 100644 webrtc/src/opcodes/Version.ts diff --git a/bundle/package.json b/bundle/package.json index 0b00b3252..aedd963b7 100644 --- a/bundle/package.json +++ b/bundle/package.json @@ -112,4 +112,4 @@ "typescript-json-schema": "^0.50.1", "ws": "^7.4.2" } -} +} \ No newline at end of file diff --git a/util/src/util/Constants.ts b/util/src/util/Constants.ts index d53157674..42a2c2748 100644 --- a/util/src/util/Constants.ts +++ b/util/src/util/Constants.ts @@ -77,8 +77,9 @@ export const VoiceOPCodes = { RESUME: 7, HELLO: 8, RESUMED: 9, - CLIENT_CONNECT: 12, - CLIENT_DISCONNECT: 13, + CLIENT_CONNECT: 12, // incorrect, op 12 is probably used for video + CLIENT_DISCONNECT: 13, // incorrect + VERSION: 16, //not documented }; export const Events = { diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 0145a2219..1d18d6d19 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -6,6 +6,8 @@ import { setHeartbeat } from "./util"; import * as mediasoup from "mediasoup"; import { types as MediasoupTypes } from "mediasoup"; +import Net from "net"; + var port = Number(process.env.PORT); if (isNaN(port)) port = 3004; @@ -13,7 +15,7 @@ export class Server { public ws: WebSocketServer; public mediasoupWorkers: MediasoupTypes.Worker[] = []; public mediasoupRouters: MediasoupTypes.Router[] = []; - public mediasoupTransports: MediasoupTypes.Transport[] = []; + public mediasoupTransports: MediasoupTypes.WebRtcTransport[] = []; constructor() { this.ws = new WebSocketServer({ @@ -26,7 +28,7 @@ export class Server { socket.on("message", async (message: string) => { const payload: Payload = JSON.parse(message); - console.log(payload); + // console.log(payload); if (OPCodeHandlers[payload.op]) try { @@ -68,9 +70,13 @@ export class Server { this.mediasoupRouters.push(router); - router.observer.on("newtransport", async (transport: MediasoupTypes.Transport) => { + router.observer.on("newtransport", async (transport: MediasoupTypes.WebRtcTransport) => { console.log("new transport created [id:%s]", transport.id); + transport.observer.on("sctpstatechange", (state) => { + console.log(state) + }); + await transport.enableTraceEvent(); transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => { diff --git a/webrtc/src/opcodes/Heartbeat.ts b/webrtc/src/opcodes/Heartbeat.ts index 06d6bcb1a..47f33f762 100644 --- a/webrtc/src/opcodes/Heartbeat.ts +++ b/webrtc/src/opcodes/Heartbeat.ts @@ -1,8 +1,8 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; -import { setHeartbeat } from "./../util"; +import { setHeartbeat } from "../util"; import { Server } from "../Server" export async function onHeartbeat(this: Server, socket: WebSocket, data: Payload) { - await setHeartbeat(socket); + await setHeartbeat(socket, data.d); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index e965e3de2..d7da5c7c5 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -28,12 +28,12 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify } ); const user = session.user; - const guild = await Guild.findOneOrFail({ id: data.d.server_id }); + const guild = await Guild.findOneOrFail({ id: data.d.server_id }, { relations: ["members"] }); if (!guild.members.find(x => x.id === user.id)) return socket.close(CLOSECODES.Invalid_intent); - var transport = await this.mediasoupRouters[0].createWebRtcTransport({ + var transport = this.mediasoupTransports[0] || await this.mediasoupRouters[0].createWebRtcTransport({ listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }], enableUdp: true, enableTcp: true, @@ -66,13 +66,39 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify } */ + + + /* + { + "streams": [ + { "type": "video", "ssrc": 129861, "rtx_ssrc": 129862, "rid": "100", "quality": 100, "active": false } + ], + "ssrc": 129860, + "port": 50003, + "modes": [ + "aead_aes256_gcm_rtpsize", + "aead_aes256_gcm", + "xsalsa20_poly1305_lite_rtpsize", + "xsalsa20_poly1305_lite", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305" + ], + "ip": "109.200.213.251", + "experiments": [ + "bwe_conservative_link_estimate", + "bwe_remote_locus_client", + "fixed_keyframe_interval" + ]; + }; + */ + socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { - streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: 1311886, ssrc: 1311885, active: false, }))], - ssrc: 1, + streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: Math.floor(Math.random() * 10000), ssrc: Math.floor(Math.random() * 10000), active: false, }))], + ssrc: Math.floor(Math.random() * 10000), ip: transport.iceCandidates[0].ip, - port: transport.iceCandidates[0].port, + port: "50001", modes: [ "aead_aes256_gcm_rtpsize", "aead_aes256_gcm", @@ -81,7 +107,6 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify "xsalsa20_poly1305_suffix", "xsalsa20_poly1305" ], - heartbeat_interval: 1, experiments: [], }, })); diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index 36527a8be..a957e14f9 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -87,42 +87,82 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa })), */ + const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video")?.mimeType + const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio") + if (!test_hasMadeProducer) { const producer = await transport.produce({ kind: "audio", rtpParameters: { mid: "audio", codecs: [{ - clockRate: 48000, - payloadType: 111, - mimeType: "audio/opus", - channels: 2, + clockRate: audioCodec!.clockRate, + payloadType: audioCodec!.preferredPayloadType as number, + mimeType: audioCodec!.mimeType, + channels: audioCodec?.channels, }], headerExtensions: res.ext?.map(x => ({ id: x.value, uri: x.uri, - })) + })), }, paused: false, }); - + const consumer = await transport.consume({ producerId: producer.id, - paused: false, + paused: true, rtpCapabilities, - }) - + }); + test_hasMadeProducer = true; } + /* server sends sdp: + + m=audio 50021 ICE/SDP //same port as sent in READY + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + c=IN IP4 109.200.213.132 //same IP as sent in READY + a=rtcp:50021 //same port? + a=ice-ufrag:rTmX + a=ice-pwd:M+ncqWK6SEdHhirOjG2VFA + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + a=candidate:1 1 UDP 4261412862 109.200.213.132 50021 typ host //same IP and PORT + + */ + + + var test = { + "video_codec": "H264", + "sdp": ` + m=audio 50011 ICE/SDP\n + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n + c=IN IP4 109.200.214.156\n + a=rtcp:50011\n + a=ice-ufrag:d0aZ\n + a=ice-pwd:51ubWYu7GSkQRqlH/apTSZ\n + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n + a=candidate:1 1 UDP 4261412862 109.200.214.156 50011 typ host\n`, + "media_session_id": "9e18c981687f2de5399edd5cb3f3babf", + "audio_codec": "opus" + }; + + socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { - video_codec: data.d.codecs.find((x: any) => x.type === "video").name, - secret_key: new Array(32).fill(null).map(x => Math.random() * 256), - mode: "xsalsa20_poly1305", - media_session_id: this.mediasoupTransports[0].id, - audio_codec: data.d.codecs.find((x: any) => x.type === "audio").name, + video_codec: videoCodec?.substring(6) || undefined, + // mode: "xsalsa20_poly1305", + media_session_id: transport.id, + audio_codec: audioCodec?.mimeType.substring(6), + sdp: `m=audio ${transport.iceCandidates[0].port} ICE/SDP\n` + + `a=fingerprint:sha-256 ${transport.dtlsParameters.fingerprints.find(x => x.algorithm === "sha-256")?.value}\n` + + `c=IN IPV4 ${transport.iceCandidates[0].ip}\n` + + `a=rtcp:${transport.iceCandidates[0].port}\n` + + `a=ice-ufrag:${transport.iceParameters.usernameFragment}\n` + + `a=ice-pwd:${transport.iceParameters.password}\n` + + `a=fingerprint:sha-1 ${transport.dtlsParameters.fingerprints[0].value}\n` + + `a=candidate:1 1 ${transport.iceCandidates[0].protocol} ${transport.iceCandidates[0].priority} ${transport.iceCandidates[0].ip} ${transport.iceCandidates[0].port} typ ${transport.iceCandidates[0].type}` } })); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Version.ts b/webrtc/src/opcodes/Version.ts new file mode 100644 index 000000000..0ea6eb4d5 --- /dev/null +++ b/webrtc/src/opcodes/Version.ts @@ -0,0 +1,14 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; +import { setHeartbeat } from "../util"; +import { Server } from "../Server" + +export async function onVersion(this: Server, socket: WebSocket, data: Payload) { + socket.send(JSON.stringify({ + op: 16, + d: { + voice: "0.8.31", //version numbers? + rtc_worker: "0.3.18", + } + })) +} \ No newline at end of file diff --git a/webrtc/src/opcodes/index.ts b/webrtc/src/opcodes/index.ts index 9b1eb270e..d0f40bc21 100644 --- a/webrtc/src/opcodes/index.ts +++ b/webrtc/src/opcodes/index.ts @@ -15,6 +15,8 @@ import { onSpeaking } from "./Speaking"; import { onResume } from "./Resume"; import { onConnect } from "./Connect"; +import { onVersion } from "./Version"; + export type OPCodeHandler = (this: WebSocket, data: Payload) => any; export default { @@ -34,4 +36,5 @@ export default { //op 13? //op 15? //op 16? empty data on client send but server sends {"voice":"0.8.24+bugfix.voice.streams.opt.branch-ffcefaff7","rtc_worker":"0.3.14-crypto-collision-copy"} + [VoiceOPCodes.VERSION]: onVersion, }; \ No newline at end of file diff --git a/webrtc/src/start.ts b/webrtc/src/start.ts index 299bfce8e..98f06ad52 100644 --- a/webrtc/src/start.ts +++ b/webrtc/src/start.ts @@ -1,10 +1,10 @@ +//testing +process.env.DATABASE = "../bundle/database.db"; + import { config } from "dotenv"; config(); import { Server } from "./Server"; -//testing -process.env.DATABASE = "../bundle/database.db"; - const server = new Server(); server.listen(); \ No newline at end of file diff --git a/webrtc/src/util/Heartbeat.ts b/webrtc/src/util/Heartbeat.ts index 7b5ed9cd4..8c5e3a7a4 100644 --- a/webrtc/src/util/Heartbeat.ts +++ b/webrtc/src/util/Heartbeat.ts @@ -1,18 +1,23 @@ import { WebSocket, CLOSECODES } from "@fosscord/gateway"; import { VoiceOPCodes } from "@fosscord/util"; -export async function setHeartbeat(socket: WebSocket) { +export async function setHeartbeat(socket: WebSocket, nonce?: Number) { if (socket.heartbeatTimeout) clearTimeout(socket.heartbeatTimeout); socket.heartbeatTimeout = setTimeout(() => { return socket.close(CLOSECODES.Session_timed_out); }, 1000 * 45); - socket.send(JSON.stringify({ - op: VoiceOPCodes.HEARTBEAT_ACK, - d: { - v: 6, - heartbeat_interval: 13750, - } - })); + if (!nonce) { + socket.send(JSON.stringify({ + op: VoiceOPCodes.HELLO, + d: { + v: 5, + heartbeat_interval: 13750, + } + })); + } + else { + socket.send(JSON.stringify({ op: VoiceOPCodes.HEARTBEAT_ACK, d: nonce })); + } } \ No newline at end of file From 69bcbf0475c4c5a84477d3e4cb862894db3052da Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Mon, 7 Mar 2022 22:57:37 +1100 Subject: [PATCH 08/10] VOICE CONNECTS!!! Dtls stuck on "connecting" state + currently no way to edit/inspect packets received or use own packet format in mediasoup ( fork? ) --- webrtc/src/Server.ts | 23 ++- webrtc/src/opcodes/Connect.ts | 32 ++++- webrtc/src/opcodes/Identify.ts | 63 ++------- webrtc/src/opcodes/Resume.ts | 20 ++- webrtc/src/opcodes/SelectProtocol.ts | 200 ++++++++++++--------------- 5 files changed, 165 insertions(+), 173 deletions(-) diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 1d18d6d19..42b82c6a4 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -6,7 +6,7 @@ import { setHeartbeat } from "./util"; import * as mediasoup from "mediasoup"; import { types as MediasoupTypes } from "mediasoup"; -import Net from "net"; +import udp from "dgram"; var port = Number(process.env.PORT); if (isNaN(port)) port = 3004; @@ -16,6 +16,8 @@ export class Server { public mediasoupWorkers: MediasoupTypes.Worker[] = []; public mediasoupRouters: MediasoupTypes.Router[] = []; public mediasoupTransports: MediasoupTypes.WebRtcTransport[] = []; + public mediasoupProducers: MediasoupTypes.Producer[] = []; + public mediasoupConsumers: MediasoupTypes.Consumer[] = []; constructor() { this.ws = new WebSocketServer({ @@ -28,8 +30,6 @@ export class Server { socket.on("message", async (message: string) => { const payload: Payload = JSON.parse(message); - // console.log(payload); - if (OPCodeHandlers[payload.op]) try { await OPCodeHandlers[payload.op].call(this, socket, payload); @@ -44,6 +44,7 @@ export class Server { } }); }); + } async listen(): Promise { @@ -73,18 +74,26 @@ export class Server { router.observer.on("newtransport", async (transport: MediasoupTypes.WebRtcTransport) => { console.log("new transport created [id:%s]", transport.id); - transport.observer.on("sctpstatechange", (state) => { - console.log(state) - }); - await transport.enableTraceEvent(); + transport.on("connect", () => { + console.log("transport connect") + }) + transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => { console.log("new producer created [id:%s]", producer.id); + + this.mediasoupProducers.push(producer); }); transport.observer.on("newconsumer", (consumer: MediasoupTypes.Consumer) => { console.log("new consumer created [id:%s]", consumer.id); + + this.mediasoupConsumers.push(consumer); + + consumer.on("rtp", (rtpPacket) => { + console.log(rtpPacket); + }); }); transport.observer.on("newdataproducer", (dataProducer) => { diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts index b312d6f2e..1f874a441 100644 --- a/webrtc/src/opcodes/Connect.ts +++ b/webrtc/src/opcodes/Connect.ts @@ -2,8 +2,38 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { Server } from "../Server" +/* +Sent by client: + +{ + "op": 12, + "d": { + "audio_ssrc": 0, + "video_ssrc": 0, + "rtx_ssrc": 0, + "streams": [ + { + "type": "video", + "rid": "100", + "ssrc": 0, + "active": false, + "quality": 100, + "rtx_ssrc": 0, + "max_bitrate": 2500000, + "max_framerate": 20, + "max_resolution": { + "type": "fixed", + "width": 1280, + "height": 720 + } + } + ] + } +} +*/ + export async function onConnect(this: Server, socket: WebSocket, data: Payload) { - socket.send(JSON.stringify({ + socket.send(JSON.stringify({ //what is op 15? op: 15, d: { any: 100 } })) diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index d7da5c7c5..9baa16e3a 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -34,71 +34,20 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify return socket.close(CLOSECODES.Invalid_intent); var transport = this.mediasoupTransports[0] || await this.mediasoupRouters[0].createWebRtcTransport({ - listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }], + listenIps: [{ ip: "10.22.64.69" }], enableUdp: true, enableTcp: true, preferUdp: true, + enableSctp: true, }); - /* - //discord proper sends: - { - "streams": [ - { "type": "video", "ssrc": 1311885, "rtx_ssrc": 1311886, "rid": "50", "quality": 50, "active": false }, - { "type": "video", "ssrc": 1311887, "rtx_ssrc": 1311888, "rid": "100", "quality": 100, "active": false } - ], - "ssrc": 1311884, - "port": 50008, - "modes": [ - "aead_aes256_gcm_rtpsize", - "aead_aes256_gcm", - "xsalsa20_poly1305_lite_rtpsize", - "xsalsa20_poly1305_lite", - "xsalsa20_poly1305_suffix", - "xsalsa20_poly1305" - ], - "ip": "109.200.214.158", - "experiments": [ - "bwe_conservative_link_estimate", - "bwe_remote_locus_client", - "fixed_keyframe_interval" - ] - } - */ - - - - /* - { - "streams": [ - { "type": "video", "ssrc": 129861, "rtx_ssrc": 129862, "rid": "100", "quality": 100, "active": false } - ], - "ssrc": 129860, - "port": 50003, - "modes": [ - "aead_aes256_gcm_rtpsize", - "aead_aes256_gcm", - "xsalsa20_poly1305_lite_rtpsize", - "xsalsa20_poly1305_lite", - "xsalsa20_poly1305_suffix", - "xsalsa20_poly1305" - ], - "ip": "109.200.213.251", - "experiments": [ - "bwe_conservative_link_estimate", - "bwe_remote_locus_client", - "fixed_keyframe_interval" - ]; - }; - */ - socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: Math.floor(Math.random() * 10000), ssrc: Math.floor(Math.random() * 10000), active: false, }))], ssrc: Math.floor(Math.random() * 10000), ip: transport.iceCandidates[0].ip, - port: "50001", + port: transport.iceCandidates[0].port, modes: [ "aead_aes256_gcm_rtpsize", "aead_aes256_gcm", @@ -107,7 +56,11 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify "xsalsa20_poly1305_suffix", "xsalsa20_poly1305" ], - experiments: [], + experiments: [ + "bwe_conservative_link_estimate", + "bwe_remote_locus_client", + "fixed_keyframe_interval" + ] }, })); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Resume.ts b/webrtc/src/opcodes/Resume.ts index dcd4f4cda..856b550c3 100644 --- a/webrtc/src/opcodes/Resume.ts +++ b/webrtc/src/opcodes/Resume.ts @@ -1,6 +1,24 @@ -import { WebSocket } from "@fosscord/gateway"; +import { CLOSECODES, WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { Server } from "../Server" +import { Guild, Session, VoiceOPCodes } from "@fosscord/util"; export async function onResume(this: Server, socket: WebSocket, data: Payload) { + const session = await Session.findOneOrFail( + { session_id: data.d.session_id, }, + { + where: { user_id: data.d.user_id }, + relations: ["user"] + } + ); + const user = session.user; + const guild = await Guild.findOneOrFail({ id: data.d.server_id }, { relations: ["members"] }); + + if (!guild.members.find(x => x.id === user.id)) + return socket.close(CLOSECODES.Invalid_intent); + + socket.send(JSON.stringify({ + op: VoiceOPCodes.RESUMED, + d: null, + })) } \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index a957e14f9..dc9d2b88e 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -6,15 +6,18 @@ import * as mediasoup from "mediasoup"; import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; import * as sdpTransform from 'sdp-transform'; + /* - { - op: 1, - d: { - protocol: "webrtc", - data: " + + Sent by client: +{ + "op": 1, + "d": { + "protocol": "webrtc", + "data": " a=extmap-allow-mixed - a=ice-ufrag:ilWh - a=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ + a=ice-ufrag:vNxb + a=ice-pwd:tZvpbVPYEKcnW0gGRPq0OOnh a=ice-options:trickle a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time @@ -32,43 +35,63 @@ import * as sdpTransform from 'sdp-transform'; a=rtpmap:96 VP8/90000 a=rtpmap:97 rtx/90000 ", - sdp: "same data as in d.data? also not documented by discord", - codecs: [ - { - name: "opus", - type: "audio", - priority: 1000, - payload_type: 111, - rtx_payload_type: null, - }, - { - name: "H264", - type: "video", - priority: 1000, - payload_type: 102, - rtx_payload_type: 121, - }, - { - name: "VP8", - type: "video", - priority: 2000, - payload_type: 96, - rtx_payload_type: 97, - }, - { - name: "VP9", - type: "video", - priority: 3000, - payload_type: 98, - rtx_payload_type: 99, - }, + "codecs": [ + { + "name": "opus", + "type": "audio", + "priority": 1000, + "payload_type": 111, + "rtx_payload_type": null + }, + { + "name": "H264", + "type": "video", + "priority": 1000, + "payload_type": 102, + "rtx_payload_type": 121 + }, + { + "name": "VP8", + "type": "video", + "priority": 2000, + "payload_type": 96, + "rtx_payload_type": 97 + }, + { + "name": "VP9", + "type": "video", + "priority": 3000, + "payload_type": 98, + "rtx_payload_type": 99 + } ], - rtc_connection_id: "b3c8628a-edb5-49ae-b860-ab0d2842b104", - }, + "rtc_connection_id": "3faa0b80-b3e2-4bae-b291-273801fbb7ab" } +} + +Sent by server: + +{ + "op": 4, + "d": { + "video_codec": "H264", + "sdp": " + m=audio 50001 ICE/SDP + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + c=IN IP4 109.200.214.158 + a=rtcp:50001 + a=ice-ufrag:CLzn + a=ice-pwd:qEmIcNwigd07mu46Ok0XCh + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + a=candidate:1 1 UDP 4261412862 109.200.214.158 50001 typ host + ", + "media_session_id": "807955cb953e98c5b90704cf048e81ec", + "audio_codec": "opus" + } +} + */ -var test_hasMadeProducer = false; export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; @@ -78,87 +101,46 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa const res = sdpTransform.parse(data.d.sdp); - /* - res.media.map(x => x.rtp).flat(1).map(x => ({ - codec: x.codec, - payloadType: x.payload, - clockRate: x.rate as number, - mimeType: `audio/${x.codec}`, + const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video"); + const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio"); + + const producer = this.mediasoupProducers[0] || await transport.produce({ + kind: "audio", + rtpParameters: { + mid: "audio", + codecs: [{ + clockRate: audioCodec!.clockRate, + payloadType: audioCodec!.preferredPayloadType as number, + mimeType: audioCodec!.mimeType, + channels: audioCodec?.channels, + }], + headerExtensions: res.ext?.map(x => ({ + id: x.value, + uri: x.uri, })), - */ + }, + paused: false, + }); - const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video")?.mimeType - const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio") - - if (!test_hasMadeProducer) { - const producer = await transport.produce({ - kind: "audio", - rtpParameters: { - mid: "audio", - codecs: [{ - clockRate: audioCodec!.clockRate, - payloadType: audioCodec!.preferredPayloadType as number, - mimeType: audioCodec!.mimeType, - channels: audioCodec?.channels, - }], - headerExtensions: res.ext?.map(x => ({ - id: x.value, - uri: x.uri, - })), - }, - paused: false, - }); - - const consumer = await transport.consume({ - producerId: producer.id, - paused: true, - rtpCapabilities, - }); - - test_hasMadeProducer = true; - } - - /* server sends sdp: - - m=audio 50021 ICE/SDP //same port as sent in READY - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 - c=IN IP4 109.200.213.132 //same IP as sent in READY - a=rtcp:50021 //same port? - a=ice-ufrag:rTmX - a=ice-pwd:M+ncqWK6SEdHhirOjG2VFA - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 - a=candidate:1 1 UDP 4261412862 109.200.213.132 50021 typ host //same IP and PORT - - */ - - - var test = { - "video_codec": "H264", - "sdp": ` - m=audio 50011 ICE/SDP\n - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n - c=IN IP4 109.200.214.156\n - a=rtcp:50011\n - a=ice-ufrag:d0aZ\n - a=ice-pwd:51ubWYu7GSkQRqlH/apTSZ\n - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n - a=candidate:1 1 UDP 4261412862 109.200.214.156 50011 typ host\n`, - "media_session_id": "9e18c981687f2de5399edd5cb3f3babf", - "audio_codec": "opus" - }; + console.log("can consume: " + this.mediasoupRouters[0].canConsume({ producerId: producer.id, rtpCapabilities: rtpCapabilities })); + const consumer = this.mediasoupConsumers[0] || await transport.consume({ + producerId: producer.id, + paused: false, + rtpCapabilities, + }); socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { - video_codec: videoCodec?.substring(6) || undefined, - // mode: "xsalsa20_poly1305", + video_codec: videoCodec?.mimeType?.substring(6) || undefined, + mode: "xsalsa20_poly1305_lite", media_session_id: transport.id, audio_codec: audioCodec?.mimeType.substring(6), sdp: `m=audio ${transport.iceCandidates[0].port} ICE/SDP\n` + `a=fingerprint:sha-256 ${transport.dtlsParameters.fingerprints.find(x => x.algorithm === "sha-256")?.value}\n` + `c=IN IPV4 ${transport.iceCandidates[0].ip}\n` - + `a=rtcp:${transport.iceCandidates[0].port}\n` + + `a=rtcp: ${transport.iceCandidates[0].port}\n` + `a=ice-ufrag:${transport.iceParameters.usernameFragment}\n` + `a=ice-pwd:${transport.iceParameters.password}\n` + `a=fingerprint:sha-1 ${transport.dtlsParameters.fingerprints[0].value}\n` From d200d83066bcd49de549742ce0987120a6a5d27d Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Tue, 8 Mar 2022 21:22:02 +1100 Subject: [PATCH 09/10] Changing Member.premium_since back from Date to number fixes an error in the Discord electron client related to rendering premium status. Client throws "Invalid time value", so I'm guessing it's something to do with premium_since not being the date format they want ( allegedly ISO8601, but works with a plain number, so wtf ) --- util/src/entities/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/entities/Member.ts b/util/src/entities/Member.ts index 3c5f9db03..b74068813 100644 --- a/util/src/entities/Member.ts +++ b/util/src/entities/Member.ts @@ -86,7 +86,7 @@ export class Member extends BaseClassWithoutId { joined_at: Date; @Column({ nullable: true }) - premium_since?: Date; + premium_since?: number; @Column() deaf: boolean; From fac020f3fb89b499401c64703ac0100a0f9dfa31 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Tue, 8 Mar 2022 21:57:20 +1100 Subject: [PATCH 10/10] Added preferred_region optional property of VoiceStateUpdateSchema to allow electron client to connect to voice without crashing --- gateway/src/schema/VoiceStateUpdateSchema.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gateway/src/schema/VoiceStateUpdateSchema.ts b/gateway/src/schema/VoiceStateUpdateSchema.ts index c046600d9..f6480414c 100644 --- a/gateway/src/schema/VoiceStateUpdateSchema.ts +++ b/gateway/src/schema/VoiceStateUpdateSchema.ts @@ -4,6 +4,7 @@ export const VoiceStateUpdateSchema = { self_mute: Boolean, self_deaf: Boolean, $self_video: Boolean, //required in docs but bots don't always send it + $preferred_region: String, }; export interface VoiceStateUpdateSchema { @@ -12,4 +13,5 @@ export interface VoiceStateUpdateSchema { self_mute: boolean; self_deaf: boolean; self_video?: boolean; + preferred_region?: string; } \ No newline at end of file