feat(webrtc+websockets): logic structure + video and audio streams

frontend/package-lock.json

@@ -2899,6 +2899,11 @@
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
     },
+    "base64-js": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
+    },
     "binary-extensions": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -2940,6 +2945,15 @@
         }
       }
     },
+    "buffer": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
+      "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
+      "requires": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.2.1"
+      }
+    },
     "buffer-from": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-0.1.2.tgz",
@@ -3281,6 +3295,11 @@
         "tapable": "^2.2.0"
       }
     },
+    "err-code": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/err-code/-/err-code-3.0.1.tgz",
+      "integrity": "sha512-GiaH0KJUewYok+eeY05IIgjtAe4Yltygk9Wqp1V5yVWLdhf0hYZchRjNIT9bb0mSwRcIusT3cx7PJUf3zEIfUA=="
+    },
     "error-ex": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
@@ -3907,6 +3926,11 @@
       "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
       "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="
     },
+    "get-browser-rtc": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/get-browser-rtc/-/get-browser-rtc-1.1.0.tgz",
+      "integrity": "sha512-MghbMJ61EJrRsDe7w1Bvqt3ZsBuqhce5nrn/XAwgwOXhcsz53/ltdxOse1h/8eKXj5slzxdsz56g5rzOFSGwfQ=="
+    },
     "get-intrinsic": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz",
@@ -4099,6 +4123,11 @@
         "through2": "~0.4.1"
       }
     },
+    "ieee754": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
+    },
     "ignore": {
       "version": "5.2.4",
       "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz",
@@ -4875,8 +4904,15 @@
     "queue-microtask": {
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
-      "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
+      "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="
-      "dev": true
+    },
+    "randombytes": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
+      "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==",
+      "requires": {
+        "safe-buffer": "^5.1.0"
+      }
     },
     "react": {
       "version": "18.2.0",
@@ -5130,6 +5166,45 @@
         "object-inspect": "^1.9.0"
       }
     },
+    "simple-peer": {
+      "version": "9.11.1",
+      "resolved": "https://registry.npmjs.org/simple-peer/-/simple-peer-9.11.1.tgz",
+      "integrity": "sha512-D1SaWpOW8afq1CZGWB8xTfrT3FekjQmPValrqncJMX7QFl8YwhrPTZvMCANLtgBwwdS+7zURyqxDDEmY558tTw==",
+      "requires": {
+        "buffer": "^6.0.3",
+        "debug": "^4.3.2",
+        "err-code": "^3.0.1",
+        "get-browser-rtc": "^1.1.0",
+        "queue-microtask": "^1.2.3",
+        "randombytes": "^2.1.0",
+        "readable-stream": "^3.6.0"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "3.6.2",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
+          "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        },
+        "safe-buffer": {
+          "version": "5.2.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+          "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
+        },
+        "string_decoder": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
+          "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
+          "requires": {
+            "safe-buffer": "~5.2.0"
+          }
+        }
+      }
+    },
     "simplebar-core": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/simplebar-core/-/simplebar-core-1.2.1.tgz",

frontend/package.json

@@ -32,6 +32,7 @@
     "react": "18.2.0",
     "react-apexcharts": "1.4.0",
     "react-dom": "18.2.0",
+    "simple-peer": "^9.11.1",
     "simplebar-react": "^3.2.1",
     "socket.io-client": "^4.6.2",
     "styled-components": "^6.0.0-rc.3",

frontend/src/contexts/socketio-context.js

@@ -1,7 +1,9 @@
-import { createContext, useContext, useEffect, useState } from 'react';
+import { createContext, useContext, useEffect, useState, useRef } from 'react';
 import PropTypes from 'prop-types';
 import io from 'socket.io-client';
 import { useAuth } from 'src/hooks/use-auth';
+import Peer from "simple-peer";
+
 
 // The role of this context is to propagate socketio io state through app tree
 export const WsContext = createContext({ undefined });
@@ -9,10 +11,20 @@ export const WsContext = createContext({ undefined });
 export const WsProvider = (props) => {
   const { children } = props;
   const [users, setUsers] = useState(null)
+  const [callAccepted, setCallAccepted] = useState(false);
+	const [callEnded, setCallEnded] = useState(false);
+	const [stream, setStream] = useState();
+	const [name, setName] = useState("");
+	const [call, setCall] = useState({});
+
+	const myVideo = useRef();
+	const userVideo = useRef();
+	const connectionRef = useRef();
   const auth = useAuth()
+  const socket = io(process.env.NEXT_PUBLIC_WS_ENPOINT)
+
   const initialize = async () => {
     // Prevent from calling twice in development mode with React.StrictMode enable
-    const socket = io(process.env.NEXT_PUBLIC_WS_ENPOINT)
 
     socket.on('connect', () => {
         console.log('[IO] Connect => A new connection has been established')
@@ -23,12 +35,93 @@ export const WsProvider = (props) => {
         })
 
         socket.emit("newuser",{
-            id:socket.id,
+            id: socket.id,
             name: window.sessionStorage.getItem("email")
         })
+
+      socket.on("callUser", ({ from, name: callerName, signal }) => {
+        console.log("you're receiving call brow")
+        setCall({ isReceivingCall: true, from, name: callerName, signal });
+      });
+
+      socket.on('disconnect', function(){
+        
+    });
+
     })
   };
 
+  const answerCall = () => {
+		setCallAccepted(true);
+
+		const peer = new Peer({ 
+      initiator: false, 
+      trickle: false, 
+      stream: stream,
+      config: {
+        iceServers: [
+            {url:'stun:stun.l.google.com:19302'},
+            {url:'stun:stun1.l.google.com:19302'},
+        ]
+    },
+    });
+
+		peer.on("signal", (data) => {
+			socket.emit("answerCall", { signal: data, to: call.from });
+		});
+
+		peer.on("stream", (currentStream) => {
+			userVideo.current.srcObject = currentStream;
+		});
+
+		peer.signal(call.signal);
+
+		connectionRef.current = peer;
+	};
+
+
+	const callUser = (id) => {
+
+    console.log("calling user ",id)
+		const peer = new Peer({ initiator: true, trickle: false, stream:stream, config: {
+      iceServers: [
+          {url:'stun:stun.l.google.com:19302'},
+          {url:'stun:stun1.l.google.com:19302'},
+          {url:'stun:stun2.l.google.com:19302'},
+          {url:'stun:stun3.l.google.com:19302'},
+          {url:'stun:stun4.l.google.com:19302'},
+      ]
+  }, });
+
+		peer.on("signal", (data) => {
+			socket.emit("callUser", {
+				userToCall: id,
+				signalData: data,
+				from: window.sessionStorage.getItem("email"),
+			});
+		});
+
+		peer.on("stream", (currentStream) => {
+			userVideo.current.srcObject = currentStream;
+		});
+
+		socket.on("callAccepted", (signal) => {
+			setCallAccepted(true);
+
+			peer.signal(signal);
+		});
+
+		connectionRef.current = peer;
+	};
+
+  const leaveCall = () => {
+		setCallEnded(true);
+
+		connectionRef.current.destroy();
+
+		window.location.reload();
+	};
+
   useEffect(
     () => {
         if(auth.isAuthenticated){
@@ -43,6 +136,16 @@ export const WsProvider = (props) => {
     <WsContext.Provider
       value={{
         users,
+        call,
+				callAccepted,
+				myVideo,
+				userVideo,
+				stream,
+				callEnded,
+				callUser,
+				leaveCall,
+				answerCall,
+        setStream
       }}
     >
       {children}

frontend/src/layouts/dashboard/top-nav.js

@@ -1,6 +1,7 @@
 import PropTypes from 'prop-types';
 import BellIcon from '@heroicons/react/24/solid/BellIcon';
 import UsersIcon from '@heroicons/react/24/solid/UsersIcon';
+import PhoneIcon from '@heroicons/react/24/solid/PhoneIcon';
 import Bars3Icon from '@heroicons/react/24/solid/Bars3Icon';
 import MagnifyingGlassIcon from '@heroicons/react/24/solid/MagnifyingGlassIcon';
 import {
@@ -11,12 +12,21 @@ import {
   Stack,
   SvgIcon,
   Tooltip,
-  useMediaQuery
+  useMediaQuery,
+  Dialog,
+  DialogTitle,
+  DialogActions,
+  DialogContent,
+  DialogContentText,
+  Button
 } from '@mui/material';
+import XMarkIcon from '@heroicons/react/24/outline/XMarkIcon';
 import { alpha } from '@mui/material/styles';
 import { usePopover } from 'src/hooks/use-popover';
 import { AccountPopover } from './account-popover';
 import { useAuth } from 'src/hooks/use-auth';
+import { WsContext } from 'src/contexts/socketio-context';
+import { useContext, useEffect } from 'react';
 
 const SIDE_NAV_WIDTH = 280;
 const TOP_NAV_HEIGHT = 64;
@@ -26,6 +36,7 @@ export const TopNav = (props) => {
   const lgUp = useMediaQuery((theme) => theme.breakpoints.up('lg'));
   const accountPopover = usePopover();
   const auth = useAuth();
+  const { answerCall, call, callAccepted } = useContext(WsContext);
 
   return ( auth.user &&
     <>
@@ -113,6 +124,57 @@ export const TopNav = (props) => {
           </Stack>
         </Stack>
       </Box>
+      {call.isReceivingCall && !callAccepted &&
+      <Dialog
+        fullWidth={ true } 
+        maxWidth={"sm"}
+        open={true}
+        //scroll={scroll}
+        aria-labelledby="scroll-dialog-title"
+        aria-describedby="scroll-dialog-description"
+      >
+        <DialogContent dividers={scroll === 'paper'}>
+        <Box display="flex" alignItems="center" justifyContent={'center'}>
+              <Box sx={{margin:"30px",textAlign:'center'}}>
+                <Avatar
+                  sx={{
+                      height: 150,
+                      width: 150,
+                  }}
+                  src={"/assets/avatars/default-avatar.png"}
+                  />
+                  <Box flexGrow={1} >{call.from}</Box>
+              </Box>
+        </Box>
+        <Box display="flex" alignItems="center" justifyContent={'center'}>
+        <IconButton>
+          <Tooltip title="Refuse" placement="left" onClick={()=>{}}>
+              <SvgIcon
+              sx={{cursor:'pointer'}}
+              style={{transform: "scale(1.5,1.5)"}}
+              >
+                <PhoneIcon 
+                color={"#CB1E02"}
+                />
+              </SvgIcon>
+          </Tooltip>
+        </IconButton>
+          <div style={{width:'15%'}}></div>
+          <IconButton>
+            <Tooltip title="Accept" placement="right" onClick={()=>{}}>
+                <SvgIcon
+                sx={{cursor:'pointer'}}
+                style={{transform: "scale(1.5,1.5) scale(-1,1)"}}
+                >
+                  <PhoneIcon 
+                  color={"#17A000"}
+                  />
+                </SvgIcon>
+            </Tooltip>
+          </IconButton>
+        </Box>
+        </DialogContent>
+      </Dialog>}
       <AccountPopover
         anchorEl={accountPopover.anchorRef.current}
         open={accountPopover.open}

frontend/src/pages/chat.js

@@ -10,9 +10,10 @@ import {
     SvgIcon,
     CircularProgress,
     Avatar,
-    Backdrop,
+    Tooltip
 } from "@mui/material";
 import { WsContext } from "src/contexts/socketio-context";
+import { useRouter } from "next/router";
 
 const Page = () => {
 
@@ -21,6 +22,7 @@ const Page = () => {
     //const [onlineUsers, setOnlineUsers] = useState([])
 
     const ws = useContext(WsContext)
+    const router = useRouter()
 
     useEffect(()=>{
         var myHeaders = new Headers();
@@ -75,7 +77,7 @@ const Page = () => {
 
                             if (x.email !== window.sessionStorage.getItem("email")){
                                 return (
-                                    <Box sx={{margin:"30px",textAlign:'center'}}>
+                                    <Box sx={{margin:"30px",textAlign:'center'}} key={x.email}>
                                         <Avatar
                                         sx={{
                                             height: 150,
@@ -86,27 +88,32 @@ const Page = () => {
                                         />
                                         <div style={{marginTop:'10px'}}>
                                         </div>
-                                        <SvgIcon
-                                        sx={{cursor:'pointer'}}
-                                        >
                                             {status === "online" ?
-                                                <PhoneIcon 
+                                            <Tooltip title="Call" placement="right" onClick={()=>{
-                                                color={color}
+                                                router.push({
-                                                onClick={()=>{
+                                                    pathname:"chat/room",
-                                                    console.log("call", x.email)
+                                                    query: {user: x.email}
-                                                }}
+                                                    })
-                                                title={"call"}
+                                                }}>
-                                                />
+                                                <SvgIcon
+                                                sx={{cursor:'pointer'}}
+                                                >
+                                                    <PhoneIcon 
+                                                    color={color}
+                                                    />
+                                                </SvgIcon>
+                                            </Tooltip>
                                             :
-                                            <PhoneXMarkIcon 
+                                            <Tooltip title="Offline" placement="right">
-                                            color={color}
+                                                <SvgIcon
-                                            onClick={()=>{
+                                                sx={{cursor:'default'}}
-                                                console.log("call", x.email)
+                                                >
-                                            }}
+                                                    <PhoneXMarkIcon 
-                                            title={"offline"}
+                                                    color={color}
-                                            />
+                                                    />
+                                                </SvgIcon>
+                                            </Tooltip>
                                             }
-                                        </SvgIcon>
                                         <p style={{marginTop:'-2.5px'}}>{x.email}</p>
                                     </Box> 
                                 )   

frontend/src/pages/chat/room.js

@@ -0,0 +1,70 @@
+import React, { useEffect, useState, useContext } from "react";
+import { Layout as DashboardLayout } from 'src/layouts/dashboard/layout';
+import { 
+    Card,
+    Box,
+    CardContent,
+    Container,
+    SvgIcon,
+    CircularProgress,
+    Avatar,
+    Tooltip
+} from "@mui/material";
+import { WsContext } from "src/contexts/socketio-context";
+import { useRouter } from "next/router";
+
+const Page = (props) => {
+
+    const { callUser, callAccepted, myVideo, userVideo, callEnded, stream, call, setStream } = 
+    useContext(WsContext);
+    const router = useRouter()
+
+    const stopCamera = () => {
+        // stream.getTracks().forEach(function(track) {
+        //     track.stop();
+        //   });
+        //console.log(stream)
+        window.location.reload() //TODO: find better way to stop recording user
+    }
+
+    useEffect(()=>{
+        callUser(router.query.user)
+        navigator.mediaDevices
+        .getUserMedia({ video: true, audio: true })
+        .then((currentStream) => {
+          setStream(currentStream);
+          if (myVideo.current) {
+            myVideo.current.srcObject = currentStream;
+          }
+        })
+        .catch((err)=>{
+            console.log('You cannot place/ receive a call without granting video and audio permissions! Please change your settings to use Oktopus calls.')
+            console.log(err)
+        })
+
+        return(stopCamera)
+    },[])
+
+    return (
+        <Card>
+            <CardContent>
+            { myVideo && 
+            <video 
+            className="userVideo" 
+            playsInline 
+            muted 
+            ref={myVideo} 
+            autoPlay />
+            }
+            </CardContent>
+        </Card>
+    )
+}
+
+Page.getLayout = (page) => (
+    <DashboardLayout>
+        {page}
+    </DashboardLayout>
+);
+
+export default Page;