From dccdc9c11630c76ad5634b91a1e83eea87ddaf94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leandro=20Ant=C3=B4nio=20Farias=20Machado?= Date: Mon, 26 Jun 2023 01:09:10 -0300 Subject: [PATCH] feat(webrtc+websockets): logic structure + video and audio streams --- frontend/package-lock.json | 79 +++++++++++++++- frontend/package.json | 1 + frontend/src/contexts/socketio-context.js | 109 +++++++++++++++++++++- frontend/src/layouts/dashboard/top-nav.js | 64 ++++++++++++- frontend/src/pages/chat.js | 47 ++++++---- frontend/src/pages/chat/room.js | 70 ++++++++++++++ 6 files changed, 344 insertions(+), 26 deletions(-) create mode 100644 frontend/src/pages/chat/room.js diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 2c95a85..693adcc 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -2899,6 +2899,11 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -2940,6 +2945,15 @@ } } }, + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "buffer-from": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-0.1.2.tgz", @@ -3281,6 +3295,11 @@ "tapable": "^2.2.0" } }, + "err-code": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-3.0.1.tgz", + "integrity": "sha512-GiaH0KJUewYok+eeY05IIgjtAe4Yltygk9Wqp1V5yVWLdhf0hYZchRjNIT9bb0mSwRcIusT3cx7PJUf3zEIfUA==" + }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -3907,6 +3926,11 @@ "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==" }, + "get-browser-rtc": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/get-browser-rtc/-/get-browser-rtc-1.1.0.tgz", + "integrity": "sha512-MghbMJ61EJrRsDe7w1Bvqt3ZsBuqhce5nrn/XAwgwOXhcsz53/ltdxOse1h/8eKXj5slzxdsz56g5rzOFSGwfQ==" + }, "get-intrinsic": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", @@ -4099,6 +4123,11 @@ "through2": "~0.4.1" } }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, "ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", @@ -4875,8 +4904,15 @@ "queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" + }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "requires": { + "safe-buffer": "^5.1.0" + } }, "react": { "version": "18.2.0", @@ -5130,6 +5166,45 @@ "object-inspect": "^1.9.0" } }, + "simple-peer": { + "version": "9.11.1", + "resolved": "https://registry.npmjs.org/simple-peer/-/simple-peer-9.11.1.tgz", + "integrity": "sha512-D1SaWpOW8afq1CZGWB8xTfrT3FekjQmPValrqncJMX7QFl8YwhrPTZvMCANLtgBwwdS+7zURyqxDDEmY558tTw==", + "requires": { + "buffer": "^6.0.3", + "debug": "^4.3.2", + "err-code": "^3.0.1", + "get-browser-rtc": "^1.1.0", + "queue-microtask": "^1.2.3", + "randombytes": "^2.1.0", + "readable-stream": "^3.6.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + } + } + }, "simplebar-core": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/simplebar-core/-/simplebar-core-1.2.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index c3aec4e..8c5dac6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -32,6 +32,7 @@ "react": "18.2.0", "react-apexcharts": "1.4.0", "react-dom": "18.2.0", + "simple-peer": "^9.11.1", "simplebar-react": "^3.2.1", "socket.io-client": "^4.6.2", "styled-components": "^6.0.0-rc.3", diff --git a/frontend/src/contexts/socketio-context.js b/frontend/src/contexts/socketio-context.js index ff1305f..e1996cb 100644 --- a/frontend/src/contexts/socketio-context.js +++ b/frontend/src/contexts/socketio-context.js @@ -1,7 +1,9 @@ -import { createContext, useContext, useEffect, useState } from 'react'; +import { createContext, useContext, useEffect, useState, useRef } from 'react'; import PropTypes from 'prop-types'; import io from 'socket.io-client'; import { useAuth } from 'src/hooks/use-auth'; +import Peer from "simple-peer"; + // The role of this context is to propagate socketio io state through app tree export const WsContext = createContext({ undefined }); @@ -9,10 +11,20 @@ export const WsContext = createContext({ undefined }); export const WsProvider = (props) => { const { children } = props; const [users, setUsers] = useState(null) + const [callAccepted, setCallAccepted] = useState(false); + const [callEnded, setCallEnded] = useState(false); + const [stream, setStream] = useState(); + const [name, setName] = useState(""); + const [call, setCall] = useState({}); + + const myVideo = useRef(); + const userVideo = useRef(); + const connectionRef = useRef(); const auth = useAuth() + const socket = io(process.env.NEXT_PUBLIC_WS_ENPOINT) + const initialize = async () => { // Prevent from calling twice in development mode with React.StrictMode enable - const socket = io(process.env.NEXT_PUBLIC_WS_ENPOINT) socket.on('connect', () => { console.log('[IO] Connect => A new connection has been established') @@ -23,12 +35,93 @@ export const WsProvider = (props) => { }) socket.emit("newuser",{ - id:socket.id, + id: socket.id, name: window.sessionStorage.getItem("email") }) + + socket.on("callUser", ({ from, name: callerName, signal }) => { + console.log("you're receiving call brow") + setCall({ isReceivingCall: true, from, name: callerName, signal }); + }); + + socket.on('disconnect', function(){ + + }); + }) }; + const answerCall = () => { + setCallAccepted(true); + + const peer = new Peer({ + initiator: false, + trickle: false, + stream: stream, + config: { + iceServers: [ + {url:'stun:stun.l.google.com:19302'}, + {url:'stun:stun1.l.google.com:19302'}, + ] + }, + }); + + peer.on("signal", (data) => { + socket.emit("answerCall", { signal: data, to: call.from }); + }); + + peer.on("stream", (currentStream) => { + userVideo.current.srcObject = currentStream; + }); + + peer.signal(call.signal); + + connectionRef.current = peer; + }; + + + const callUser = (id) => { + + console.log("calling user ",id) + const peer = new Peer({ initiator: true, trickle: false, stream:stream, config: { + iceServers: [ + {url:'stun:stun.l.google.com:19302'}, + {url:'stun:stun1.l.google.com:19302'}, + {url:'stun:stun2.l.google.com:19302'}, + {url:'stun:stun3.l.google.com:19302'}, + {url:'stun:stun4.l.google.com:19302'}, + ] + }, }); + + peer.on("signal", (data) => { + socket.emit("callUser", { + userToCall: id, + signalData: data, + from: window.sessionStorage.getItem("email"), + }); + }); + + peer.on("stream", (currentStream) => { + userVideo.current.srcObject = currentStream; + }); + + socket.on("callAccepted", (signal) => { + setCallAccepted(true); + + peer.signal(signal); + }); + + connectionRef.current = peer; + }; + + const leaveCall = () => { + setCallEnded(true); + + connectionRef.current.destroy(); + + window.location.reload(); + }; + useEffect( () => { if(auth.isAuthenticated){ @@ -43,6 +136,16 @@ export const WsProvider = (props) => { {children} diff --git a/frontend/src/layouts/dashboard/top-nav.js b/frontend/src/layouts/dashboard/top-nav.js index 39188ed..59b9670 100644 --- a/frontend/src/layouts/dashboard/top-nav.js +++ b/frontend/src/layouts/dashboard/top-nav.js @@ -1,6 +1,7 @@ import PropTypes from 'prop-types'; import BellIcon from '@heroicons/react/24/solid/BellIcon'; import UsersIcon from '@heroicons/react/24/solid/UsersIcon'; +import PhoneIcon from '@heroicons/react/24/solid/PhoneIcon'; import Bars3Icon from '@heroicons/react/24/solid/Bars3Icon'; import MagnifyingGlassIcon from '@heroicons/react/24/solid/MagnifyingGlassIcon'; import { @@ -11,12 +12,21 @@ import { Stack, SvgIcon, Tooltip, - useMediaQuery + useMediaQuery, + Dialog, + DialogTitle, + DialogActions, + DialogContent, + DialogContentText, + Button } from '@mui/material'; +import XMarkIcon from '@heroicons/react/24/outline/XMarkIcon'; import { alpha } from '@mui/material/styles'; import { usePopover } from 'src/hooks/use-popover'; import { AccountPopover } from './account-popover'; import { useAuth } from 'src/hooks/use-auth'; +import { WsContext } from 'src/contexts/socketio-context'; +import { useContext, useEffect } from 'react'; const SIDE_NAV_WIDTH = 280; const TOP_NAV_HEIGHT = 64; @@ -26,6 +36,7 @@ export const TopNav = (props) => { const lgUp = useMediaQuery((theme) => theme.breakpoints.up('lg')); const accountPopover = usePopover(); const auth = useAuth(); + const { answerCall, call, callAccepted } = useContext(WsContext); return ( auth.user && <> @@ -113,6 +124,57 @@ export const TopNav = (props) => { + {call.isReceivingCall && !callAccepted && + + + + + + {call.from} + + + + + {}}> + + + + + +
+ + {}}> + + + + + + + + } { @@ -21,6 +22,7 @@ const Page = () => { //const [onlineUsers, setOnlineUsers] = useState([]) const ws = useContext(WsContext) + const router = useRouter() useEffect(()=>{ var myHeaders = new Headers(); @@ -75,7 +77,7 @@ const Page = () => { if (x.email !== window.sessionStorage.getItem("email")){ return ( - + { />
- {status === "online" ? - { - console.log("call", x.email) - }} - title={"call"} - /> + { + router.push({ + pathname:"chat/room", + query: {user: x.email} + }) + }}> + + + + : - { - console.log("call", x.email) - }} - title={"offline"} - /> + + + + + } -

{x.email}

 ) diff --git a/frontend/src/pages/chat/room.js b/frontend/src/pages/chat/room.js new file mode 100644 index 0000000..11c80f7 --- /dev/null +++ b/frontend/src/pages/chat/room.js @@ -0,0 +1,70 @@ +import React, { useEffect, useState, useContext } from "react"; +import { Layout as DashboardLayout } from 'src/layouts/dashboard/layout'; +import { + Card, + Box, + CardContent, + Container, + SvgIcon, + CircularProgress, + Avatar, + Tooltip +} from "@mui/material"; +import { WsContext } from "src/contexts/socketio-context"; +import { useRouter } from "next/router"; + +const Page = (props) => { + + const { callUser, callAccepted, myVideo, userVideo, callEnded, stream, call, setStream } = + useContext(WsContext); + const router = useRouter() + + const stopCamera = () => { + // stream.getTracks().forEach(function(track) { + // track.stop(); + // }); + //console.log(stream) + window.location.reload() //TODO: find better way to stop recording user + } + + useEffect(()=>{ + callUser(router.query.user) + navigator.mediaDevices + .getUserMedia({ video: true, audio: true }) + .then((currentStream) => { + setStream(currentStream); + if (myVideo.current) { + myVideo.current.srcObject = currentStream; + } + }) + .catch((err)=>{ + console.log('You cannot place/ receive a call without granting video and audio permissions! Please change your settings to use Oktopus calls.') + console.log(err) + }) + + return(stopCamera) + },[]) + + return ( + + + { myVideo && + + + ) +} + +Page.getLayout = (page) => ( + + {page} + +); + +export default Page; \ No newline at end of file