Skip to content

Commit 70be7f0

Browse files
GoldinGuyGoldinGuy
committed
persist bg removal effect across streams and input types
1 parent d1f4805 commit 70be7f0

7 files changed

Lines changed: 130 additions & 243 deletions

File tree

‎example/index.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ import Draggable from 'react-draggable';
77
const App = () => {
88
return (
99
<>
10-
<GeneralTest />
11-
{/* <HoursTest /> */}
10+
{/* <GeneralTest /> */}
11+
<HoursTest />
1212
</>
1313
);
1414
};

‎src/CatalystChat.tsx‎

Lines changed: 53 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -81,47 +81,59 @@ const CatalystChat = ({
8181

8282
useEffect(() => {
8383
if (ready && token.length < 1) {
84-
// set client ID
85-
const uniqueClientIdentifier =
86-
cookies.PERSISTENT_CLIENT_ID || generateUUID();
87-
if (!cookies.PERSISTENT_CLIENT_ID)
88-
setCookie('PERSISTENT_CLIENT_ID', uniqueClientIdentifier, {
89-
expires: new Date(Date.now() + 1000 * 60 * 60 * 24 * 365),
90-
});
91-
// obtain user token
92-
fetch(
93-
`${AUTH_ADDRESS}?participantName=${userName}&appId=${appId}&roomName=${room}&uniqueClientIdentifier=${uniqueClientIdentifier}`,
94-
{
95-
method: 'GET',
96-
headers: {
97-
'Content-Type': 'application/json',
98-
mode: 'no-cors',
99-
},
100-
}
101-
)
102-
.then(response => {
103-
if (!isMounted()) return;
104-
if (response.status === 200) {
105-
response.json().then((user: CatalystUserData) => {
106-
if (!isMounted()) return;
107-
setToken(user.token);
108-
if (handleUserData) handleUserData(user);
109-
// console.log(user);
110-
return user.token;
111-
});
112-
}
113-
if (response.status === 500) {
114-
console.log(
115-
'There is no user record corresponding to the provided identifier'
116-
);
117-
setToken('INVALID');
118-
}
119-
return token;
120-
})
121-
.catch(err => {
122-
console.log(err);
123-
setToken('INVALID');
124-
});
84+
setTimeout(() => {
85+
if (!isMounted()) return;
86+
// set client ID
87+
const uniqueClientIdentifier =
88+
cookies.PERSISTENT_CLIENT_ID || generateUUID();
89+
if (!cookies.PERSISTENT_CLIENT_ID)
90+
setCookie(
91+
'PERSISTENT_CLIENT_ID',
92+
uniqueClientIdentifier,
93+
{
94+
expires: new Date(
95+
Date.now() + 1000 * 60 * 60 * 24 * 365
96+
),
97+
}
98+
);
99+
// obtain user token
100+
fetch(
101+
`${AUTH_ADDRESS}?participantName=${userName}&appId=${appId}&roomName=${room}&uniqueClientIdentifier=${uniqueClientIdentifier}`,
102+
{
103+
method: 'GET',
104+
headers: {
105+
'Content-Type': 'application/json',
106+
mode: 'no-cors',
107+
},
108+
}
109+
)
110+
.then(response => {
111+
if (!isMounted()) return;
112+
if (response.status === 200) {
113+
response
114+
.json()
115+
.then((user: CatalystUserData) => {
116+
if (!isMounted()) return;
117+
setToken(user.token);
118+
if (handleUserData) handleUserData(user);
119+
// console.log(user);
120+
return user.token;
121+
});
122+
}
123+
if (response.status === 500) {
124+
console.log(
125+
'There is no user record corresponding to the provided identifier'
126+
);
127+
setToken('INVALID');
128+
}
129+
return token;
130+
})
131+
.catch(err => {
132+
console.log(err);
133+
setToken('INVALID');
134+
});
135+
}, 500)
136+
125137
}
126138
}, [ready]);
127139

‎src/components/toolbar/Toolbar.tsx‎

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ import { ChatMessage } from '../../typings/interfaces';
5050
import Tippy from '@tippyjs/react';
5151
import useIsMounted from '../../hooks/useIsMounted';
5252
import {
53-
applyBgEffectToTrack,
5453
initBgFilter,
5554
applyBgFilterToTrack,
5655
} from '../../utils/bg_removal';
@@ -225,10 +224,12 @@ const Toolbar = ({
225224
if (video) {
226225
unpublishTrack(video.track as LocalVideoTrack);
227226
}
227+
let newTrack: LocalVideoTrack | MediaStreamTrack = track;
228228
if (bgRemovalEffect && bgFilter) {
229-
await applyBgFilterToTrack(track, bgFilter);
229+
await applyBgFilterToTrack(track.mediaStreamTrack, bgFilter);
230+
newTrack = await bgFilter.getOutputTrack();
230231
}
231-
room.localParticipant.publishTrack(track);
232+
room.localParticipant.publishTrack(newTrack);
232233
})
233234
.catch((err: Error) => {
234235
console.log(err);
@@ -259,13 +260,12 @@ const Toolbar = ({
259260
});
260261
// console.log('init filter');
261262
} else if (filter) {
262-
263263
if (bgRemovalEffect === 'none') {
264264
const filterDisabledStream = await filter.disable();
265265
if (filterDisabledStream) {
266266
track = filterDisabledStream.getVideoTracks()[0];
267267
if (track) {
268-
console.log(track);
268+
// console.log(track);
269269
if (video) {
270270
unpublishTrack(video.track as LocalVideoTrack);
271271
}
@@ -276,12 +276,12 @@ const Toolbar = ({
276276
} else {
277277
if (!filterEnabled) {
278278
const filterEnabledStream = await filter.enable();
279-
console.log('filterEnabledStream', filterEnabledStream);
279+
// console.log('filterEnabledStream', filterEnabledStream);
280280
if (filterEnabledStream) {
281281
// track = filterEnabledStream.getVideoTracks()[0];
282282
track = await filter.getOutputTrack();
283283
if (track) {
284-
console.log(track);
284+
// console.log(track);
285285
if (video) {
286286
unpublishTrack(video.track as LocalVideoTrack);
287287
}
@@ -292,23 +292,16 @@ const Toolbar = ({
292292
}
293293
if (bgRemovalEffect === 'blur' && filter.background !== 'blur') {
294294
await filter.changeBackground('blur');
295-
// await filter.changeBlurRadius(7);
296-
} else if (
297-
filter.background !==
298-
'https://demo.vectorly.io/virtual-backgrounds/1.jpg'
299-
) {
300-
await filter.changeBackground(
301-
'https://demo.vectorly.io/virtual-backgrounds/1.jpg'
302-
);
303-
// await filter.changeBlurRadius(0);
295+
} else if (filter.background !== bgRemovalEffect) {
296+
await filter.changeBackground(bgRemovalEffect);
304297
}
305298
}
306299
}
307300

308301
};
309302

310303
useEffect(() => {
311-
console.log(bgRemovalEffect);
304+
// console.log(bgRemovalEffect);
312305
handleBgEffect();
313306
}, [bgRemovalEffect]);
314307

@@ -321,10 +314,12 @@ const Toolbar = ({
321314
if (videoDevice) options.deviceId = videoDevice?.deviceId;
322315
createLocalVideoTrack(options)
323316
.then(async (track: LocalVideoTrack) => {
317+
let newTrack: LocalVideoTrack | MediaStreamTrack = track;
324318
if (bgRemovalEffect && bgFilter) {
325-
await applyBgFilterToTrack(track, bgFilter);
319+
await applyBgFilterToTrack(track.mediaStreamTrack, bgFilter);
320+
newTrack = await bgFilter.getOutputTrack();
326321
}
327-
room.localParticipant.publishTrack(track);
322+
room.localParticipant.publishTrack(newTrack);
328323
})
329324
.catch((err: Error) => {
330325
console.log(err);
@@ -395,8 +390,8 @@ const Toolbar = ({
395390
bgRemovalEffect={bgRemovalEffect}
396391
setBgRemovalEffect={setBgRemovalEffect}
397392
/>
398-
{/* Screen Share Button */}
399-
{/* TODO: screen share on mobile */}
393+
{/* Screen Share Button
394+
TODO: screen share on mobile */}
400395
{!isMobile && (
401396
<ToolbarButton
402397
type="screenshare"

‎src/components/toolbar/ToolbarButton.tsx‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,12 +95,10 @@ const ToolbarButton = React.memo(
9595
!bgRemovalEffect ||
9696
bgRemovalEffect === 'none' ||
9797
bgRemovalEffect === 'blur'
98-
? BUILT_IN_BACKGROUNDS[0]
98+
? BUILT_IN_BACKGROUNDS[Math.floor(Math.random() * BUILT_IN_BACKGROUNDS.length)]
9999
: bgRemovalEffect,
100100
]);
101101

102-
// const [showBgRemovalOps, setShowBgRemovalOps] = useState(false);
103-
104102
const handleOnIpDeviceClick = (id: CatalystDev) => {
105103
if (onIpDeviceClick) {
106104
onIpDeviceClick(id);

‎src/utils/bg_removal.ts‎

Lines changed: 9 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -11,38 +11,17 @@ export const initBgFilter = async (
1111
token: 'BG_REMOVAL_TOKEN',
1212
background: effectType,
1313
blurRadius: 6
14-
// 'https://terrigen-cdn-dev.marvel.com/content/prod/1x/333.jpg', //'blur'
1514
});
1615
return filter;
1716
// console.log('bg', bgRemovedTrack);
1817
}
1918
}
2019
return null;
21-
};
22-
23-
export const applyBgFilterEffect = async (
24-
effectType: string,
25-
filter: BackgroundFilter,
26-
): Promise<BackgroundFilter> => {
27-
switch (effectType) {
28-
case 'none':
29-
await filter.changeBackground('blur');
30-
await filter.changeBlurRadius(0);
31-
// filter.disable();
32-
break;
33-
case 'blur':
34-
await filter.changeBlurRadius(7);
35-
// filter.enable();
36-
break;
37-
default:
38-
await filter.changeBackground(effectType);
39-
await filter.changeBlurRadius(0);
40-
// filter.enable();
41-
}
42-
console.log('bg', filter);
43-
return filter;
4420
};
4521

22+
23+
24+
4625
export const applyBgFilterToTracks = async (
4726
tracks: LocalTrack[],
4827
filter: BackgroundFilter,
@@ -59,58 +38,10 @@ export const applyBgFilterToTracks = async (
5938
}
6039

6140
export const applyBgFilterToTrack = async (
62-
track: LocalTrack,
63-
filter: BackgroundFilter
64-
): Promise<LocalTrack> => {
65-
if (filter) {
66-
await filter.changeInput(track.mediaStreamTrack);
67-
}
68-
return track;
69-
};
70-
71-
export const applyBgEffectToTracks = async (
72-
tracks: LocalTrack[],
73-
effectType: string
74-
): Promise<LocalTrack[]> => {
75-
if (effectType && effectType.length > 0 && effectType !== 'none') {
76-
const vidTrack = tracks.find(track => track.kind === 'video');
77-
if (vidTrack?.mediaStreamTrack) {
78-
const filter = new BackgroundFilter(vidTrack.mediaStreamTrack, {
79-
token: 'BG_REMOVAL_TOKEN',
80-
background: effectType,
81-
blurRadius: 6,
82-
// 'https://terrigen-cdn-dev.marvel.com/content/prod/1x/333.jpg', //'blur'
83-
});
84-
const bgRemovedTrack = await filter.getOutput();
85-
// console.log('bg', bgRemovedTrack);
86-
tracks.forEach(track => {
87-
if (track.kind === 'video' && !track.name.includes('screen')) {
88-
track.mediaStreamTrack = bgRemovedTrack.getVideoTracks()[0];
89-
}
90-
});
91-
}
92-
}
93-
return tracks;
94-
};
95-
96-
export const applyBgEffectToTrack = async (
97-
track: LocalVideoTrack,
98-
effectType?: string
99-
): Promise<LocalVideoTrack> => {
100-
if (effectType && effectType.length > 0 && effectType !== 'none') {
101-
console.log('filtering', effectType);
102-
if (track) {
103-
const filter = new BackgroundFilter(track.mediaStreamTrack, {
104-
token: 'BG_REMOVAL_TOKEN',
105-
background: effectType,
106-
blurRadius: 6,
107-
});
108-
const bgRemovedTrack = await filter.getOutput();
109-
// console.log('bg', bgRemovedTrack);
110-
if (track.kind === 'video') {
111-
track.mediaStreamTrack = bgRemovedTrack.getVideoTracks()[0];
112-
}
113-
}
41+
track: MediaStreamTrack,
42+
filter: BackgroundFilter
43+
): Promise<void> => {
44+
if (filter) {
45+
await filter.changeInput(track);
11446
}
115-
return track;
116-
};
47+
};

‎src/utils/globals.ts‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,20 @@ export const THEMES = {
7474
},
7575
};
7676
export const BUILT_IN_BACKGROUNDS = [
77-
'https://demo.vectorly.io/virtual-backgrounds/1.jpg',
78-
'https://user-images.githubusercontent.com/47064842/134439146-4df72e66-6f56-4bcd-8ac3-d71fae56e755.png',
79-
'https://user-images.githubusercontent.com/47064842/134443664-e46ee13f-eda7-4a85-942a-30e39eb2748c.png',
80-
'https://user-images.githubusercontent.com/47064842/134443692-c09b982c-5009-4312-9745-3ed07af769a8.png',
81-
'https://user-images.githubusercontent.com/47064842/134443673-c1858afb-1263-4ea6-822e-b84658ce726f.png',
82-
'https://user-images.githubusercontent.com/47064842/134443701-6f97b098-c6ba-4261-93f1-83188783a044.png',
83-
'https://user-images.githubusercontent.com/47064842/134443845-ea83f32d-5833-4f3f-8002-5507bc4b03ae.jpg',
84-
'https://user-images.githubusercontent.com/47064842/134443897-2f1c7b4f-32dd-46cd-9104-cf41f4d623a0.jpg',
77+
'https://demo.vectorly.io/virtual-backgrounds/1.jpg',
78+
'https://demo.vectorly.io/virtual-backgrounds/2.jpg',
79+
'https://demo.vectorly.io/virtual-backgrounds/3.jpg',
80+
'https://demo.vectorly.io/virtual-backgrounds/4.jpg',
81+
'https://demo.vectorly.io/virtual-backgrounds/5.jpg',
82+
'https://demo.vectorly.io/virtual-backgrounds/6.jpg',
83+
'https://demo.vectorly.io/virtual-backgrounds/7.jpg',
84+
'https://demo.vectorly.io/virtual-backgrounds/8.jpg',
85+
'https://demo.vectorly.io/virtual-backgrounds/9.jpg',
86+
// 'https://user-images.githubusercontent.com/47064842/134439146-4df72e66-6f56-4bcd-8ac3-d71fae56e755.png',
87+
// 'https://user-images.githubusercontent.com/47064842/134443664-e46ee13f-eda7-4a85-942a-30e39eb2748c.png',
88+
// 'https://user-images.githubusercontent.com/47064842/134443692-c09b982c-5009-4312-9745-3ed07af769a8.png',
89+
// 'https://user-images.githubusercontent.com/47064842/134443673-c1858afb-1263-4ea6-822e-b84658ce726f.png',
90+
// 'https://user-images.githubusercontent.com/47064842/134443701-6f97b098-c6ba-4261-93f1-83188783a044.png',
91+
// 'https://user-images.githubusercontent.com/47064842/134443845-ea83f32d-5833-4f3f-8002-5507bc4b03ae.jpg',
92+
// 'https://user-images.githubusercontent.com/47064842/134443897-2f1c7b4f-32dd-46cd-9104-cf41f4d623a0.jpg',
8593
];

0 commit comments

Comments
 (0)