From b9daa0cff5bad80d696a847b45dfa9f0dc76fbdd Mon Sep 17 00:00:00 2001 From: pierrick Date: Wed, 6 May 2026 14:20:54 +0000 Subject: [PATCH] feat: add meet-live-agent sample --- meet-live-agent/.gcloudignore | 3 + meet-live-agent/.gitignore | 25 + meet-live-agent/Dockerfile | 32 + meet-live-agent/README.md | 192 ++ meet-live-agent/deploy.sh | 50 + meet-live-agent/deployment.json | 21 + meet-live-agent/index.css | 14 + meet-live-agent/index.html | 23 + meet-live-agent/index.tsx | 688 +++++++ .../channel_handlers/channel_logger.ts | 58 + .../media_entries_channel_handler.ts | 401 ++++ .../media_stats_channel_handler.ts | 258 +++ .../participants_channel_handler.ts | 241 +++ .../session_control_channel_handler.ts | 155 ++ .../video_assignment_channel_handler.ts | 318 ++++ .../default_communication_protocol_impl.ts | 79 + .../internal_meet_stream_track_impl.ts | 128 ++ meet-live-agent/internal/internal_types.ts | 91 + .../internal/meet_stream_track_impl.ts | 40 + .../internal/meetmediaapiclient_impl.ts | 455 +++++ meet-live-agent/internal/subscribable_impl.ts | 80 + meet-live-agent/internal/utils.ts | 115 ++ meet-live-agent/metadata.json | 8 + meet-live-agent/package-lock.json | 1615 ++++++++++++++++ meet-live-agent/package.json | 21 + .../public/pcm-recorder-processor.js | 13 + meet-live-agent/sample.env | 5 + meet-live-agent/server/package-lock.json | 1667 +++++++++++++++++ meet-live-agent/server/package.json | 21 + .../server/public/service-worker.js | 128 ++ .../server/public/websocket-interceptor.js | 65 + meet-live-agent/server/server.js | 350 ++++ meet-live-agent/tsconfig.json | 29 + .../types/communication_protocol.d.ts | 43 + meet-live-agent/types/datachannels.d.ts | 903 +++++++++ meet-live-agent/types/enums.ts | 47 + .../types/mediacapture_transform.d.ts | 149 ++ meet-live-agent/types/mediatypes.d.ts | 309 +++ meet-live-agent/types/meetmediaapiclient.d.ts | 115 ++ meet-live-agent/types/subscribable.d.ts | 45 + meet-live-agent/vite.config.ts | 24 + 41 files changed, 9024 insertions(+) create mode 100644 meet-live-agent/.gcloudignore create mode 100644 meet-live-agent/.gitignore create mode 100644 meet-live-agent/Dockerfile create mode 100644 meet-live-agent/README.md create mode 100755 meet-live-agent/deploy.sh create mode 100644 meet-live-agent/deployment.json create mode 100644 meet-live-agent/index.css create mode 100644 meet-live-agent/index.html create mode 100644 meet-live-agent/index.tsx create mode 100644 meet-live-agent/internal/channel_handlers/channel_logger.ts create mode 100644 meet-live-agent/internal/channel_handlers/media_entries_channel_handler.ts create mode 100644 meet-live-agent/internal/channel_handlers/media_stats_channel_handler.ts create mode 100644 meet-live-agent/internal/channel_handlers/participants_channel_handler.ts create mode 100644 meet-live-agent/internal/channel_handlers/session_control_channel_handler.ts create mode 100644 meet-live-agent/internal/channel_handlers/video_assignment_channel_handler.ts create mode 100644 meet-live-agent/internal/communication_protocols/default_communication_protocol_impl.ts create mode 100644 meet-live-agent/internal/internal_meet_stream_track_impl.ts create mode 100644 meet-live-agent/internal/internal_types.ts create mode 100644 meet-live-agent/internal/meet_stream_track_impl.ts create mode 100644 meet-live-agent/internal/meetmediaapiclient_impl.ts create mode 100644 meet-live-agent/internal/subscribable_impl.ts create mode 100644 meet-live-agent/internal/utils.ts create mode 100644 meet-live-agent/metadata.json create mode 100644 meet-live-agent/package-lock.json create mode 100644 meet-live-agent/package.json create mode 100644 meet-live-agent/public/pcm-recorder-processor.js create mode 100644 meet-live-agent/sample.env create mode 100644 meet-live-agent/server/package-lock.json create mode 100644 meet-live-agent/server/package.json create mode 100644 meet-live-agent/server/public/service-worker.js create mode 100644 meet-live-agent/server/public/websocket-interceptor.js create mode 100644 meet-live-agent/server/server.js create mode 100644 meet-live-agent/tsconfig.json create mode 100644 meet-live-agent/types/communication_protocol.d.ts create mode 100644 meet-live-agent/types/datachannels.d.ts create mode 100644 meet-live-agent/types/enums.ts create mode 100644 meet-live-agent/types/mediacapture_transform.d.ts create mode 100644 meet-live-agent/types/mediatypes.d.ts create mode 100644 meet-live-agent/types/meetmediaapiclient.d.ts create mode 100644 meet-live-agent/types/subscribable.d.ts create mode 100644 meet-live-agent/vite.config.ts diff --git a/meet-live-agent/.gcloudignore b/meet-live-agent/.gcloudignore new file mode 100644 index 0000000..1945caf --- /dev/null +++ b/meet-live-agent/.gcloudignore @@ -0,0 +1,3 @@ +package-lock.json +dist/ +node_modules/ \ No newline at end of file diff --git a/meet-live-agent/.gitignore b/meet-live-agent/.gitignore new file mode 100644 index 0000000..438657a --- /dev/null +++ b/meet-live-agent/.gitignore @@ -0,0 +1,25 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local +.env + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/meet-live-agent/Dockerfile b/meet-live-agent/Dockerfile new file mode 100644 index 0000000..0e76ca4 --- /dev/null +++ b/meet-live-agent/Dockerfile @@ -0,0 +1,32 @@ +# Stage 1: Build the frontend, and install server dependencies +FROM node:22 AS builder + +WORKDIR /app + +# Copy all files from the current directory +COPY . ./ + + +# Install server dependencies +WORKDIR /app/server +RUN npm install + +# Install dependencies and build the frontend +WORKDIR /app +RUN mkdir dist +RUN bash -c 'if [ -f package.json ]; then npm install && npm run build; fi' + + +# Stage 2: Build the final server image +FROM node:22 + +WORKDIR /app + +#Copy server files +COPY --from=builder /app/server . +# Copy built frontend assets from the builder stage +COPY --from=builder /app/dist ./dist + +EXPOSE 3000 + +CMD ["node", "server.js"] diff --git a/meet-live-agent/README.md b/meet-live-agent/README.md new file mode 100644 index 0000000..0024654 --- /dev/null +++ b/meet-live-agent/README.md @@ -0,0 +1,192 @@ +# Meet Live Agent - Multimodal AI in Google Meet + +This sample project demonstrates how to integrate Google Meet with Gemini Live to create a multimodal AI agent that can participate in a meeting. The agent can listen to participants, see the meeting video, and respond in real-time with audio. It also provides live transcription and scene description in the Meet side panel. + +> [!NOTE] +> The Google Meet Add-on SDK, the Meet Media API, and the Gemini Live model `gemini-3.1-flash-live-preview` are all currently in preview. You need to request access to the [Google Workspace Developer Preview Program (DPP)](https://developers.google.com/workspace/preview). + +## Design Overview + +The application consists of a frontend built with Lit web components and a Node.js backend. + +- **Frontend**: Uses the `@googleworkspace/meet-addons` SDK to integrate with Google Meet and the `@google/genai` SDK to connect to Gemini Live. It captures audio and video from the meeting via the Meet Media API and streams it to Gemini. +- **Backend**: An Express server that serves the static frontend files and acts as a secure reverse proxy for Gemini API calls (both HTTP and WebSockets). This allows the application to use the Gemini API without exposing the API key in the browser. It automatically injects an interceptor script to route SDK calls through the proxy. + +## Main Features + +- **Real-time Bidirectional Audio**: Speak to Gemini and hear it respond in real-time within the meeting. +- **Visual Grounding**: The agent receives video frames from the meeting, allowing it to "see" and comment on what's happening. +- **Live Transcription**: Displays transcripts of what participants say and what Gemini says. +- **Scene Description**: Periodically generates a description of the visual scene using `gemini-2.5-flash`. +- **Secure Proxy**: Protects your Gemini API key by routing requests through the backend. + +## Prerequisites + +Before you begin, ensure you have: + +1. **Google Cloud Project**: A project with billing enabled that has been granted access to the Meet Media API Developer Preview Program (DPP). +2. **gcloud CLI**: Installed and authenticated. [Install guide](https://cloud.google.com/sdk/docs/install). +3. **Gemini API Key**: Get one from [Google AI Studio](https://aistudio.google.com/). +4. **Google Workspace Account**: With permissions to create and use Meet Add-ons. +5. **Node.js**: Version >22 (required if you plan to build locally). + +--- + +## Deployment Instructions + +Follow these steps to deploy the Meet Live Agent as a Google Meet add-on. + +### 1. Enable Required APIs + + Enable the necessary Google Cloud APIs using the command line: + + ```bash + gcloud services enable meet.googleapis.com \ + artifactregistry.googleapis.com \ + run.googleapis.com \ + cloudbuild.googleapis.com \ + appsmarket.googleapis.com \ + appsmarket-component.googleapis.com \ + gsuiteaddons.googleapis.com + ``` + +### 2. Configure OAuth + +Before creating the client, you need to configure branding: + +1. Go to the **APIs & Services > OAuth consent screen** page in the Google Cloud Console. +2. Click **Get started** +3. Set **App name** to **Meet Live Agent** and **User support email** to your support email, then click **Next**. +4. Select **Internal** for the User Type (this is sufficient for testing within your organization), then click **Next**. +5. Set **Email addresses** to your support email, then click **Next**. +6. Review and check **I agree to the Google API Services: User Data Policy**, then click **Continue** and **Create**. + +Then you need to set the data access for the OAuth: + +1. Navigate to **Data Access**. +2. Click **Add or remove scopes**. +3. Under **Manually add scopes**, paste the following: `https://www.googleapis.com/auth/meetings.space.readonly https://www.googleapis.com/auth/meetings.conference.media.readonly` +4. Click **Add to table**, **Update** and **Save**. + +### 3. Create OAuth Client + +To allow the add-on to authenticate with the Meet Media API: + +1. Go to the **APIs & Services > Credentials** page in the Google Cloud Console. +2. Navigate to **Clients**. +3. Click **+ Create client**. +4. Select **Web application** as the application type. +5. Set **Name** to `Meet Live Agent`. +6. Click **Create**. +7. Note the **Client ID**. + +### 4. Configure Environment Variables + +Copy the sample environment file and fill in your details: + +```bash +cp sample.env .env +``` + +Edit the `.env` file and provide values for: + +* `PROJECT_ID`: Your Google Cloud Project ID. +* `REGION`: The region to deploy to (e.g., `us-central1`). +* `GEMINI_API_KEY`: Your Gemini API key from AI Studio. +* `CLOUD_PROJECT_NUMBER`: Your Google Cloud Project Number (found in Project Settings). +* `CLIENT_ID`: Your OAuth 2.0 Client ID (see step 3). + +You can use the following commands to retrieve some of the required values: + +```bash +# Get Project ID +gcloud config get-value project + +# Get Project Number +gcloud projects describe $(gcloud config get-value project) --format="value(projectNumber)" +``` + +### 5. Deploy to Cloud Run + +1. Run the provided deployment script. This script builds the Docker image and deploys it to Cloud Run, passing the environment variables securely. + + ```bash + chmod +x deploy.sh + ./deploy.sh + ``` + +2. Once the deployment completes, the script will output the URL of your Cloud Run service, copy it. + +### 6. Update OAuth Redirect URIs + +1. Go back to the **APIs & Services > Credentials** page in the Google Cloud Console. +2. Edit the OAuth client you initialized and named **Meet Live Agent** in Step 3. +3. Add the Cloud Run URL you copied in Step 5 to the **Authorized JavaScript origins** list by clicking **+ Add URI**. +4. Click **Save**. + +### 7. Configure Google Workspace Add-on and Marketplace SDK + + To make the app appear in Google Meet, you need to configure both the Workspace Add-on deployment and the Marketplace SDK. + + #### 7.1 Configure Google Workspace Add-on (HTTP Deployment) + + 1. Open the `deployment.json` file in the root of the project. + 2. Update the `addOnOrigins` and `sidePanelUrl` fields, replacing the placeholder `https://YOUR_CLOUD_RUN_URL` with your actual Cloud Run service URL (obtained in Step 5). + 3. Run the following command to create the deployment using the `gcloud` CLI: + + ```bash + gcloud workspace-add-ons deployments create meet-live-agent \ + --deployment-file=deployment.json + ``` + + 4. The **Deployment ID** will be `meet-live-agent`. You will need this in the next step. + + #### 7.2 Configure Google Workspace Marketplace SDK + + 1. Search and select **Google Workspace Marketplace SDK** in the Google Cloud Console. + 2. Click **Manage** then select the **App Configuration** tab. + 3. Set **App Visibility** to **Private** for testing. + 4. Set **Installation Settings** to **Individual + Admin Install**. + 5. Under **App Integrations** select **Google Workspace add-on**, select **HTTP or other deployments**, and select the deployment ID **meet-live-agent**. + 6. Under **Developer Information**, set the **Developer Name**, **Developer Website URL**, and **Developer Email** to your own information. + 7. Click **Save Draft**. + + #### 7.3 Install the Add-on Deployment + + To install the add-on for your account so you can see it in Google Meet, run the following command: + + ```bash + gcloud workspace-add-ons deployments install meet-live-agent + ``` + + ## Testing the Add-on in Google Meet + +After completing the deployment and configuration, you can test the add-on in a live meeting: + +1. Go to [Google Meet](https://meet.google.com) and start a new **instant meeting**. +2. Click the **Meeting tools** icon in the bottom right corner then select the **Add-ons** tab. +3. You should see your add-on **Meet Live Agent** listed as installed. +4. Click on it to open the side panel. +5. Click **Connect to Meet Media API** to start the agent. +6. Go through the OAuth flow and grant all the permissions requested by the add-on. +7. Click **Connect to Meet Media API** in the side panel. +8. Click **Start Meet Live Agent** in the pop-up window to share audio and video of the meeting to the add-on. +9. The side panel should display real-time audio volume and transcripts, and a scene description. +10. You can talk and present in the meeting to test interacting with Gemini Live. + +## Building Locally + + If you want to build the project locally (e.g., to verify the build before deploying): + + 1. Install the dependencies: + ```bash + npm install + ``` + 2. Run the build command: + ```bash + npm run build + ``` + + This will generate the static assets in the `dist` directory. + + *Note: You do not need to build locally to deploy, as the `deploy.sh` script triggers Cloud Build to handle the build process in the cloud.* diff --git a/meet-live-agent/deploy.sh b/meet-live-agent/deploy.sh new file mode 100755 index 0000000..8c09cb5 --- /dev/null +++ b/meet-live-agent/deploy.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status +set -e + +# Load environment variables from .env if it exists +if [ -f .env ]; then + # Export variables ignoring comments and empty lines + export $(grep -v '^#' .env | xargs) +fi + +# Check required environment variables +if [ -z "$PROJECT_ID" ]; then + echo "Error: PROJECT_ID is not set. Please set it in your environment or .env file." + exit 1 +fi + +if [ -z "$GEMINI_API_KEY" ]; then + echo "Error: GEMINI_API_KEY is not set. Please set it in your environment or .env file." + exit 1 +fi + +if [ -z "$CLOUD_PROJECT_NUMBER" ]; then + echo "Error: CLOUD_PROJECT_NUMBER is not set. Please set it in your environment or .env file." + exit 1 +fi + +if [ -z "$CLIENT_ID" ]; then + echo "Error: CLIENT_ID is not set. Please set it in your environment or .env file." + exit 1 +fi + + +# Set default region if not specified +REGION=${REGION:-us-west1} + +echo "Deploying meet-live-agent to Cloud Run in project $PROJECT_ID and region $REGION..." + +gcloud run deploy meet-live-agent \ + --source . \ + --region "$REGION" \ + --project "$PROJECT_ID" \ + --allow-unauthenticated \ + --min-instances 0 \ + --max-instances 3 \ + --memory 1Gi \ + --cpu 1000m \ + --set-env-vars "GEMINI_API_KEY=$GEMINI_API_KEY,CLOUD_PROJECT_NUMBER=$CLOUD_PROJECT_NUMBER,CLIENT_ID=$CLIENT_ID" \ + --clear-base-image \ + --port 3000 diff --git a/meet-live-agent/deployment.json b/meet-live-agent/deployment.json new file mode 100644 index 0000000..bf469ab --- /dev/null +++ b/meet-live-agent/deployment.json @@ -0,0 +1,21 @@ +{ + "addOns": { + "common": { + "name": "Meet Live Agent", + "logoUrl": "https://developers.google.com/chat/images/quickstart-app-avatar.png" + }, + "meet": { + "web": { + "logoUrl": "https://developers.google.com/chat/images/quickstart-app-avatar.png", + "addOnOrigins": [ + "https://YOUR_CLOUD_RUN_URL" + ], + "sidePanelUrl": "https://YOUR_CLOUD_RUN_URL", + "darkModeLogoUrl": "https://developers.google.com/chat/images/quickstart-app-avatar.png" + } + }, + "httpOptions": { + "granularOauthPermissionSupport": "OPT_IN" + } + } +} \ No newline at end of file diff --git a/meet-live-agent/index.css b/meet-live-agent/index.css new file mode 100644 index 0000000..09e8c17 --- /dev/null +++ b/meet-live-agent/index.css @@ -0,0 +1,14 @@ +html, body { + margin: 0; + padding: 0; + background-color: #121212; + color: white; + width: 100%; + height: 100%; +} +body { + display: flex; + align-items: center; + justify-content: center; + font-family: sans-serif; +} diff --git a/meet-live-agent/index.html b/meet-live-agent/index.html new file mode 100644 index 0000000..193967c --- /dev/null +++ b/meet-live-agent/index.html @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/meet-live-agent/index.tsx b/meet-live-agent/index.tsx new file mode 100644 index 0000000..a422afa --- /dev/null +++ b/meet-live-agent/index.tsx @@ -0,0 +1,688 @@ +import { LitElement, css, html } from 'lit'; +import { customElement, state } from 'lit/decorators.js'; +import { meet } from '@googleworkspace/meet-addons'; +import { MeetMediaApiClientImpl } from './internal/meetmediaapiclient_impl'; +import { MeetConnectionState } from './types/enums'; +import { GoogleGenAI, Modality, Session } from '@google/genai'; + +const CLOUD_PROJECT_NUMBER = process.env.CLOUD_PROJECT_NUMBER; +const CLIENT_ID = process.env.CLIENT_ID; + +@customElement('gdm-live-audio') +export class GdmLiveAudio extends LitElement { + @state() connected = false; + @state() connecting = false; + @state() initialized = false; + @state() error = ''; + @state() volume = 0; + @state() transcript = ''; + @state() outputTranscript = ''; + @state() sceneDescription = ''; + + private meetClient: MeetMediaApiClientImpl | null = null; + private isAddonInitialized = false; + private accessToken = ''; + private activeTrackIds = new Set(); + + private audioContext: AudioContext | null = null; + private outputAudioContext: AudioContext | null = null; + private analyser: AnalyserNode | null = null; + private dataArray: Uint8Array | null = null; + private animationFrameId: number | null = null; + + private ai: GoogleGenAI | null = null; + private session: Session | null = null; + private workletNode: AudioWorkletNode | null = null; + + private accumulatedInputData: Float32Array | null = null; + + private accumulatedResponseChunks: Uint8Array[] = []; + private responseTranscriptionTimer: number | null = null; + + private nextStartTime = 0; + private sources = new Set(); + + private videoEl: HTMLVideoElement | null = null; + private canvasEl: HTMLCanvasElement | null = null; + private videoIntervalId: number | null = null; + + static styles = css` + :host { + display: flex; + flex-direction: column; + align-items: center; + justify-content: flex-start; + height: 100%; + box-sizing: border-box; + font-family: sans-serif; + background: #121212; + color: white; + padding: 10px; + } + button { + padding: 15px 30px; + font-size: 18px; + cursor: pointer; + background-color: #007bff; + color: white; + border: none; + border-radius: 5px; + transition: background-color 0.3s; + } + button:hover { + background-color: #0056b3; + } + button[disabled] { + background-color: #555; + cursor: not-allowed; + } + .message { + font-size: 20px; + color: #4caf50; + } + .error { + color: #f44336; + margin-top: 10px; + } + .volume-bar { + width: 200px; + height: 20px; + background-color: #333; + border-radius: 10px; + overflow: hidden; + margin-top: 20px; + } + .volume-level { + height: 100%; + background-color: #4caf50; + transition: width 0.1s ease; + } + .transcript-area { + width: 95%; + flex-grow: 1; + margin-top: 10px; + background-color: #222; + color: #ccc; + border: 1px solid #444; + border-radius: 5px; + padding: 10px; + font-family: monospace; + resize: none; + } + .label { + align-self: flex-start; + margin-left: 5%; + margin-top: 15px; + font-weight: bold; + color: #aaa; + } + .hidden-video { + display: none; + } + `; + + constructor() { + super(); + this.unloadHandler = this.unloadHandler.bind(this); + } + + connectedCallback() { + super.connectedCallback(); + window.addEventListener('unload', this.unloadHandler); + } + + disconnectedCallback() { + super.disconnectedCallback(); + window.removeEventListener('unload', this.unloadHandler); + this.disconnect(); + } + + private unloadHandler() { + this.disconnect(); + } + + firstUpdated() { + this.initializeSession(); + } + + private initializeSession() { + const google = (window as any).google; + if (!google) { + this.error = "Google Identity Services not loaded"; + return; + } + + const client = google.accounts.oauth2.initTokenClient({ + client_id: CLIENT_ID, + scope: 'https://www.googleapis.com/auth/meetings.space.created https://www.googleapis.com/auth/meetings.conference.media.readonly https://www.googleapis.com/auth/meetings.space.readonly', + callback: async (tokenResponse: any) => { + this.accessToken = tokenResponse.access_token; + await this.initializeAddon(); + const meetingId = (window as any).meetingId; + if (!meetingId) { + this.error = "Meeting ID not found"; + return; + } + this.initialized = true; + }, + error_callback: (errorResponse: any) => { + this.error = "Authentication failed"; + }, + }); + + client.requestAccessToken(); + } + + private async initializeAddon() { + if (this.isAddonInitialized) return; + const session = await meet.addon.createAddonSession({ + cloudProjectNumber: CLOUD_PROJECT_NUMBER, + }); + const sidePanelClient = await session.createSidePanelClient(); + const meetingInfo = await sidePanelClient.getMeetingInfo(); + (window as any).meetingId = meetingInfo.meetingId; + this.isAddonInitialized = true; + } + + private async connect() { + if (!this.initialized || !this.accessToken) return; + + this.connecting = true; + this.error = ''; + const meetingId = (window as any).meetingId; + + try { + // Initialize AudioContexts. Gemini expects 16kHz input and returns 24kHz output. + this.audioContext = new AudioContext({ sampleRate: 16000 }); + this.outputAudioContext = new AudioContext({ sampleRate: 24000 }); + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 256; + this.dataArray = new Uint8Array(this.analyser.frequencyBinCount); + + this.nextStartTime = this.outputAudioContext.currentTime; + + // Load the AudioWorklet that captures raw PCM audio data. + await this.audioContext.audioWorklet.addModule('/pcm-recorder-processor.js'); + this.workletNode = new AudioWorkletNode(this.audioContext, 'pcm-recorder-processor'); + + this.workletNode.port.onmessage = (e) => { + const inputData = e.data; // Float32Array + + // Accumulate for transcription (approx 5 seconds at 16kHz = 80000 samples) + if (!this.accumulatedInputData) { + this.accumulatedInputData = inputData; + } else { + const newArray = new Float32Array(this.accumulatedInputData.length + inputData.length); + newArray.set(this.accumulatedInputData); + newArray.set(inputData, this.accumulatedInputData.length); + this.accumulatedInputData = newArray; + } + + if (this.accumulatedInputData.length >= 80000) { + const dataToTranscribe = this.accumulatedInputData; + this.accumulatedInputData = null; // Reset buffer + this.transcribeInputAudio(dataToTranscribe); + } + + const pcmBuffer = this.floatTo16BitPCM(inputData); + const base64Data = this.arrayBufferToBase64(pcmBuffer); + + if (this.session) { + try { + this.session.sendRealtimeInput({ + audio: { + mimeType: "audio/pcm;rate=16000", + data: base64Data + } + }); + if (Math.random() < 0.01) { + console.log("Sent audio chunk to Gemini"); + } + } catch (err) { + console.error("Error sending audio to Gemini:", err); + } + } + }; + + // Initialize Gemini Live session. + this.ai = new GoogleGenAI({ apiKey: process.env.GEMINI_API_KEY }); + const model = 'gemini-3.1-flash-live-preview'; // Use the live preview model + + this.session = await this.ai.live.connect({ + model: model, + config: { + responseModalities: [Modality.AUDIO], + }, + callbacks: { + onopen: () => { + console.log("Gemini Live: Session opened."); + }, + onmessage: (message) => { + const parts = message.serverContent?.modelTurn?.parts; + if (parts) { + for (const part of parts) { + if (part.inlineData) { + const audio = part.inlineData; + const pcmBytes = this.base64ToUint8Array(audio.data); + + this.accumulatedResponseChunks.push(pcmBytes); + + if (this.responseTranscriptionTimer) { + clearTimeout(this.responseTranscriptionTimer); + } + this.responseTranscriptionTimer = window.setTimeout(() => { + this.transcribeResponseAudio(); + }, 1000); + + this.nextStartTime = Math.max( + this.nextStartTime, + this.outputAudioContext!.currentTime, + ); + + const audioBuffer = this.outputAudioContext!.createBuffer(1, pcmBytes.length / 2, 24000); + const channelData = audioBuffer.getChannelData(0); + const view = new DataView(pcmBytes.buffer); + for (let i = 0; i < channelData.length; i++) { + channelData[i] = view.getInt16(i * 2, true) / 0x7FFF; + } + + const source = this.outputAudioContext!.createBufferSource(); + source.buffer = audioBuffer; + source.connect(this.outputAudioContext!.destination); + source.addEventListener('ended', () => { + this.sources.delete(source); + }); + + source.start(this.nextStartTime); + this.nextStartTime = this.nextStartTime + audioBuffer.duration; + this.sources.add(source); + } + } + } + if (Math.random() < 0.01) { + console.log("Received message from Gemini:", message); + } + }, + onerror: (e) => { + console.error("Gemini Live error:", e); + }, + onclose: (e) => { + console.log("Gemini Live closed:", e.reason); + this.session = null; + } + } + }); + + // Initialize the Meet Media API client. + this.meetClient = new MeetMediaApiClientImpl({ + meetingSpaceId: meetingId, + numberOfVideoStreams: 1, + enableAudioStreams: true, + accessToken: this.accessToken, + logsCallback: (event) => console.log(`Meet Media API [${event.sourceType}]:`, event.logString), + }); + + this.meetClient.sessionStatus.subscribe((status) => { + if (status.connectionState === MeetConnectionState.JOINED) { + this.connected = true; + this.connecting = false; + this.startVolumeAnalysis(); + + console.log("Applying layout to receive video..."); + const mediaLayout = this.meetClient!.createMediaLayout({ width: 768, height: 768 }); + this.meetClient!.applyLayout([{ mediaLayout }]).catch(e => console.error("Error applying layout:", e)); + } + }); + + this.meetClient.meetStreamTracks.subscribe((tracks) => { + tracks.forEach((meetTrack) => { + const track = meetTrack.mediaStreamTrack; + if (track.kind === 'audio' && !this.activeTrackIds.has(track.id)) { + console.log("Connecting audio track:", track.id); + + const audioEl = document.createElement('audio'); + audioEl.muted = true; + audioEl.srcObject = new MediaStream([track]); + audioEl.play().catch(e => console.error("Error playing wakeup audio:", e)); + (this as any)[`wakeupAudio_${track.id}`] = audioEl; + + const source = this.audioContext!.createMediaStreamSource(new MediaStream([track])); + source.connect(this.analyser!); + source.connect(this.workletNode!); + this.activeTrackIds.add(track.id); + } else if (track.kind === 'video') { + console.log("Connecting video track:", track.id); + this.videoEl = document.createElement('video'); + this.videoEl.srcObject = new MediaStream([track]); + this.videoEl.muted = true; + this.videoEl.setAttribute('playsinline', 'true'); + + // Make it invisible but in the DOM + this.videoEl.style.position = 'absolute'; + this.videoEl.style.width = '0'; + this.videoEl.style.height = '0'; + this.videoEl.style.opacity = '0'; + this.videoEl.style.pointerEvents = 'none'; + + this.shadowRoot!.appendChild(this.videoEl); + + this.videoEl.play().catch(e => console.error("Error playing video:", e)); + + this.canvasEl = document.createElement('canvas'); + this.canvasEl.width = 768; + this.canvasEl.height = 768; + + this.startVideoProcessing(); + } + }); + }); + + await this.meetClient.joinMeeting(); + } catch (e: any) { + this.error = `Failed to connect: ${e.message || e}`; + this.connecting = false; + } + } + + private startVideoProcessing() { + this.videoIntervalId = window.setInterval(() => { + this.captureAndProcessFrame(); + }, 5000); // Every 5 seconds + } + + private async captureAndProcessFrame() { + if (!this.videoEl || !this.canvasEl || !this.session) return; + + const ctx = this.canvasEl.getContext('2d'); + if (!ctx) return; + + // Draw the video frame to the canvas (resizing to 768x768) + ctx.drawImage(this.videoEl, 0, 0, this.canvasEl.width, this.canvasEl.height); + + // Get base64 JPEG + const base64Data = this.canvasEl.toDataURL('image/jpeg', 0.8).split(',')[1]; + + // Send to Gemini Live + try { + this.session.sendRealtimeInput({ + video: { + mimeType: "image/jpeg", + data: base64Data + } + }); + console.log("Sent video frame to Gemini Live"); + } catch (e) { + console.error("Error sending video to Gemini Live:", e); + } + + // Send to Description Model + if (!this.ai) return; + try { + const response = await this.ai.models.generateContent({ + model: 'gemini-2.5-flash', + contents: [ + { + inlineData: { + mimeType: 'image/jpeg', + data: base64Data + } + }, + "Describe what is seen in this image in one or two sentences." + ] + }); + this.sceneDescription = response.text || 'No description available'; + console.log("Updated scene description."); + } catch (e) { + console.error("Failed to get scene description:", e); + } + } + + private startVolumeAnalysis() { + const updateVolume = () => { + if (this.analyser && this.dataArray) { + this.analyser.getByteFrequencyData(this.dataArray as any); + let sum = 0; + for (let i = 0; i < this.dataArray.length; i++) { + sum += this.dataArray[i]; + } + const average = sum / this.dataArray.length; + this.volume = average; + this.animationFrameId = requestAnimationFrame(updateVolume); + } + }; + updateVolume(); + } + + private async transcribeInputAudio(float32Array: Float32Array) { + console.log("Transcribing accumulated input audio..."); + try { + const pcmBuffer = this.floatTo16BitPCM(float32Array); + const wavBytes = this.addWavHeader(new Uint8Array(pcmBuffer), 16000); + const base64Wav = this.arrayBufferToBase64(wavBytes.buffer); + + if (!this.ai) return; + + const response = await this.ai.models.generateContent({ + model: 'gemini-2.5-flash', + contents: [ + { + inlineData: { + mimeType: 'audio/wav', + data: base64Wav + } + }, + "Please provide a transcript of this audio. If it is only noise, say 'Noise'." + ] + }); + + const text = response.text || ''; + this.transcript = (text.trim().toLowerCase() === 'noise' || text.trim().toLowerCase() === 'noise.') ? '' : text; + } catch (e) { + console.error("Failed to transcribe input audio:", e); + this.transcript = 'Failed to transcribe audio.'; + } + } + + private concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array { + let totalLength = 0; + for (const arr of arrays) { + totalLength += arr.length; + } + const result = new Uint8Array(totalLength); + let offset = 0; + for (const arr of arrays) { + result.set(arr, offset); + offset += arr.length; + } + return result; + } + + private async transcribeResponseAudio() { + if (this.accumulatedResponseChunks.length === 0) return; + + const chunks = this.accumulatedResponseChunks; + this.accumulatedResponseChunks = []; + this.responseTranscriptionTimer = null; + + console.log("Transcribing accumulated response audio..."); + try { + const pcmBytes = this.concatenateUint8Arrays(chunks); + const wavBytes = this.addWavHeader(pcmBytes, 24000); + const base64Wav = this.arrayBufferToBase64(wavBytes.buffer); + + if (!this.ai) return; + + const response = await this.ai.models.generateContent({ + model: 'gemini-2.5-flash', + contents: [ + { + inlineData: { + mimeType: 'audio/wav', + data: base64Wav + } + }, + "Please provide a transcript of this audio. If you cannot hear anything, say 'Silence'." + ] + }); + + const text = response.text || ''; + this.outputTranscript = (text.trim().toLowerCase() === 'silence' || text.trim().toLowerCase() === 'silence.') ? '' : text; + } catch (e) { + console.error("Failed to transcribe response audio:", e); + this.outputTranscript = 'Failed to transcribe response.'; + } + } + + private addWavHeader(pcmData: Uint8Array, sampleRate: number): Uint8Array { + const header = new ArrayBuffer(44); + const view = new DataView(header); + + const writeString = (offset: number, string: string) => { + for (let i = 0; i < string.length; i++) { + view.setUint8(offset + i, string.charCodeAt(i)); + } + }; + + writeString(0, 'RIFF'); + view.setUint32(4, 36 + pcmData.length, true); + writeString(8, 'WAVE'); + writeString(12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + writeString(36, 'data'); + view.setUint32(40, pcmData.length, true); + + const wav = new Uint8Array(44 + pcmData.length); + wav.set(new Uint8Array(header), 0); + wav.set(pcmData, 44); + + return wav; + } + + private async disconnect() { + if (this.animationFrameId) { + cancelAnimationFrame(this.animationFrameId); + this.animationFrameId = null; + } + if (this.videoIntervalId) { + clearInterval(this.videoIntervalId); + this.videoIntervalId = null; + } + if (this.audioContext) { + await this.audioContext.close(); + this.audioContext = null; + } + if (this.outputAudioContext) { + await this.outputAudioContext.close(); + this.outputAudioContext = null; + } + if (this.session) { + this.session.close(); + this.session = null; + } + if (this.meetClient) { + try { + await this.meetClient.leaveMeeting(); + } catch (e) { + console.error("Error leaving meeting:", e); + } + this.meetClient = null; + } + + this.activeTrackIds.forEach(id => { + const audioEl = (this as any)[`wakeupAudio_${id}`]; + if (audioEl) { + audioEl.srcObject = null; + delete (this as any)[`wakeupAudio_${id}`]; + } + }); + this.activeTrackIds.clear(); + + if (this.videoEl) { + this.videoEl.srcObject = null; + this.videoEl.remove(); + this.videoEl = null; + } + this.canvasEl = null; + + this.connected = false; + this.connecting = false; + this.volume = 0; + this.transcript = ''; + this.outputTranscript = ''; + this.sceneDescription = ''; + this.accumulatedResponseChunks = []; + + this.sources.forEach(source => source.stop()); + this.sources.clear(); + this.nextStartTime = 0; + } + + private floatTo16BitPCM(float32Array: Float32Array): ArrayBuffer { + const buffer = new ArrayBuffer(float32Array.length * 2); + const view = new DataView(buffer); + let offset = 0; + for (let i = 0; i < float32Array.length; i++, offset += 2) { + let s = Math.max(-1, Math.min(1, float32Array[i])); + view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true); + } + return buffer; + } + + private arrayBufferToBase64(buffer: ArrayBufferLike): string { + let binary = ''; + const bytes = new Uint8Array(buffer); + const len = bytes.byteLength; + for (let i = 0; i < len; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); + } + + private base64ToUint8Array(base64: string): Uint8Array { + const binaryString = atob(base64); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes; + } + + render() { + const volumePercentage = (this.volume / 255) * 100; + return html` + ${!this.initialized ? html`
Initializing...
` : ''} + + ${this.initialized && !this.connected && !this.connecting ? html` + + ` : ''} + + ${this.connecting ? html`
Connecting...
` : ''} + + ${this.connected ? html` +
Connected successfully!
+
Volume: ${Math.round(volumePercentage)}%
+
+
+
+ +
Input Transcript:
+ + +
Output Transcript:
+ + +
Scene Description:
+ + ` : ''} + + ${this.error ? html`
${this.error}
` : ''} + `; + } +} diff --git a/meet-live-agent/internal/channel_handlers/channel_logger.ts b/meet-live-agent/internal/channel_handlers/channel_logger.ts new file mode 100644 index 0000000..1a54bfb --- /dev/null +++ b/meet-live-agent/internal/channel_handlers/channel_logger.ts @@ -0,0 +1,58 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview A helper class that allows user to logs events to a specified + * function. + */ + +import { + DeletedResource, + MediaApiRequest, + MediaApiResponse, + ResourceSnapshot, +} from '../../types/datachannels'; +import {LogLevel} from '../../types/enums'; +import {LogEvent, LogSourceType} from '../../types/mediatypes'; + +/** + * Helper class that helps log channel resources, updates or errors. + */ +export class ChannelLogger { + constructor( + private readonly logSourceType: LogSourceType, + // @ts-ignore + private readonly callback = (logEvent: LogEvent) => {}, + ) {} + + log( + level: LogLevel, + logString: string, + relevantObject?: + | Error + | DeletedResource + | ResourceSnapshot + | MediaApiResponse + | MediaApiRequest, + ) { + this.callback({ + sourceType: this.logSourceType, + level, + logString, + relevantObject, + }); + } +} diff --git a/meet-live-agent/internal/channel_handlers/media_entries_channel_handler.ts b/meet-live-agent/internal/channel_handlers/media_entries_channel_handler.ts new file mode 100644 index 0000000..eead69a --- /dev/null +++ b/meet-live-agent/internal/channel_handlers/media_entries_channel_handler.ts @@ -0,0 +1,401 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Handles Media entries + */ + +import { + DeletedMediaEntry, + MediaEntriesChannelToClient, + MediaEntryResource, +} from '../../types/datachannels'; +import {LogLevel} from '../../types/enums'; +import { + MediaEntry, + MediaLayout, + MeetStreamTrack, + Participant, +} from '../../types/mediatypes'; +import { + InternalMediaEntry, + InternalMediaLayout, + InternalMeetStreamTrack, + InternalParticipant, +} from '../internal_types'; +import {SubscribableDelegate} from '../subscribable_impl'; +import {createMediaEntry} from '../utils'; +import {ChannelLogger} from './channel_logger'; + +/** + * Helper class to handle the media entries channel. + */ +export class MediaEntriesChannelHandler { + constructor( + private readonly channel: RTCDataChannel, + private readonly mediaEntriesDelegate: SubscribableDelegate, + private readonly idMediaEntryMap: Map, + private readonly internalMediaEntryMap = new Map< + MediaEntry, + InternalMediaEntry + >(), + private readonly internalMeetStreamTrackMap = new Map< + MeetStreamTrack, + InternalMeetStreamTrack + >(), + private readonly internalMediaLayoutMap = new Map< + MediaLayout, + InternalMediaLayout + >(), + private readonly participantsDelegate: SubscribableDelegate, + private readonly nameParticipantMap: Map, + private readonly idParticipantMap: Map, + private readonly internalParticipantMap: Map< + Participant, + InternalParticipant + >, + private readonly presenterDelegate: SubscribableDelegate< + MediaEntry | undefined + >, + private readonly screenshareDelegate: SubscribableDelegate< + MediaEntry | undefined + >, + private readonly channelLogger?: ChannelLogger, + ) { + this.channel.onmessage = (event) => { + this.onMediaEntriesMessage(event); + }; + this.channel.onopen = () => { + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Media entries channel: opened', + ); + }; + this.channel.onclose = () => { + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Media entries channel: closed', + ); + }; + } + + private onMediaEntriesMessage(message: MessageEvent) { + const data = JSON.parse(message.data) as MediaEntriesChannelToClient; + let mediaEntryArray = this.mediaEntriesDelegate.get(); + + // Delete media entries. + data.deletedResources?.forEach((deletedResource: DeletedMediaEntry) => { + this.channelLogger?.log( + LogLevel.RESOURCES, + 'Media entries channel: resource deleted', + deletedResource, + ); + const deletedMediaEntry = this.idMediaEntryMap.get(deletedResource.id); + if (deletedMediaEntry) { + mediaEntryArray = mediaEntryArray.filter( + (mediaEntry) => mediaEntry !== deletedMediaEntry, + ); + // If we find the media entry in the id map, it should exist in the + // internal map. + const internalMediaEntry = + this.internalMediaEntryMap.get(deletedMediaEntry); + // Remove relationship between media entry and media layout. + const mediaLayout: MediaLayout | undefined = + internalMediaEntry!.mediaLayout.get(); + if (mediaLayout) { + const internalMediaLayout = + this.internalMediaLayoutMap.get(mediaLayout); + if (internalMediaLayout) { + internalMediaLayout.mediaEntry.set(undefined); + } + } + + // Remove relationship between media entry and meet stream tracks. + const videoMeetStreamTrack = + internalMediaEntry!.videoMeetStreamTrack.get(); + if (videoMeetStreamTrack) { + const internalVideoStreamTrack = + this.internalMeetStreamTrackMap.get(videoMeetStreamTrack); + internalVideoStreamTrack!.mediaEntry.set(undefined); + } + + const audioMeetStreamTrack = + internalMediaEntry!.audioMeetStreamTrack.get(); + if (audioMeetStreamTrack) { + const internalAudioStreamTrack = + this.internalMeetStreamTrackMap.get(audioMeetStreamTrack); + internalAudioStreamTrack!.mediaEntry.set(undefined); + } + + // Remove relationship between media entry and participant. + const participant = internalMediaEntry!.participant.get(); + if (participant) { + const internalParticipant = + this.internalParticipantMap.get(participant); + const newMediaEntries: MediaEntry[] = + internalParticipant!.mediaEntries + .get() + .filter((mediaEntry) => mediaEntry !== deletedMediaEntry); + internalParticipant!.mediaEntries.set(newMediaEntries); + internalMediaEntry!.participant.set(undefined); + } + + // Remove from maps + this.idMediaEntryMap.delete(deletedResource.id); + this.internalMediaEntryMap.delete(deletedMediaEntry); + + if (this.screenshareDelegate.get() === deletedMediaEntry) { + this.screenshareDelegate.set(undefined); + } + if (this.presenterDelegate.get() === deletedMediaEntry) { + this.presenterDelegate.set(undefined); + } + } + }); + + // Update or add media entries. + const addedMediaEntries: MediaEntry[] = []; + data.resources?.forEach((resource: MediaEntryResource) => { + this.channelLogger?.log( + LogLevel.RESOURCES, + 'Media entries channel: resource added', + resource, + ); + + let internalMediaEntry: InternalMediaEntry | undefined; + let mediaEntry: MediaEntry | undefined; + let videoCsrc = 0; + if ( + resource.mediaEntry.videoCsrcs && + resource.mediaEntry.videoCsrcs.length > 0 + ) { + // We expect there to only be one video Csrcs. There is possibility + // for this to be more than value in WebRTC but unlikely in Meet. + // TODO : Explore making video csrcs field singluar. + videoCsrc = resource.mediaEntry.videoCsrcs[0]; + } else { + this.channelLogger?.log( + LogLevel.ERRORS, + 'Media entries channel: more than one video Csrc in media entry', + resource, + ); + } + + if (this.idMediaEntryMap.has(resource.id!)) { + // Update media entry if it already exists. + mediaEntry = this.idMediaEntryMap.get(resource.id!); + mediaEntry!.sessionName = resource.mediaEntry.sessionName; + mediaEntry!.session = resource.mediaEntry.session; + internalMediaEntry = this.internalMediaEntryMap.get(mediaEntry!); + internalMediaEntry!.audioMuted.set(resource.mediaEntry.audioMuted); + internalMediaEntry!.videoMuted.set(resource.mediaEntry.videoMuted); + internalMediaEntry!.screenShare.set(resource.mediaEntry.screenshare); + internalMediaEntry!.isPresenter.set(resource.mediaEntry.presenter); + internalMediaEntry!.audioCsrc = resource.mediaEntry.audioCsrc; + internalMediaEntry!.videoCsrc = videoCsrc; + } else { + // Create new media entry if it does not exist. + const mediaEntryElement = createMediaEntry({ + audioMuted: resource.mediaEntry.audioMuted, + videoMuted: resource.mediaEntry.videoMuted, + screenShare: resource.mediaEntry.screenshare, + isPresenter: resource.mediaEntry.presenter, + id: resource.id!, + audioCsrc: resource.mediaEntry.audioCsrc, + videoCsrc, + sessionName: resource.mediaEntry.sessionName, + session: resource.mediaEntry.session, + }); + internalMediaEntry = mediaEntryElement.internalMediaEntry; + mediaEntry = mediaEntryElement.mediaEntry; + this.internalMediaEntryMap.set(mediaEntry, internalMediaEntry); + this.idMediaEntryMap.set(internalMediaEntry.id, mediaEntry); + addedMediaEntries.push(mediaEntry); + } + + // Assign meet streams to media entry if they are not already assigned + // correctly. + if ( + !mediaEntry!.audioMuted.get() && + internalMediaEntry!.audioCsrc && + !this.isMediaEntryAssignedToMeetStreamTrack(internalMediaEntry!) + ) { + this.assignAudioMeetStreamTrack(mediaEntry!, internalMediaEntry!); + } + + // Assign participant to media entry + let existingParticipant: Participant | undefined; + if (resource.mediaEntry.participant) { + existingParticipant = this.nameParticipantMap.get( + resource.mediaEntry.participant, + ); + } else if (resource.mediaEntry.participantKey) { + existingParticipant = Array.from( + this.internalParticipantMap.entries(), + ).find( + ([participant, _]) => + participant.participant.participantKey === + resource.mediaEntry.participantKey, + )?.[0]; + } + + if (existingParticipant) { + const internalParticipant = + this.internalParticipantMap.get(existingParticipant); + if (internalParticipant) { + const newMediaEntries: MediaEntry[] = [ + ...internalParticipant.mediaEntries.get(), + mediaEntry!, + ]; + internalParticipant.mediaEntries.set(newMediaEntries); + } + internalMediaEntry!.participant.set(existingParticipant); + } else if ( + resource.mediaEntry.participant || + resource.mediaEntry.participantKey + ) { + // This is unexpected behavior, but technically possible. We expect + // that the participants are received from the participants channel + // before the media entries channel but this is not guaranteed. + this.channelLogger?.log( + LogLevel.RESOURCES, + 'Media entries channel: participant not found in name participant map' + + ' creating participant', + ); + const subscribableDelegate = new SubscribableDelegate([ + mediaEntry!, + ]); + const newParticipant: Participant = { + participant: { + name: resource.mediaEntry.participant, + anonymousUser: {}, + participantKey: resource.mediaEntry.participantKey, + }, + mediaEntries: subscribableDelegate.getSubscribable(), + }; + // TODO: Use participant resource name instead of id. + // tslint:disable-next-line:deprecation + const ids: Set = resource.mediaEntry.participantId + ? // tslint:disable-next-line:deprecation + new Set([resource.mediaEntry.participantId]) + : new Set(); + const internalParticipant: InternalParticipant = { + name: resource.mediaEntry.participant ?? '', + ids, + mediaEntries: subscribableDelegate, + }; + if (resource.mediaEntry.participant) { + this.nameParticipantMap.set( + resource.mediaEntry.participant, + newParticipant, + ); + } + this.internalParticipantMap.set(newParticipant, internalParticipant); + // TODO: Use participant resource name instead of id. + // tslint:disable-next-line:deprecation + if (resource.mediaEntry.participantId) { + this.idParticipantMap.set( + // TODO: Use participant resource name instead of id. + // tslint:disable-next-line:deprecation + resource.mediaEntry.participantId, + newParticipant, + ); + } + const participantArray = this.participantsDelegate.get(); + this.participantsDelegate.set([...participantArray, newParticipant]); + internalMediaEntry!.participant.set(newParticipant); + } + if (resource.mediaEntry.presenter) { + this.presenterDelegate.set(mediaEntry); + } else if ( + !resource.mediaEntry.presenter && + this.presenterDelegate.get() === mediaEntry + ) { + this.presenterDelegate.set(undefined); + } + if (resource.mediaEntry.screenshare) { + this.screenshareDelegate.set(mediaEntry); + } else if ( + !resource.mediaEntry.screenshare && + this.screenshareDelegate.get() === mediaEntry + ) { + this.screenshareDelegate.set(undefined); + } + }); + + // Update media entry collection. + if ( + (data.resources && data.resources.length > 0) || + (data.deletedResources && data.deletedResources.length > 0) + ) { + const newMediaEntryArray = [...mediaEntryArray, ...addedMediaEntries]; + this.mediaEntriesDelegate.set(newMediaEntryArray); + } + } + + private isMediaEntryAssignedToMeetStreamTrack( + internalMediaEntry: InternalMediaEntry, + ): boolean { + const audioStreamTrack = internalMediaEntry.audioMeetStreamTrack.get(); + if (!audioStreamTrack) return false; + const internalAudioMeetStreamTrack = + this.internalMeetStreamTrackMap.get(audioStreamTrack); + // This is not expected. Map should be comprehensive of all meet stream + // tracks. + if (!internalAudioMeetStreamTrack) return false; + // The Audio CRSCs changed and therefore need to be checked if the current + // audio csrc is in the contributing sources. + const contributingSources: RTCRtpContributingSource[] = + internalAudioMeetStreamTrack.receiver.getContributingSources(); + + for (const contributingSource of contributingSources) { + if (contributingSource.source === internalMediaEntry.audioCsrc) { + // Audio Csrc found in contributing sources. + return true; + } + } + // Audio Csrc not found in contributing sources, unassign audio meet stream + // track. + internalMediaEntry.audioMeetStreamTrack.set(undefined); + return false; + } + + private assignAudioMeetStreamTrack( + mediaEntry: MediaEntry, + internalMediaEntry: InternalMediaEntry, + ) { + for (const [ + meetStreamTrack, + internalMeetStreamTrack, + ] of this.internalMeetStreamTrackMap.entries()) { + // Only audio tracks are assigned here. + if (meetStreamTrack.mediaStreamTrack.kind !== 'audio') continue; + const receiver = internalMeetStreamTrack.receiver; + const contributingSources: RTCRtpContributingSource[] = + receiver.getContributingSources(); + for (const contributingSource of contributingSources) { + if (contributingSource.source === internalMediaEntry.audioCsrc) { + internalMediaEntry.audioMeetStreamTrack.set(meetStreamTrack); + internalMeetStreamTrack.mediaEntry.set(mediaEntry); + return; + } + } + // If Audio Csrc is not found in contributing sources, fall back to + // polling frames for assignment. + internalMeetStreamTrack.maybeAssignMediaEntryOnFrame(mediaEntry, 'audio'); + } + } +} diff --git a/meet-live-agent/internal/channel_handlers/media_stats_channel_handler.ts b/meet-live-agent/internal/channel_handlers/media_stats_channel_handler.ts new file mode 100644 index 0000000..166c1e8 --- /dev/null +++ b/meet-live-agent/internal/channel_handlers/media_stats_channel_handler.ts @@ -0,0 +1,258 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview A class to handle the media stats channel. + */ + +import { + MediaApiResponseStatus, + MediaStatsChannelFromClient, + MediaStatsChannelToClient, + MediaStatsResource, + StatsSectionData, + UploadMediaStatsRequest, + UploadMediaStatsResponse, +} from '../../types/datachannels'; +import { LogLevel } from '../../types/enums'; +import { ChannelLogger } from './channel_logger'; + +type SupportedMediaStatsTypes = + | 'codec' + | 'candidate-pair' + | 'media-playout' + | 'transport' + | 'local-candidate' + | 'remote-candidate' + | 'inbound-rtp'; + +const STATS_TYPE_CONVERTER: { [key: string]: string } = { + 'codec': 'codec', + 'candidate-pair': 'candidate_pair', + 'media-playout': 'media_playout', + 'transport': 'transport', + 'local-candidate': 'local_candidate', + 'remote-candidate': 'remote_candidate', + 'inbound-rtp': 'inbound_rtp', +}; + +/** + * Helper class to handle the media stats channel. This class is responsible + * for sending media stats to the backend and receiving configuration updates + * from the backend. For realtime metrics when debugging manually, use + * chrome://webrtc-internals. + */ +export class MediaStatsChannelHandler { + /** + * A map of allowlisted sections. The key is the section type, and the value + * is the keys that are allowlisted for that section. + */ + private readonly allowlist = new Map(); + private requestId = 1; + private readonly pendingRequestResolveMap = new Map< + number, + (value: MediaApiResponseStatus) => void + >(); + /** Id for the interval to send media stats. */ + private intervalId = 0; + + constructor( + private readonly channel: RTCDataChannel, + private readonly peerConnection: RTCPeerConnection, + private readonly channelLogger?: ChannelLogger, + ) { + this.channel.onmessage = (event) => { + this.onMediaStatsMessage(event); + }; + this.channel.onclose = () => { + clearInterval(this.intervalId); + this.intervalId = 0; + this.channelLogger?.log(LogLevel.MESSAGES, 'Media stats channel: closed'); + // Resolve all pending requests with an error. + for (const [, resolve] of this.pendingRequestResolveMap) { + resolve({ code: 400, message: 'Channel closed', details: [] }); + } + this.pendingRequestResolveMap.clear(); + }; + this.channel.onopen = () => { + this.channelLogger?.log(LogLevel.MESSAGES, 'Media stats channel: opened'); + }; + } + + private onMediaStatsMessage(message: MessageEvent) { + const data = JSON.parse(message.data) as MediaStatsChannelToClient; + if (data.response) { + this.onMediaStatsResponse(data.response); + } + if (data.resources) { + this.onMediaStatsResources(data.resources); + } + } + + private onMediaStatsResponse(response: UploadMediaStatsResponse) { + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Media stats channel: response received', + response, + ); + const resolve = this.pendingRequestResolveMap.get(response.requestId); + if (resolve) { + resolve(response.status); + this.pendingRequestResolveMap.delete(response.requestId); + } + } + + private onMediaStatsResources(resources: MediaStatsResource[]) { + // We expect only one resource to be sent. + if (resources.length > 1) { + resources.forEach((resource) => { + this.channelLogger?.log( + LogLevel.ERRORS, + 'Media stats channel: more than one resource received', + resource, + ); + }); + } + const resource = resources[0]; + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Media stats channel: resource received', + resource, + ); + if (resource.configuration) { + for (const [key, value] of Object.entries( + resource.configuration.allowlist, + )) { + this.allowlist.set(key, value.keys); + } + // We want to stop the interval if the upload interval is zero + if ( + this.intervalId && + resource.configuration.uploadIntervalSeconds === 0 + ) { + clearInterval(this.intervalId); + this.intervalId = 0; + } + // We want to start the interval if the upload interval is not zero. + if (resource.configuration.uploadIntervalSeconds) { + // We want to reset the interval if the upload interval has changed. + if (this.intervalId) { + clearInterval(this.intervalId); + } + this.intervalId = window.setInterval( + this.sendMediaStats.bind(this), + resource.configuration.uploadIntervalSeconds * 1000, + ); + } + } else { + this.channelLogger?.log( + LogLevel.ERRORS, + 'Media stats channel: resource received without configuration', + ); + } + } + + async sendMediaStats(): Promise { + const stats: RTCStatsReport = await this.peerConnection.getStats(); + const requestStats: StatsSectionData[] = []; + + stats.forEach( + ( + report: + | RTCTransportStats + | RTCIceCandidatePairStats + | RTCOutboundRtpStreamStats + | RTCInboundRtpStreamStats, + ) => { + const statsType = report.type as SupportedMediaStatsTypes; + if (statsType && this.allowlist.has(report.type)) { + const filteredMediaStats: { [key: string]: string | number } = {}; + Object.entries(report).forEach((entry) => { + // id is not accepted with other stats. It is populated in the top + // level section. + if ( + this.allowlist.get(report.type)?.includes(entry[0]) && + entry[0] !== 'id' + ) { + // We want to convert the camel case to underscore. + filteredMediaStats[this.camelToUnderscore(entry[0])] = entry[1]; + } + }); + const filteredMediaStatsDictionary = { + 'id': report.id, + [STATS_TYPE_CONVERTER[report.type as string]]: filteredMediaStats, + }; + const filteredStatsSectionData = + filteredMediaStatsDictionary as StatsSectionData; + + requestStats.push(filteredStatsSectionData); + } + }, + ); + + if (!requestStats.length) { + this.channelLogger?.log( + LogLevel.ERRORS, + 'Media stats channel: no media stats to send', + ); + return { code: 400, message: 'No media stats to send', details: [] }; + } + + if (this.channel.readyState === 'open') { + const mediaStatsRequest: UploadMediaStatsRequest = { + requestId: this.requestId, + uploadMediaStats: { sections: requestStats }, + }; + + const request: MediaStatsChannelFromClient = { + request: mediaStatsRequest, + }; + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Media stats channel: sending request', + mediaStatsRequest, + ); + try { + this.channel.send(JSON.stringify(request)); + } catch (e) { + this.channelLogger?.log( + LogLevel.ERRORS, + 'Media stats channel: Failed to send request with error', + e as Error, + ); + throw e; + } + + this.requestId++; + const requestPromise = new Promise((resolve) => { + this.pendingRequestResolveMap.set(mediaStatsRequest.requestId, resolve); + }); + return requestPromise; + } else { + clearInterval(this.intervalId); + this.intervalId = 0; + this.channelLogger?.log( + LogLevel.ERRORS, + 'Media stats channel: handler tried to send message when channel was closed', + ); + return { code: 400, message: 'Channel is not open', details: [] }; + } + } + + private camelToUnderscore(text: string): string { + return text.replace(/([A-Z])/g, '_$1').toLowerCase(); + } +} diff --git a/meet-live-agent/internal/channel_handlers/participants_channel_handler.ts b/meet-live-agent/internal/channel_handlers/participants_channel_handler.ts new file mode 100644 index 0000000..2d8be3d --- /dev/null +++ b/meet-live-agent/internal/channel_handlers/participants_channel_handler.ts @@ -0,0 +1,241 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Handles participants data channel updates + */ + +import { + DeletedParticipant, + ParticipantResource, + ParticipantsChannelToClient, +} from '../../types/datachannels'; +import {LogLevel} from '../../types/enums'; +import { + Participant as LocalParticipant, + MediaEntry, +} from '../../types/mediatypes'; +import {InternalMediaEntry, InternalParticipant} from '../internal_types'; +import {SubscribableDelegate} from '../subscribable_impl'; +import {ChannelLogger} from './channel_logger'; + +/** + * Handler for participants channel + */ +export class ParticipantsChannelHandler { + constructor( + private readonly channel: RTCDataChannel, + private readonly participantsDelegate: SubscribableDelegate< + LocalParticipant[] + >, + private readonly idParticipantMap = new Map(), + private readonly nameParticipantMap = new Map(), + private readonly internalParticipantMap = new Map< + LocalParticipant, + InternalParticipant + >(), + private readonly internalMediaEntryMap = new Map< + MediaEntry, + InternalMediaEntry + >(), + private readonly channelLogger?: ChannelLogger, + ) { + this.channel.onmessage = (event) => { + this.onParticipantsMessage(event); + }; + this.channel.onopen = () => { + this.onParticipantsOpened(); + }; + this.channel.onclose = () => { + this.onParticipantsClosed(); + }; + } + + private onParticipantsOpened() { + this.channelLogger?.log(LogLevel.MESSAGES, 'Participants channel: opened'); + } + + private onParticipantsMessage(event: MessageEvent) { + const data = JSON.parse(event.data) as ParticipantsChannelToClient; + let participants = this.participantsDelegate.get(); + data.deletedResources?.forEach((deletedResource: DeletedParticipant) => { + this.channelLogger?.log( + LogLevel.RESOURCES, + 'Participants channel: deleted resource', + deletedResource, + ); + const participant = this.idParticipantMap.get(deletedResource.id); + if (!participant) { + return; + } + this.idParticipantMap.delete(deletedResource.id); + const deletedParticipant = this.internalParticipantMap.get(participant); + if (!deletedParticipant) { + return; + } + deletedParticipant.ids.delete(deletedResource.id); + if (deletedParticipant.ids.size !== 0) { + return; + } + if (participant.participant.name) { + this.nameParticipantMap.delete(participant.participant.name); + } + participants = participants.filter((p) => p !== participant); + this.internalParticipantMap.delete(participant); + deletedParticipant.mediaEntries.get().forEach((mediaEntry) => { + const internalMediaEntry = this.internalMediaEntryMap.get(mediaEntry); + if (internalMediaEntry) { + internalMediaEntry.participant.set(undefined); + } + }); + }); + + const addedParticipants: LocalParticipant[] = []; + data.resources?.forEach((resource: ParticipantResource) => { + this.channelLogger?.log( + LogLevel.RESOURCES, + 'Participants channel: added resource', + resource, + ); + if (!resource.id) { + // We expect all participants to have an id. If not, we log an error + // and ignore the participant. + this.channelLogger?.log( + LogLevel.ERRORS, + 'Participants channel: participant resource has no id', + resource, + ); + return; + } + // We do not expect that the participant resource already exists. + // However, it is possible that the media entries channel references it + // before we receive the participant resource. In this case, we update + // the participant resource with the type and maintain the media entry + // relationship. + let existingMediaEntriesDelegate: + | SubscribableDelegate + | undefined; + let existingParticipant: LocalParticipant | undefined; + let existingIds: Set | undefined; + if (this.idParticipantMap.has(resource.id)) { + existingParticipant = this.idParticipantMap.get(resource.id); + } else if ( + resource.participant.name && + this.nameParticipantMap.has(resource.participant.name) + ) { + existingParticipant = this.nameParticipantMap.get( + resource.participant.name, + ); + } else if (resource.participant.participantKey) { + existingParticipant = Array.from( + this.internalParticipantMap.entries(), + ).find( + ([participant, _]) => + participant.participant.participantKey === + resource.participant.participantKey, + )?.[0]; + } + + if (existingParticipant) { + const internalParticipant = + this.internalParticipantMap.get(existingParticipant); + if (internalParticipant) { + existingMediaEntriesDelegate = internalParticipant.mediaEntries; + // (TODO: Remove this once we are using participant + // names as identifiers. Right now, it is possible for a participant to + // have multiple ids due to updates being treated as new resources. + existingIds = internalParticipant.ids; + existingIds.forEach((id) => { + this.idParticipantMap.delete(id); + }); + } + if (existingParticipant.participant.name) { + this.nameParticipantMap.delete(existingParticipant.participant.name); + } + this.internalParticipantMap.delete(existingParticipant); + participants = participants.filter((p) => p !== existingParticipant); + this.channelLogger?.log( + LogLevel.ERRORS, + 'Participants channel: participant resource already exists', + resource, + ); + } + + const participantElement = createParticipant( + resource, + existingMediaEntriesDelegate, + existingIds, + ); + const participant = participantElement.participant; + const internalParticipant = participantElement.internalParticipant; + participantElement.internalParticipant.ids.forEach((id) => { + this.idParticipantMap.set(id, participant); + }); + if (resource.participant.name) { + this.nameParticipantMap.set(resource.participant.name, participant); + } + + this.internalParticipantMap.set(participant, internalParticipant); + addedParticipants.push(participant); + }); + + // Update participant collection. + if (data.resources?.length || data.deletedResources?.length) { + const newParticipants = [...participants, ...addedParticipants]; + this.participantsDelegate.set(newParticipants); + } + } + + private onParticipantsClosed() { + this.channelLogger?.log(LogLevel.MESSAGES, 'Participants channel: closed'); + } +} + +interface InternalParticipantElement { + participant: LocalParticipant; + internalParticipant: InternalParticipant; +} + +/** + * Creates a new participant. + * @return The new participant and its internal representation. + */ +function createParticipant( + resource: ParticipantResource, + mediaEntriesDelegate = new SubscribableDelegate([]), + existingIds = new Set(), +): InternalParticipantElement { + if (!resource.id) { + throw new Error('Participant resource must have an id'); + } + + const participant: LocalParticipant = { + participant: resource.participant, + mediaEntries: mediaEntriesDelegate.getSubscribable(), + }; + + existingIds.add(resource.id); + + const internalParticipant: InternalParticipant = { + name: resource.participant.name ?? '', + ids: existingIds, + mediaEntries: mediaEntriesDelegate, + }; + return { + participant, + internalParticipant, + }; +} diff --git a/meet-live-agent/internal/channel_handlers/session_control_channel_handler.ts b/meet-live-agent/internal/channel_handlers/session_control_channel_handler.ts new file mode 100644 index 0000000..1182aea --- /dev/null +++ b/meet-live-agent/internal/channel_handlers/session_control_channel_handler.ts @@ -0,0 +1,155 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Handles the session control channel. + */ + +import { + LeaveRequest, + SessionControlChannelFromClient, + SessionControlChannelToClient, +} from '../../types/datachannels'; +import { + LogLevel, + MeetConnectionState, + MeetDisconnectReason, +} from '../../types/enums'; +import {MeetSessionStatus} from '../../types/meetmediaapiclient'; +import {SubscribableDelegate} from '../subscribable_impl'; +import {ChannelLogger} from './channel_logger'; + +const DISCONNECT_REASON_MAP = new Map([ + ['REASON_CLIENT_LEFT', MeetDisconnectReason.CLIENT_LEFT], + ['REASON_USER_STOPPED', MeetDisconnectReason.USER_STOPPED], + ['REASON_CONFERENCE_ENDED', MeetDisconnectReason.CONFERENCE_ENDED], + ['REASON_SESSION_UNHEALTHY', MeetDisconnectReason.SESSION_UNHEALTHY], +]); + +/** + * Helper class to handles the session control channel. + */ +export class SessionControlChannelHandler { + private requestId = 1; + private leaveSessionPromise: (() => void) | undefined; + + constructor( + private readonly channel: RTCDataChannel, + private readonly sessionStatusDelegate: SubscribableDelegate, + private readonly channelLogger?: ChannelLogger, + ) { + this.channel.onmessage = (event) => { + this.onSessionControlMessage(event); + }; + this.channel.onopen = () => { + this.onSessionControlOpened(); + }; + this.channel.onclose = () => { + this.onSessionControlClosed(); + }; + } + + private onSessionControlOpened() { + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Session control channel: opened', + ); + this.sessionStatusDelegate.set({ + connectionState: MeetConnectionState.WAITING, + }); + } + + private onSessionControlMessage(event: MessageEvent) { + const message = event.data; + const json = JSON.parse(message) as SessionControlChannelToClient; + if (json?.response) { + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Session control channel: response recieved', + json.response, + ); + this.leaveSessionPromise?.(); + } + if (json?.resources && json.resources.length > 0) { + const sessionStatus = json.resources[0].sessionStatus; + this.channelLogger?.log( + LogLevel.RESOURCES, + 'Session control channel: resource recieved', + json.resources[0], + ); + if (sessionStatus.connectionState === 'STATE_WAITING') { + this.sessionStatusDelegate.set({ + connectionState: MeetConnectionState.WAITING, + }); + } else if (sessionStatus.connectionState === 'STATE_JOINED') { + this.sessionStatusDelegate.set({ + connectionState: MeetConnectionState.JOINED, + }); + } else if (sessionStatus.connectionState === 'STATE_DISCONNECTED') { + this.sessionStatusDelegate.set({ + connectionState: MeetConnectionState.DISCONNECTED, + disconnectReason: + DISCONNECT_REASON_MAP.get(sessionStatus.disconnectReason || '') ?? + MeetDisconnectReason.SESSION_UNHEALTHY, + }); + } + } + } + private onSessionControlClosed() { + // If the channel is closed, we should resolve the leave session promise. + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Session control channel: closed', + ); + this.leaveSessionPromise?.(); + if ( + this.sessionStatusDelegate.get().connectionState !== + MeetConnectionState.DISCONNECTED + ) { + this.sessionStatusDelegate.set({ + connectionState: MeetConnectionState.DISCONNECTED, + disconnectReason: MeetDisconnectReason.UNKNOWN, + }); + } + } + + leaveSession(): Promise { + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Session control channel: leave session request sent', + ); + try { + this.channel.send( + JSON.stringify({ + request: { + requestId: this.requestId++, + leave: {}, + } as LeaveRequest, + } as SessionControlChannelFromClient), + ); + } catch (e) { + this.channelLogger?.log( + LogLevel.ERRORS, + 'Session control channel: Failed to send leave request with error', + e as Error, + ); + throw e; + } + return new Promise((resolve) => { + this.leaveSessionPromise = resolve; + }); + } +} diff --git a/meet-live-agent/internal/channel_handlers/video_assignment_channel_handler.ts b/meet-live-agent/internal/channel_handlers/video_assignment_channel_handler.ts new file mode 100644 index 0000000..8dd8dbb --- /dev/null +++ b/meet-live-agent/internal/channel_handlers/video_assignment_channel_handler.ts @@ -0,0 +1,318 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Video assignment channel handler. + */ + +import { + MediaApiCanvas, + MediaApiResponseStatus, + SetVideoAssignmentRequest, + SetVideoAssignmentResponse, + VideoAssignmentChannelFromClient, + VideoAssignmentChannelToClient, + VideoAssignmentResource, +} from '../../types/datachannels'; +import {LogLevel} from '../../types/enums'; +import { + MediaEntry, + MediaLayout, + MediaLayoutRequest, + MeetStreamTrack, +} from '../../types/mediatypes'; +import { + InternalMediaEntry, + InternalMediaLayout, + InternalMeetStreamTrack, +} from '../internal_types'; +import {SubscribableDelegate} from '../subscribable_impl'; +import {createMediaEntry} from '../utils'; +import {ChannelLogger} from './channel_logger'; + +// We request the highest possible resolution by default. +const MAX_RESOLUTION = { + height: 1080, + width: 1920, + frameRate: 30, +}; + +/** + * Helper class to handle the video assignment channel. + */ +export class VideoAssignmentChannelHandler { + private requestId = 1; + private readonly mediaLayoutLabelMap = new Map(); + private readonly pendingRequestResolveMap = new Map< + number, + (value: MediaApiResponseStatus) => void + >(); + + constructor( + private readonly channel: RTCDataChannel, + private readonly idMediaEntryMap: Map, + private readonly internalMediaEntryMap = new Map< + MediaEntry, + InternalMediaEntry + >(), + private readonly idMediaLayoutMap = new Map(), + private readonly internalMediaLayoutMap = new Map< + MediaLayout, + InternalMediaLayout + >(), + private readonly mediaEntriesDelegate: SubscribableDelegate, + private readonly internalMeetStreamTrackMap = new Map< + MeetStreamTrack, + InternalMeetStreamTrack + >(), + private readonly channelLogger?: ChannelLogger, + ) { + this.channel.onmessage = (event) => { + this.onVideoAssignmentMessage(event); + }; + this.channel.onclose = () => { + // Resolve all pending requests with an error. + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Video assignment channel: closed', + ); + for (const [, resolve] of this.pendingRequestResolveMap) { + resolve({code: 400, message: 'Channel closed', details: []}); + } + this.pendingRequestResolveMap.clear(); + }; + this.channel.onopen = () => { + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Video assignment channel: opened', + ); + }; + } + + private onVideoAssignmentMessage(message: MessageEvent) { + const data = JSON.parse(message.data) as VideoAssignmentChannelToClient; + if (data.response) { + this.onVideoAssignmentResponse(data.response); + } + if (data.resources) { + this.onVideoAssignmentResources(data.resources); + } + } + + private onVideoAssignmentResponse(response: SetVideoAssignmentResponse) { + // Users should listen on the video assignment channel for actual video + // assignments. These responses signify that the request was expected. + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Video assignment channel: recieved response', + response, + ); + this.pendingRequestResolveMap.get(response.requestId)?.(response.status); + } + + private onVideoAssignmentResources(resources: VideoAssignmentResource[]) { + resources.forEach((resource) => { + this.channelLogger?.log( + LogLevel.RESOURCES, + 'Video assignment channel: resource added', + resource, + ); + if (resource.videoAssignment.canvases) { + this.onVideoAssignment(resource); + } + }); + } + + private onVideoAssignment(videoAssignment: VideoAssignmentResource) { + const canvases = videoAssignment.videoAssignment.canvases; + canvases.forEach( + (canvas: {canvasId: number; ssrc?: number; mediaEntryId: number}) => { + const mediaLayout = this.idMediaLayoutMap.get(canvas.canvasId); + // We expect that the media layout is already created. + let internalMediaEntry; + if (mediaLayout) { + const assignedMediaEntry = mediaLayout.mediaEntry.get(); + let mediaEntry; + // if association already exists, we need to either update the video + // ssrc or remove the association if the ids don't match. + if ( + assignedMediaEntry && + this.internalMediaEntryMap.get(assignedMediaEntry)?.id === + canvas.mediaEntryId + ) { + // We expect the internal media entry to be already created if the media entry exists. + internalMediaEntry = + this.internalMediaEntryMap.get(assignedMediaEntry); + // If the media canvas is already associated with a media entry, we + // need to update the video ssrc. + // Expect the media entry to be created, without assertion, TS + // complains it can be undefined. + // tslint:disable:no-unnecessary-type-assertion + internalMediaEntry!.videoSsrc = canvas.ssrc; + mediaEntry = assignedMediaEntry; + } else { + // If asssocation does not exist, we will attempt to retreive the + // media entry from the map. + const existingMediaEntry = this.idMediaEntryMap.get( + canvas.mediaEntryId, + ); + // Clear existing association if it exists. + if (assignedMediaEntry) { + this.internalMediaEntryMap + .get(assignedMediaEntry) + ?.mediaLayout.set(undefined); + this.internalMediaLayoutMap + .get(mediaLayout) + ?.mediaEntry.set(undefined); + } + if (existingMediaEntry) { + // If the media entry exists, need to create the media canvas association. + internalMediaEntry = + this.internalMediaEntryMap.get(existingMediaEntry); + internalMediaEntry!.videoSsrc = canvas.ssrc; + internalMediaEntry!.mediaLayout.set(mediaLayout); + mediaEntry = existingMediaEntry; + } else { + // If the media entry doewsn't exist, we need to create it and + // then create the media canvas association. + // We don't expect to hit this expression, but since data channels + // don't guarantee order, we do this to be safe. + const mediaEntryElement = createMediaEntry({ + id: canvas.mediaEntryId, + mediaLayout, + videoSsrc: canvas.ssrc, + }); + this.internalMediaEntryMap.set( + mediaEntryElement.mediaEntry, + mediaEntryElement.internalMediaEntry, + ); + internalMediaEntry = mediaEntryElement.internalMediaEntry; + const newMediaEntry = mediaEntryElement.mediaEntry; + this.idMediaEntryMap.set(canvas.mediaEntryId, newMediaEntry); + const newMediaEntries = [ + ...this.mediaEntriesDelegate.get(), + newMediaEntry, + ]; + this.mediaEntriesDelegate.set(newMediaEntries); + mediaEntry = newMediaEntry; + } + this.internalMediaLayoutMap + .get(mediaLayout) + ?.mediaEntry.set(mediaEntry); + this.internalMediaEntryMap + + .get(mediaEntry!) + ?.mediaLayout.set(mediaLayout); + } + if ( + !this.isMediaEntryAssignedToMeetStreamTrack( + mediaEntry!, + internalMediaEntry!, + ) + ) { + this.assignVideoMeetStreamTrack(mediaEntry!); + } + } + // tslint:enable:no-unnecessary-type-assertion + this.channelLogger?.log( + LogLevel.ERRORS, + 'Video assignment channel: server sent a canvas that was not created by the client', + ); + }, + ); + } + + sendRequests( + mediaLayoutRequests: MediaLayoutRequest[], + ): Promise { + const label = Date.now().toString(); + const canvases: MediaApiCanvas[] = []; + mediaLayoutRequests.forEach((request) => { + this.mediaLayoutLabelMap.set(request.mediaLayout, label); + canvases.push({ + id: this.internalMediaLayoutMap.get(request.mediaLayout)!.id, + dimensions: request.mediaLayout.canvasDimensions, + relevant: {}, + }); + }); + const request: SetVideoAssignmentRequest = { + requestId: this.requestId++, + setAssignment: { + layoutModel: { + label, + canvases, + }, + maxVideoResolution: MAX_RESOLUTION, + }, + }; + this.channelLogger?.log( + LogLevel.MESSAGES, + 'Video Assignment channel: Sending request', + request, + ); + try { + this.channel.send( + JSON.stringify({ + request, + } as VideoAssignmentChannelFromClient), + ); + } catch (e) { + this.channelLogger?.log( + LogLevel.ERRORS, + 'Video Assignment channel: Failed to send request with error', + e as Error, + ); + throw e; + } + + const requestPromise = new Promise((resolve) => { + this.pendingRequestResolveMap.set(request.requestId, resolve); + }); + return requestPromise; + } + + private isMediaEntryAssignedToMeetStreamTrack( + mediaEntry: MediaEntry, + internalMediaEntry: InternalMediaEntry, + ): boolean { + const videoMeetStreamTrack = mediaEntry.videoMeetStreamTrack.get(); + if (!videoMeetStreamTrack) return false; + const internalMeetStreamTrack = + this.internalMeetStreamTrackMap.get(videoMeetStreamTrack); + + if (internalMeetStreamTrack!.videoSsrc === internalMediaEntry.videoSsrc) { + return true; + } else { + // ssrcs can change, if the video ssrc is not the same, we need to remove + // the relationship between the media entry and the meet stream track. + internalMediaEntry.videoMeetStreamTrack.set(undefined); + internalMeetStreamTrack?.mediaEntry.set(undefined); + return false; + } + } + + private assignVideoMeetStreamTrack(mediaEntry: MediaEntry) { + for (const [meetStreamTrack, internalMeetStreamTrack] of this + .internalMeetStreamTrackMap) { + if (meetStreamTrack.mediaStreamTrack.kind === 'video') { + internalMeetStreamTrack.maybeAssignMediaEntryOnFrame( + mediaEntry, + 'video', + ); + } + } + } +} diff --git a/meet-live-agent/internal/communication_protocols/default_communication_protocol_impl.ts b/meet-live-agent/internal/communication_protocols/default_communication_protocol_impl.ts new file mode 100644 index 0000000..a2d25f3 --- /dev/null +++ b/meet-live-agent/internal/communication_protocols/default_communication_protocol_impl.ts @@ -0,0 +1,79 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview The default communication protocol for the Media API client + * with Meet API. + */ + +import {MeetMediaClientRequiredConfiguration} from '../../types/mediatypes'; + +import { + MediaApiCommunicationProtocol, + MediaApiCommunicationResponse, +} from '../../types/communication_protocol'; + +const MEET_API_URL = 'https://meet.googleapis.com/v2beta/'; + +/** + * The HTTP communication protocol for communication with Meet API. + */ +export class DefaultCommunicationProtocolImpl + implements MediaApiCommunicationProtocol +{ + constructor( + private readonly requiredConfiguration: MeetMediaClientRequiredConfiguration, + private readonly meetApiUrl: string = MEET_API_URL, + ) {} + + async connectActiveConference( + sdpOffer: string, + ): Promise { + // Call to Meet API + const spaceId = this.requiredConfiguration.meetingSpaceId.startsWith('spaces/') ? this.requiredConfiguration.meetingSpaceId : `spaces/${this.requiredConfiguration.meetingSpaceId}`; + const connectUrl = `${this.meetApiUrl}${spaceId}:connectActiveConference`; + console.log('[DefaultCommunicationProtocolImpl] SDP Offer:', sdpOffer); + const response = await fetch(connectUrl, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${this.requiredConfiguration.accessToken}`, + }, + body: JSON.stringify({ + 'offer': sdpOffer, + }), + }); + if (!response.ok) { + const bodyReader = response.body?.getReader(); + let error = ''; + if (bodyReader) { + const decoder = new TextDecoder(); + let readingDone = false; + while (!readingDone) { + const {done, value} = await bodyReader?.read(); + if (done) { + readingDone = true; + break; + } + error += decoder.decode(value); + } + } + const errorJson = JSON.parse(error); + throw new Error(`${JSON.stringify(errorJson, null, 2)}`); + } + const payload = await response.json(); + return {answer: payload['answer']} as MediaApiCommunicationResponse; + } +} diff --git a/meet-live-agent/internal/internal_meet_stream_track_impl.ts b/meet-live-agent/internal/internal_meet_stream_track_impl.ts new file mode 100644 index 0000000..9dcfe81 --- /dev/null +++ b/meet-live-agent/internal/internal_meet_stream_track_impl.ts @@ -0,0 +1,128 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Implementation of InternalMeetStreamTrack. + */ + +import {MediaEntry, MeetStreamTrack} from '../types/mediatypes'; +import {SubscribableDelegate} from './subscribable_impl'; + +import {InternalMediaEntry, InternalMeetStreamTrack} from './internal_types'; + +/** + * Implementation of InternalMeetStreamTrack. + */ +export class InternalMeetStreamTrackImpl implements InternalMeetStreamTrack { + private readonly reader: ReadableStreamDefaultReader; + videoSsrc?: number; + + constructor( + readonly receiver: RTCRtpReceiver, + readonly mediaEntry: SubscribableDelegate, + private readonly meetStreamTrack: MeetStreamTrack, + private readonly internalMediaEntryMap: Map, + ) { + const mediaStreamTrack = meetStreamTrack.mediaStreamTrack; + let mediaStreamTrackProcessor; + if (mediaStreamTrack.kind === 'audio') { + mediaStreamTrackProcessor = new MediaStreamTrackProcessor({ + track: mediaStreamTrack as MediaStreamAudioTrack, + }); + } else { + mediaStreamTrackProcessor = new MediaStreamTrackProcessor({ + track: mediaStreamTrack as MediaStreamVideoTrack, + }); + } + this.reader = mediaStreamTrackProcessor.readable.getReader(); + } + + async maybeAssignMediaEntryOnFrame( + mediaEntry: MediaEntry, + kind: 'audio' | 'video', + ): Promise { + // Only want to check the media entry if it has the correct csrc type + // for this meet stream track. + if ( + !this.mediaStreamTrackSrcPresent(mediaEntry) || + this.meetStreamTrack.mediaStreamTrack.kind !== kind + ) { + return; + } + // Loop through the frames until media entry is assigned by either this + // meet stream track or another meet stream track. + while (!this.mediaEntryTrackAssigned(mediaEntry, kind)) { + const frame = await this.reader.read(); + if (frame.done) break; + if (kind === 'audio') { + await this.onAudioFrame(mediaEntry); + } else if (kind === 'video') { + this.onVideoFrame(mediaEntry); + } + frame.value.close(); + } + return; + } + + private async onAudioFrame(mediaEntry: MediaEntry): Promise { + const internalMediaEntry = this.internalMediaEntryMap.get(mediaEntry); + const contributingSources: RTCRtpContributingSource[] = + this.receiver.getContributingSources(); + for (const contributingSource of contributingSources) { + if (contributingSource.source === internalMediaEntry!.audioCsrc) { + internalMediaEntry!.audioMeetStreamTrack.set(this.meetStreamTrack); + this.mediaEntry.set(mediaEntry); + } + } + } + + private onVideoFrame(mediaEntry: MediaEntry): void { + const internalMediaEntry = this.internalMediaEntryMap.get(mediaEntry); + const synchronizationSources: RTCRtpSynchronizationSource[] = + this.receiver.getSynchronizationSources(); + for (const syncSource of synchronizationSources) { + if (syncSource.source === internalMediaEntry!.videoSsrc) { + this.videoSsrc = syncSource.source; + internalMediaEntry!.videoMeetStreamTrack.set(this.meetStreamTrack); + this.mediaEntry.set(mediaEntry); + } + } + return; + } + + private mediaEntryTrackAssigned( + mediaEntry: MediaEntry, + kind: 'audio' | 'video', + ): boolean { + if ( + (kind === 'audio' && mediaEntry.audioMeetStreamTrack.get()) || + (kind === 'video' && mediaEntry.videoMeetStreamTrack.get()) + ) { + return true; + } + return false; + } + + private mediaStreamTrackSrcPresent(mediaEntry: MediaEntry): boolean { + const internalMediaEntry = this.internalMediaEntryMap.get(mediaEntry); + if (this.meetStreamTrack.mediaStreamTrack.kind === 'audio') { + return !!internalMediaEntry?.audioCsrc; + } else if (this.meetStreamTrack.mediaStreamTrack.kind === 'video') { + return !!internalMediaEntry?.videoSsrc; + } + return false; + } +} diff --git a/meet-live-agent/internal/internal_types.ts b/meet-live-agent/internal/internal_types.ts new file mode 100644 index 0000000..f9305c9 --- /dev/null +++ b/meet-live-agent/internal/internal_types.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview types are for use in the internal client and should not be + * used outside of the client. + */ + +import { + MediaEntry, + MediaLayout, + MeetStreamTrack, + Participant, +} from '../types/mediatypes'; + +import {SubscribableDelegate} from './subscribable_impl'; + +/** + * Internal representation of a media entry. Has handles on the + * subscribable delegates. + */ +export interface InternalMediaEntry { + audioCsrc?: number; + videoCsrc?: number; + videoSsrc?: number; + readonly id: number; + readonly audioMuted: SubscribableDelegate; + readonly videoMuted: SubscribableDelegate; + readonly screenShare: SubscribableDelegate; + readonly isPresenter: SubscribableDelegate; + readonly mediaLayout: SubscribableDelegate; + readonly videoMeetStreamTrack: SubscribableDelegate< + MeetStreamTrack | undefined + >; + readonly audioMeetStreamTrack: SubscribableDelegate< + MeetStreamTrack | undefined + >; + readonly participant: SubscribableDelegate; +} + +/** + * Internal representation of a media layout. Has handles on the + * subscribable delegates. This only relates to video. + */ +export interface InternalMediaLayout { + videoSsrc?: number; + readonly id: number; + readonly mediaEntry: SubscribableDelegate; +} + +/** + * Internal representation of a meet stream track. Has handles on the + * subscribable delegates. + */ +export interface InternalMeetStreamTrack { + readonly mediaEntry: SubscribableDelegate; + readonly receiver: RTCRtpReceiver; + videoSsrc?: number; + maybeAssignMediaEntryOnFrame: ( + mediaEntry: MediaEntry, + kind: 'audio' | 'video', + ) => Promise; +} + +/** + * Internal representation of a participant. Has handles on the + * subscribable delegates. + */ +export interface InternalParticipant { + // TODO - Remove this once we are using participant names as + // identifiers. Right now, it is possible for a participant to have multiple + // ids due to updates being treated as new resources. This occurance should + // become less frequent once we ignore child participants but can will still + // be possible until participant names are used as identifiers. + readonly ids: Set; + readonly name: string; + readonly mediaEntries: SubscribableDelegate; +} diff --git a/meet-live-agent/internal/meet_stream_track_impl.ts b/meet-live-agent/internal/meet_stream_track_impl.ts new file mode 100644 index 0000000..6a06417 --- /dev/null +++ b/meet-live-agent/internal/meet_stream_track_impl.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Implementation of MeetStreamTrack. + */ + +import {MediaEntry, MeetStreamTrack} from '../types/mediatypes'; +import {Subscribable} from '../types/subscribable'; + +import {SubscribableDelegate} from './subscribable_impl'; + +/** + * The implementation of MeetStreamTrack. + */ +export class MeetStreamTrackImpl implements MeetStreamTrack { + readonly mediaEntry: Subscribable; + + constructor( + readonly mediaStreamTrack: MediaStreamTrack, + private readonly mediaEntryDelegate: SubscribableDelegate< + MediaEntry | undefined + >, + ) { + this.mediaEntry = this.mediaEntryDelegate.getSubscribable(); + } +} diff --git a/meet-live-agent/internal/meetmediaapiclient_impl.ts b/meet-live-agent/internal/meetmediaapiclient_impl.ts new file mode 100644 index 0000000..2664cf3 --- /dev/null +++ b/meet-live-agent/internal/meetmediaapiclient_impl.ts @@ -0,0 +1,455 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + MediaApiCommunicationProtocol, + MediaApiCommunicationResponse, +} from '../types/communication_protocol'; +import { MediaApiResponseStatus } from '../types/datachannels'; +import { MeetConnectionState } from '../types/enums'; +import { + CanvasDimensions, + MediaEntry, + MediaLayout, + MediaLayoutRequest, + MeetMediaClientRequiredConfiguration, + MeetStreamTrack, + Participant, +} from '../types/mediatypes'; +import { + MeetMediaApiClient, + MeetSessionStatus, +} from '../types/meetmediaapiclient'; +import { Subscribable } from '../types/subscribable'; +import { ChannelLogger } from './channel_handlers/channel_logger'; +import { MediaEntriesChannelHandler } from './channel_handlers/media_entries_channel_handler'; +import { MediaStatsChannelHandler } from './channel_handlers/media_stats_channel_handler'; +import { ParticipantsChannelHandler } from './channel_handlers/participants_channel_handler'; +import { SessionControlChannelHandler } from './channel_handlers/session_control_channel_handler'; +import { VideoAssignmentChannelHandler } from './channel_handlers/video_assignment_channel_handler'; +import { DefaultCommunicationProtocolImpl } from './communication_protocols/default_communication_protocol_impl'; +import { InternalMeetStreamTrackImpl } from './internal_meet_stream_track_impl'; +import { + InternalMediaEntry, + InternalMediaLayout, + InternalMeetStreamTrack, + InternalParticipant, +} from './internal_types'; +import { MeetStreamTrackImpl } from './meet_stream_track_impl'; +import { SubscribableDelegate, SubscribableImpl } from './subscribable_impl'; + +// Meet only supports 3 audio virtual ssrcs. If disabled, there will be no +// audio. +const NUMBER_OF_AUDIO_VIRTUAL_SSRC = 3; + +const MINIMUM_VIDEO_STREAMS = 0; +const MAXIMUM_VIDEO_STREAMS = 3; + +/** + * Implementation of MeetMediaApiClient. + */ +export class MeetMediaApiClientImpl implements MeetMediaApiClient { + // Public properties + readonly sessionStatus: Subscribable; + readonly meetStreamTracks: Subscribable; + readonly mediaEntries: Subscribable; + readonly participants: Subscribable; + readonly presenter: Subscribable; + readonly screenshare: Subscribable; + + // Private properties + private readonly sessionStatusDelegate: SubscribableDelegate; + private readonly meetStreamTracksDelegate: SubscribableDelegate< + MeetStreamTrack[] + >; + private readonly mediaEntriesDelegate: SubscribableDelegate; + private readonly participantsDelegate: SubscribableDelegate; + private readonly presenterDelegate: SubscribableDelegate< + MediaEntry | undefined + >; + private readonly screenshareDelegate: SubscribableDelegate< + MediaEntry | undefined + >; + + private readonly peerConnection: RTCPeerConnection; + + private sessionControlChannel: RTCDataChannel | undefined; + private sessionControlChannelHandler: + | SessionControlChannelHandler + | undefined; + + private videoAssignmentChannel: RTCDataChannel | undefined; + private videoAssignmentChannelHandler: + | VideoAssignmentChannelHandler + | undefined; + + private mediaEntriesChannel: RTCDataChannel | undefined; + private mediaStatsChannel: RTCDataChannel | undefined; + private participantsChannel: RTCDataChannel | undefined; + + /* tslint:disable:no-unused-variable */ + // This is unused because it is receive only. + // @ts-ignore + private mediaEntriesChannelHandler: MediaEntriesChannelHandler | undefined; + + // @ts-ignore + private mediaStatsChannelHandler: MediaStatsChannelHandler | undefined; + + // @ts-ignore + private participantsChannelHandler: ParticipantsChannelHandler | undefined; + /* tslint:enable:no-unused-variable */ + + private mediaLayoutId = 1; + + // Media layout retrieval by id. Needed by the video assignment channel handler + // to update the media layout. + private readonly idMediaLayoutMap = new Map(); + + // Used to update media layouts. + private readonly internalMediaLayoutMap = new Map< + MediaLayout, + InternalMediaLayout + >(); + + // Media entry retrieval by id. Needed by the video assignment channel handler + // to update the media entry. + private readonly idMediaEntryMap = new Map(); + + // Used to update media entries. + private readonly internalMediaEntryMap = new Map< + MediaEntry, + InternalMediaEntry + >(); + + // Used to update meet stream tracks. + private readonly internalMeetStreamTrackMap = new Map< + MeetStreamTrack, + InternalMeetStreamTrack + >(); + + private readonly idParticipantMap = new Map(); + private readonly nameParticipantMap = new Map(); + private readonly internalParticipantMap = new Map< + Participant, + InternalParticipant + >(); + + constructor( + private readonly requiredConfiguration: MeetMediaClientRequiredConfiguration, + ) { + this.validateConfiguration(); + + this.sessionStatusDelegate = new SubscribableDelegate({ + connectionState: MeetConnectionState.UNKNOWN, + }); + this.sessionStatus = this.sessionStatusDelegate.getSubscribable(); + this.meetStreamTracksDelegate = new SubscribableDelegate( + [], + ); + this.meetStreamTracks = this.meetStreamTracksDelegate.getSubscribable(); + this.mediaEntriesDelegate = new SubscribableDelegate([]); + this.mediaEntries = this.mediaEntriesDelegate.getSubscribable(); + this.participantsDelegate = new SubscribableDelegate([]); + this.participants = this.participantsDelegate.getSubscribable(); + this.presenterDelegate = new SubscribableDelegate( + undefined, + ); + this.presenter = this.presenterDelegate.getSubscribable(); + this.screenshareDelegate = new SubscribableDelegate( + undefined, + ); + this.screenshare = this.screenshareDelegate.getSubscribable(); + + const configuration = { + sdpSemantics: 'unified-plan', + bundlePolicy: 'max-bundle' as RTCBundlePolicy, + iceServers: [{ urls: 'stun:stun.l.google.com:19302' }], + }; + + // Create peer connection + this.peerConnection = new RTCPeerConnection(configuration); + this.peerConnection.ontrack = (e) => { + if (e.track) { + this.createMeetStreamTrack(e.track, e.receiver); + } + }; + } + + private validateConfiguration(): void { + if ( + this.requiredConfiguration.numberOfVideoStreams < MINIMUM_VIDEO_STREAMS || + this.requiredConfiguration.numberOfVideoStreams > MAXIMUM_VIDEO_STREAMS + ) { + throw new Error( + `Unsupported number of video streams, must be between ${MINIMUM_VIDEO_STREAMS} and ${MAXIMUM_VIDEO_STREAMS}`, + ); + } + } + + private createMeetStreamTrack( + mediaStreamTrack: MediaStreamTrack, + receiver: RTCRtpReceiver, + ): void { + const meetStreamTracks = this.meetStreamTracks.get(); + const mediaEntryDelegate = new SubscribableDelegate( + undefined, + ); + const meetStreamTrack = new MeetStreamTrackImpl( + mediaStreamTrack, + mediaEntryDelegate, + ); + + const internalMeetStreamTrack = new InternalMeetStreamTrackImpl( + receiver, + mediaEntryDelegate, + meetStreamTrack, + this.internalMediaEntryMap, + ); + + const newStreamTrackArray = [...meetStreamTracks, meetStreamTrack]; + this.internalMeetStreamTrackMap.set( + meetStreamTrack, + internalMeetStreamTrack, + ); + this.meetStreamTracksDelegate.set(newStreamTrackArray); + } + + async joinMeeting( + communicationProtocol?: MediaApiCommunicationProtocol, + ): Promise { + // The offer must be in the order of audio, datachannels, video. + + // Create audio transceivers based on initial config. + if (this.requiredConfiguration.enableAudioStreams) { + for (let i = 0; i < NUMBER_OF_AUDIO_VIRTUAL_SSRC; i++) { + // Integrating clients must support and negotiate the OPUS codec in + // the SDP offer. + // This is the default for WebRTC. + // https://developer.mozilla.org/en-US/docs/Web/Media/Formats/WebRTC_codecs. + this.peerConnection.addTransceiver('audio', { direction: 'recvonly' }); + } + } + + // ---- UTILITY DATA CHANNELS ----- + + // All data channels must be reliable and ordered. + const dataChannelConfig = { + ordered: true, + reliable: true, + }; + + // Always create the session and media stats control channel. + this.sessionControlChannel = this.peerConnection.createDataChannel( + 'session-control', + dataChannelConfig, + ); + let sessionControlchannelLogger; + if (this.requiredConfiguration?.logsCallback) { + sessionControlchannelLogger = new ChannelLogger( + 'session-control', + this.requiredConfiguration.logsCallback, + ); + } + this.sessionControlChannelHandler = new SessionControlChannelHandler( + this.sessionControlChannel, + this.sessionStatusDelegate, + sessionControlchannelLogger, + ); + + this.mediaStatsChannel = this.peerConnection.createDataChannel( + 'media-stats', + dataChannelConfig, + ); + let mediaStatsChannelLogger; + if (this.requiredConfiguration?.logsCallback) { + mediaStatsChannelLogger = new ChannelLogger( + 'media-stats', + this.requiredConfiguration.logsCallback, + ); + } + this.mediaStatsChannelHandler = new MediaStatsChannelHandler( + this.mediaStatsChannel, + this.peerConnection, + mediaStatsChannelLogger, + ); + + // ---- CONDITIONAL DATA CHANNELS ----- + + // We only need the video assignment channel if we are requesting video. + if (this.requiredConfiguration.numberOfVideoStreams > 0) { + this.videoAssignmentChannel = this.peerConnection.createDataChannel( + 'video-assignment', + dataChannelConfig, + ); + let videoAssignmentChannelLogger; + if (this.requiredConfiguration?.logsCallback) { + videoAssignmentChannelLogger = new ChannelLogger( + 'video-assignment', + this.requiredConfiguration.logsCallback, + ); + } + this.videoAssignmentChannelHandler = new VideoAssignmentChannelHandler( + this.videoAssignmentChannel, + this.idMediaEntryMap, + this.internalMediaEntryMap, + this.idMediaLayoutMap, + this.internalMediaLayoutMap, + this.mediaEntriesDelegate, + this.internalMeetStreamTrackMap, + videoAssignmentChannelLogger, + ); + } + + if ( + this.requiredConfiguration.numberOfVideoStreams > 0 || + this.requiredConfiguration.enableAudioStreams + ) { + this.mediaEntriesChannel = this.peerConnection.createDataChannel( + 'media-entries', + dataChannelConfig, + ); + let mediaEntriesChannelLogger; + if (this.requiredConfiguration?.logsCallback) { + mediaEntriesChannelLogger = new ChannelLogger( + 'media-entries', + this.requiredConfiguration.logsCallback, + ); + } + this.mediaEntriesChannelHandler = new MediaEntriesChannelHandler( + this.mediaEntriesChannel, + this.mediaEntriesDelegate, + this.idMediaEntryMap, + this.internalMediaEntryMap, + this.internalMeetStreamTrackMap, + this.internalMediaLayoutMap, + this.participantsDelegate, + this.nameParticipantMap, + this.idParticipantMap, + this.internalParticipantMap, + this.presenterDelegate, + this.screenshareDelegate, + mediaEntriesChannelLogger, + ); + + this.participantsChannel = + this.peerConnection.createDataChannel('participants'); + let participantsChannelLogger; + if (this.requiredConfiguration?.logsCallback) { + participantsChannelLogger = new ChannelLogger( + 'participants', + this.requiredConfiguration.logsCallback, + ); + } + + this.participantsChannelHandler = new ParticipantsChannelHandler( + this.participantsChannel, + this.participantsDelegate, + this.idParticipantMap, + this.nameParticipantMap, + this.internalParticipantMap, + this.internalMediaEntryMap, + participantsChannelLogger, + ); + } + + this.sessionStatusDelegate.subscribe((status) => { + if (status.connectionState === MeetConnectionState.DISCONNECTED) { + this.mediaStatsChannel?.close(); + this.videoAssignmentChannel?.close(); + this.mediaEntriesChannel?.close(); + } + }); + + // Local description has to be set before adding video transceivers to + // preserve the order of audio, datachannels, video. + let pcOffer = await this.peerConnection.createOffer(); + await this.peerConnection.setLocalDescription(pcOffer); + + for (let i = 0; i < this.requiredConfiguration.numberOfVideoStreams; i++) { + // Integrating clients must support and negotiate AV1, VP9, and VP8 codecs + // in the SDP offer. + // The default for WebRTC is VP8. + // https://developer.mozilla.org/en-US/docs/Web/Media/Formats/WebRTC_codecs. + this.peerConnection.addTransceiver('video', { direction: 'recvonly' }); + } + + pcOffer = await this.peerConnection.createOffer(); + await this.peerConnection.setLocalDescription(pcOffer); + const protocol: MediaApiCommunicationProtocol = + communicationProtocol ?? + new DefaultCommunicationProtocolImpl(this.requiredConfiguration); + const response: MediaApiCommunicationResponse = + await protocol.connectActiveConference(pcOffer.sdp ?? ''); + if (response?.answer) { + await this.peerConnection.setRemoteDescription({ + type: 'answer', + sdp: response?.answer, + }); + } else { + // We do not expect this to happen and therefore it is an internal + // error. + throw new Error('Internal error, no answer in response'); + } + return; + } + + leaveMeeting(): Promise { + if (this.sessionControlChannelHandler) { + return this.sessionControlChannelHandler?.leaveSession(); + } else { + throw new Error('You must connect to a meeting before leaving it'); + } + } + + // The promise resolving on the request does not mean the layout has been + // applied. It means that the request has been accepted and you may need to + // wait a short amount of time for these layouts to be applied. + applyLayout(requests: MediaLayoutRequest[]): Promise { + if (!this.videoAssignmentChannelHandler) { + throw new Error( + 'You must connect to a meeting with video before applying a layout', + ); + } + requests.forEach((request) => { + if (!request.mediaLayout) { + throw new Error('The request must include a media layout'); + } + if (!this.internalMediaLayoutMap.has(request.mediaLayout)) { + throw new Error( + 'The media layout must be created using the client before it can be applied', + ); + } + }); + return this.videoAssignmentChannelHandler.sendRequests(requests); + } + + createMediaLayout(canvasDimensions: CanvasDimensions): MediaLayout { + const mediaEntryDelegate = new SubscribableDelegate( + undefined, + ); + const mediaEntry = new SubscribableImpl( + mediaEntryDelegate, + ); + const mediaLayout: MediaLayout = { canvasDimensions, mediaEntry }; + this.internalMediaLayoutMap.set(mediaLayout, { + id: this.mediaLayoutId, + mediaEntry: mediaEntryDelegate, + }); + this.idMediaLayoutMap.set(this.mediaLayoutId, mediaLayout); + this.mediaLayoutId++; + return mediaLayout; + } +} diff --git a/meet-live-agent/internal/subscribable_impl.ts b/meet-live-agent/internal/subscribable_impl.ts new file mode 100644 index 0000000..32e4348 --- /dev/null +++ b/meet-live-agent/internal/subscribable_impl.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Implementation of the Subscribable interface. + */ + +import {Subscribable} from '../types/subscribable'; + +/** + * Implementation of the Subscribable interface. + */ +export class SubscribableImpl implements Subscribable { + constructor(private readonly subscribableDelegate: SubscribableDelegate) {} + + get(): T { + return this.subscribableDelegate.get(); + } + + subscribe(callback: (value: T) => void): () => void { + this.subscribableDelegate.subscribe(callback); + return () => { + this.subscribableDelegate.unsubscribe(callback); + }; + } + + unsubscribe(callback: (value: T) => void): boolean { + return this.subscribableDelegate.unsubscribe(callback); + } +} + +/** + * Helper class to update a subscribable value. + */ +export class SubscribableDelegate { + private readonly subscribers = new Set<(value: T) => void>(); + private readonly subscribable: Subscribable = new SubscribableImpl( + this, + ); + + constructor(private value: T) {} + + set(newValue: T) { + if (this.value !== newValue) { + this.value = newValue; + for (const callback of this.subscribers) { + callback(newValue); + } + } + } + + get(): T { + return this.value; + } + + subscribe(callback: (value: T) => void): void { + this.subscribers.add(callback); + } + + unsubscribe(callback: (value: T) => void): boolean { + return this.subscribers.delete(callback); + } + + getSubscribable(): Subscribable { + return this.subscribable; + } +} diff --git a/meet-live-agent/internal/utils.ts b/meet-live-agent/internal/utils.ts new file mode 100644 index 0000000..70d5004 --- /dev/null +++ b/meet-live-agent/internal/utils.ts @@ -0,0 +1,115 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Utility functions for the MeetMediaApiClient. + */ + +import { + MediaEntry, + MediaLayout, + MeetStreamTrack, + Participant, +} from '../types/mediatypes'; + +import {InternalMediaEntry} from './internal_types'; +import {SubscribableDelegate} from './subscribable_impl'; + +interface InternalMediaEntryElement { + mediaEntry: MediaEntry; + internalMediaEntry: InternalMediaEntry; +} + +/** + * Creates a new media entry. + * @return The new media entry and its internal representation. + */ +export function createMediaEntry({ + audioMuted = false, + videoMuted = false, + screenShare = false, + isPresenter = false, + participant, + mediaLayout, + videoMeetStreamTrack, + audioMeetStreamTrack, + audioCsrc, + videoCsrc, + videoSsrc, + id, + session = '', + sessionName = '', +}: { + id: number; + audioMuted?: boolean; + videoMuted?: boolean; + screenShare?: boolean; + isPresenter?: boolean; + participant?: Participant; + mediaLayout?: MediaLayout; + audioMeetStreamTrack?: MeetStreamTrack; + videoMeetStreamTrack?: MeetStreamTrack; + videoCsrc?: number; + audioCsrc?: number; + videoSsrc?: number; + session?: string; + sessionName?: string; +}): InternalMediaEntryElement { + const participantDelegate = new SubscribableDelegate( + participant, + ); + const audioMutedDelegate = new SubscribableDelegate(audioMuted); + const videoMutedDelegate = new SubscribableDelegate(videoMuted); + const screenShareDelegate = new SubscribableDelegate(screenShare); + const isPresenterDelegate = new SubscribableDelegate(isPresenter); + const mediaLayoutDelegate = new SubscribableDelegate( + mediaLayout, + ); + const audioMeetStreamTrackDelegate = new SubscribableDelegate< + MeetStreamTrack | undefined + >(audioMeetStreamTrack); + const videoMeetStreamTrackDelegate = new SubscribableDelegate< + MeetStreamTrack | undefined + >(videoMeetStreamTrack); + + const mediaEntry: MediaEntry = { + participant: participantDelegate.getSubscribable(), + audioMuted: audioMutedDelegate.getSubscribable(), + videoMuted: videoMutedDelegate.getSubscribable(), + screenShare: screenShareDelegate.getSubscribable(), + isPresenter: isPresenterDelegate.getSubscribable(), + mediaLayout: mediaLayoutDelegate.getSubscribable(), + audioMeetStreamTrack: audioMeetStreamTrackDelegate.getSubscribable(), + videoMeetStreamTrack: videoMeetStreamTrackDelegate.getSubscribable(), + sessionName, + session, + }; + const internalMediaEntry: InternalMediaEntry = { + id, + audioMuted: audioMutedDelegate, + videoMuted: videoMutedDelegate, + screenShare: screenShareDelegate, + isPresenter: isPresenterDelegate, + mediaLayout: mediaLayoutDelegate, + audioMeetStreamTrack: audioMeetStreamTrackDelegate, + videoMeetStreamTrack: videoMeetStreamTrackDelegate, + participant: participantDelegate, + videoSsrc, + audioCsrc, + videoCsrc, + }; + return {mediaEntry, internalMediaEntry}; +} diff --git a/meet-live-agent/metadata.json b/meet-live-agent/metadata.json new file mode 100644 index 0000000..8fbe5ee --- /dev/null +++ b/meet-live-agent/metadata.json @@ -0,0 +1,8 @@ +{ + "name": "Audio Orb", + "description": "Speak, and the orb responds. An interactive experience powered by the Live Audio API.", + "requestFramePermissions": [ + "microphone" + ], + "prompt": "" +} \ No newline at end of file diff --git a/meet-live-agent/package-lock.json b/meet-live-agent/package-lock.json new file mode 100644 index 0000000..eb55e3c --- /dev/null +++ b/meet-live-agent/package-lock.json @@ -0,0 +1,1615 @@ +{ + "name": "meet-live-agent", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "meet-live-agent", + "version": "0.0.0", + "dependencies": { + "@google/genai": "^1.51.0", + "@googleworkspace/meet-addons": "^1.2.0", + "lit": "^3.3.0" + }, + "devDependencies": { + "@types/node": "^22.14.0", + "typescript": "~5.8.2", + "vite": "^6.2.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@google/genai": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.52.0.tgz", + "integrity": "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==", + "hasInstallScript": true, + "dependencies": { + "google-auth-library": "^10.3.0", + "p-retry": "^4.6.2", + "protobufjs": "^7.5.4", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, + "node_modules/@googleworkspace/meet-addons": { + "version": "1.2.0", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@googleworkspace/meet-addons/-/meet-addons-1.2.0.tgz", + "integrity": "sha512-UNyyJZ5x+NiD51OKATw7n6iUmVyaoTyLmMwKR+Bvwe9YkPw0Y45SZwa1Gev8YkJaiQJ/zo23Zhv0cH/QYnxuOQ==", + "license": "SEE LICENSE IN LICENSE" + }, + "node_modules/@lit-labs/ssr-dom-shim": { + "version": "1.5.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@lit-labs/ssr-dom-shim/-/ssr-dom-shim-1.5.1.tgz", + "integrity": "sha512-Aou5UdlSpr5whQe8AA/bZG0jMj96CoJIWbGfZ91qieWu5AWUMKw8VR/pAkQkJYvBNhmCcWnZlyyk5oze8JIqYA==", + "license": "BSD-3-Clause" + }, + "node_modules/@lit/reactive-element": { + "version": "2.1.2", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@lit/reactive-element/-/reactive-element-2.1.2.tgz", + "integrity": "sha512-pbCDiVMnne1lYUIaYNN5wrwQXDtHaYtg7YEFPeW+hws6U47WeFvISGUWekPGKWOP1ygrs0ef0o1VJMk1exos5A==", + "license": "BSD-3-Clause", + "dependencies": { + "@lit-labs/ssr-dom-shim": "^1.5.0" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.1.tgz", + "integrity": "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew==" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.57.1.tgz", + "integrity": "sha512-A6ehUVSiSaaliTxai040ZpZ2zTevHYbvu/lDoeAteHI8QnaosIzm4qwtezfRg1jOYaUmnzLX1AOD6Z+UJjtifg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.57.1.tgz", + "integrity": "sha512-dQaAddCY9YgkFHZcFNS/606Exo8vcLHwArFZ7vxXq4rigo2bb494/xKMMwRRQW6ug7Js6yXmBZhSBRuBvCCQ3w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.57.1.tgz", + "integrity": "sha512-crNPrwJOrRxagUYeMn/DZwqN88SDmwaJ8Cvi/TN1HnWBU7GwknckyosC2gd0IqYRsHDEnXf328o9/HC6OkPgOg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.57.1.tgz", + "integrity": "sha512-Ji8g8ChVbKrhFtig5QBV7iMaJrGtpHelkB3lsaKzadFBe58gmjfGXAOfI5FV0lYMH8wiqsxKQ1C9B0YTRXVy4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.57.1.tgz", + "integrity": "sha512-R+/WwhsjmwodAcz65guCGFRkMb4gKWTcIeLy60JJQbXrJ97BOXHxnkPFrP+YwFlaS0m+uWJTstrUA9o+UchFug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.57.1.tgz", + "integrity": "sha512-IEQTCHeiTOnAUC3IDQdzRAGj3jOAYNr9kBguI7MQAAZK3caezRrg0GxAb6Hchg4lxdZEI5Oq3iov/w/hnFWY9Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.57.1.tgz", + "integrity": "sha512-F8sWbhZ7tyuEfsmOxwc2giKDQzN3+kuBLPwwZGyVkLlKGdV1nvnNwYD0fKQ8+XS6hp9nY7B+ZeK01EBUE7aHaw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.57.1.tgz", + "integrity": "sha512-rGfNUfn0GIeXtBP1wL5MnzSj98+PZe/AXaGBCRmT0ts80lU5CATYGxXukeTX39XBKsxzFpEeK+Mrp9faXOlmrw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.57.1.tgz", + "integrity": "sha512-MMtej3YHWeg/0klK2Qodf3yrNzz6CGjo2UntLvk2RSPlhzgLvYEB3frRvbEF2wRKh1Z2fDIg9KRPe1fawv7C+g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.57.1.tgz", + "integrity": "sha512-1a/qhaaOXhqXGpMFMET9VqwZakkljWHLmZOX48R0I/YLbhdxr1m4gtG1Hq7++VhVUmf+L3sTAf9op4JlhQ5u1Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.57.1.tgz", + "integrity": "sha512-QWO6RQTZ/cqYtJMtxhkRkidoNGXc7ERPbZN7dVW5SdURuLeVU7lwKMpo18XdcmpWYd0qsP1bwKPf7DNSUinhvA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.57.1.tgz", + "integrity": "sha512-xpObYIf+8gprgWaPP32xiN5RVTi/s5FCR+XMXSKmhfoJjrpRAjCuuqQXyxUa/eJTdAE6eJ+KDKaoEqjZQxh3Gw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.57.1.tgz", + "integrity": "sha512-4BrCgrpZo4hvzMDKRqEaW1zeecScDCR+2nZ86ATLhAoJ5FQ+lbHVD3ttKe74/c7tNT9c6F2viwB3ufwp01Oh2w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.57.1.tgz", + "integrity": "sha512-NOlUuzesGauESAyEYFSe3QTUguL+lvrN1HtwEEsU2rOwdUDeTMJdO5dUYl/2hKf9jWydJrO9OL/XSSf65R5+Xw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.57.1.tgz", + "integrity": "sha512-ptA88htVp0AwUUqhVghwDIKlvJMD/fmL/wrQj99PRHFRAG6Z5nbWoWG4o81Nt9FT+IuqUQi+L31ZKAFeJ5Is+A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.57.1.tgz", + "integrity": "sha512-S51t7aMMTNdmAMPpBg7OOsTdn4tySRQvklmL3RpDRyknk87+Sp3xaumlatU+ppQ+5raY7sSTcC2beGgvhENfuw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.57.1.tgz", + "integrity": "sha512-Bl00OFnVFkL82FHbEqy3k5CUCKH6OEJL54KCyx2oqsmZnFTR8IoNqBF+mjQVcRCT5sB6yOvK8A37LNm/kPJiZg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.57.1.tgz", + "integrity": "sha512-ABca4ceT4N+Tv/GtotnWAeXZUZuM/9AQyCyKYyKnpk4yoA7QIAuBt6Hkgpw8kActYlew2mvckXkvx0FfoInnLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.57.1.tgz", + "integrity": "sha512-HFps0JeGtuOR2convgRRkHCekD7j+gdAuXM+/i6kGzQtFhlCtQkpwtNzkNj6QhCDp7DRJ7+qC/1Vg2jt5iSOFw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.57.1.tgz", + "integrity": "sha512-H+hXEv9gdVQuDTgnqD+SQffoWoc0Of59AStSzTEj/feWTBAnSfSD3+Dql1ZruJQxmykT/JVY0dE8Ka7z0DH1hw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.57.1.tgz", + "integrity": "sha512-4wYoDpNg6o/oPximyc/NG+mYUejZrCU2q+2w6YZqrAs2UcNUChIZXjtafAiiZSUc7On8v5NyNj34Kzj/Ltk6dQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.57.1.tgz", + "integrity": "sha512-O54mtsV/6LW3P8qdTcamQmuC990HDfR71lo44oZMZlXU4tzLrbvTii87Ni9opq60ds0YzuAlEr/GNwuNluZyMQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.57.1.tgz", + "integrity": "sha512-P3dLS+IerxCT/7D2q2FYcRdWRl22dNbrbBEtxdWhXrfIMPP9lQhb5h4Du04mdl5Woq05jVCDPCMF7Ub0NAjIew==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.57.1.tgz", + "integrity": "sha512-VMBH2eOOaKGtIJYleXsi2B8CPVADrh+TyNxJ4mWPnKfLB/DBUmzW+5m1xUrcwWoMfSLagIRpjUFeW5CO5hyciQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.57.1.tgz", + "integrity": "sha512-mxRFDdHIWRxg3UfIIAwCm6NzvxG0jDX/wBN6KsQFTvKFqqg9vTrWUE68qEjHt19A5wwx5X5aUi2zuZT7YR0jrA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.19.7", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@types/node/-/node-22.19.7.tgz", + "integrity": "sha512-MciR4AKGHWl7xwxkBa6xUGxQJ4VBOmPTF7sL+iGzuahOFaO0jHCsuEfS80pan1ef4gWId1oWOweIhrDEYLuaOw==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==" + }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT" + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "engines": { + "node": "*" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gaxios": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", + "integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-auth-library": { + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", + "integrity": "sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw==", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "engines": { + "node": ">=14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/lit": { + "version": "3.3.2", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/lit/-/lit-3.3.2.tgz", + "integrity": "sha512-NF9zbsP79l4ao2SNrH3NkfmFgN/hBYSQo90saIVI1o5GpjAdCPVstVzO1MrLOakHoEhYkrtRjPK6Ob521aoYWQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@lit/reactive-element": "^2.1.0", + "lit-element": "^4.2.0", + "lit-html": "^3.3.0" + } + }, + "node_modules/lit-element": { + "version": "4.2.2", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/lit-element/-/lit-element-4.2.2.tgz", + "integrity": "sha512-aFKhNToWxoyhkNDmWZwEva2SlQia+jfG0fjIWV//YeTaWrVnOxD89dPKfigCUspXFmjzOEUQpOkejH5Ly6sG0w==", + "license": "BSD-3-Clause", + "dependencies": { + "@lit-labs/ssr-dom-shim": "^1.5.0", + "@lit/reactive-element": "^2.1.0", + "lit-html": "^3.3.0" + } + }, + "node_modules/lit-html": { + "version": "3.3.2", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/lit-html/-/lit-html-3.3.2.tgz", + "integrity": "sha512-Qy9hU88zcmaxBXcc10ZpdK7cOLXvXpRoBxERdtqV9QOrfpMZZ6pSYP91LhpPtap3sFMUiL7Tw2RImbe0Al2/kw==", + "license": "BSD-3-Clause", + "dependencies": { + "@types/trusted-types": "^2.0.2" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/protobufjs": { + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.6.tgz", + "integrity": "sha512-M71sTMB146U3u0di3yup8iM+zv8yPRNQVr1KK4tyBitl3qFvEGucq/rGDRShD2rsJhtN02RJaJ7j5X5hmy8SJg==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.1", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/rollup": { + "version": "4.57.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/rollup/-/rollup-4.57.1.tgz", + "integrity": "sha512-oQL6lgK3e2QZeQ7gcgIkS2YZPg5slw37hYufJ3edKlfQSGGm8ICoxswK15ntSzF/a8+h7ekRy7k7oWc3BQ7y8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.57.1", + "@rollup/rollup-android-arm64": "4.57.1", + "@rollup/rollup-darwin-arm64": "4.57.1", + "@rollup/rollup-darwin-x64": "4.57.1", + "@rollup/rollup-freebsd-arm64": "4.57.1", + "@rollup/rollup-freebsd-x64": "4.57.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.57.1", + "@rollup/rollup-linux-arm-musleabihf": "4.57.1", + "@rollup/rollup-linux-arm64-gnu": "4.57.1", + "@rollup/rollup-linux-arm64-musl": "4.57.1", + "@rollup/rollup-linux-loong64-gnu": "4.57.1", + "@rollup/rollup-linux-loong64-musl": "4.57.1", + "@rollup/rollup-linux-ppc64-gnu": "4.57.1", + "@rollup/rollup-linux-ppc64-musl": "4.57.1", + "@rollup/rollup-linux-riscv64-gnu": "4.57.1", + "@rollup/rollup-linux-riscv64-musl": "4.57.1", + "@rollup/rollup-linux-s390x-gnu": "4.57.1", + "@rollup/rollup-linux-x64-gnu": "4.57.1", + "@rollup/rollup-linux-x64-musl": "4.57.1", + "@rollup/rollup-openbsd-x64": "4.57.1", + "@rollup/rollup-openharmony-arm64": "4.57.1", + "@rollup/rollup-win32-arm64-msvc": "4.57.1", + "@rollup/rollup-win32-ia32-msvc": "4.57.1", + "@rollup/rollup-win32-x64-gnu": "4.57.1", + "@rollup/rollup-win32-x64-msvc": "4.57.1", + "fsevents": "~2.3.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/typescript": { + "version": "5.8.3", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/vite": { + "version": "6.4.1", + "resolved": "https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/vite/-/vite-6.4.1.tgz", + "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/ws": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", + "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/meet-live-agent/package.json b/meet-live-agent/package.json new file mode 100644 index 0000000..a5e87b1 --- /dev/null +++ b/meet-live-agent/package.json @@ -0,0 +1,21 @@ +{ + "name": "meet-live-agent", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "@google/genai": "^1.51.0", + "@googleworkspace/meet-addons": "^1.2.0", + "lit": "^3.3.0" + }, + "devDependencies": { + "@types/node": "^22.14.0", + "typescript": "~5.8.2", + "vite": "^6.2.0" + } +} diff --git a/meet-live-agent/public/pcm-recorder-processor.js b/meet-live-agent/public/pcm-recorder-processor.js new file mode 100644 index 0000000..3262899 --- /dev/null +++ b/meet-live-agent/public/pcm-recorder-processor.js @@ -0,0 +1,13 @@ +class PCMRecorderProcessor extends AudioWorkletProcessor { + process(inputs, outputs, parameters) { + const input = inputs[0]; + if (input && input.length > 0) { + const channelData = input[0]; + // Send the audio data to the main thread as a Float32Array + this.port.postMessage(channelData); + } + return true; + } +} + +registerProcessor('pcm-recorder-processor', PCMRecorderProcessor); diff --git a/meet-live-agent/sample.env b/meet-live-agent/sample.env new file mode 100644 index 0000000..ef734c7 --- /dev/null +++ b/meet-live-agent/sample.env @@ -0,0 +1,5 @@ +PROJECT_ID= +REGION= +GEMINI_API_KEY= +CLOUD_PROJECT_NUMBER= +CLIENT_ID= diff --git a/meet-live-agent/server/package-lock.json b/meet-live-agent/server/package-lock.json new file mode 100644 index 0000000..0238619 --- /dev/null +++ b/meet-live-agent/server/package-lock.json @@ -0,0 +1,1667 @@ +{ + "name": "appletserver", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "appletserver", + "version": "0.0.0", + "dependencies": { + "@google/genai": "^1.52.0", + "axios": "^1.6.7", + "dotenv": "^16.4.5", + "express": "^4.18.2", + "express-rate-limit": "^7.5.0", + "ws": "^8.17.0" + }, + "devDependencies": { + "@types/node": "^22.14.0", + "nodemon": "^3.1.0" + } + }, + "node_modules/@google/genai": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.52.0.tgz", + "integrity": "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==", + "hasInstallScript": true, + "dependencies": { + "google-auth-library": "^10.3.0", + "p-retry": "^4.6.2", + "protobufjs": "^7.5.4", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.1.tgz", + "integrity": "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew==" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==" + }, + "node_modules/@types/node": { + "version": "22.19.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz", + "integrity": "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==" + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.0.tgz", + "integrity": "sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==", + "dependencies": { + "follow-redirects": "^1.16.0", + "form-data": "^4.0.5", + "proxy-from-env": "^2.1.0" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "engines": { + "node": "*" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/body-parser": { + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/qs": { + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "dev": true, + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dotenv": { + "version": "16.6.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", + "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "7.5.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.5.1.tgz", + "integrity": "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw==", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/follow-redirects": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", + "integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/google-auth-library": { + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", + "integrity": "sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw==", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "engines": { + "node": ">=14" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", + "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/https-proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", + "dev": true + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "dev": true, + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/nodemon": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.14.tgz", + "integrity": "sha512-jakjZi93UtB3jHMWsXL68FXSAosbLfY0In5gtKq3niLSkrWznrVBzXFNOEMJUfc9+Ke7SHWoAZsiMkNP3vq6Jw==", + "dev": true, + "dependencies": { + "chokidar": "^3.5.2", + "debug": "^4", + "ignore-by-default": "^1.0.1", + "minimatch": "^10.2.1", + "pstree.remy": "^1.1.8", + "semver": "^7.5.3", + "simple-update-notifier": "^2.0.0", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "bin": { + "nodemon": "bin/nodemon.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nodemon" + } + }, + "node_modules/nodemon/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/nodemon/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==" + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/protobufjs": { + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.6.tgz", + "integrity": "sha512-M71sTMB146U3u0di3yup8iM+zv8yPRNQVr1KK4tyBitl3qFvEGucq/rGDRShD2rsJhtN02RJaJ7j5X5hmy8SJg==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.1", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-from-env": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", + "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true + }, + "node_modules/qs": { + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/simple-update-notifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", + "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", + "dev": true, + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/touch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.1.tgz", + "integrity": "sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==", + "dev": true, + "bin": { + "nodetouch": "bin/nodetouch.js" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/ws": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", + "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/meet-live-agent/server/package.json b/meet-live-agent/server/package.json new file mode 100644 index 0000000..652ff75 --- /dev/null +++ b/meet-live-agent/server/package.json @@ -0,0 +1,21 @@ +{ + "name": "appletserver", + "private": true, + "version": "0.0.0", + "scripts": { + "start": "node server.js", + "dev": "nodemon server.js" + }, + "dependencies": { + "@google/genai": "^1.52.0", + "axios": "^1.6.7", + "dotenv": "^16.4.5", + "express": "^4.18.2", + "express-rate-limit": "^7.5.0", + "ws": "^8.17.0" + }, + "devDependencies": { + "@types/node": "^22.14.0", + "nodemon": "^3.1.0" + } +} diff --git a/meet-live-agent/server/public/service-worker.js b/meet-live-agent/server/public/service-worker.js new file mode 100644 index 0000000..1c4ef63 --- /dev/null +++ b/meet-live-agent/server/public/service-worker.js @@ -0,0 +1,128 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ +// service-worker.js + +// Define the target URL that we want to intercept and proxy. +const TARGET_URL_PREFIX = 'https://generativelanguage.googleapis.com'; + +// Installation event: +self.addEventListener('install', (event) => { + try { + console.log('Service Worker: Installing...'); + event.waitUntil(self.skipWaiting()); + } catch (error) { + console.error('Service Worker: Error during install event:', error); + // If skipWaiting fails, the new SW might get stuck in a waiting state. + } +}); + +// Activation event: +self.addEventListener('activate', (event) => { + try { + console.log('Service Worker: Activating...'); + event.waitUntil(self.clients.claim()); + } catch (error) { + console.error('Service Worker: Error during activate event:', error); + // If clients.claim() fails, the SW might not control existing pages until next nav. + } +}); + +// Fetch event: +self.addEventListener('fetch', (event) => { + try { + const requestUrl = event.request.url; + + if (requestUrl.startsWith(TARGET_URL_PREFIX)) { + console.log(`Service Worker: Intercepting request to ${requestUrl}`); + + const remainingPathAndQuery = requestUrl.substring(TARGET_URL_PREFIX.length); + const proxyUrl = `${self.location.origin}/api-proxy${remainingPathAndQuery}`; + + console.log(`Service Worker: Proxying to ${proxyUrl}`); + + // Construct headers for the request to the proxy + const newHeaders = new Headers(); + // Copy essential headers from the original request + // For OPTIONS (preflight) requests, Access-Control-Request-* are critical. + // For actual requests (POST, GET), Content-Type, Accept etc. + const headersToCopy = [ + 'Content-Type', + 'Accept', + 'Access-Control-Request-Method', + 'Access-Control-Request-Headers', + ]; + + for (const headerName of headersToCopy) { + if (event.request.headers.has(headerName)) { + newHeaders.set(headerName, event.request.headers.get(headerName)); + } + } + + if (event.request.method === 'POST') { + + // Ensure Content-Type is set for POST requests to the proxy, defaulting to application/json + if (!newHeaders.has('Content-Type')) { + console.warn("Service Worker: POST request to proxy was missing Content-Type in newHeaders. Defaulting to application/json."); + newHeaders.set('Content-Type', 'application/json'); + } else { + console.log(`Service Worker: POST request to proxy has Content-Type: ${newHeaders.get('Content-Type')}`); + } + } + + const requestOptions = { + method: event.request.method, + headers: newHeaders, // Use simplified headers + body: event.request.body, // Still use the original body stream + mode: event.request.mode, + credentials: event.request.credentials, + cache: event.request.cache, + redirect: event.request.redirect, + referrer: event.request.referrer, + integrity: event.request.integrity, + }; + + // Only set duplex if there's a body and it's a relevant method + if (event.request.method !== 'GET' && event.request.method !== 'HEAD' && event.request.body ) { + requestOptions.duplex = 'half'; + } + + const promise = fetch(new Request(proxyUrl, requestOptions)) + .then((response) => { + console.log(`Service Worker: Successfully proxied request to ${proxyUrl}, Status: ${response.status}`); + return response; + }) + .catch((error) => { + // Log more error details + console.error(`Service Worker: Error proxying request to ${proxyUrl}. Message: ${error.message}, Name: ${error.name}, Stack: ${error.stack}`); + return new Response( + JSON.stringify({ error: 'Proxying failed', details: error.message, name: error.name, proxiedUrl: proxyUrl }), + { + status: 502, // Bad Gateway is appropriate for proxy errors + headers: { 'Content-Type': 'application/json' } + } + ); + }); + + event.respondWith(promise); + + } else { + // If the request URL doesn't match our target, let it proceed as normal. + event.respondWith(fetch(event.request)); + } + } catch (error) { + // Log more error details for unhandled errors too + console.error('Service Worker: Unhandled error in fetch event handler. Message:', error.message, 'Name:', error.name, 'Stack:', error.stack); + event.respondWith( + new Response( + JSON.stringify({ error: 'Service worker fetch handler failed', details: error.message, name: error.name }), + { + status: 500, + headers: { 'Content-Type': 'application/json' } + } + ) + ); + } +}); diff --git a/meet-live-agent/server/public/websocket-interceptor.js b/meet-live-agent/server/public/websocket-interceptor.js new file mode 100644 index 0000000..8d559ed --- /dev/null +++ b/meet-live-agent/server/public/websocket-interceptor.js @@ -0,0 +1,65 @@ +(function() { + const TARGET_WS_HOST = 'generativelanguage.googleapis.com'; // Host to intercept + const originalWebSocket = window.WebSocket; + + if (!originalWebSocket) { + console.error('[WebSocketInterceptor] Original window.WebSocket not found. Cannot apply interceptor.'); + return; + } + + const handler = { + construct(target, args) { + let [url, protocols] = args; + //stringify url's if necessary for parsing + let newUrlString = typeof url === 'string' ? url : (url && typeof url.toString === 'function' ? url.toString() : null); + //get ready to check for host to proxy + let isTarget = false; + + if (newUrlString) { + try { + // For full URLs, parse string and check the host + if (newUrlString.startsWith('ws://') || newUrlString.startsWith('wss://')) { + //URL object again + const parsedUrl = new URL(newUrlString); + if (parsedUrl.host === TARGET_WS_HOST) { + isTarget = true; + //use wss if https, else ws + const proxyScheme = window.location.protocol === 'https:' ? 'wss' : 'ws'; + const proxyHost = window.location.host; + newUrlString = `${proxyScheme}://${proxyHost}/api-proxy${parsedUrl.pathname}${parsedUrl.search}`; + } + } + } catch (e) { + console.warn('[WebSocketInterceptor-Proxy] Error parsing WebSocket URL, using original:', url, e); + } + } else { + console.warn('[WebSocketInterceptor-Proxy] WebSocket URL is not a string or stringifiable. Using original.'); + } + + if (isTarget) { + console.log('[WebSocketInterceptor-Proxy] Original WebSocket URL:', url); + console.log('[WebSocketInterceptor-Proxy] Redirecting to proxy URL:', newUrlString); + } + + // Call the original constructor with potentially modified arguments + // Reflect.construct ensures 'new target(...)' behavior and correct prototype chain + if (protocols) { + return Reflect.construct(target, [newUrlString, protocols]); + } else { + return Reflect.construct(target, [newUrlString]); + } + }, + get(target, prop, receiver) { + // Forward static property access (e.g., WebSocket.OPEN, WebSocket.CONNECTING) + // and prototype access to the original WebSocket constructor/prototype + if (prop === 'prototype') { + return target.prototype; + } + return Reflect.get(target, prop, receiver); + } + }; + + window.WebSocket = new Proxy(originalWebSocket, handler); + + console.log('[WebSocketInterceptor-Proxy] Global WebSocket constructor has been wrapped using Proxy.'); +})(); diff --git a/meet-live-agent/server/server.js b/meet-live-agent/server/server.js new file mode 100644 index 0000000..5d1c521 --- /dev/null +++ b/meet-live-agent/server/server.js @@ -0,0 +1,350 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +require('dotenv').config(); +const express = require('express'); +const fs = require('fs'); +const axios = require('axios'); +const https = require('https'); +const path = require('path'); +const WebSocket = require('ws'); +const { URLSearchParams, URL } = require('url'); +const rateLimit = require('express-rate-limit'); + +const app = express(); +const port = process.env.PORT || 3000; +const externalApiBaseUrl = 'https://generativelanguage.googleapis.com'; +const externalWsBaseUrl = 'wss://generativelanguage.googleapis.com'; +// Support either API key env-var variant +const apiKey = process.env.GEMINI_API_KEY || process.env.API_KEY; + +const staticPath = path.join(__dirname,'dist'); +const publicPath = path.join(__dirname,'public'); + + +if (!apiKey) { + // Only log an error, don't exit. The server will serve apps without proxy functionality + console.error("Warning: GEMINI_API_KEY or API_KEY environment variable is not set! Proxy functionality will be disabled."); +} +else { + console.log("API KEY FOUND (proxy will use this)") +} + +// Limit body size to 50mb +app.use(express.json({ limit: '50mb' })); +app.use(express.urlencoded({extended: true, limit: '50mb'})); +app.set('trust proxy', 1 /* number of proxies between user and server */) + +// Rate limiter for the proxy +const proxyLimiter = rateLimit({ + windowMs: 15 * 60 * 1000, // Set ratelimit window at 15min (in ms) + max: 100, // Limit each IP to 100 requests per window + message: 'Too many requests from this IP, please try again after 15 minutes', + standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers + legacyHeaders: false, // no `X-RateLimit-*` headers + handler: (req, res, next, options) => { + console.warn(`Rate limit exceeded for IP: ${req.ip}. Path: ${req.path}`); + res.status(options.statusCode).send(options.message); + } +}); + +// Apply the rate limiter to the /api-proxy route before the main proxy logic +app.use('/api-proxy', proxyLimiter); + +// Proxy route for Gemini API calls (HTTP) +app.use('/api-proxy', async (req, res, next) => { + console.log(req.ip); + // If the request is an upgrade request, it's for WebSockets, so pass to next middleware/handler + if (req.headers.upgrade && req.headers.upgrade.toLowerCase() === 'websocket') { + return next(); // Pass to the WebSocket upgrade handler + } + + // Handle OPTIONS request for CORS preflight + if (req.method === 'OPTIONS') { + res.setHeader('Access-Control-Allow-Origin', '*'); // Adjust as needed for security + res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS'); + res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, X-Goog-Api-Key'); + res.setHeader('Access-Control-Max-Age', '86400'); // Cache preflight response for 1 day + return res.sendStatus(200); + } + + if (req.body) { // Only log body if it exists + console.log(" Request Body (from frontend):", req.body); + } + try { + // Construct the target URL by taking the part of the path after /api-proxy/ + const targetPath = req.url.startsWith('/') ? req.url.substring(1) : req.url; + const apiUrl = `${externalApiBaseUrl}/${targetPath}`; + console.log(`HTTP Proxy: Forwarding request to ${apiUrl}`); + + // Prepare headers for the outgoing request + const outgoingHeaders = {}; + // Copy most headers from the incoming request + for (const header in req.headers) { + // Exclude host-specific headers and others that might cause issues upstream + if (!['host', 'connection', 'content-length', 'transfer-encoding', 'upgrade', 'sec-websocket-key', 'sec-websocket-version', 'sec-websocket-extensions'].includes(header.toLowerCase())) { + outgoingHeaders[header] = req.headers[header]; + } + } + + // Set the actual API key in the appropriate header + outgoingHeaders['X-Goog-Api-Key'] = apiKey; + + // Set Content-Type from original request if present (for relevant methods) + if (req.headers['content-type'] && ['POST', 'PUT', 'PATCH'].includes(req.method.toUpperCase())) { + outgoingHeaders['Content-Type'] = req.headers['content-type']; + } else if (['POST', 'PUT', 'PATCH'].includes(req.method.toUpperCase())) { + // Default Content-Type to application/json if no content type for post/put/patch + outgoingHeaders['Content-Type'] = 'application/json'; + } + + // For GET or DELETE requests, ensure Content-Type is NOT sent, + // even if the client erroneously included it. + if (['GET', 'DELETE'].includes(req.method.toUpperCase())) { + delete outgoingHeaders['Content-Type']; // Case-sensitive common practice + delete outgoingHeaders['content-type']; // Just in case + } + + // Ensure 'accept' is reasonable if not set + if (!outgoingHeaders['accept']) { + outgoingHeaders['accept'] = '*/*'; + } + + + const axiosConfig = { + method: req.method, + url: apiUrl, + headers: outgoingHeaders, + responseType: 'stream', + validateStatus: function (status) { + return true; // Accept any status code, we'll pipe it through + }, + }; + + if (['POST', 'PUT', 'PATCH'].includes(req.method.toUpperCase())) { + axiosConfig.data = req.body; + } + // For GET, DELETE, etc., axiosConfig.data will remain undefined, + // and axios will not send a request body. + + const apiResponse = await axios(axiosConfig); + + // Pass through response headers from Gemini API to the client + for (const header in apiResponse.headers) { + res.setHeader(header, apiResponse.headers[header]); + } + res.status(apiResponse.status); + + + apiResponse.data.on('data', (chunk) => { + res.write(chunk); + }); + + apiResponse.data.on('end', () => { + res.end(); + }); + + apiResponse.data.on('error', (err) => { + console.error('Error during streaming data from target API:', err); + if (!res.headersSent) { + res.status(500).json({ error: 'Proxy error during streaming from target' }); + } else { + // If headers already sent, we can't send a JSON error, just end the response. + res.end(); + } + }); + + } catch (error) { + console.error('Proxy error before request to target API:', error); + if (!res.headersSent) { + if (error.response) { + const errorData = { + status: error.response.status, + message: error.response.data?.error?.message || 'Proxy error from upstream API', + details: error.response.data?.error?.details || null + }; + res.status(error.response.status).json(errorData); + } else { + res.status(500).json({ error: 'Proxy setup error', message: error.message }); + } + } + } +}); + +const webSocketInterceptorScriptTag = ``; + +// Prepare service worker registration script content +const serviceWorkerRegistrationScript = ` + +`; + +// Serve index.html or placeholder based on API key and file availability +app.get('/', (req, res) => { + const placeholderPath = path.join(publicPath, 'placeholder.html'); + + // Try to serve index.html + console.log("LOG: Route '/' accessed. Attempting to serve index.html."); + const indexPath = path.join(staticPath, 'index.html'); + + fs.readFile(indexPath, 'utf8', (err, indexHtmlData) => { + if (err) { + // index.html not found or unreadable, serve the original placeholder + console.log('LOG: index.html not found or unreadable. Falling back to original placeholder.'); + return res.sendFile(placeholderPath); + } + + // If API key is not set, serve original HTML without injection + if (!apiKey) { + console.log("LOG: API key not set. Serving original index.html without script injections."); + return res.sendFile(indexPath); + } + + // index.html found and apiKey set, inject scripts + console.log("LOG: index.html read successfully. Injecting scripts."); + let injectedHtml = indexHtmlData; + + + if (injectedHtml.includes('')) { + // Inject WebSocket interceptor first, then service worker script + injectedHtml = injectedHtml.replace( + '', + `${webSocketInterceptorScriptTag}${serviceWorkerRegistrationScript}` + ); + console.log("LOG: Scripts injected into ."); + } else { + console.warn("WARNING: tag not found in index.html. Prepending scripts to the beginning of the file as a fallback."); + injectedHtml = `${webSocketInterceptorScriptTag}${serviceWorkerRegistrationScript}${indexHtmlData}`; + } + res.send(injectedHtml); + }); +}); + +app.get('/service-worker.js', (req, res) => { + return res.sendFile(path.join(publicPath, 'service-worker.js')); +}); + +app.use('/public', express.static(publicPath)); +app.use(express.static(staticPath)); + +// Start the HTTP server +const server = app.listen(port, () => { + console.log(`Server listening on port ${port}`); + console.log(`HTTP proxy active on /api-proxy/**`); + console.log(`WebSocket proxy active on /api-proxy/**`); +}); + +// Create WebSocket server and attach it to the HTTP server +const wss = new WebSocket.Server({ noServer: true }); + +server.on('upgrade', (request, socket, head) => { + const requestUrl = new URL(request.url, `http://${request.headers.host}`); + const pathname = requestUrl.pathname; + + if (pathname.startsWith('/api-proxy/')) { + if (!apiKey) { + console.error("WebSocket proxy: API key not configured. Closing connection."); + socket.destroy(); + return; + } + + wss.handleUpgrade(request, socket, head, (clientWs) => { + console.log('Client WebSocket connected to proxy for path:', pathname); + + const targetPathSegment = pathname.substring('/api-proxy'.length); + const clientQuery = new URLSearchParams(requestUrl.search); + clientQuery.set('key', apiKey); + const targetGeminiWsUrl = `${externalWsBaseUrl}${targetPathSegment}?${clientQuery.toString()}`; + console.log(`Attempting to connect to target WebSocket: ${targetGeminiWsUrl}`); + + const geminiWs = new WebSocket(targetGeminiWsUrl, { + protocol: request.headers['sec-websocket-protocol'], + }); + + const messageQueue = []; + + geminiWs.on('open', () => { + console.log('Proxy connected to Gemini WebSocket'); + // Send any queued messages + while (messageQueue.length > 0) { + const message = messageQueue.shift(); + if (geminiWs.readyState === WebSocket.OPEN) { + console.log('Sending queued message from client -> Gemini'); + geminiWs.send(message); + } else { + // Should not happen if we are in 'open' event, but good for safety + console.warn('Gemini WebSocket not open when trying to send queued message. Re-queuing.'); + messageQueue.unshift(message); // Add it back to the front + break; // Stop processing queue for now + } + } + }); + + geminiWs.on('message', (message) => { + console.log('Message from Gemini -> client'); + if (clientWs.readyState === WebSocket.OPEN) { + clientWs.send(message); + } + }); + + geminiWs.on('close', (code, reason) => { + console.log(`Gemini WebSocket closed: ${code} ${reason.toString()}`); + if (clientWs.readyState === WebSocket.OPEN || clientWs.readyState === WebSocket.CONNECTING) { + clientWs.close(code, reason.toString()); + } + }); + + geminiWs.on('error', (error) => { + console.error('Error on Gemini WebSocket connection:', error); + if (clientWs.readyState === WebSocket.OPEN || clientWs.readyState === WebSocket.CONNECTING) { + clientWs.close(1011, 'Upstream WebSocket error'); + } + }); + + clientWs.on('message', (message) => { + if (geminiWs.readyState === WebSocket.OPEN) { + console.log('Message from client -> Gemini'); + geminiWs.send(message); + } else if (geminiWs.readyState === WebSocket.CONNECTING) { + console.log('Queueing message from client -> Gemini (Gemini still connecting)'); + messageQueue.push(message); + } else { + console.warn('Client sent message but Gemini WebSocket is not open or connecting. Message dropped.'); + } + }); + + clientWs.on('close', (code, reason) => { + console.log(`Client WebSocket closed: ${code} ${reason.toString()}`); + if (geminiWs.readyState === WebSocket.OPEN || geminiWs.readyState === WebSocket.CONNECTING) { + geminiWs.close(code, reason.toString()); + } + }); + + clientWs.on('error', (error) => { + console.error('Error on client WebSocket connection:', error); + if (geminiWs.readyState === WebSocket.OPEN || geminiWs.readyState === WebSocket.CONNECTING) { + geminiWs.close(1011, 'Client WebSocket error'); + } + }); + }); + } else { + console.log(`WebSocket upgrade request for non-proxy path: ${pathname}. Closing connection.`); + socket.destroy(); + } +}); diff --git a/meet-live-agent/tsconfig.json b/meet-live-agent/tsconfig.json new file mode 100644 index 0000000..2c6eed5 --- /dev/null +++ b/meet-live-agent/tsconfig.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + "target": "ES2022", + "experimentalDecorators": true, + "useDefineForClassFields": false, + "module": "ESNext", + "lib": [ + "ES2022", + "DOM", + "DOM.Iterable" + ], + "skipLibCheck": true, + "types": [ + "node" + ], + "moduleResolution": "bundler", + "isolatedModules": true, + "moduleDetection": "force", + "allowJs": true, + "jsx": "react-jsx", + "paths": { + "@/*": [ + "./*" + ] + }, + "allowImportingTsExtensions": true, + "noEmit": true + } +} \ No newline at end of file diff --git a/meet-live-agent/types/communication_protocol.d.ts b/meet-live-agent/types/communication_protocol.d.ts new file mode 100644 index 0000000..2a3c11b --- /dev/null +++ b/meet-live-agent/types/communication_protocol.d.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview An abstract communication protocol. + */ + +/** + * An abstract communication protocol. + */ +export interface MediaApiCommunicationProtocol { + /** + * Connects to the active conference with the given SDP offer. + * @param sdpOffer The SDP offer to connect to the active conference. + * @return A promise that resolves to the communication response. + */ + connectActiveConference( + sdpOffer: string, + ): Promise; +} + +/** + * The response from the communication protocol. + */ +export declare interface MediaApiCommunicationResponse { + /** + * The WebRTC answer to the offer. Format is SDP. + */ + answer: string; +} diff --git a/meet-live-agent/types/datachannels.d.ts b/meet-live-agent/types/datachannels.d.ts new file mode 100644 index 0000000..9cd6299 --- /dev/null +++ b/meet-live-agent/types/datachannels.d.ts @@ -0,0 +1,903 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Data channel interfaces for the Google Meet Media API. + */ + +// COMMON + +/** + * Base interface for all requests. + */ +export declare interface MediaApiRequest { + /** + * A unique client-generated identifier for this request. Different requests + * must never have the same request ID in the scope of one data channel. + */ + requestId: number; +} + +/** + * Base status for a response. + */ +export declare interface MediaApiResponseStatus { + /** Status code for the response. */ + code: number; + /** Message for the response. */ + message: string; + /** Additional details for the response. */ + // tslint:disable-next-line:no-any + details: any[]; +} + +/** + * Base interface for all responses. + */ +export declare interface MediaApiResponse { + /** + * ID of the associated request. + */ + requestId: number; + /** + * Response status for the request. + */ + status: MediaApiResponseStatus; +} + +/** + * Base interface for all resource snapshots provided by the server. + */ +export declare interface ResourceSnapshot { + /** + * The resource ID of the resource being updated. For singleton resources, + * this is unset. + */ + id?: number; +} + +/** + * Base interface for all deleted resources. + */ +export declare interface DeletedResource { + /** + * ID of the deleted resource. + */ + id: number; +} + +// SESSION CONTROL + +/** + * Session control data channel message from the client to the server. + */ +export declare interface SessionControlChannelFromClient { + /** Request to leave the session. */ + request: LeaveRequest; +} + +/** + * Session control data channel message from the server to the client. + */ +export declare interface SessionControlChannelToClient { + /** An optional response to an incoming request. */ + response?: LeaveResponse; + /** + * List of resource snapshots managed by the server, with no implied order. + */ + resources?: SessionStatusResource[]; +} + +/** + * Tells the server the client is about to disconnect. After receiving the + * response, the client should not expect to receive any other messages or media + * RTP. + */ +export declare interface LeaveRequest extends MediaApiRequest { + /** + * Leave field, always empty. + */ + leave: {}; +} + +/** + * Response to a leave request from the server. + */ +export declare interface LeaveResponse extends MediaApiResponse { + /** + * Leave field, always empty. + */ + leave: {}; +} + +/** + * Singleton resource containing the status of the media session. + */ +export declare interface SessionStatusResource extends ResourceSnapshot { + sessionStatus: SessionStatus; +} + +/** + * Session status. + */ +export declare interface SessionStatus { + /** + * The connection state of the session. + * + * - `STATE_WAITING`: Session is waiting to be admitted into the meeting. + * The client may never observe this state if it was admitted or rejected + * quickly. + * + * - `STATE_JOINED`: Session has fully joined the meeting. + * + * - `STATE_DISCONNECTED`: Session is not connected to the meeting. + */ + connectionState: 'STATE_WAITING' | 'STATE_JOINED' | 'STATE_DISCONNECTED'; + + /** + * The reason for the disconnection from the meeting. Only set if the + * `connectionState` is `STATE_DISCONNECTED`. + * + * - `REASON_CLIENT_LEFT`: The Media API client sent a leave request. + * + * - `REASON_USER_STOPPED`: A user explicitly stopped the Media API session. + * + * - `REASON_CONFERENCE_ENDED`: The conference ended. + * + * - `REASON_SESSION_UNHEALTHY`: Something else went wrong with the session. + */ + disconnectReason?: + | 'REASON_CLIENT_LEFT' + | 'REASON_USER_STOPPED' + | 'REASON_CONFERENCE_ENDED' + | 'REASON_SESSION_UNHEALTHY'; +} + +// PARTICIPANTS + +/** + * Participants data channel message from the server to the client. + */ +export declare interface ParticipantsChannelToClient { + /** + * List of resource snapshots managed by the server, with no implied order. + */ + resources?: ParticipantResource[]; + /** List of deleted resources with no implied order. */ + deletedResources?: DeletedParticipant[]; +} + +// (-- LINT.IfChange --) + +/** + * Base participant resource type + */ +export declare interface ParticipantResource extends ResourceSnapshot { + participant: BaseParticipant; +} + +/** + * Singleton resource containing participant information. + * There will be exactly one of signedInUser, anonymousUser, or phoneUser fields + * set to determine the type of participant. + */ +export declare interface BaseParticipant extends ResourceSnapshot { + /** + * Resource name of the participant. + * Format: `conferenceRecords/{conferenceRecord}/participants/{participant}` + * + * You can use this to retrieve additional information about the participant + * from the {@link https://developers.google.com/meet/api/reference/rest/v2/conferenceRecords.participants | Meet REST API - Participants} resource. + * + * Unused for now. Use participantKey instead. + */ + name?: string; + /** + * Participant key of associated participant. + * Format is `participants/{participant}`. + * + * You can use this to retrieve additional information about the participant + * from the {@link https://developers.google.com/meet/api/reference/rest/v2/conferenceRecords.participants | Meet REST API - Participants} resource. + * + * `Note`: This has to be in the format of `conferenceRecords/{conference_record}/participants/{participant}`. + * + * You can retrieve the conference record from the {@link https://developers.google.com/meet/api/guides/conferences | Meet REST API - Conferences} resource. + * + */ + participantKey?: string; + /** + * Participant id for internal usage. + */ + participantId: number; + /** + * If set, the participant is a signed in user. Provides a unique ID and + * display name. + */ + signedInUser?: SignedInUser; + /** + * If set, the participant is an anonymous user. Provides a display name. + */ + anonymousUser?: AnonymousUser; + /** + * If set, the participant is a dial-in user. Provides a partially redacted + * phone number. + */ + phoneUser?: PhoneUser; +} + +/** + * Signed in user type, always has a unique id and display name. + */ +export declare interface SignedInUser { + /** + * Unique ID for the user. Interoperable with {@link https://developers.google.com/admin-sdk/directory/reference/rest/v1/users | Admin SDK API } and {@link https://developers.google.com/people/api/rest/v1/people | People API}. + * Format: `users/{user}` + */ + user: string; + + /** + * For a personal device, it's the user's first name and last name. + * For a robot account, it's the administrator-specified device name. For + * example, "Altostrat Room". + */ + displayName: string; +} + +/** + * Anonymous user type, requires display name to be set. + */ +export declare interface AnonymousUser { + /** User provided name when they join a conference anonymously. */ + displayName: string; +} + +/** + * Phone user type, always has a display name. User dialing in from a phone + * where the user's identity is unknown because they haven't signed in with a + * Google Account. + */ +export declare interface PhoneUser { + /** Partially redacted user's phone number. */ + displayName: string; +} +// (-- +// LINT.ThenChange(//depot/google3/google/apps/meet/v2main/resource.proto) +// --) + +/** + * Deleted resource for a participant. + */ +export declare interface DeletedParticipant extends DeletedResource { + /** + * Set to true if the participant is successfully deleted. + */ + participant: boolean; +} + +// MEDIA ENTRIES + +/** + * Media entries data channel message from the server to the client. + */ +export declare interface MediaEntriesChannelToClient { + /** + * List of resource snapshots managed by the server, with no implied order. + */ + resources?: MediaEntryResource[]; + /** List of deleted resources with no implied order. */ + deletedResources?: DeletedMediaEntry[]; +} + +/** + * Resource snapshot for a media entry. + */ +export declare interface MediaEntryResource extends ResourceSnapshot { + /** + * Media entry resource. + */ + mediaEntry: MediaEntry; +} + +/** + * Media Entry interface. + */ +export declare interface MediaEntry { + /** + * Participant ID for the media entry. + * @deprecated Use participant key instead. + */ + participantId: number; + + /** + * Resource name of the participant. + * Format: `conferenceRecords/{conferenceRecord}/participants/{participant}` + * + * You can use this to retrieve additional information about the participant + * from the {@link https://developers.google.com/meet/api/reference/rest/v2/conferenceRecords.participants | Meet REST API - Participants} resource. + * + * Unused for now. Use participantKey instead. + */ + participant?: string; + + /** + * Participant key of associated participant. + * Format is `participants/{participant}`. + * + * You can use this to retrieve additional information about the participant + * from the {@link https://developers.google.com/meet/api/reference/rest/v2/conferenceRecords.participants | Meet REST API - Participants} resource. + * + * `Note`: This has to be in the format of `conferenceRecords/{conference_record}/participants/{participant}`. + * + * You can retrieve the conference record from the {@link https://developers.google.com/meet/api/guides/conferences | Meet REST API - Conferences} resource. + * + */ + participantKey?: string; + + /** + * Participant session name. There should be a one to one mapping of session + * to Media Entry. You can use this to retrieve additional information about + * the participant session from the + * {@link https://developers.google.com/meet/api/reference/rest/v2/conferenceRecords.participants.participantSessions | Meet REST API - ParticipantSessions} resource + * + * Format is + * `conferenceRecords/{conference_record}/participants/{participant}/participantSessions/{participant_session}` + * Unused for now. Use sessionName instead. + */ + session?: string; + + /** + * The session ID of the media entry. + * + * Format is + * `participants/{participant}/participantSessions/{participant_session}` + * + * You can use this to retrieve additional information about + * the participant session from the + * {@link https://developers.google.com/meet/api/reference/rest/v2/conferenceRecords.participants.participantSessions | Meet REST API - ParticipantSessions} resource. + * + * `Note`: This has to be in the format of `conferenceRecords/{conference_record}/participants/{participant}/participantSessions/{participant_session}`. + * + * You can retrieve the conference record from the {@link https://developers.google.com/meet/api/guides/conferences | Meet REST API - Conferences} resource. + */ + sessionName?: string; + /** + * CSRC for any audio stream contributed by this participant. + */ + audioCsrc?: number; + /** + * CSRCs for any video streams contributed by this participant. + */ + videoCsrcs?: number[]; + /** + * Whether the current entry is presentating. + */ + presenter: boolean; + /** + * Whether the current entry is a screenshare. + */ + screenshare: boolean; + /** + * Whether this participant muted their audio stream. + */ + audioMuted: boolean; + /** + * Whether this participant muted their video stream. + */ + videoMuted: boolean; +} + +/** + * Deleted resource for a media entry. + */ +export declare interface DeletedMediaEntry extends DeletedResource { + /** + * Set to true if the media entry is successfully deleted. + */ + mediaEntry: boolean; +} + +// VIDEO ASSIGNMENT + +/** + * Video assignment data channel message from the server to the client. + */ +export declare interface VideoAssignmentChannelToClient { + /** An optional response to an incoming request. */ + response?: SetVideoAssignmentResponse; + /** + * Resource snapshots managed by the server. + */ + resources?: VideoAssignmentResource[]; +} + +/** + * Video assignment data channel message from the client to the server. + */ +export declare interface VideoAssignmentChannelFromClient { + /** + * Request to set video assignment. + */ + request: SetVideoAssignmentRequest; +} + +/** + * Dimensions of a canvas. + */ +export declare interface CanvasDimensions { + /** + * Height in square pixels. For cameras that can change orientation, + * height refers to the measurement on the vertical axis. + */ + height: number; + /** + * Width in square pixels. For cameras that can change orientation, + * width refers to the measurement on the horizontal axis. + */ + width: number; +} + +/** + * Video canvas for video assignment. + */ +export declare interface MediaApiCanvas { + /** + * ID for the video canvas. This is required and must be unique within + * the containing layout model. Clients should prudently reuse these + * IDs, as this allows the backend to keep assigning video streams to + * the same canvas as much as possible. + */ + id: number; + /** + * Dimensions of the canvas. + */ + dimensions: CanvasDimensions; + /** + * Tells the server to choose the best video stream for this canvas. + * This is the only supported mode for now. + */ + relevant: {}; +} + +/** + * Request to set video assignment. In order to get video streams, the client + * must set a video assignment. + */ +export declare interface SetVideoAssignmentRequest extends MediaApiRequest { + /** + * Set video assignment. + */ + setAssignment: { + /** Layout model for the video assignment. */ + layoutModel: LayoutModel; + /** + * Maximum video resolution the client wants to receive for any video + * feeds. + */ + maxVideoResolution: VideoAssignmentMaxResolution; + }; +} + +/** + * Maximum video resolution the client wants to receive for any video feeds. + */ +export declare interface VideoAssignmentMaxResolution { + /** + * Height in square pixels. For cameras that can change orientation, + * height refers to the measurement on the vertical axis. + */ + height: number; + /** + * Width in square pixels. For cameras that can change orientation, + * width refers to the measurement on the horizontal axis. + */ + width: number; + /** Frames per second. */ + frameRate: number; +} + +/** Layout model for the video assignment. */ +export declare interface LayoutModel { + /** + * Label of the layout model. This is used to identify the layout model + * when requesting video assignment. + */ + label: string; + /** + * Canvases to assign videos to virtual SSRCs. Providing more + * canvases than exists virtual streams will result in an error status. + * Virtual video SSRCs are allocated during initialization of the client + * and the number of virtual SSRCs is fixed to the initial number of requested video streams. + */ + canvases: MediaApiCanvas[]; +} + +/** + * Response to a set video assignment request from the server. + */ +export declare interface SetVideoAssignmentResponse extends MediaApiResponse { + /** + * Set video assignment. This is always empty. + */ + setAssignment: {}; +} + +/** + * Singleton resource describing how video streams are assigned to video + * canvases specified in the client's video layout model. + */ +export declare interface VideoAssignmentResource extends ResourceSnapshot { + videoAssignment: VideoAssignmentLayoutModel; +} + +/** + * Video assignment for a layout model. + */ +export declare interface VideoAssignmentLayoutModel { + /** Label of the layout model. */ + label: string; + /** Canvas assignments, with no implied order. */ + canvases: CanvasAssignment[]; +} + +/** + * Video assignment for a single canvas. + */ +export declare interface CanvasAssignment { + /** + * The video canvas the video should be shown in. + */ + canvasId: number; + /** + * The virtual video SSRC that the video will be sent over, or + * unset if no video from the participant. + */ + ssrc?: number; + /** + * ID of the media entry associated with the video stream. + */ + mediaEntryId: number; +} + +// MEDIA STATS + +/** + * Media stats data channel message from the server to the client. + */ +export declare interface MediaStatsChannelToClient { + /** An optional response to an incoming request. */ + response?: UploadMediaStatsResponse; + /** + * Resource snapshots managed by the server. + */ + resources?: MediaStatsResource[]; +} + +/** + * Resource snapshot for media stats. Managed by the server. + */ +export declare interface MediaStatsResource extends ResourceSnapshot { + /** + * Configuration for media stats provided by the server and has to be used by + * the client to upload media stats. + */ + configuration: MediaStatsConfiguration; +} + +/** + * Configuration for media stats. Provided by the server and has to be used by + * the client to upload media stats. + */ +export declare interface MediaStatsConfiguration { + /** + * The interval between each upload of media stats. If this is zero, the + * client should not upload any media stats. + */ + uploadIntervalSeconds: number; + /** + * A map of allow listed sections. The key is the section type, and the value + * is the keys that are allow listed for that section. Fields can be found in + * {@link https://developer.mozilla.org/en-US/docs/Web/API/RTCStatsReport | RTCStatsReport} + */ + allowlist: Map; +} + +/** + * Media stats data channel message from the client to the server. + */ +export declare interface MediaStatsChannelFromClient { + /** + * Request to upload media stats. + */ + request: UploadMediaStatsRequest; +} + +/** + * Uploads media stats from the client to the server. The stats are + * retrieved from WebRTC by calling {@link https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/getStats | RTCPeerConnection.getStats()}. The + * returned {@link https://developer.mozilla.org/en-US/docs/Web/API/RTCStatsReport | RTCStatsReport} can be mapped to the sections below. + */ +export declare interface UploadMediaStatsRequest extends MediaApiRequest { + /** + * Upload media stats. + */ + uploadMediaStats: UploadMediaStats; +} + +/** + * Upload media stats. + */ +export declare interface UploadMediaStats { + /** + * Represents the entries in + * {@link https://developer.mozilla.org/en-US/docs/Web/API/RTCStatsReport | RTCStatsReport}. + * Formatted as an array of objects with an id and a type. + * The value of the id is a string WebRTC id of the section. + * The value of the type is the section. + */ + sections: StatsSectionData[]; +} + +/** + * A base section of media stats. All sections have an id. + */ +export declare interface StatsSection { + id: string; +} + +// STATS SECTION TYPES +// Mapped Types are being used instead of interfaces to allow for objects to be +// indexed by string. + +/** + * Stats section types. There are defined by the {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype | WebRTC spec.} + */ +export declare interface StatTypes { + // Need to use underscore naming to conform to expected structure for data + // channel. + // tslint:disable: enforce-name-casing + /** + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-candidate-pair | ICE candidate pair stats } related to RTCIceTransport. + */ + candidate_pair: CandidatePairSection; + /** + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-codec | Codec stats } that is currently being used by RTP streams being received by RTCPeerConnection. + */ + codec: CodecSection; + /** + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-inbound-rtp | RTP stats } for inbound stream that is currently received by RTCPeerConnection. + */ + inbound_rtp: InboundRtpSection; + /** + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-media-playout | Media playout stats } related to RTCPeerConnection. + */ + media_playout: MediaPlayoutSection; + /** + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-transport | Transport stats } related to RTCPeerConnection. + */ + transport: TransportSection; + /** + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-local-candidate | ICE candidate stats } for the local candidate related to RTCPeerConnection. + */ + local_candidate: IceCandidateSection; + /** + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-remote-candidate | ICE candidate stats } for the remote candidate related to RTCPeerConnection. + */ + remote_candidate: IceCandidateSection; + // tslint:enable: enforce-name-casing +} + +/** + * A section of media stats. Used to map the {@link https://developer.mozilla.org/en-US/docs/Web/API/RTCStatsReport | RTCStatsReport} to the expected + * structure for the data channel. All sections have an id and a type. + * For fields in a specific type, please see the StatTypes interface. + */ +export declare type StatsSectionData = StatsSection & { + [key in keyof StatTypes]?: StatTypes[key]; +}; + +/** + * Code section string fields. + * @ignore + */ +type CodeSectionStringFields = 'mime_type'; +/** + * Codec section numeric fields. + * @ignore + */ +type CodecSectionNumberFields = 'payload_type'; + +/** + * Codec fields. + * @ignore + */ +export declare type CodecSectionFields = + | CodeSectionStringFields + | CodecSectionNumberFields; + +/** + * Codec field mapping. + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-codec | WebRTC spec.} + * @ignore + */ +export declare type CodecSection = { + [key in CodecSectionFields]?: key extends CodeSectionStringFields + ? string + : number; +}; + +/** + * Inbound RTP section string fields. + * @ignore + */ +type InboundRtpSectionStringFields = 'codec_id' | 'kind'; + +/** + * Inbound RTP section number fields. + * @ignore + */ +type InboundRtpSectionNumberFields = + | 'ssrc' + | 'jitter' + | 'packets_lost' + | 'packets_received' + | 'bytes_received' + | 'jitter_buffer_delay' + | 'jitter_buffer_emitted_count' + | 'jitter_buffer_minimum_delay' + | 'jitter_buffer_target_delay' + | 'total_audio_energy' + | 'fir_count' + | 'frame_height' + | 'frame_width' + | 'frames_decoded' + | 'frames_dropped' + | 'frames_per_second' + | 'frames_received' + | 'freeze_count' + | 'key_frames_decoded' + | 'nack_count' + | 'pli_count' + | 'retransmitted_packets_received' + | 'total_freezes_duration' + | 'total_pauses_duration' + | 'audio_level' + | 'concealed_samples' + | 'total_samples_received' + | 'total_samples_duration'; + +/** + * Inbound Rtp fields. + * @ignore + */ +export type InboundRtpSectionFields = + | InboundRtpSectionStringFields + | InboundRtpSectionNumberFields; + +/** + * Inbound RTP field mapping. + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-inbound-rtp | WebRTC spec.} + * @ignore + */ +export declare type InboundRtpSection = { + [key in InboundRtpSectionFields]?: key extends InboundRtpSectionStringFields + ? string + : number; +}; + +/** + * Candidate pair fields. + * @ignore + */ +export type CandidatePairSectionFields = + | 'available_outgoing_bitrate' + | 'bytes_received' + | 'bytes_sent' + | 'current_round_trip_time' + | 'packets_received' + | 'packets_sent' + | 'total_round_trip_time'; + +/** + * Candidate pair field mapping. + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-candidate-pair | WebRTC spec.} + * @ignore + */ +export declare type CandidatePairSection = { + [key in CandidatePairSectionFields]: number; +}; + +/** + * Media playout fields. + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcaudioplayoutstats | WebRTC spec.} + * @ignore + */ +export type MediaPlayoutSectionFields = + | 'synthesized_samples_duration' + | 'synthesized_samples_events' + | 'total_samples_duration' + | 'total_playout_delay' + | 'total_samples_count'; + +/** + * Media playout field mapping. + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-media-playout | WebRTC spec.} + * @ignore + */ +export declare type MediaPlayoutSection = { + [key in MediaPlayoutSectionFields]?: number; +}; + +/** + * Transport fields. + * {@link https://www.w3.org/TR/webrtc-stats/#transportstats-dict | WebRTC spec.} + * @ignore + */ +export type TransportSectionFields = 'selected_candidate_pair_id'; + +/** + * Transport field mapping. + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-transport. | WebRTC spec.} + * @ignore + */ +export declare type TransportSection = { + [key in TransportSectionFields]?: string; +}; + +/** + * Ice candidate string fields. + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcicecandidatestats | WebRTC spec.} + * @ignore + */ +type IceCandidateSectionStringFields = + | 'address' + | 'candidate_type' + | 'protocol' + | 'network_type'; + +/** + * Ice candidate number fields. + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcicecandidatestats | WebRTC spec.} + * @ignore + */ +type IceCandidateSectionNumberFields = 'port'; + +/** + * Ice candidate fields. + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcicecandidatestats | WebRTC spec.} + * @ignore + */ +export declare type IceCandidateSectionFields = + | IceCandidateSectionStringFields + | IceCandidateSectionNumberFields; + +/** + * Ice candidate field mapping. + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-local-candidate | Local candidate WebRTC spec.} or + * {@link https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-remote-candidate | Remote candidate WebRTC spec.}. + * @ignore + */ +export declare type IceCandidateSection = { + [key in IceCandidateSectionFields]?: key extends IceCandidateSectionStringFields + ? string + : number; +}; + +/** + * Response to a media stats upload request. + */ +export declare interface UploadMediaStatsResponse extends MediaApiResponse { + uploadMediaStats: {}; +} diff --git a/meet-live-agent/types/enums.ts b/meet-live-agent/types/enums.ts new file mode 100644 index 0000000..ff959f9 --- /dev/null +++ b/meet-live-agent/types/enums.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Enums for the Media API Web Client. Since other files are + * using the .d.ts file, we need to keep the enums in this file. + */ + +/** + * Log level for each data channel. + */ +export enum LogLevel { + UNKNOWN = 0, + ERRORS = 1, + RESOURCES = 2, + MESSAGES = 3, +} + +/** Connection state of the Meet Media API session. */ +export enum MeetConnectionState { + UNKNOWN = 0, + WAITING = 1, + JOINED = 2, + DISCONNECTED = 3, +} + +/** Reasons for the Meet Media API session to disconnect. */ +export enum MeetDisconnectReason { + UNKNOWN = 0, + CLIENT_LEFT = 1, + USER_STOPPED = 2, + CONFERENCE_ENDED = 3, + SESSION_UNHEALTHY = 4, +} diff --git a/meet-live-agent/types/mediacapture_transform.d.ts b/meet-live-agent/types/mediacapture_transform.d.ts new file mode 100644 index 0000000..6de5af6 --- /dev/null +++ b/meet-live-agent/types/mediacapture_transform.d.ts @@ -0,0 +1,149 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Type definitions for non-npm package MediaStreamTrack Insertable Media Processing using Streams 0.1 +// Project: https://w3c.github.io/mediacapture-transform/ +// Definitions by: Ben Wagner +// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped +// Minimum TypeScript Version: 3.9 + +// In general, these types are only available behind a command line flag or an origin trial in +// Chrome 90+. + +// Versioning: +// Until the above-mentioned spec is finalized, the major version number is 0. Although not +// necessary for version 0, consider incrementing the minor version number for breaking changes. + +// The following modify existing DOM types to allow defining type-safe APIs on audio and video tracks. + +/** Specialize MediaStreamTrack so that we can refer specifically to an audio track. */ +interface MediaStreamAudioTrack extends MediaStreamTrack { + readonly kind: 'audio'; + clone(): MediaStreamAudioTrack; +} + +/** Specialize MediaStreamTrack so that we can refer specifically to a video track. */ +interface MediaStreamVideoTrack extends MediaStreamTrack { + readonly kind: 'video'; + clone(): MediaStreamVideoTrack; +} + +/** Assert that getAudioTracks and getVideoTracks return the tracks with the appropriate kind. */ +interface MediaStream { + getAudioTracks(): MediaStreamAudioTrack[]; + getVideoTracks(): MediaStreamVideoTrack[]; +} + +// The following were originally generated from the spec using +// https://github.com/microsoft/TypeScript-DOM-lib-generator, then heavily modified. + +/** + * A track sink that is capable of exposing the unencoded frames from the track to a + * ReadableStream, and exposes a control channel for signals going in the oppposite direction. + */ +interface MediaStreamTrackProcessor { + /** + * Allows reading the frames flowing through the MediaStreamTrack provided to the constructor. + */ + readonly readable: ReadableStream; + /** Allows sending control signals to the MediaStreamTrack provided to the constructor. */ + readonly writableControl: WritableStream; +} + +declare var MediaStreamTrackProcessor: { + // tslint:disable-next-line no-any + prototype: MediaStreamTrackProcessor; + + /** Constructor overrides based on the type of track. */ + new ( + init: MediaStreamTrackProcessorInit & {track: MediaStreamAudioTrack}, + ): MediaStreamTrackProcessor; + new ( + init: MediaStreamTrackProcessorInit & {track: MediaStreamVideoTrack}, + ): MediaStreamTrackProcessor; +}; + +interface MediaStreamTrackProcessorInit { + track: MediaStreamTrack; + /** + * If media frames are not read from MediaStreamTrackProcessor.readable quickly enough, the + * MediaStreamTrackProcessor will internally buffer up to maxBufferSize of the frames produced + * by the track. If the internal buffer is full, each time the track produces a new frame, the + * oldest frame in the buffer will be dropped and the new frame will be added to the buffer. + */ + maxBufferSize?: number | undefined; +} + +/** + * Takes video frames as input, and emits control signals that result from subsequent processing. + */ +interface MediaStreamTrackGenerator + extends MediaStreamTrack { + /** + * Allows writing media frames to the MediaStreamTrackGenerator, which is itself a + * MediaStreamTrack. When a frame is written to writable, the frame’s close() method is + * automatically invoked, so that its internal resources are no longer accessible from + * JavaScript. + */ + readonly writable: WritableStream; + /** + * Allows reading control signals sent from any sinks connected to the + * MediaStreamTrackGenerator. + */ + readonly readableControl: ReadableStream; +} + +type MediaStreamAudioTrackGenerator = MediaStreamTrackGenerator & + MediaStreamAudioTrack; +type MediaStreamVideoTrackGenerator = MediaStreamTrackGenerator & + MediaStreamVideoTrack; + +declare var MediaStreamTrackGenerator: { + // tslint:disable-next-line no-any + prototype: MediaStreamTrackGenerator; + + /** Constructor overrides based on the type of track. */ + new ( + init: MediaStreamTrackGeneratorInit & { + kind: 'audio'; + signalTarget?: MediaStreamAudioTrack | undefined; + }, + ): MediaStreamAudioTrackGenerator; + new ( + init: MediaStreamTrackGeneratorInit & { + kind: 'video'; + signalTarget?: MediaStreamVideoTrack | undefined; + }, + ): MediaStreamVideoTrackGenerator; +}; + +interface MediaStreamTrackGeneratorInit { + kind: MediaStreamTrackGeneratorKind; + /** + * (Optional) track to which the MediaStreamTrackGenerator will automatically forward control + * signals. If signalTarget is provided and signalTarget.kind and kind do not match, the + * MediaStreamTrackGenerator’s constructor will raise an exception. + */ + signalTarget?: MediaStreamTrack | undefined; +} + +type MediaStreamTrackGeneratorKind = 'audio' | 'video'; + +type MediaStreamTrackSignalType = 'request-frame'; + +interface MediaStreamTrackSignal { + signalType: MediaStreamTrackSignalType; +} diff --git a/meet-live-agent/types/mediatypes.d.ts b/meet-live-agent/types/mediatypes.d.ts new file mode 100644 index 0000000..1697b5e --- /dev/null +++ b/meet-live-agent/types/mediatypes.d.ts @@ -0,0 +1,309 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Meet Media Api types and interfaces. + */ + +import { + DeletedResource, + MediaApiRequest, + MediaApiResponse, + ResourceSnapshot, +} from './datachannels'; +import {LogLevel} from './enums'; +import {Subscribable} from './subscribable'; + +/** + * Serves as the central relational object between the + * participant, media canvas and meet stream. This object represents media in + * a Meet call and holds metadata for the media. + */ +export interface MediaEntry { + /** + * Participant abstraction associated with this media entry. + * participant is immutable. + */ + readonly participant: Subscribable; + /** + * Whether this participant muted their audio stream. + */ + readonly audioMuted: Subscribable; + /** + * Whether this participant muted their video stream. + */ + readonly videoMuted: Subscribable; + /** + * Whether the current entry is a screenshare. + */ + readonly screenShare: Subscribable; + /** + * Whether the current entry is a presenter self-view. + */ + readonly isPresenter: Subscribable; + /** + * The media layout associated with this media entry. + */ + readonly mediaLayout: Subscribable; + /** + * The video meet stream track associated with this media entry. Contains the + * webrtc media stream track. + */ + readonly videoMeetStreamTrack: Subscribable; + /** + * The audio meet stream track associated with this media entry. Contains the + * webrtc media stream track. + */ + readonly audioMeetStreamTrack: Subscribable; + /** + * The session ID of the media entry. + * + * Format is + * `participants/{participant}/participantSessions/{participant_session}` + * + * You can use this to retrieve additional information about + * the participant session from the + * {@link https://developers.google.com/meet/api/reference/rest/v2/conferenceRecords.participants.participantSessions | Meet REST API - ParticipantSessions} resource. + * + * `Note`: This has to be in the format of `conferenceRecords/{conference_record}/participants/{participant}/participantSessions/{participant_session}`. + * + * You can retrieve the conference record from the {@link https://developers.google.com/meet/api/guides/conferences | Meet REST API - Conferences} resource. + */ + sessionName?: string; + /** + * Participant session name. There should be a one to one mapping of session + * to Media Entry. You can use this to retrieve additional information about + * the participant session from the + * {@link https://developers.google.com/meet/api/reference/rest/v2/conferenceRecords.participants.participantSessions | Meet REST API - ParticipantSessions} resource + * + * Format is + * `conferenceRecords/{conference_record}/participants/{participant}/participantSessions/{participant_session}` + * Unused for now. Use sessionName instead. + */ + session?: string; +} + +/** + * An abstraction that represents a participant in a Meet call. Contains + * the participant object and the media entries associated with this participant. + */ +export interface Participant { + /** + * Participant abstraction associated with this participant. + */ + participant: BaseParticipant; + /** + * The media entries associated with this participant. These can be + * transient. There is one participant to many media entries relationship. + */ + readonly mediaEntries: Subscribable; +} + +/** + * Base participant type. Only one of signedInUser, anonymousUser, or phoneUser + * fields will be set to determine the type of participant. + */ +export interface BaseParticipant { + /** + * Resource name of the participant. + * Format: `conferenceRecords/{conferenceRecord}/participants/{participant}` + * + * You can use this to retrieve additional information about the participant + * from the {@link https://developers.google.com/meet/api/reference/rest/v2/conferenceRecords.participants | Meet REST API - Participants} resource. + * + * Unused for now. Use participantKey instead. + */ + name?: string; + /** + * Participant key of associated participant. + * Format is `participants/{participant}`. + * + * You can use this to retrieve additional information about the participant + * from the {@link https://developers.google.com/meet/api/reference/rest/v2/conferenceRecords.participants | Meet REST API - Participants} resource. + * + * `Note`: This has to be in the format of `conferenceRecords/{conference_record}/participants/{participant}`. + * + * You can retrieve the conference record from the {@link https://developers.google.com/meet/api/guides/conferences | Meet REST API - Conferences} resource. + * + */ + participantKey?: string; + /** + * If set, the participant is a signed in user. Provides a unique ID and + * display name. + */ + signedInUser?: SignedInUser; + /** + * If set, the participant is an anonymous user. Provides a display name. + */ + anonymousUser?: AnonymousUser; + /** + * If set, the participant is a dial-in user. Provides a partially redacted + * phone number. + */ + phoneUser?: PhoneUser; +} + +/** + * A signed in user in a Meet call. + */ +export interface SignedInUser { + /** + * Unique ID for the user. Interoperable with {@link https://developers.google.com/admin-sdk/directory/reference/rest/v1/users | Admin SDK API} and {@link https://developers.google.com/people/api/rest/v1/people | People API.} + * Format: `users/{user}` + */ + readonly user: string; + /** + * Display name of the user. First and last name for a personal device. Admin + * defined name for a robot account. + */ + readonly displayName: string; +} + +/** + * An anonymous user in a Meet call. + */ +export interface AnonymousUser { + /** User specified display name. */ + readonly displayName?: string; +} + +/** + * A dial-in user in a Meet call. + */ +export interface PhoneUser { + /** Partially redacted user's phone number. */ + readonly displayName: string; +} + +/** + * An abstraction of a track in a Meet stream. This is used to represent + * both audio and video tracks and their relationship to Media Entries. + */ +export interface MeetStreamTrack { + /** + * The {@link https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack | WebRTC MediaStreamTrack} interface of the Media Capture and Streams API represents a single audio or video media track within a stream. + */ + readonly mediaStreamTrack: MediaStreamTrack; + /** + * The media entry associated with this track. This a one to one + * relationship. + */ + readonly mediaEntry: Subscribable; +} + +/** + * The dimensions of the canvas for video streams. + */ +export interface CanvasDimensions { + /** + * Width measured in pixels. This can be changed by the user. + */ + width: number; + /** + * Height measured in pixels. This can be changed by the user. + */ + height: number; +} + +/** + * A Media layout for the Media API Web client. This must be created by the + * Media API client to be valid. This is used to request a video stream. + */ +export interface MediaLayout { + /** + * The dimensions of the layout. + */ + readonly canvasDimensions: CanvasDimensions; + /** + * The media entry associated with this layout. + */ + readonly mediaEntry: Subscribable; +} + +/** + * A request for a {@link https://developers.google.com/meet/media-api/reference/web/media_api_web.medialayout | MediaLayout}. This is required to be able to request a video + * stream. + */ +export interface MediaLayoutRequest { + /** + * The {@link https://developers.google.com/meet/media-api/reference/web/media_api_web.medialayout | MediaLayout} to request. + */ + mediaLayout: MediaLayout; +} + +/** + * Required configuration for the {@link https://developers.google.com/meet/media-api/reference/web/media_api_web.meetmediaapiclient | MeetMediaApiClient}. + */ +export interface MeetMediaClientRequiredConfiguration { + /** + * The meeting space ID to connect to. + */ + meetingSpaceId: string; + /** + * The number of video streams to request. + */ + numberOfVideoStreams: number; + /** + * Number of audio streams is not configurable. False maps to 0 and True maps + * to 3. + */ + enableAudioStreams: boolean; + /** + * The access token to use for authentication. + */ + accessToken: string; + /** + * The callback to use for logging events. + */ + logsCallback?: (logEvent: LogEvent) => void; +} + +/** + * List of log source types. + */ +export type LogSourceType = + | 'session-control' + | 'participants' + | 'media-entries' + | 'video-assignment' + | 'media-stats'; + +/** + * Log event that is propagated to the callback. + */ +export interface LogEvent { + /** + * The level of the log event. + */ + level: LogLevel; + /** + * The log string of the event. + */ + logString: string; + /** + * The source type of the event. + */ + sourceType: LogSourceType; + /** + * The relevant object of the event. + */ + relevantObject?: + | Error + | DeletedResource + | ResourceSnapshot + | MediaApiResponse + | MediaApiRequest; +} diff --git a/meet-live-agent/types/meetmediaapiclient.d.ts b/meet-live-agent/types/meetmediaapiclient.d.ts new file mode 100644 index 0000000..0819813 --- /dev/null +++ b/meet-live-agent/types/meetmediaapiclient.d.ts @@ -0,0 +1,115 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview interface for MeetMediaApiClient. + */ + +import {MediaApiCommunicationProtocol} from './communication_protocol'; +import {MediaApiResponseStatus} from './datachannels'; +import {MeetConnectionState, MeetDisconnectReason} from './enums'; +import { + CanvasDimensions, + MediaEntry, + MediaLayout, + MediaLayoutRequest, + MeetStreamTrack, + Participant, +} from './mediatypes'; +import {Subscribable} from './subscribable'; + +/** + * The status of the Meet Media API session. + */ +export interface MeetSessionStatus { + connectionState: MeetConnectionState; + disconnectReason?: MeetDisconnectReason; +} + +/** + * Interface for the MeetMediaApiClient. Takes a required configuration + * and provides a set of subscribables to the client. + * Takes a {@link https://developers.google.com/meet/media-api/reference/web/media_api_web.meetmediaclientrequiredconfiguration | MeetMediaClientRequiredConfiguration} as a constructor + * parameter. + */ +export interface MeetMediaApiClient { + /** + * The status of the session. Subscribable to changes in the session status. + */ + readonly sessionStatus: Subscribable; + /** + * The meet stream tracks in the meeting. Subscribable to changes in the meet + * stream track collection. + */ + readonly meetStreamTracks: Subscribable; + /** + * The media entries in the meeting. Subscribable to changes in the media + * entry collection. + */ + readonly mediaEntries: Subscribable; + /** + * The participants in the meeting. Subscribable to changes in the + * participant collection. + */ + readonly participants: Subscribable; + /** + * The presenter in the meeting. Subscribable to changes in the presenter. + */ + readonly presenter: Subscribable; + /** + * The screenshare in the meeting. Subscribable to changes in the screenshare. + */ + readonly screenshare: Subscribable; + + /** + * Joins the meeting. + * @param communicationProtocol The communication protocol to use. If not + * provided, a default {@link https://developers.google.com/meet/media-api/reference/web/media_api_web.mediaapicommunicationprotocol | MediaApiCommunicationProtocol} will be used. + */ + joinMeeting( + communicationProtocol?: MediaApiCommunicationProtocol, + ): Promise; + + /** + * Leaves the meeting. + */ + leaveMeeting(): Promise; + + /** + * Applies the given media layout requests. This is required to be able to + * request a video stream. Only accepts media layouts that have been + * created with the {@link https://developers.google.com/meet/media-api/reference/web/media_api_web.meetmediaapiclient.createmedialayout | createMediaLayout} function. + * @param requests The requests to apply. + * @return A promise that resolves when the request has been accepted. NOTE: + * The promise resolving on the request does not mean the layout has been + * applied. It means that the request has been accepted and you may need + * to wait a short amount of time for these layouts to be applied. + */ + applyLayout(requests: MediaLayoutRequest[]): Promise; + + /** + * Creates a new media layout. Only media layouts that are created with this + * function can be applied. Otherwise, the {@link https://developers.google.com/meet/media-api/reference/web/media_api_web.meetmediaapiclient.applylayout.md | applyLayout} function will + * throw an error. Once the media layout has been created, you can construct a + * request and apply it with the {@link https://developers.google.com/meet/media-api/reference/web/media_api_web.meetmediaapiclient.applylayout.md | applyLayout} function. These media + * layout objects are meant to be reused (can be reassigned to a different + * request) but are distinct per stream (need to be created for each stream). + * @param canvasDimensions The dimensions of the canvas to render the layout + * on. + * @return The new media layout. + */ + createMediaLayout(canvasDimensions: CanvasDimensions): MediaLayout; +} diff --git a/meet-live-agent/types/subscribable.d.ts b/meet-live-agent/types/subscribable.d.ts new file mode 100644 index 0000000..fb9cce7 --- /dev/null +++ b/meet-live-agent/types/subscribable.d.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview A helper class that implements generic getter and subscriber + * functions. Only the owner of the class can update the value. + */ + +/** + * A helper class that can be used to get and subscribe to updates on a + * value. + */ +export interface Subscribable { + /** + * @return the current value. + */ + get(): T; + + /** + * Allows a callback to be added. This callback will be called whenever the + * value is updated. + * @return An unsubscribe function. + */ + subscribe(callback: (value: T) => void): () => void; + + /** + * Removes the callback from the list of subscribers. The original callback + * instance must be passed in as an argument. + * @return True if the callback was removed, false if it was not found. + */ + unsubscribe(callback: (value: T) => void): boolean; +} diff --git a/meet-live-agent/vite.config.ts b/meet-live-agent/vite.config.ts new file mode 100644 index 0000000..9313ff4 --- /dev/null +++ b/meet-live-agent/vite.config.ts @@ -0,0 +1,24 @@ +import path from 'path'; +import { defineConfig, loadEnv } from 'vite'; + +export default defineConfig(({ mode }) => { + const env = loadEnv(mode, '.', ''); + console.log("Loaded env:", env); + return { + server: { + port: 3000, + host: '0.0.0.0', + }, + plugins: [], + define: { + 'process.env.GEMINI_API_KEY': JSON.stringify(env.GEMINI_API_KEY), + 'process.env.CLIENT_ID': JSON.stringify(env.CLIENT_ID), + 'process.env.CLOUD_PROJECT_NUMBER': JSON.stringify(env.CLOUD_PROJECT_NUMBER) + }, + resolve: { + alias: { + '@': path.resolve(__dirname, '.'), + } + } + }; +});