Make editor audio-only proof

Make sure the editor deals reasonably well with
audio-only tracks.
Includes:
- Displaying a little image for the player if there is
no video stream.
- Make sure the track selection makes sense (don't display
a video stream for an audio only track,
don't allow users to deselect all tracks)
- Disable thumbnail generation for audio-only tracks, but keep
thumbnail generation.

Author: Arnei

src/i18n/locales/en-US.json

@@ -233,8 +233,7 @@
       "unauthorizedError-text": "You are not allowed to edit this video",
       "comError-text": "A problem occurred during communication with Opencast.",
       "loadError-text": "An error has occurred loading this video.",
-      "durationError-text": "Opencast failed to provide the video duration.",
-      "noVideoError-text": "The editor does not support audio files yet!"
+      "durationError-text": "Opencast failed to provide the video duration."
     },
 
     "landing": {

src/img/video-off.png

@@ -18,7 +18,6 @@ import {
   setIsPlayPreview,
   jumpToPreviousSegment,
   jumpToNextSegment,
-  selectVideos,
   cut,
   mergeAll,
   mergeLeft,
@@ -48,7 +47,6 @@ const Cutting: React.FC = () => {
     state.videoState.status);
   const error = useAppSelector((state: { videoState: { error: httpRequestState["error"]; }; }) =>
     state.videoState.error);
-  const videos = useAppSelector(selectVideos);
   const duration = useAppSelector(selectDuration);
   const theme = useTheme();
   const errorReason = useAppSelector((state: { videoState: { errorReason: httpRequestState["errorReason"]; }; }) =>
@@ -75,14 +73,6 @@ const Cutting: React.FC = () => {
         }));
       }
     } else if (videoURLStatus === "success") {
-      // Editor can not handle events with no videos/audio-only atm
-      if (videos === null || videos.length === 0) {
-        dispatch(setError({
-          error: true,
-          errorMessage: t("error.noVideoError-text"),
-          errorDetails: error,
-        }));
-      }
       if (duration === null) {
         dispatch(setError({
           error: true,
@@ -91,7 +81,7 @@ const Cutting: React.FC = () => {
         }));
       }
     }
-  }, [videoURLStatus, dispatch, error, t, errorReason, duration, videos]);
+  }, [videoURLStatus, dispatch, error, t, errorReason, duration]);
 
   // Already try fetching Metadata to reduce wait time
   useEffect(() => {

src/main/Cutting.tsx

@@ -3,7 +3,7 @@ import { css } from "@emotion/react";
 import { RootState, ThunkApiConfig, useAppSelector } from "../redux/store";
 import {
   selectIsMuted,
-  selectVideos,
+  selectTracks,
   selectVolume,
   selectJumpTriggered,
   setIsMuted,
@@ -63,7 +63,7 @@ const SubtitleVideoArea: React.FC<{
   setCurrentlyAtAndTriggerPreview,
 }) => {
 
-  const tracks = useAppSelector(selectVideos);
+  const tracks = useAppSelector(selectTracks);
   const subtitle = useAppSelector(selectSelectedSubtitleById);
   const [selectedFlavor, setSelectedFlavor] = useState<Flavor>();
   const [subtitleUrl, setSubtitleUrl] = useState("");
@@ -103,6 +103,14 @@ const SubtitleVideoArea: React.FC<{
     }
   };
 
+  const isAudioOnly = () => {
+    for (const track of tracks) {
+      if (track.flavor.type === selectedFlavor?.type && track.flavor.subtype === selectedFlavor?.subtype) {
+        return !track.video_stream.available;
+      }
+    }
+  };
+
   // Parse subtitles to something the video player understands
   useEffect(() => {
     if (subtitle?.cues) {
@@ -143,6 +151,7 @@ const SubtitleVideoArea: React.FC<{
             subtitleUrl={subtitleUrl}
             first={true}
             last={true}
+            audioOnly={isAudioOnly()}
             selectIsPlaying={selectIsPlaying}
             selectIsMuted={selectIsMuted}
             selectCurrentlyAtInSeconds={selectCurrentlyAtInSeconds}

src/main/SubtitleVideoArea.tsx

@@ -11,7 +11,6 @@ import {
 import { Theme, useTheme } from "../themes";
 import {
   selectOriginalThumbnails,
-  selectVideos,
   selectTracks,
   setHasChanges,
   setThumbnail,
@@ -27,7 +26,7 @@ import { setIndex, setIsDisplayEditView } from "../redux/thumbnailSlice";
  */
 const ThumbnailSelect: React.FC = () => {
 
-  const videoTracks = useAppSelector(selectVideos);
+  const tracks = useAppSelector(selectTracks);
 
   const thumbnailSelectStyle = css({
     display: "flex",
@@ -42,7 +41,7 @@ const ThumbnailSelect: React.FC = () => {
 
   return (
     <div css={thumbnailSelectStyle}>
-      {videoTracks.map((track: Track, index: number) => (
+      {tracks.map((track: Track, index: number) => (
         <ThumbnailSelector
           key={index}
           track={track}
@@ -177,10 +176,12 @@ const ThumbnailButtons: React.FC<{
         track={track}
         index={0}
       />
-      <ToGenerationButton
-        trackIndex={trackIndex}
-        index={1}
-      />
+      {track.video_stream.available &&
+        <ToGenerationButton
+          trackIndex={trackIndex}
+          index={1}
+        />
+      }
       <DiscardButton
         track={track}
         index={2}

src/main/ThumbnailSelect.tsx

@@ -9,7 +9,7 @@ import { useAppDispatch, useAppSelector } from "../redux/store";
 import { Segment, httpRequestState } from "../types";
 import {
   selectDuration,
-  selectVideoURL,
+  selectTrackURLs,
   selectWaveformImages,
   setWaveformImages,
   selectTimelineZoom,
@@ -687,7 +687,7 @@ export const Waveforms: React.FC<{ timelineHeight: number; topOffset?: number }>
   const { t } = useTranslation();
 
   const dispatch = useAppDispatch();
-  const videoURLs = useAppSelector(selectVideoURL);
+  const videoURLs = useAppSelector(selectTrackURLs);
   const videoURLStatus = useAppSelector((state: { videoState: { status: httpRequestState["status"]; }; }) =>
     state.videoState.status);
   const theme = useTheme();

src/main/Timeline.tsx

@@ -6,7 +6,9 @@ import ReactPlayer from "react-player";
 
 import { Track } from "../types";
 import {
+  selectAudiosOnly,
   selectCustomizedTrackSelection,
+  selectTracks,
   selectVideos,
   selectWaveformImages,
   setAudioEnabled,
@@ -37,27 +39,35 @@ const TrackSelection: React.FC = () => {
   const dispatch = useAppDispatch();
 
   // Generate list of tracks
-  const tracks = useAppSelector(selectVideos);
+  const tracks = useAppSelector(selectTracks);
+  const videos = useAppSelector(selectVideos);
+  const audioOnlys = useAppSelector(selectAudiosOnly);
+
   let enabledCount = 0;
   if (settings.trackSelection.atLeastOneVideo) {
     // Only care about at least one video stream being enabled
-    enabledCount = tracks.reduce(
+    enabledCount = videos.reduce(
       (memo: number, track: Track) =>
         memo + (track.video_stream.enabled ? 1 : 0),
       0,
     );
   } else {
     // Make sure that at least one track remains enabled
-    enabledCount = tracks.reduce(
+    enabledCount += videos.reduce(
       (memo: number, track: Track) =>
         memo + (track.video_stream.enabled ? 1 : 0) + (track.audio_stream.enabled ? 1 : 0),
       0,
     );
+    enabledCount += audioOnlys.reduce(
+      (memo: number, track: Track) =>
+        memo + (track.audio_stream.enabled ? 1 : 0),
+      0,
+    );
   }
   const images = useAppSelector(selectWaveformImages);
   const customizedTrackSelection = !!useAppSelector(selectCustomizedTrackSelection);
 
-  const videoTrackItems = tracks.map(
+  const videoTrackItems = videos.map(
     (track: Track) => (
       <VideoTrackItem
         key={track.id}
@@ -67,7 +77,7 @@ const TrackSelection: React.FC = () => {
       />),
   );
 
-  const audioTrackItems = tracks.map(
+  const audioTrackItems = [...videos, ...audioOnlys].map(
     (track: Track, index: number) => (
       <AudioTrackItem
         key={track.id}

src/main/TrackSelection.tsx

@@ -9,7 +9,7 @@ import {
   setIsPlaying,
   selectIsMuted,
   selectVolume,
-  selectVideoCount,
+  selectTrackCount,
   selectDurationInSeconds,
   setPreviewTriggered,
   selectPreviewTriggered,
@@ -20,7 +20,7 @@ import {
   setJumpTriggered,
   selectJumpTriggered,
   setCurrentlyAt,
-  selectVideos,
+  selectTracks,
 } from "../redux/videoSlice";
 
 import ReactPlayer, { Config } from "react-player";
@@ -39,6 +39,7 @@ import { useTheme } from "../themes";
 import { backgroundBoxStyle } from "../cssStyles";
 import { BaseReactPlayerProps } from "react-player/base";
 import { ErrorBox } from "@opencast/appkit";
+import VideoOffImage from "../img/video-off.png?url";
 
 const VideoPlayers: React.FC<{
   refs?: React.MutableRefObject<(VideoPlayerForwardRef | null)[]>,
@@ -50,10 +51,10 @@ const VideoPlayers: React.FC<{
   maxHeightInPixel = 300,
 }) => {
 
-  const videos = useAppSelector(selectVideos);
+  const videos = useAppSelector(selectTracks);
   let primaryIndex = videos.findIndex(e => e.audio_stream.available === true);
   primaryIndex = primaryIndex < 0 ? 0 : primaryIndex;
-  const videoCount = useAppSelector(selectVideoCount);
+  const videoCount = useAppSelector(selectTrackCount);
 
   const [videoPlayers, setVideoPlayers] = useState<JSX.Element[]>([]);
 
@@ -81,6 +82,7 @@ const VideoPlayers: React.FC<{
           subtitleUrl={""}
           first={i === 0}
           last={i === videoCount - 1}
+          audioOnly={!videos[i].video_stream.available}
           selectIsPlaying={selectIsPlaying}
           selectIsMuted={selectIsMuted}
           selectVolume={selectVolume}
@@ -126,6 +128,7 @@ interface VideoPlayerProps {
   first: boolean,
   last: boolean,
   overwritePlayerCSS?: SerializedStyles,
+  audioOnly?: boolean,
   selectIsPlaying: (state: RootState) => boolean,
   selectIsMuted: (state: RootState) => boolean,
   selectVolume: (state: RootState) => number,
@@ -169,6 +172,7 @@ export const VideoPlayer = React.forwardRef<VideoPlayerForwardRef, VideoPlayerPr
       first,
       last,
       overwritePlayerCSS,
+      audioOnly = false,
       selectCurrentlyAtInSeconds,
       selectPreviewTriggered,
       selectClickTriggered,
@@ -316,6 +320,7 @@ export const VideoPlayer = React.forwardRef<VideoPlayerForwardRef, VideoPlayerPr
           // Skip player when navigating page with keyboard
           tabIndex: "-1",
           crossOrigin: "anonymous",    // allow thumbnail generation
+          poster: audioOnly && VideoOffImage, // Show image when there is no video stream
         },
         tracks: [
           { kind: "subtitles", src: subtitleUrl, srcLang: "en", default: true, label: "I am irrelevant" },
@@ -394,6 +399,7 @@ export const VideoPlayer = React.forwardRef<VideoPlayerForwardRef, VideoPlayerPr
 
     const reactPlayerStyle = css({
       aspectRatio: "16 / 9",    // Hard-coded for now because there are problems with updating this value at runtime
+      padding: audioOnly ? "0px" : "20px",
 
       overflow: "hidden", // Required for borderRadius to show
       ...first && {

src/main/VideoPlayers.tsx

@@ -1,8 +1,8 @@
 import reducer, { initialState, setIsPlaying, selectIsPlaying, setCurrentlyAt,
   selectCurrentlyAt, selectActiveSegmentIndex, selectPreviewTriggered,
   selectDuration, video, cut, selectSegments, markAsDeletedOrAlive, mergeRight,
-  fetchVideoInformation, selectVideoURL, selectTitle,
-  selectTracks, selectWorkflows } from "../videoSlice";
+  fetchVideoInformation, selectTitle,
+  selectTracks, selectWorkflows, selectTrackURLs } from "../videoSlice";
 import cloneDeep from "lodash/cloneDeep";
 import { httpRequestState } from "../../types";
 
@@ -300,12 +300,12 @@ describe("Video reducer", () => {
     // Arrange
     const resultStatus: httpRequestState = { status: "success", error: undefined, errorReason: "unknown" };
     const segments = [{ start: 0, end: 42, deleted: false }];
-    const videoURLs: video["videoURLs"] = ["video/url"];
+    const trackURLs: video["trackURLs"] = ["video/url"];
     const dur: video["duration"] = 42;
     const title: video["title"] = "Video Title";
     // const presenters: video["presenters"] = [ "Otto Opencast" ]    // Currently missing from the API
     const tracks: video["tracks"] = [{
-      id: "id", uri: videoURLs[0], flavor: { subtype: "prepared", type: "presenter" },
+      id: "id", uri: trackURLs[0], flavor: { subtype: "prepared", type: "presenter" },
       /* eslint-disable camelcase */
       video_stream: { available: true, enabled: true, thumbnail_uri: "thumb/url" },
       audio_stream: { available: true, enabled: true, thumbnail_uri: "thumb/url" },
@@ -333,7 +333,7 @@ describe("Video reducer", () => {
     expect(rootState.videoState).toMatchObject(resultStatus);
 
     expect(selectSegments(rootState)).toMatchObject(segments);
-    expect(selectVideoURL(rootState)).toMatchObject(videoURLs);
+    expect(selectTrackURLs(rootState)).toMatchObject(trackURLs);
     expect(selectDuration(rootState)).toEqual(dur);
     expect(selectTitle(rootState)).toEqual(title);
     expect(selectTracks(rootState)).toMatchObject(tracks);

src/redux/__tests__/videoSlice.test.ts

@@ -29,8 +29,8 @@ export interface video {
   waveformImages: string[];
   originalThumbnails: { id: Track["id"], uri: Track["thumbnailUri"]; }[];
 
-  videoURLs: string[],  // Links to each video
-  videoCount: number,   // Total number of videos
+  trackURLs: string[],  // Links to each track
+  trackCount: number,   // Total number of tracks
   duration: number,     // Video duration in milliseconds. Can be null due to Opencast internal error
   title: string,
   presenters: string[],
@@ -69,8 +69,8 @@ export const initialState: video & httpRequestState = {
   waveformImages: [],
   originalThumbnails: [],
 
-  videoURLs: [],
-  videoCount: 0,
+  trackURLs: [],
+  trackCount: 0,
   duration: 0,
   title: "",
   presenters: [],
@@ -358,9 +358,8 @@ const videoSlice = createSlice({
             }
             return track;
           });
-        const videos = state.tracks.filter((track: Track) => track.video_stream.available === true);
-        state.videoURLs = videos.reduce((a: string[], o: { uri: string; }) => (a.push(o.uri), a), []);
-        state.videoCount = state.videoURLs.length;
+        state.trackURLs = state.tracks.reduce((a: string[], o: { uri: string; }) => (a.push(o.uri), a), []);
+        state.trackCount = state.trackURLs.length;
         state.subtitlesFromOpencast = payload.subtitles ?
           state.subtitlesFromOpencast = payload.subtitles : [];
         state.chaptersFromOpencast = payload.chapters ?
@@ -413,8 +412,10 @@ const videoSlice = createSlice({
     selectOriginalThumbnails: state => state.originalThumbnails,
     // Selectors mainly pertaining to the information fetched from Opencast
     selectVideos: state => state.tracks.filter((track: Track) => track.video_stream.available === true),
-    selectVideoURL: state => state.videoURLs,
-    selectVideoCount: state => state.videoCount,
+    selectTrackURLs: state => state.trackURLs,
+    selectTrackCount: state => state.trackCount,
+    selectAudiosOnly: state => state.tracks.filter((track: Track) =>
+      !track.video_stream.available === true && track.audio_stream.available),
     selectDuration: state => state.duration,
     selectDurationInSeconds: state => state.duration / 1000,
     selectTitle: state => state.title,
@@ -628,8 +629,8 @@ export const {
   selectTimelineZoom,
   selectWaveformImages,
   selectOriginalThumbnails,
-  selectVideoURL,
-  selectVideoCount,
+  selectTrackURLs,
+  selectTrackCount,
   selectDuration,
   selectDurationInSeconds,
   selectTitle,
@@ -641,6 +642,7 @@ export const {
   selectChaptersFromOpencast,
   selectChaptersFromOpencastById,
   selectVideos,
+  selectAudiosOnly,
   selectDisplayDuration,
   selectPrimaryThumbnailTrack,
 } = videoSlice.selectors;