POC - AWS Bedrock

package.json

@@ -34,7 +34,7 @@
     "@babel/core": "^7.26.0",
     "@ndla/preset-panda": "^0.0.48",
     "@ndla/scripts": "^2.1.3",
-    "@ndla/types-backend": "^1.0.1",
+    "@ndla/types-backend": "^1.0.15",
     "@ndla/types-embed": "^5.0.6-alpha.0",
     "@ndla/types-taxonomy": "^1.0.30",
     "@pandacss/dev": "^0.48.0",
@@ -71,11 +71,14 @@
     "tsx": "^4.19.2",
     "typescript": "^5.7.2",
     "typescript-eslint": "^8.16.0",
+    "vercel": "^33.6.2",
     "vite": "^6.0.1",
     "vitest": "^2.1.6"
   },
   "dependencies": {
     "@ark-ui/react": "^4.1.2",
+    "@aws-sdk/client-bedrock-runtime": "^3.670.0",
+    "@aws-sdk/client-transcribe": "^3.699.0",
     "@dnd-kit/core": "^6.0.8",
     "@dnd-kit/modifiers": "^6.0.1",
     "@dnd-kit/sortable": "^7.0.2",
@@ -98,6 +101,7 @@
     "@ndla/video-search": "^8.0.70-alpha.0",
     "@tanstack/react-query": "5.62.3",
     "auth0-js": "^9.22.1",
+    "buffer": "^6.0.3",
     "compression": "^1.7.4",
     "cross-fetch": "^3.1.5",
     "date-fns": "2.30.0",

src/components/LLM/helpers.ts

@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) 2024-present, NDLA.
+ *
+ * This source code is licensed under the GPLv3 license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+
+import { Buffer } from "buffer";
+
+export const claudeHaikuDefaults = { top_p: 0.7, top_k: 100, temperature: 0.9 };
+
+interface modelProps {
+  prompt: string;
+  image?: {
+    base64: string;
+    fileType: string;
+  };
+  max_tokens?: number;
+}
+
+export const invokeModel = async ({ prompt, image, max_tokens = 2000, ...rest }: modelProps) => {
+  if (!prompt) {
+    // console.error("No prompt provided to invokeModel");
+    return null;
+  }
+
+  const payload: any = { prompt, max_tokens, ...rest };
+  if (image) {
+    payload.image = image;
+  }
+
+  const response = await fetch("/invoke-model", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(payload),
+  });
+
+  if (!response.ok) {
+    // console.error("Failed to get a response from the model");
+    return null;
+  }
+
+  const responseBody = await response.json();
+  return parseResponse(responseBody.content[0].text);
+};
+
+export const getTextFromHTML = (html: string) => {
+  const parser = new DOMParser();
+  const doc = parser.parseFromString(html, "text/html");
+  return doc.body.textContent || "";
+};
+
+const parseResponse = (response: string) => {
+  return response.split("<answer>")[1].split("</answer>")[0].trim();
+};
+
+export const convertBufferToBase64 = (buffer: ArrayBuffer) => {
+  return Buffer.from(buffer).toString("base64");
+};

src/components/SlateEditor/PlainTextEditor.tsx

@@ -74,7 +74,7 @@ const PlainTextEditor = forwardRef<HTMLTextAreaElement, Props>(
     const { status, setStatus } = useFormikContext<ArticleFormType>();
 
     useEffect(() => {
-      if (status?.status === "revertVersion") {
+      if (status?.status === "revertVersion" || status?.status === "acceptGenerated") {
         ReactEditor.deselect(editor);
         editor.children = value;
         setStatus((prevStatus: FormikStatus) => ({

src/components/SlateEditor/RichTextEditor.tsx

@@ -10,7 +10,7 @@ import { useFormikContext } from "formik";
 import { isKeyHotkey } from "is-hotkey";
 import isEqual from "lodash/isEqual";
 import { FocusEvent, KeyboardEvent, useCallback, useEffect, useMemo, useRef, useState } from "react";
-import { createEditor, Descendant, Editor, NodeEntry, Range, Transforms } from "slate";
+import { BaseRange, createEditor, Descendant, Editor, NodeEntry, Range, Transforms } from "slate";
 import { withHistory } from "slate-history";
 import { Slate, Editable, withReact, RenderElementProps, RenderLeafProps, ReactEditor } from "slate-react";
 import { EditableProps } from "slate-react/dist/components/editable";
@@ -28,6 +28,7 @@ import { onDragOver, onDragStart, onDrop } from "./plugins/DND";
 import { TYPE_HEADING } from "./plugins/heading/types";
 import { TYPE_LIST } from "./plugins/list/types";
 import { TYPE_PARAGRAPH } from "./plugins/paragraph/types";
+import RephraseModal from "./plugins/rephrase/RephraseModal";
 import { TYPE_TABLE } from "./plugins/table/types";
 import { SlateToolbar } from "./plugins/toolbar";
 import { AreaFilters, CategoryFilters } from "./plugins/toolbar/toolbarState";
@@ -93,6 +94,7 @@ const RichTextEditor = ({
 }: RichTextEditorProps) => {
   const [editor] = useState(() => withPlugins(withReact(withHistory(createEditor())), plugins));
   const [isFirstNormalize, setIsFirstNormalize] = useState(true);
+  const [rephraseSelection, setRephraseSelection] = useState<BaseRange | null>(null);
   const [labelledBy, setLabelledBy] = useState<string | undefined>(undefined);
   const prevSubmitted = useRef(submitted);
   const field = useFieldContext();
@@ -138,7 +140,11 @@ const RichTextEditor = ({
 
   useEffect(() => {
     // When form is submitted or form content has been revert to a previous version, the editor has to be reinitialized.
-    if ((!submitted && prevSubmitted.current) || status === "revertVersion") {
+    if (
+      (!submitted && prevSubmitted.current) ||
+      status?.status === "revertVersion" ||
+      status?.status === "acceptGenerated"
+    ) {
       if (isFirstNormalize) {
         return;
       }
@@ -319,6 +325,10 @@ const RichTextEditor = ({
     [additionalOnKeyDown, editor],
   );
 
+  const selectors = useMemo(() => {
+    return { rephrase: setRephraseSelection };
+  }, []);
+
   return (
     <article className={noArticleStyling ? undefined : "ndla-article"}>
       <ArticleLanguageProvider language={language}>
@@ -329,7 +339,12 @@ const RichTextEditor = ({
                 <Spinner />
               ) : (
                 <>
-                  <SlateToolbar options={toolbarOptions} areaOptions={toolbarAreaFilters} hideToolbar={hideToolbar} />
+                  <SlateToolbar
+                    options={toolbarOptions}
+                    areaOptions={toolbarAreaFilters}
+                    hideToolbar={hideToolbar}
+                    selectors={selectors}
+                  />
                   {!hideBlockPicker && (
                     <SlateBlockPicker
                       editor={editor}
@@ -338,6 +353,7 @@ const RichTextEditor = ({
                       {...createBlockpickerOptions(blockpickerOptions)}
                     />
                   )}
+                  <RephraseModal selection={rephraseSelection} setSelection={setRephraseSelection} />
                   <StyledEditable
                     {...fieldProps}
                     aria-labelledby={labelledBy}

src/components/SlateEditor/plugins/framedContent/SlateFramedContent.tsx

@@ -6,19 +6,22 @@
  *
  */
 
-import { useMemo } from "react";
+import { useMemo, useState } from "react";
 import { useTranslation } from "react-i18next";
 import { Editor, Element, NodeEntry, Transforms } from "slate";
 import { ReactEditor, RenderElementProps } from "slate-react";
-import { BrushLine, CopyrightLine } from "@ndla/icons";
+import { BrushLine, CopyrightLine, FileListLine } from "@ndla/icons";
 import { IconButton } from "@ndla/primitives";
 import { styled } from "@ndla/styled-system/jsx";
 import { ContentTypeFramedContent, EmbedWrapper } from "@ndla/ui";
 import { FramedContentElement } from ".";
 import { TYPE_FRAMED_CONTENT } from "./types";
+import { editorValueToPlainText } from "../../../../util/articleContentConverter";
 import { useArticleContentType } from "../../../ContentTypeProvider";
 import DeleteButton from "../../../DeleteButton";
+import { claudeHaikuDefaults, invokeModel } from "../../../LLM/helpers";
 import MoveContentButton from "../../../MoveContentButton";
+import { useArticleLanguage } from "../../ArticleLanguageProvider";
 import { TYPE_COPYRIGHT } from "../copyright/types";
 import { defaultCopyrightBlock } from "../copyright/utils";
 import { StyledFigureButtons } from "../embed/FigureButtons";
@@ -38,6 +41,8 @@ interface Props extends RenderElementProps {
 const SlateFramedContent = (props: Props) => {
   const { element, editor, attributes, children } = props;
   const { t } = useTranslation();
+  const language = useArticleLanguage();
+  const [isLoading, setIsLoading] = useState(false);
   const variant = element.data?.variant ?? "neutral";
   const contentType = useArticleContentType();
   const hasSlateCopyright = useMemo(() => {
@@ -81,9 +86,45 @@ const SlateFramedContent = (props: Props) => {
     Transforms.insertNodes(editor, defaultCopyrightBlock(), { at: path.concat(node.children.length) });
   };
 
+  const generateQuestions = async () => {
+    const articleText = editorValueToPlainText(editor.children);
+    if (!articleText) {
+      // console.error("No article content provided to generate meta description");
+      return;
+    }
+    setIsLoading(true);
+    try {
+      const generatedText = await invokeModel({
+        prompt: t("textGeneration.reflectionQuestions.prompt", {
+          article: articleText,
+          language: t(`languages.${language}`),
+        }),
+        ...claudeHaikuDefaults,
+      });
+      if (generatedText) {
+        editor.insertText(generatedText);
+      }
+      // generatedText ? editor.insertText(generatedText) : console.error("No generated text");
+    } catch (error) {
+      // console.error("Error generating reflection questions", error);
+    } finally {
+      setIsLoading(false);
+    }
+  };
+
   return (
     <EmbedWrapper draggable {...attributes}>
       <FigureButtons contentEditable={false}>
+        <IconButton
+          variant={variant === "colored" ? "primary" : "secondary"}
+          size="small"
+          title={t("textGeneration.reflectionQuestions.button")}
+          aria-label={t("textGeneration.reflectionQuestions.button")}
+          onClick={generateQuestions}
+          loading={isLoading}
+        >
+          <FileListLine />
+        </IconButton>
         {!hasSlateCopyright && (
           <IconButton
             variant="tertiary"

src/components/SlateEditor/plugins/image/ImageEmbedForm.tsx

@@ -7,10 +7,10 @@
  */
 
 import { Formik, useFormikContext } from "formik";
-import { useMemo } from "react";
+import { useMemo, useState } from "react";
 import { useTranslation } from "react-i18next";
 import { Descendant } from "slate";
-import { CheckLine } from "@ndla/icons";
+import { FileListLine, CheckLine } from "@ndla/icons";
 import {
   Button,
   CheckboxControl,
@@ -22,10 +22,12 @@ import {
   FieldRoot,
   FieldErrorMessage,
   FieldTextArea,
+  Spinner,
 } from "@ndla/primitives";
 import { styled } from "@ndla/styled-system/jsx";
 import { IImageMetaInformationV3DTO } from "@ndla/types-backend/image-api";
 import { ImageEmbedData } from "@ndla/types-embed";
+import { convertBufferToBase64, claudeHaikuDefaults, invokeModel } from "../../../../components/LLM/helpers";
 import { InlineField } from "../../../../containers/FormikForm/InlineField";
 import ImageEditor from "../../../../containers/ImageEditor/ImageEditor";
 import { inlineContentToEditorValue, inlineContentToHTML } from "../../../../util/articleContentConverter";
@@ -146,6 +148,12 @@ const InputWrapper = styled("div", {
   },
 });
 
+const StyledButton = styled(Button, {
+  base: {
+    alignSelf: "flex-start",
+  },
+});
+
 const EmbedForm = ({
   onClose,
   language,
@@ -156,6 +164,31 @@ const EmbedForm = ({
   const inGrid = useInGrid();
   const { values, initialValues, isValid, setFieldValue, dirty, isSubmitting } =
     useFormikContext<ImageEmbedFormValues>();
+  const [isLoading, setIsLoading] = useState<boolean>(false);
+
+  const generateAltText = async () => {
+    setIsLoading(true);
+    if (!image?.image.imageUrl) {
+      return null;
+    }
+
+    const response = await fetch(image?.image.imageUrl);
+    const responseContentType = response.headers.get("Content-Type");
+    const buffer = await response.arrayBuffer();
+    const base64 = convertBufferToBase64(buffer);
+
+    const result = await invokeModel({
+      prompt: t("textGeneration.altText.prompt", { language: t(`languages.${language}`) }),
+      image: {
+        base64,
+        fileType: responseContentType ?? "",
+      },
+      max_tokens: 2000,
+      ...claudeHaikuDefaults,
+    });
+    setIsLoading(false);
+    return result;
+  };
 
   const formIsDirty = isFormikFormDirty({
     values,
@@ -182,13 +215,25 @@ const EmbedForm = ({
             </FieldRoot>
           )}
         </FormField>
-
         {!values.isDecorative && (
           <FormField name="alt">
-            {({ field, meta }) => (
+            {({ field, meta, helpers }) => (
               <FieldRoot invalid={!!meta.error}>
                 <FieldLabel>{t("form.image.alt.label")}</FieldLabel>
                 <FieldTextArea {...field} placeholder={t("form.image.alt.placeholder")} />
+                <StyledButton
+                  onClick={async () => {
+                    const text = await generateAltText();
+                    if (text && text.length > 0) {
+                      helpers.setValue(text);
+                    }
+                  }}
+                  size="small"
+                  title={t("textGeneration.altText.title")}
+                >
+                  {t("textGeneration.altText.button")}
+                  {isLoading ? <Spinner size="small" /> : <FileListLine />}
+                </StyledButton>
                 <FieldErrorMessage>{meta.error}</FieldErrorMessage>
               </FieldRoot>
             )}