fix: LSDV-4864: LEAP-148: Magic Wand doesn't work with MIG

e2e/tests/helpers.js

@@ -608,7 +608,7 @@ const switchRegionTreeView = (viewName) => {
   Htx.annotationStore.selected.regionStore.setView(viewName);
 };
 
-const serialize = () => window.Htx.annotationStore.selected.serializeAnnotation();
+const serialize = async () => await window.Htx.annotationStore.selected.serializeAnnotation();
 
 const selectText = async ({ selector, rangeStart, rangeEnd }) => {
   let [doc, win] = [document, window];

e2e/tests/smart-tools.history.test.js

@@ -22,7 +22,7 @@ function createRectangleConfig(params = {}) {
 
 const IMAGE = 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/df/Html_headers.png/640px-Html_headers.png';
 
-function getRectangleSuggestions(reg, group) {
+async function getRectangleSuggestions(reg, group) {
   const allSuggestions = [{
     'original_width': 640,
     'original_height': 507,
@@ -451,7 +451,7 @@ function getRectangleSuggestions(reg, group) {
   }];
   const annotation = window.labelStudio.store.annotationStore.selected;
   const ids = group.map(r => r.id);
-  const results = annotation.serializeAnnotation().filter((res) => ids.includes(res.id));
+  const results = (await annotation.serializeAnnotation()).filter((res) => ids.includes(res.id));
   const suggestions = allSuggestions.filter(predictionResult => {
     const targetCenterX = predictionResult.value.x + predictionResult.value.width / 2;
     const targetCenterY = predictionResult.value.y + predictionResult.value.height / 2;
@@ -473,6 +473,9 @@ function getRectangleSuggestions(reg, group) {
 }
 
 Scenario('Undo regions auto-annotated from predictions', async function({ I, LabelStudio, AtImageView, AtSidebar }) {
+  LabelStudio.setFeatureFlags({
+    fflag_fix_front_dev_1284_auto_detect_undo_281022_short: true,
+  });
   I.amOnPage('/');
   LabelStudio.init({
     config: createRectangleConfig({
@@ -492,9 +495,6 @@ Scenario('Undo regions auto-annotated from predictions', async function({ I, Lab
       forceAutoAcceptSuggestions: true,
     },
   });
-  LabelStudio.setFeatureFlags({
-    fflag_fix_front_dev_1284_auto_detect_undo_281022_short: true,
-  });
   AtImageView.waitForImage();
   AtSidebar.seeRegions(0);
   await AtImageView.lookForStage();

package.json

@@ -143,7 +143,7 @@
     "emoji-regex": "^7.0.3",
     "enzyme": "^3.11.0",
     "enzyme-to-json": "^3.5.0",
-    "eslint": "^8.28.0",
+    "eslint": "^8.38.0",
     "eslint-webpack-plugin": "^3.0.1",
     "file-loader": "^6.2.0",
     "html-webpack-plugin": "^5.3.1",
@@ -192,7 +192,7 @@
     "svg.js": "^2.7.0",
     "terser-webpack-plugin": "^5.1.1",
     "ts-jest": "^29.0.3",
-    "typescript": "^4.2.3",
+    "typescript": "^5.0.4",
     "url-loader": "^4.1.1",
     "wavesurfer.js": "^6.0.1",
     "webpack": "^5.30.0",

public/index.html

@@ -159,7 +159,7 @@
             }
           })
           ls.on("updateAnnotation", (_, annotation) => {
-            console.log(annotation.serializeAnnotation());
+            annotation.serializeAnnotation().then(r => console.log(r));
           })
           ls.on("regionFinishedDrawing", (region, list) => {
             console.log("finish drawing", {region, list})

src/components/Debug.js

@@ -3,9 +3,9 @@ import { Button, Form } from 'antd';
 
 import { observer } from 'mobx-react';
 
-const toJSON = (annotation) => {
+const toJSON = async (annotation) => {
   const id = annotation.pk || annotation.id;
-  const result = annotation.serializeAnnotation();
+  const result = await annotation.serializeAnnotation();
   const draft = annotation.versions.draft;
   const json = { id, result };
 
@@ -32,22 +32,22 @@ const DebugComponent = ({ store }) => {
     if (cs.annotations.length) cs.selectAnnotation(cs.annotations[0].id);
   }, []);
 
-  const serializeCurrent = useCallback(() => {
+  const serializeCurrent = useCallback(async () => {
     const input = refAnnotations.current;
 
     if (!input) return;
     const annotation = store.annotationStore.selected;
-    const json = [toJSON(annotation)];
+    const json = await Promise.all([toJSON(annotation)]);
 
     input.value = JSON.stringify(json, null, 2);
   }, []);
 
-  const serializeAll = useCallback(() => {
+  const serializeAll = useCallback(async () => {
     const input = refAnnotations.current;
 
     if (!input) return;
     const { annotations, predictions } = store.annotationStore;
-    const json = [...annotations, ...predictions].map(toJSON);
+    const json = await Promise.all([...annotations, ...predictions].map(toJSON));
 
     input.value = JSON.stringify(json, null, 2);
   }, []);

src/components/ImageView/Image.js

@@ -12,6 +12,7 @@ export const Image = observer(forwardRef(({
   updateImageSize,
   usedValue,
   size,
+  overlay,
 }, ref) => {
   const imageSize = useMemo(() => {
     return {
@@ -27,6 +28,7 @@ export const Image = observer(forwardRef(({
 
   return (
     <Block name="image" style={imageSize}>
+      {overlay}
       <ImageProgress
         downloading={imageEntity.downloading}
         progress={imageEntity.progress}
@@ -58,7 +60,7 @@ const ImageProgress = observer(({
   return downloading ? (
     <Block name="image-progress">
       <Elem name="message">Downloading image</Elem>
-      <Elem tag="progress" name="bar" value={progress} min="0" max={1} step={0.0001}/>
+      <Elem tag="progress" name="bar" value={progress} min="0" max={1} step={0.0001} />
     </Block>
   ) : error ? (
     <ImageLoadingError src={src} value={usedValue} />
@@ -103,6 +105,6 @@ const ImageLoadingError = ({ src, value }) => {
   }, [src]);
 
   return (
-    <ErrorMessage error={error}/>
+    <ErrorMessage error={error} />
   );
 };

src/components/ImageView/ImageView.js

@@ -89,7 +89,7 @@ const Regions = memo(({ regions, useLayers = true, chunkSize = 15, suggestion =
 
 const DrawingRegion = observer(({ item }) => {
   const { drawingRegion } = item;
-  
+
   if (!drawingRegion) return null;
   if (item.multiImage && item.currentImage !== drawingRegion.item_index) return null;
 
@@ -280,7 +280,7 @@ const SelectedRegions = observer(({ item, selectedRegions }) => {
 
   return (
     <>
-      <TransformerBack item={item}/>
+      <TransformerBack item={item} />
       {brushRegions.length > 0 && (
         <Regions
           key="brushes"
@@ -326,7 +326,7 @@ const SelectionLayer = observer(({ item, selectionArea }) => {
       window.removeEventListener('mousedown', dragHandler);
       window.removeEventListener('mouseup', dragHandler);
     };
-  },[]);
+  }, []);
 
   const disableTransform = item.zoomScale > 1 && (shift || isPanTool || isMouseWheelClick);
 
@@ -375,7 +375,7 @@ const Selection = observer(({ item, selectionArea, ...triggeredOnResize }) => {
   return (
     <>
       <SelectedRegions item={item} selectedRegions={item.selectedRegions} {...triggeredOnResize} />
-      <SelectionLayer item={item} selectionArea={selectionArea}/>
+      <SelectionLayer item={item} selectionArea={selectionArea} />
     </>
   );
 });
@@ -458,6 +458,22 @@ const Crosshair = memo(forwardRef(({ width, height }, ref) => {
   );
 }));
 
+/**
+ * Component that creates an overlay on top
+ * of the image to support Magic Wand tool
+ */
+const CanvasOverlay = observer(({ item }) => {
+  return isFF(FF_DEV_4081) ? (
+    <canvas
+      className={styles.overlay}
+      ref={ref => {
+        item.setOverlayRef(ref);
+      }}
+      style={item.imageTransform}
+    />
+  ) : null;
+});
+
 export default observer(
   class ImageView extends Component {
     // stored position of canvas before creating region
@@ -534,7 +550,7 @@ export default observer(
 
       const handleMouseDown = () => {
         if (
-        // create regions over another regions with Cmd/Ctrl pressed
+          // create regions over another regions with Cmd/Ctrl pressed
           item.getSkipInteractions() ||
           e.target === item.stageRef ||
           findClosestParent(
@@ -944,6 +960,7 @@ export default observer(
                 imageTransform={item.imageTransform}
                 updateImageSize={item.updateImageSize}
                 size={item.canvasSize}
+                overlay={<CanvasOverlay item={item} />}
               />
             ) : (
               <div
@@ -969,15 +986,7 @@ export default observer(
                   crossOrigin={item.imageCrossOrigin}
                   alt="LS"
                 />
-                {isFF(FF_DEV_4081) ? (
-                  <canvas
-                    className={styles.overlay}
-                    ref={ref => {
-                      item.setOverlayRef(ref);
-                    }}
-                    style={item.imageTransform}
-                  />
-                ) : null}
+                <CanvasOverlay item={item} />
               </div>
             )}
             {/* @todo this is dirty hack; rewrite to proper async waiting for data to load */}

src/regions/BrushRegion.js

@@ -403,7 +403,7 @@ const Model = types
        * @param {boolean} [options.fast] Saving only touches, without RLE
        * @return {BrushRegionResult}
        */
-      serialize(options) {
+      async serialize(options) {
         const object = self.object;
         const value = { format: 'rle' };
 
@@ -413,7 +413,7 @@ const Model = types
           if (self.touches.length) value.touches = self.touches;
           if (self.maskDataURL) value.maskDataURL = self.maskDataURL;
         } else {
-          const rle = Canvas.Region2RLE(self, object);
+          const rle = await Canvas.Region2RLE(self, object);
 
           if (!rle || !rle.length) return null;
 

src/regions/Result.js

@@ -262,10 +262,10 @@ const Result = types
     // label, becuase it takes color from the label
     updateAppearenceFromState() { },
 
-    serialize(options) {
+    async serialize(options) {
       const { type, score, value, ...sn } = getSnapshot(self);
       const { valueType } = self.from_name;
-      const data = self.area ? self.area.serialize(options) : {};
+      const data = self.area ? await self.area.serialize(options) : {};
       // cut off annotation id
       const id = self.area?.cleanId;
       const from_name = Tree.cleanUpId(sn.from_name);

src/stores/Annotation/Annotation.js

@@ -181,16 +181,6 @@ export const Annotation = types
       return results;
     },
 
-    get serialized() {
-      // Dirty hack to force MST track changes
-      self.areas.toJSON();
-
-      return self.results
-        .map(r => r.serialize())
-        .filter(Boolean)
-        .concat(self.relationStore.serializeAnnotation());
-    },
-
     get serializedSelection() {
       // Dirty hack to force MST track changes
       self.areas.toJSON();
@@ -666,11 +656,11 @@ export const Annotation = types
       onSnapshot(self.areas, self.autosave);
     }),
 
-    saveDraft() {
+    saveDraft: flow(function*() {
       // if this is now a history item or prediction don't save it
       if (!self.editable) return;
 
-      const result = self.serializeAnnotation({ fast: true });
+      const result = yield self.serializeAnnotation({ fast: true });
       // if this is new annotation and no regions added yet
 
       if (!isFF(FF_LSDV_3009) && !self.pk && !result.length) return;
@@ -680,7 +670,7 @@ export const Annotation = types
       self.setDraftSaving(true);
 
       self.store.submitDraft(self).then(self.onDraftSaved);
-    },
+    }),
 
     saveDraftImmediately() {
       if (self.autosave) self.autosave.flush();
@@ -908,15 +898,17 @@ export const Annotation = types
       return self.regionStore.regions.slice(prevSize);
     },
 
-    serializeAnnotation(options) {
-      // return self.serialized;
-
+    async serializeAnnotation(options) {
       document.body.style.cursor = 'wait';
+      let result = [];
+
+      for (const singleResult of self.results) {
+        const serialized = await singleResult.serialize(); 
+
+        if (serialized) result.push(serialized);
+      }
 
-      const result = self.results
-        .map(r => r.serialize(options))
-        .filter(Boolean)
-        .concat(self.relationStore.serializeAnnotation(options));
+      result = result.concat(self.relationStore.serializeAnnotation(options));      
 
       document.body.style.cursor = 'default';
 

src/tools/MagicWand.js

@@ -250,7 +250,7 @@ const _Tool = types
           msg = 'The Magic Wand is not supported on rotated images';
         } else {
           msg = 'The Magic Wand is not supported if the crosshair is turned on';
-        } 
+        }
 
         alert(msg);
         throw msg;
@@ -408,7 +408,7 @@ const _Tool = types
      * once the user is done with the Magic Wand by releasing the mouse button.
      */
     initCurrentRegion() {
-      if (self.isFirstWand){
+      if (self.isFirstWand) {
         const regionOpts = {
           id: guidGenerator(),
           strokewidth: 1,

src/utils/canvas.js

@@ -173,8 +173,9 @@ function RLE2Region(item, { color = Constants.FILL_COLOR } = {}) {
 /**
 * Exports region using canvas. Doesn't require Konva#Stage access
 * @param {Region} region Brush region
+* @return {Promise<Uint8Array>} RLE encoded data
 */
-function exportRLE(region) {
+async function exportRLE(region) {
   const {
     naturalWidth,
     naturalHeight,
@@ -198,6 +199,21 @@ function exportRLE(region) {
 
   document.body.appendChild(canvas);
 
+  if (region.maskDataURL) {
+    await new Promise((resolve) => {
+      const image = new Image();
+
+      image.onload = () => {
+        console.log('image loaded');
+        ctx.drawImage(image, 0, 0);
+        resolve();
+      };
+
+      image.src = region.maskDataURL;
+      image.load();
+    });
+  }
+
   // Restore original RLE if available
   if (region.rle && region.rle.length > 0) {
     // Apply RLE to existing image data
@@ -263,9 +279,9 @@ function exportRLE(region) {
  * Given a brush region return the RLE encoded array.
  * @param {BrushRegion} region BrushRegtion to turn into RLE array.
  * @param {tags.object.Image} image Image the region will be interacting with.
- * @returns {string} RLE encoded contents.
+ * @returns {Promise<Uint8Array>} RLE encoded contents.
  */
-function Region2RLE(region) {
+async function Region2RLE(region) {
   // New way of exporting brush regions
   if (isFF(FF_LSDV_4583)) return exportRLE(region);