Semantic matching basics #4788

detox/detox.d.ts

@@ -1094,6 +1094,8 @@ declare global {
         interface NativeElement extends NativeElementActions {
         }
 
+        type SemanticMatchingTypes = 'image' | 'input-field' | 'text' | 'button' | 'scrollview' | 'list' | 'switch' | 'slider' | 'picker' | 'activity-indicator' | 'progress';
+
         interface ByFacade {
             /**
              * by.id will match an id that is given to the view via testID prop.
@@ -1123,10 +1125,19 @@ declare global {
             label(label: string | RegExp): NativeMatcher;
 
             /**
-             * Find an element by native view type.
-             * @example await element(by.type('RCTImageView'));
+             * Find an element by native view type OR semantic type.
+             * Automatically detects if the input is a semantic type or regular class name.
+             * @example 
+             * // Semantic types (cross-platform):
+             * await element(by.type('image'));
+             * await element(by.type('button'));
+             * await element(by.type('input-field'));
+             * 
+             * // Native class names (platform-specific):
+             * await element(by.type('RCTImageView'));
+             * await element(by.type('android.widget.Button'));
              */
-            type(nativeViewType: string): NativeMatcher;
+            type(typeOrSemanticType: SemanticMatchingTypes | string): NativeMatcher;
 
             /**
              * Find an element with an accessibility trait. (iOS only)

detox/src/android/matchers/native.js

@@ -1,9 +1,16 @@
 const DetoxRuntimeError = require('../../errors/DetoxRuntimeError');
 const invoke = require('../../invoke');
+const semanticTypes = require('../../matchers/semanticTypes');
 const { isRegExp } = require('../../utils/isRegExp');
 const { NativeMatcher } = require('../core/NativeMatcher');
 const DetoxMatcherApi = require('../espressoapi/DetoxMatcher');
 
+const createClassMatcher = (className) =>
+  new NativeMatcher(invoke.callDirectly(DetoxMatcherApi.matcherForClass(className)));
+
+const combineWithOr = (matchers) =>
+  matchers.reduce((acc, matcher) => acc?.or(matcher) ?? matcher, null);
+
 class LabelMatcher extends NativeMatcher {
   constructor(value) {
     super();
@@ -29,9 +36,29 @@ class IdMatcher extends NativeMatcher {
 }
 
 class TypeMatcher extends NativeMatcher {
-  constructor(value) {
+  constructor(typeOrSemanticType) {
     super();
-    this._call = invoke.callDirectly(DetoxMatcherApi.matcherForClass(value));
+    if (semanticTypes.includes(typeOrSemanticType)) {
+      const classNames = semanticTypes.getClasses(typeOrSemanticType, 'android');
+
+      const matchers = classNames.map(item => {
+        if (typeof item === 'string') return createClassMatcher(item);
+        if (!item.className || !item.excludes) return createClassMatcher(item);
+
+        const includeMatcher = createClassMatcher(item.className);
+        const excludeCombined = combineWithOr(item.excludes.map(createClassMatcher));
+
+        return includeMatcher.and(excludeCombined.not);
+      });
+
+      const combinedMatcher = combineWithOr(matchers);
+      if (!combinedMatcher) {
+        throw new DetoxRuntimeError(`No class names found for semantic type: ${typeOrSemanticType}`);
+      }
+      this._call = combinedMatcher._call;
+    } else {
+      this._call = invoke.callDirectly(DetoxMatcherApi.matcherForClass(typeOrSemanticType));
+    }
   }
 }
 

detox/src/android/matchers/native.test.js

@@ -0,0 +1,84 @@
+// @ts-nocheck
+// Mock the semanticTypes module before importing anything that depends on it
+jest.mock('../../matchers/semanticTypes', () => ({
+  getTypes: jest.fn(),
+  getClasses: jest.fn(),
+  includes: jest.fn()
+}));
+
+
+const semanticTypes = require('../../matchers/semanticTypes');
+
+const { TypeMatcher } = require('./native');
+
+describe('Native Matchers', () => {
+  describe('TypeMatcher', () => {
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it('should handle regular class names', () => {
+      semanticTypes.includes.mockReturnValue(false);
+
+      expect(() => {
+        new TypeMatcher('com.example.CustomView');
+      }).not.toThrow();
+    });
+
+    it('should handle semantic types automatically', () => {
+      semanticTypes.includes.mockReturnValue(true);
+      semanticTypes.getClasses.mockReturnValue([
+        'android.widget.ImageView',
+        'com.facebook.react.views.image.ReactImageView'
+      ]);
+
+      expect(() => {
+        new TypeMatcher('image');
+      }).not.toThrow();
+    });
+
+    it('should handle exclusion objects for semantic types', () => {
+      semanticTypes.includes.mockReturnValue(true);
+      semanticTypes.getClasses.mockReturnValue([
+        {
+          className: 'android.widget.ProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        },
+        {
+          className: 'androidx.core.widget.ContentLoadingProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('activity-indicator');
+      }).not.toThrow();
+    });
+
+    it('should handle mixed string and exclusion objects', () => {
+      semanticTypes.includes.mockReturnValue(true);
+      semanticTypes.getClasses.mockReturnValue([
+        {
+          className: 'android.widget.ProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        },
+        {
+          className: 'androidx.core.widget.ContentLoadingProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('progress');
+      }).not.toThrow();
+    });
+
+    it('should handle regular class names when not semantic types', () => {
+      semanticTypes.includes.mockReturnValue(false);
+
+      expect(() => {
+        new TypeMatcher('android.widget.ImageView');
+      }).not.toThrow();
+    });
+  });
+});

detox/src/ios/expectTwo.js

@@ -5,6 +5,24 @@ const fs = require('fs-extra');
 const _ = require('lodash');
 
 
+const semanticTypes = require('../matchers/semanticTypes');
+
+// Functions for semantic type predicate creation
+const createTypePredicate = (className) => ({ type: 'type', value: className });
+
+const createOrPredicate = (predicates) => ({ type: 'or', predicates });
+
+const createExclusionPredicate = (className, excludes) => ({
+  type: 'and',
+  predicates: [
+    createTypePredicate(className),
+    {
+      type: 'not',
+      predicate: createOrPredicate(excludes.map(createTypePredicate))
+    }
+  ]
+});
+
 const { assertTraceDescription, assertEnum, assertNormalized } = require('../utils/assertArgument');
 const { removeMilliseconds } = require('../utils/dateUtils');
 const { actionDescription, expectDescription } = require('../utils/invocationTraceDescriptions');
@@ -440,9 +458,21 @@ class Matcher {
     return this;
   }
 
-  type(type) {
-    if (typeof type !== 'string') throw new Error('type should be a string, but got ' + (type + (' (' + (typeof type + ')'))));
-    this.predicate = { type: 'type', value: type };
+    type(typeOrSemanticType) {
+    if (typeof typeOrSemanticType !== 'string') throw new Error('type should be a string, but got ' + (typeOrSemanticType + (' (' + (typeof typeOrSemanticType + ')'))));
+
+    if (semanticTypes.includes(typeOrSemanticType)) {
+      const classNames = semanticTypes.getClasses(typeOrSemanticType, 'ios');
+      const predicates = classNames.map(item => {
+        if (typeof item === 'string') return createTypePredicate(item);
+        if (!item.className || !item.excludes) return createTypePredicate(item);
+        return createExclusionPredicate(item.className, item.excludes);
+      });
+      this.predicate = createOrPredicate(predicates);
+    } else {
+      this.predicate = { type: 'type', value: typeOrSemanticType };
+    }
+
     return this;
   }
 

detox/src/ios/expectTwo.test.js

@@ -793,6 +793,49 @@ describe('expectTwo', () => {
     });
   });
 
+  describe('semantic types', () => {
+    it(`should parse correct JSON for regular class name using by.type()`, async () => {
+      const testCall = await e.element(e.by.type('CustomUIView')).tap();
+      const jsonOutput = {
+        invocation: {
+          type: 'action',
+          action: 'tap',
+          predicate: {
+            type: 'type',
+            value: 'CustomUIView'
+          }
+        }
+      };
+
+      expect(testCall).toDeepEqual(jsonOutput);
+    });
+
+    it(`should parse correct JSON for semantic type 'image' using by.type()`, async () => {
+      const testCall = await e.element(e.by.type('image')).tap();
+      const jsonOutput = {
+        invocation: {
+          type: 'action',
+          action: 'tap',
+          predicate: {
+            type: 'or',
+            predicates: [
+              {
+                type: 'type',
+                value: 'RCTImageView'
+              },
+              {
+                type: 'type',
+                value: 'UIImageView'
+              }
+            ]
+          }
+        }
+      };
+
+      expect(testCall).toDeepEqual(jsonOutput);
+    });
+  });
+
   describe('web views', () => {
     it(`should parse expect(web(by.id('webViewId').element(web(by.label('tapMe')))).toExist()`, async () => {
       const testCall = await e.expect(e.web(e.by.id('webViewId')).atIndex(1).element(e.by.web.label('tapMe')).atIndex(2)).toExist();