diff --git a/core/public/painting_ctx_platform_impl.h b/core/public/painting_ctx_platform_impl.h index 83b473b18b..b2d498aaca 100644 --- a/core/public/painting_ctx_platform_impl.h +++ b/core/public/painting_ctx_platform_impl.h @@ -86,8 +86,6 @@ struct InitialLynxUITreeNodeForReplay { int id = 0; std::string tag; fml::RefPtr painting_data; - bool flatten = false; - uint32_t node_index = 0; bool has_parent = false; int parent = 0; diff --git a/core/renderer/dom/element_manager.cc b/core/renderer/dom/element_manager.cc index fd10412622..f7efdd73fa 100644 --- a/core/renderer/dom/element_manager.cc +++ b/core/renderer/dom/element_manager.cc @@ -86,8 +86,6 @@ void CollectElementContainerForReplay( node.id = element->impl_id(); node.tag = element->GetPlatformNodeTag().str(); node.painting_data = element->GetPropBundleForRecording(); - node.flatten = element->TendToFlatten(); - node.node_index = element->NodeIndex(); const auto layout = container->CalculateCurrentPlatformLayout(); if (ui_parent != nullptr) { node.has_parent = true; diff --git a/core/renderer/ui_wrapper/painting/android/painting_context_android.cc b/core/renderer/ui_wrapper/painting/android/painting_context_android.cc index b451e32f7f..f0e3b31a0c 100644 --- a/core/renderer/ui_wrapper/painting/android/painting_context_android.cc +++ b/core/renderer/ui_wrapper/painting/android/painting_context_android.cc @@ -735,8 +735,6 @@ void PaintingContextAndroid::RecordInitialLynxUITreeForReplay( std::vector signs(size); std::vector parent_signs(size); std::vector child_indexes(size); - std::vector node_indexes(size); - std::vector flattens(size); std::vector has_bounds(size); std::vector has_sticky(size); std::vector layouts(layout_size); @@ -766,8 +764,6 @@ void PaintingContextAndroid::RecordInitialLynxUITreeForReplay( signs[index] = node.id; parent_signs[index] = node.has_parent ? node.parent : -1; child_indexes[index] = node.has_parent ? node.index : -1; - node_indexes[index] = static_cast(node.node_index); - flattens[index] = static_cast(node.flatten); has_bounds[index] = static_cast(node.has_bounds); has_sticky[index] = static_cast(node.has_sticky); @@ -804,11 +800,7 @@ void PaintingContextAndroid::RecordInitialLynxUITreeForReplay( env, env->NewIntArray(java_size)); base::android::ScopedLocalJavaRef child_indexes_ref( env, env->NewIntArray(java_size)); - base::android::ScopedLocalJavaRef node_indexes_ref( - env, env->NewIntArray(java_size)); - base::android::ScopedLocalJavaRef flattens_ref( - env, env->NewBooleanArray(java_size)); base::android::ScopedLocalJavaRef has_bounds_ref( env, env->NewBooleanArray(java_size)); base::android::ScopedLocalJavaRef has_sticky_ref( @@ -822,10 +814,6 @@ void PaintingContextAndroid::RecordInitialLynxUITreeForReplay( parent_signs.data()); env->SetIntArrayRegion(child_indexes_ref.Get(), 0, java_size, child_indexes.data()); - env->SetIntArrayRegion(node_indexes_ref.Get(), 0, java_size, - node_indexes.data()); - env->SetBooleanArrayRegion(flattens_ref.Get(), 0, java_size, - flattens.data()); env->SetBooleanArrayRegion(has_bounds_ref.Get(), 0, java_size, has_bounds.data()); env->SetBooleanArrayRegion(has_sticky_ref.Get(), 0, java_size, @@ -836,9 +824,8 @@ void PaintingContextAndroid::RecordInitialLynxUITreeForReplay( Java_PaintingContext_recordInitialTreeForReplay( env, local_ref.Get(), signs_ref.Get(), tag_names.Get(), bundles.Get(), - styles.Get(), flattens_ref.Get(), node_indexes_ref.Get(), - parent_signs_ref.Get(), child_indexes_ref.Get(), layouts_ref.Get(), - has_bounds_ref.Get(), has_sticky_ref.Get()); + styles.Get(), parent_signs_ref.Get(), child_indexes_ref.Get(), + layouts_ref.Get(), has_bounds_ref.Get(), has_sticky_ref.Get()); }); } diff --git a/platform/android/lynx_android/src/android_test/java/com/lynx/tasm/recording/LynxFrameRecorderTest.java b/platform/android/lynx_android/src/android_test/java/com/lynx/tasm/recording/LynxFrameRecorderTest.java index 5a15c0f993..962bc9bf6c 100644 --- a/platform/android/lynx_android/src/android_test/java/com/lynx/tasm/recording/LynxFrameRecorderTest.java +++ b/platform/android/lynx_android/src/android_test/java/com/lynx/tasm/recording/LynxFrameRecorderTest.java @@ -4,9 +4,13 @@ package com.lynx.tasm.recording; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import com.lynx.react.bridge.JavaOnlyMap; +import com.lynx.tasm.behavior.ui.PropBundle; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -19,6 +23,8 @@ public class LynxFrameRecorderTest { private static final AtomicInteger sInstanceIds = new AtomicInteger(900000); + private static final String LYNX_BYTE_REPLAY_MASK_PROP = "lynx-data-byte" + + "replay-mask"; @Test public void pendingUIOperationsBeforeInitialTreeAreIgnored() throws Exception { @@ -33,13 +39,12 @@ public void pendingUIOperationsBeforeInitialTreeAreIgnored() throws Exception { try { recorder.startRecording(instanceId); - recorder.recordCreateNode(instanceId, 1, "old-view", null, null, false, 1); + recorder.recordCreateNode(instanceId, 1, "old-view", null, null); recorder.recordInitialTree(instanceId, new int[] {2}, new String[] {"view"}, - new Object[] {null}, new Object[] {null}, new boolean[] {false}, new int[] {2}, - new int[] {-1}, new int[] {-1}, new float[25], new boolean[] {false}, - new boolean[] {false}); + new Object[] {null}, new Object[] {null}, new int[] {-1}, new int[] {-1}, new float[25], + new boolean[] {false}, new boolean[] {false}, null); recorder.recordUpdateLayout( - instanceId, 3, 0, 0, 10, 10, new float[4], new float[4], new float[4], null, null, 0, 3); + instanceId, 3, 0, 0, 10, 10, new float[4], new float[4], new float[4], null, null, 0); recorder.stopRecording(instanceId); assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); @@ -52,6 +57,441 @@ public void pendingUIOperationsBeforeInitialTreeAreIgnored() throws Exception { } } + @Test + public void initialTreeRecordsTextExtraData() throws Exception { + int instanceId = sInstanceIds.incrementAndGet(); + LynxFrameRecorder recorder = LynxFrameRecorder.inst(); + CountDownLatch frameLatch = new CountDownLatch(1); + List frames = Collections.synchronizedList(new ArrayList<>()); + recorder.setFrameCallback(instanceId, frame -> { + frames.add(frame); + frameLatch.countDown(); + }); + + try { + recorder.startRecording(instanceId); + recorder.recordInitialTree(instanceId, new int[] {7}, new String[] {"text"}, + new Object[] {null}, new Object[] {null}, new int[] {-1}, new int[] {-1}, new float[25], + new boolean[] {false}, new boolean[] {false}, new String[] {"hello"}); + + assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); + JSONObject payload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_EXTRA_DATA, 7); + assertEquals("text", payload.getString("type")); + assertEquals("hello", payload.getString("text")); + } finally { + recorder.stopRecording(instanceId); + recorder.clearFrameCallback(instanceId); + recorder.clearFrames(instanceId); + } + } + + @Test + public void maskTrueMasksDescendantText() throws Exception { + int instanceId = sInstanceIds.incrementAndGet(); + LynxFrameRecorder recorder = LynxFrameRecorder.inst(); + CountDownLatch frameLatch = new CountDownLatch(1); + List frames = Collections.synchronizedList(new ArrayList<>()); + recorder.setFrameCallback(instanceId, frame -> { + frames.add(frame); + frameLatch.countDown(); + }); + + try { + recorder.startRecording(instanceId); + recorder.recordInitialTree(instanceId, new int[] {10, 11}, new String[] {"view", "text"}, + new Object[] {propBundle(LYNX_BYTE_REPLAY_MASK_PROP, "true"), null}, + new Object[] {null, null}, new int[] {-1, 10}, new int[] {-1, 0}, new float[50], + new boolean[] {false, false}, new boolean[] {false, false}, + new String[] {null, "secret"}); + + assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); + JSONObject textPayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_EXTRA_DATA, 11); + assertEquals("******", textPayload.getString("text")); + + JSONObject createPayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_CREATE_NODE, 10); + JSONObject props = createPayload.getJSONObject("bundle").getJSONObject("props"); + assertEquals("true", props.getString(LYNX_BYTE_REPLAY_MASK_PROP)); + } finally { + recorder.stopRecording(instanceId); + recorder.clearFrameCallback(instanceId); + recorder.clearFrames(instanceId); + } + } + + @Test + public void maskNameMasksOnlyCurrentNodeText() throws Exception { + int instanceId = sInstanceIds.incrementAndGet(); + LynxFrameRecorder recorder = LynxFrameRecorder.inst(); + CountDownLatch frameLatch = new CountDownLatch(1); + List frames = Collections.synchronizedList(new ArrayList<>()); + recorder.setFrameCallback(instanceId, frame -> { + frames.add(frame); + frameLatch.countDown(); + }); + + try { + recorder.startRecording(instanceId); + recorder.recordInitialTree(instanceId, new int[] {20, 21}, new String[] {"text", "text"}, + new Object[] {propBundle(LYNX_BYTE_REPLAY_MASK_PROP, "name"), null}, + new Object[] {null, null}, new int[] {-1, 20}, new int[] {-1, 0}, new float[50], + new boolean[] {false, false}, new boolean[] {false, false}, + new String[] {"root-secret", "child-secret"}); + + assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); + JSONObject rootText = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_EXTRA_DATA, 20); + JSONObject childText = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_EXTRA_DATA, 21); + assertEquals("***********", rootText.getString("text")); + assertEquals("child-secret", childText.getString("text")); + + JSONObject createPayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_CREATE_NODE, 20); + JSONObject props = createPayload.getJSONObject("bundle").getJSONObject("props"); + assertEquals("name", props.getString(LYNX_BYTE_REPLAY_MASK_PROP)); + } finally { + recorder.stopRecording(instanceId); + recorder.clearFrameCallback(instanceId); + recorder.clearFrames(instanceId); + } + } + + @Test + public void maskIgnoreKeepsOnlyRootBox() throws Exception { + int instanceId = sInstanceIds.incrementAndGet(); + LynxFrameRecorder recorder = LynxFrameRecorder.inst(); + CountDownLatch frameLatch = new CountDownLatch(1); + List frames = Collections.synchronizedList(new ArrayList<>()); + recorder.setFrameCallback(instanceId, frame -> { + frames.add(frame); + frameLatch.countDown(); + }); + + try { + recorder.startRecording(instanceId); + recorder.recordInitialTree(instanceId, new int[] {30, 31}, new String[] {"text", "text"}, + new Object[] {propBundle(LYNX_BYTE_REPLAY_MASK_PROP, "ignore", "class", "masked-box", + "value", "root-secret", "src", "https://example.com/private.png"), + null}, + new Object[] {null, null}, new int[] {-1, 30}, new int[] {-1, 0}, new float[50], + new boolean[] {false, false}, new boolean[] {false, false}, + new String[] {"root-secret", "child-secret"}); + + assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); + JSONObject createPayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_CREATE_NODE, 30); + assertEquals("view", createPayload.getString("tagName")); + JSONObject props = createPayload.getJSONObject("bundle").getJSONObject("props"); + assertEquals("ignore", props.getString(LYNX_BYTE_REPLAY_MASK_PROP)); + assertEquals("masked-box", props.getString("class")); + assertEquals(mask("root-secret"), props.getString("value")); + assertEquals(mask("https://example.com/private.png"), props.getString("src")); + assertTrue(containsOperation(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_LAYOUT, 30)); + assertFalse(containsOperation(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_PROPS, 30)); + assertFalse(containsOperationSign(frames, 31)); + } finally { + recorder.stopRecording(instanceId); + recorder.clearFrameCallback(instanceId); + recorder.clearFrames(instanceId); + } + } + + @Test + public void maskStrictMasksTextAndFiltersMedia() throws Exception { + int instanceId = sInstanceIds.incrementAndGet(); + LynxFrameRecorder recorder = LynxFrameRecorder.inst(); + CountDownLatch frameLatch = new CountDownLatch(1); + List frames = Collections.synchronizedList(new ArrayList<>()); + recorder.setFrameCallback(instanceId, frame -> { + frames.add(frame); + frameLatch.countDown(); + }); + + try { + recorder.startRecording(instanceId); + recorder.recordInitialTree(instanceId, new int[] {40, 41, 42}, + new String[] {"view", "text", "image"}, + new Object[] {propBundle(LYNX_BYTE_REPLAY_MASK_PROP, "strict"), null, + propBundle("src", "https://example.com/private.png", "mode", "aspectFit", "value", + "secret-media")}, + new Object[] {null, null, null}, new int[] {-1, 40, 40}, new int[] {-1, 0, 1}, + new float[75], new boolean[] {false, false, false}, new boolean[] {false, false, false}, + new String[] {null, "secret", null}); + + assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); + JSONObject textPayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_EXTRA_DATA, 41); + assertEquals("******", textPayload.getString("text")); + + JSONObject rootCreate = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_CREATE_NODE, 40); + JSONObject props = rootCreate.getJSONObject("bundle").getJSONObject("props"); + assertEquals("strict", props.getString(LYNX_BYTE_REPLAY_MASK_PROP)); + + JSONObject imageCreate = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_CREATE_NODE, 42); + assertEquals("view", imageCreate.getString("tagName")); + JSONObject imageProps = imageCreate.getJSONObject("bundle").getJSONObject("props"); + assertEquals("aspectFit", imageProps.getString("mode")); + assertEquals(mask("https://example.com/private.png"), imageProps.getString("src")); + assertEquals(mask("secret-media"), imageProps.getString("value")); + assertFalse(containsOperation(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_PROPS, 42)); + assertTrue(containsOperation(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_LAYOUT, 42)); + } finally { + recorder.stopRecording(instanceId); + recorder.clearFrameCallback(instanceId); + recorder.clearFrames(instanceId); + } + } + + @Test + public void maskIgnoreBoxRecordsSanitizedPropUpdates() throws Exception { + int instanceId = sInstanceIds.incrementAndGet(); + LynxFrameRecorder recorder = LynxFrameRecorder.inst(); + CountDownLatch frameLatch = new CountDownLatch(2); + List frames = Collections.synchronizedList(new ArrayList<>()); + recorder.setFrameCallback(instanceId, frame -> { + frames.add(frame); + frameLatch.countDown(); + }); + + try { + recorder.startRecording(instanceId); + recorder.recordInitialTree(instanceId, new int[] {45}, new String[] {"view"}, + new Object[] {propBundle(LYNX_BYTE_REPLAY_MASK_PROP, "ignore", "class", "initial")}, + new Object[] {null}, new int[] {-1}, new int[] {-1}, new float[25], new boolean[] {false}, + new boolean[] {false}, null); + recorder.recordUpdateProps(instanceId, 45, false, + propBundle( + "class", "updated", "value", "next-secret", "src", "https://example.com/updated.png"), + null); + recorder.stopRecording(instanceId); + + assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); + JSONObject updateProps = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_PROPS, 45); + JSONObject props = updateProps.getJSONObject("bundle").getJSONObject("props"); + assertEquals("updated", props.getString("class")); + assertEquals(mask("next-secret"), props.getString("value")); + assertEquals(mask("https://example.com/updated.png"), props.getString("src")); + } finally { + recorder.clearFrameCallback(instanceId); + recorder.clearFrames(instanceId); + } + } + + @Test + public void passwordInputDoesNotRecordInputValue() throws Exception { + int instanceId = sInstanceIds.incrementAndGet(); + LynxFrameRecorder recorder = LynxFrameRecorder.inst(); + CountDownLatch frameLatch = new CountDownLatch(2); + List frames = Collections.synchronizedList(new ArrayList<>()); + recorder.setFrameCallback(instanceId, frame -> { + frames.add(frame); + frameLatch.countDown(); + }); + + try { + recorder.startRecording(instanceId); + recorder.recordInitialTree(instanceId, new int[] {50}, new String[] {"input"}, + new Object[] {propBundle("type", "password", "value", "secret")}, new Object[] {null}, + new int[] {-1}, new int[] {-1}, new float[25], new boolean[] {false}, + new boolean[] {false}, null); + JavaOnlyMap params = new JavaOnlyMap(); + params.putString("value", "next-secret"); + params.putInt("cursor", 1); + recorder.recordInvoke(instanceId, 50, "setValue", params); + recorder.recordInputValue(instanceId, 50, "typed-secret", 2, 2); + recorder.stopRecording(instanceId); + + assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); + JSONObject updateProps = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_PROPS, 50); + JSONObject props = updateProps.getJSONObject("bundle").getJSONObject("props"); + assertEquals("******", props.getString("value")); + assertEquals("password", props.getString("type")); + + JSONObject invokePayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_INVOKE, 50); + JSONObject invokeParams = invokePayload.getJSONObject("params"); + assertEquals("***********", invokeParams.getString("value")); + assertEquals(1, invokeParams.getInt("cursor")); + + JSONObject inputValuePayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_INPUT_VALUE, 50); + assertEquals("************", inputValuePayload.getString("value")); + assertEquals(2, inputValuePayload.getInt("selectionStart")); + assertEquals(2, inputValuePayload.getInt("selectionEnd")); + } finally { + recorder.clearFrameCallback(instanceId); + recorder.clearFrames(instanceId); + } + } + + @Test + public void inputValueRecordsTextAndSelection() throws Exception { + int instanceId = sInstanceIds.incrementAndGet(); + LynxFrameRecorder recorder = LynxFrameRecorder.inst(); + CountDownLatch frameLatch = new CountDownLatch(2); + List frames = Collections.synchronizedList(new ArrayList<>()); + recorder.setFrameCallback(instanceId, frame -> { + frames.add(frame); + frameLatch.countDown(); + }); + + try { + recorder.startRecording(instanceId); + recorder.recordInitialTree(instanceId, new int[] {51}, new String[] {"input"}, + new Object[] {propBundle("type", "text")}, new Object[] {null}, new int[] {-1}, + new int[] {-1}, new float[25], new boolean[] {false}, new boolean[] {false}, null); + recorder.recordInputValue(instanceId, 51, "hello", 5, 5); + recorder.stopRecording(instanceId); + + assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); + JSONObject inputValuePayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_INPUT_VALUE, 51); + assertEquals("hello", inputValuePayload.getString("value")); + assertEquals(5, inputValuePayload.getInt("selectionStart")); + assertEquals(5, inputValuePayload.getInt("selectionEnd")); + } finally { + recorder.clearFrameCallback(instanceId); + recorder.clearFrames(instanceId); + } + } + + @Test + public void ignoredSubtreeTouchTargetMapsToVisibleBox() throws Exception { + int instanceId = sInstanceIds.incrementAndGet(); + LynxFrameRecorder recorder = LynxFrameRecorder.inst(); + CountDownLatch frameLatch = new CountDownLatch(2); + List frames = Collections.synchronizedList(new ArrayList<>()); + recorder.setFrameCallback(instanceId, frame -> { + frames.add(frame); + frameLatch.countDown(); + }); + + try { + recorder.startRecording(instanceId); + recorder.recordInitialTree(instanceId, new int[] {60, 61}, new String[] {"view", "text"}, + new Object[] {propBundle(LYNX_BYTE_REPLAY_MASK_PROP, "ignore"), null}, + new Object[] {null, null}, new int[] {-1, 60}, new int[] {-1, 0}, new float[50], + new boolean[] {false, false}, new boolean[] {false, false}, + new String[] {null, "secret"}); + List points = new ArrayList<>(); + points.add(new LynxFrameRecorder.TouchPointPayload(1, 2, 0, 61)); + recorder.recordTouch(instanceId, LynxFrameRecorder.EVENT_LYNX_TOUCH_START, points, 123); + + assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); + assertEquals(60, findFirstTouchTarget(frames)); + } finally { + recorder.stopRecording(instanceId); + recorder.clearFrameCallback(instanceId); + recorder.clearFrames(instanceId); + } + } + + @Test + public void uiOperationPayloadOmitsUnusedFields() throws Exception { + int instanceId = sInstanceIds.incrementAndGet(); + LynxFrameRecorder recorder = LynxFrameRecorder.inst(); + CountDownLatch frameLatch = new CountDownLatch(2); + List frames = Collections.synchronizedList(new ArrayList<>()); + recorder.setFrameCallback(instanceId, frame -> { + frames.add(frame); + frameLatch.countDown(); + }); + + try { + recorder.startRecording(instanceId); + recorder.recordInitialTree(instanceId, new int[] {8}, new String[] {"view"}, + new Object[] {null}, new Object[] {null}, new int[] {-1}, new int[] {-1}, new float[25], + new boolean[] {false}, new boolean[] {false}, null); + recorder.recordInvoke(instanceId, 8, "noop", null); + recorder.stopRecording(instanceId); + + assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); + JSONObject createPayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_CREATE_NODE, 8); + assertFalse(createPayload.has("isFlatten")); + assertFalse(createPayload.has("nodeIndex")); + + JSONObject layoutPayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_LAYOUT, 8); + assertFalse(layoutPayload.has("nodeIndex")); + + JSONObject updatePropsPayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_UPDATE_PROPS, 8); + assertFalse(updatePropsPayload.has("tendToFlatten")); + + JSONObject invokePayload = + findOperationPayload(frames, LynxFrameRecorder.EVENT_LYNX_UI_INVOKE, 8); + assertFalse(invokePayload.has("context")); + assertFalse(invokePayload.has("callback")); + } finally { + recorder.clearFrameCallback(instanceId); + recorder.clearFrames(instanceId); + } + } + + @Test + public void uiFramePayloadIncludesScreenDensity() throws Exception { + int instanceId = sInstanceIds.incrementAndGet(); + LynxFrameRecorder recorder = LynxFrameRecorder.inst(); + CountDownLatch frameLatch = new CountDownLatch(1); + List frames = Collections.synchronizedList(new ArrayList<>()); + recorder.setFrameCallback(instanceId, frame -> { + frames.add(frame); + frameLatch.countDown(); + }); + + try { + recorder.startRecording(instanceId, 3.f); + recorder.recordInitialTree(instanceId, new int[] {9}, new String[] {"view"}, + new Object[] {null}, new Object[] {null}, new int[] {-1}, new int[] {-1}, new float[25], + new boolean[] {false}, new boolean[] {false}, null); + + assertTrue(frameLatch.await(5, TimeUnit.SECONDS)); + JSONObject data = frames.get(0).getJSONObject("event").getJSONObject("data"); + assertEquals(3.0, data.getDouble("screenDensity"), 0.001); + } finally { + recorder.stopRecording(instanceId); + recorder.clearFrameCallback(instanceId); + recorder.clearFrames(instanceId); + } + } + + private static PropBundle propBundle(String... keyValues) throws Exception { + Method create = PropBundle.class.getDeclaredMethod("createPropBundle"); + create.setAccessible(true); + PropBundle bundle = (PropBundle) create.invoke(null); + for (int i = 0; i + 1 < keyValues.length; i += 2) { + bundle.putString(keyValues[i], keyValues[i + 1]); + } + return bundle; + } + + private static String mask(String value) { + int length = value.codePointCount(0, value.length()); + StringBuilder result = new StringBuilder(length); + for (int index = 0; index < length; ++index) { + result.append("*"); + } + return result.toString(); + } + + private static boolean containsOperation(List frames, int eventType, int sign) + throws Exception { + try { + findOperationPayload(frames, eventType, sign); + return true; + } catch (AssertionError error) { + return false; + } + } + private static boolean containsOperationSign(List frames, int sign) throws Exception { synchronized (frames) { for (JSONObject frame : frames) { @@ -74,4 +514,47 @@ private static boolean containsOperationSign(List frames, int sign) } return false; } + + private static JSONObject findOperationPayload(List frames, int eventType, int sign) + throws Exception { + synchronized (frames) { + for (JSONObject frame : frames) { + JSONObject event = frame.getJSONObject("event"); + JSONObject data = event.optJSONObject("data"); + if (data == null) { + continue; + } + JSONArray operations = data.optJSONArray("operations"); + if (operations == null) { + continue; + } + for (int operationIndex = 0; operationIndex < operations.length(); ++operationIndex) { + JSONObject operation = operations.getJSONObject(operationIndex); + if (operation.optInt("eventType", Integer.MIN_VALUE) != eventType) { + continue; + } + JSONObject payload = operation.optJSONObject("data"); + if (payload != null && payload.optInt("sign", Integer.MIN_VALUE) == sign) { + return payload; + } + } + } + } + throw new AssertionError("operation not found, eventType=" + eventType + ", sign=" + sign); + } + + private static int findFirstTouchTarget(List frames) throws Exception { + synchronized (frames) { + for (JSONObject frame : frames) { + JSONObject event = frame.getJSONObject("event"); + if (event.optInt("eventType", Integer.MIN_VALUE) + != LynxFrameRecorder.EVENT_LYNX_TOUCH_START) { + continue; + } + JSONArray points = event.getJSONObject("data").getJSONArray("points"); + return points.getJSONObject(0).getInt("targetId"); + } + } + throw new AssertionError("touch frame not found"); + } } diff --git a/platform/android/lynx_android/src/main/java/com/lynx/tasm/NativeFacade.java b/platform/android/lynx_android/src/main/java/com/lynx/tasm/NativeFacade.java index b23ffeae57..29490ca057 100644 --- a/platform/android/lynx_android/src/main/java/com/lynx/tasm/NativeFacade.java +++ b/platform/android/lynx_android/src/main/java/com/lynx/tasm/NativeFacade.java @@ -5,6 +5,7 @@ import android.os.Bundle; import android.text.TextUtils; +import android.util.DisplayMetrics; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import com.lynx.BuildConfig; @@ -551,8 +552,12 @@ public void invoke(Object... args) { } }; // ui_result is already checked in lynx engine - context.getLynxUIOwner().invokeUIMethodForSelectorQuery( - ui_result.getUiArray().getInt(0), method, params, callback); + int sign = ui_result.getUiArray().getInt(0); + int instanceId = context.getInstanceId(); + if (LynxFrameRecorder.inst().isRecordingEnabled(instanceId)) { + LynxFrameRecorder.inst().recordInvoke(instanceId, sign, method, params); + } + context.getLynxUIOwner().invokeUIMethodForSelectorQuery(sign, method, params, callback); } } @@ -566,11 +571,17 @@ public int getInstanceId() { @CalledByNative private void startRecording(ReadableMap params) { - int instanceId = getInstanceId(); + LynxContext context = mLynxContext != null ? mLynxContext.get() : null; + if (context == null) { + return; + } + int instanceId = context.getInstanceId(); if (instanceId == LynxContext.INSTANCE_ID_DEFAULT) { return; } - LynxFrameRecorder.inst().startRecording(instanceId); + DisplayMetrics screenMetrics = context.getScreenMetrics(); + float screenDensity = screenMetrics != null ? screenMetrics.density : 0.f; + LynxFrameRecorder.inst().startRecording(instanceId, screenDensity); } @CalledByNative diff --git a/platform/android/lynx_android/src/main/java/com/lynx/tasm/behavior/PaintingContext.java b/platform/android/lynx_android/src/main/java/com/lynx/tasm/behavior/PaintingContext.java index 352659a7dd..249fb5120c 100644 --- a/platform/android/lynx_android/src/main/java/com/lynx/tasm/behavior/PaintingContext.java +++ b/platform/android/lynx_android/src/main/java/com/lynx/tasm/behavior/PaintingContext.java @@ -7,6 +7,7 @@ import android.graphics.PointF; import android.graphics.Rect; import android.os.Build; +import android.text.Layout; import android.util.DisplayMetrics; import android.view.Display; import android.view.MotionEvent; @@ -26,9 +27,12 @@ import com.lynx.tasm.base.trace.TraceEventDef; import com.lynx.tasm.behavior.shadow.ShadowNodeType; import com.lynx.tasm.behavior.shadow.TextLayout; +import com.lynx.tasm.behavior.shadow.text.TextUpdateBundle; import com.lynx.tasm.behavior.ui.LynxBaseUI; import com.lynx.tasm.behavior.ui.PropBundle; import com.lynx.tasm.behavior.ui.list.container.UIListContainer; +import com.lynx.tasm.behavior.ui.text.FlattenUIText; +import com.lynx.tasm.behavior.ui.text.UIText; import com.lynx.tasm.behavior.ui.view.UIComponent; import com.lynx.tasm.behavior.utils.LynxUIMethodsExecutor; import com.lynx.tasm.event.EventsListener; @@ -92,6 +96,7 @@ public final class PaintingContext implements IPaintingContext { private static final String TAG = "lynx_PaintingContext"; private static final int STICKY_INFO_COUNT = 10; private static final long UI_THREAD_QUERY_TIMEOUT_MS = 1000; + private static final String RECORDER_TEXT_EXTRA_DATA_TYPE = "text"; private final LynxUIOwner mUIOwner; private TextLayout mTextLayout; @@ -130,13 +135,13 @@ private boolean isRecordingEnabled() { return LynxFrameRecorder.inst().isRecordingEnabled(getInstanceId()); } - private void recordCreateNode(int sign, String tagName, PropBundle bundle, - ReadableMapBuffer initialStyles, boolean isFlatten, int nodeIndex) { + private void recordCreateNode( + int sign, String tagName, PropBundle bundle, ReadableMapBuffer initialStyles) { if (!isRecordingEnabled()) { return; } LynxFrameRecorder.inst().recordCreateNode( - getInstanceId(), sign, tagName, bundle, initialStyles, isFlatten, nodeIndex); + getInstanceId(), sign, tagName, bundle, initialStyles); } private void recordNodeRelation(int eventType, int parentSign, int childSign, Integer index) { @@ -160,7 +165,7 @@ private void recordUpdateLayout(int sign, float x, float y, float width, float h float paddingLeft, float paddingTop, float paddingRight, float paddingBottom, float marginLeft, float marginTop, float marginRight, float marginBottom, float borderLeftWidth, float borderTopWidth, float borderRightWidth, float borderBottomWidth, - Rect bounds, float[] sticky, float maxHeight, int nodeIndex) { + Rect bounds, float[] sticky, float maxHeight) { if (!isRecordingEnabled()) { return; } @@ -169,27 +174,113 @@ private void recordUpdateLayout(int sign, float x, float y, float width, float h new float[] {marginLeft, marginTop, marginRight, marginBottom}, new float[] {borderLeftWidth, borderTopWidth, borderRightWidth, borderBottomWidth}, bounds != null ? new float[] {bounds.left, bounds.top, bounds.right, bounds.bottom} : null, - sticky, maxHeight, nodeIndex); + sticky, maxHeight); } - private void recordInvoke( - int sign, String method, ReadableMap params, long context, int callback) { + private void recordInvoke(int sign, String method, ReadableMap params) { if (!isRecordingEnabled()) { return; } - LynxFrameRecorder.inst().recordInvoke(getInstanceId(), sign, method, params, context, callback); + LynxFrameRecorder.inst().recordInvoke(getInstanceId(), sign, method, params); + } + + private void recordTextExtraData(int sign, Object data) { + if (!isRecordingEnabled()) { + return; + } + String text = snapshotTextExtraData(data); + if (text == null) { + return; + } + LynxFrameRecorder.inst().recordUpdateExtraData( + getInstanceId(), sign, RECORDER_TEXT_EXTRA_DATA_TYPE, text); } @CalledByNative private void recordInitialTreeForReplay(int[] signs, String[] tagNames, Object[] bundles, - Object[] initialStyles, boolean[] isFlattens, int[] nodeIndexes, int[] parentSigns, - int[] childIndexes, float[] layouts, boolean[] hasBounds, boolean[] hasSticky) { + Object[] initialStyles, int[] parentSigns, int[] childIndexes, float[] layouts, + boolean[] hasBounds, boolean[] hasSticky) { if (!isRecordingEnabled()) { return; } + String[] textContents = snapshotInitialTreeTextContents(signs); LynxFrameRecorder.inst().recordInitialTree(getInstanceId(), signs, tagNames, bundles, - initialStyles, isFlattens, nodeIndexes, parentSigns, childIndexes, layouts, hasBounds, - hasSticky); + initialStyles, parentSigns, childIndexes, layouts, hasBounds, hasSticky, textContents); + } + + private String[] snapshotInitialTreeTextContents(int[] signs) { + if (signs == null || signs.length == 0) { + return null; + } + String[] textContents = null; + for (int i = 0; i < signs.length; ++i) { + String text = snapshotTextFromUI(signs[i]); + if (text == null) { + continue; + } + if (textContents == null) { + textContents = new String[signs.length]; + } + textContents[i] = text; + } + return textContents; + } + + private String snapshotTextFromUI(int sign) { + String text = snapshotTextPage(mTextraPages.get(sign)); + if (text != null) { + return text; + } + LynxBaseUI ui = mUIOwner != null ? mUIOwner.getNode(sign) : null; + if (ui instanceof UIText) { + return charSequenceToString(((UIText) ui).getOriginText()); + } + if (ui instanceof FlattenUIText) { + return charSequenceToString(((FlattenUIText) ui).getOriginText()); + } + return null; + } + + private String snapshotTextExtraData(Object data) { + if (data instanceof TextUpdateBundle) { + return snapshotTextUpdateBundle((TextUpdateBundle) data); + } + if (data instanceof Page) { + return snapshotTextPage((Page) data); + } + return null; + } + + private String snapshotTextUpdateBundle(TextUpdateBundle bundle) { + if (bundle == null) { + return null; + } + CharSequence text = bundle.getOriginText(); + if (text == null) { + Layout layout = bundle.getTextLayout(); + text = layout != null ? layout.getText() : null; + } + return charSequenceToString(text); + } + + private String snapshotTextPage(Page page) { + if (page == null) { + return null; + } + try { + int length = page.getTextLength(); + if (length < 0) { + return null; + } + return page.getSelectedText(0, length); + } catch (RuntimeException e) { + LLog.e(TAG, "record text page failed: " + e); + return null; + } + } + + private String charSequenceToString(CharSequence text) { + return text != null ? text.toString() : null; } // this func will be execed on main thread. @@ -313,7 +404,7 @@ public Object createNode(final int sign, String tagName, final PropBundle bundle ReadableMap initialProps = bundle != null ? bundle.getProps() : null; ReadableArray eventListeners = bundle != null ? bundle.getEventHandlers() : null; ReadableArray gestureDetectors = bundle != null ? bundle.getGestures() : null; - recordCreateNode(sign, finalTagName, bundle, initialStyles, isFlatten, nodeIndex); + recordCreateNode(sign, finalTagName, bundle, initialStyles); final Future future = createNodeAsync(sign, finalTagName, initialProps, initialStyles, eventListeners, isFlatten, nodeIndex, gestureDetectors); return new Runnable() { @@ -369,7 +460,7 @@ private boolean needProcessDirection(String tagName) { @CalledByNative public void createPaintingNodeSync(int sign, String tagName, PropBundle bundle, ReadableMapBuffer initialStyles, boolean isFlatten, int nodeIndex) { - recordCreateNode(sign, tagName, bundle, initialStyles, isFlatten, nodeIndex); + recordCreateNode(sign, tagName, bundle, initialStyles); ReadableMap initialProps = bundle != null ? bundle.getProps() : null; ReadableArray eventListeners = bundle != null ? bundle.getEventHandlers() : null; ReadableArray gestureDetectors = bundle != null ? bundle.getGestures() : null; @@ -398,7 +489,7 @@ public Future createNodeAsync(int sign, String tagName, ReadableMap in @CalledByNative public Object createPaintingNodeAsync(int sign, String tagName, PropBundle bundle, ReadableMapBuffer initialStyles, boolean isFlatten, int nodeIndex) { - recordCreateNode(sign, tagName, bundle, initialStyles, isFlatten, nodeIndex); + recordCreateNode(sign, tagName, bundle, initialStyles); ReadableMap initialProps = bundle != null ? bundle.getProps() : null; ReadableArray eventListeners = bundle != null ? bundle.getEventHandlers() : null; ReadableArray gestureDetectors = bundle != null ? bundle.getGestures() : null; @@ -650,6 +741,7 @@ public void FinishLayoutOperation(int componentId, long operationId, boolean isF @CalledByNative public void updateExtraData(int signature, Object data) { + recordTextExtraData(signature, data); mUIOwner.updateViewExtraData(signature, data); } @@ -665,6 +757,7 @@ public void updateTextBundle(int sign, long textBundle) { return; } + recordTextExtraData(sign, page); Page old = mTextraPages.put(sign, page); if (old != null) { old.destroy(); @@ -836,7 +929,7 @@ private float[] getScrollDefaultResult(float width, float height) { @CalledByNative public void invoke( int sign, String method, ReadableMap params, final long context, final int callback) { - recordInvoke(sign, method, params, context, callback); + recordInvoke(sign, method, params); UIThreadUtils.runOnUiThreadImmediately(new Runnable() { private void cb(Object... args) { if (mDestroyed || mUIOwner.getContext() == null) { @@ -867,7 +960,7 @@ private void setLayoutData(int sign, int x, int y, int width, int height, int pa int nodeIndex) { recordUpdateLayout(sign, x, y, width, height, paddingLeft, paddingTop, paddingRight, paddingBottom, marginLeft, marginTop, marginRight, marginBottom, borderLeftWidth, - borderTopWidth, borderRightWidth, borderBottomWidth, bounds, sticky, maxHeight, nodeIndex); + borderTopWidth, borderRightWidth, borderBottomWidth, bounds, sticky, maxHeight); mUIOwner.updateLayout(sign, x, y, width, height, paddingLeft, paddingTop, paddingRight, paddingBottom, marginLeft, marginTop, marginRight, marginBottom, borderLeftWidth, borderTopWidth, borderRightWidth, borderBottomWidth, bounds, sticky, maxHeight, nodeIndex); diff --git a/platform/android/lynx_android/src/main/java/com/lynx/tasm/recording/LynxFrameRecorder.java b/platform/android/lynx_android/src/main/java/com/lynx/tasm/recording/LynxFrameRecorder.java index 7d6ec83193..fe272d9235 100644 --- a/platform/android/lynx_android/src/main/java/com/lynx/tasm/recording/LynxFrameRecorder.java +++ b/platform/android/lynx_android/src/main/java/com/lynx/tasm/recording/LynxFrameRecorder.java @@ -50,6 +50,8 @@ public final class LynxFrameRecorder { public static final int EVENT_LYNX_UI_UPDATE_PROPS = 106; public static final int EVENT_LYNX_UI_UPDATE_LAYOUT = 107; public static final int EVENT_LYNX_UI_INVOKE = 108; + public static final int EVENT_LYNX_UI_UPDATE_EXTRA_DATA = 109; + public static final int EVENT_LYNX_UI_UPDATE_INPUT_VALUE = 110; public static final int EVENT_LYNX_TOUCH_START = 201; public static final int EVENT_LYNX_TOUCH_MOVE = 202; @@ -69,6 +71,21 @@ public final class LynxFrameRecorder { private static final int INITIAL_TREE_STICKY_INDEX = 20; private static final int INITIAL_TREE_MAX_HEIGHT_INDEX = 24; private static final long NANOS_PER_MILLIS = 1000000L; + private static final String LYNX_BYTE_REPLAY_MASK_PROP = "lynx-data-byte" + + "replay-mask"; + private static final String MASK_VALUE_TRUE = "true"; + private static final String MASK_VALUE_IGNORE = "ignore"; + private static final String MASK_VALUE_NAME = "name"; + private static final String MASK_VALUE_STRICT = "strict"; + private static final String MASKED_TEXT = "*"; + private static final String TEXT_EXTRA_DATA_TYPE = "text"; + private static final String INPUT_TAG = "input"; + private static final String X_INPUT_TAG = "x-input"; + private static final String TEXTAREA_TAG = "textarea"; + private static final String X_TEXTAREA_TAG = "x-textarea"; + private static final String INPUT_TYPE_PROP = "type"; + private static final String INPUT_PASSWORD_TYPE = "password"; + private static final String VIEW_TAG = "view"; private static final LynxFrameRecorder sInstance = new LynxFrameRecorder(); @@ -81,7 +98,9 @@ public final class LynxFrameRecorder { private final ConcurrentHashMap mFrames = new ConcurrentHashMap<>(); private final ConcurrentHashMap mFrameCallbacks = new ConcurrentHashMap<>(); + private final ConcurrentHashMap mScreenDensities = new ConcurrentHashMap<>(); private final Map mPendingUIFrames = new HashMap<>(); + private final Map mTreeStates = new HashMap<>(); private final AtomicInteger mNextSessionId = new AtomicInteger(1); private LynxFrameRecorder() {} @@ -107,9 +126,18 @@ public void clearFrameCallback(int instanceId) { } public void startRecording(int instanceId) { + startRecording(instanceId, 0.f); + } + + public void startRecording(int instanceId, float screenDensity) { if (instanceId < 0) { return; } + if (screenDensity > 0.f) { + mScreenDensities.put(instanceId, screenDensity); + } else { + mScreenDensities.remove(instanceId); + } int sessionId = mNextSessionId.getAndIncrement(); mInitialTreePendingSessions.put(instanceId, sessionId); mRecordingSessions.put(instanceId, sessionId); @@ -119,6 +147,7 @@ public void startRecording(int instanceId) { if (!isRecordingSession(instanceId, sessionId)) { return; } + mTreeStates.put(instanceId, new RecorderTreeState(sessionId)); clearPendingUIFrameOnExecutor(instanceId, null); }); } @@ -130,6 +159,8 @@ public void stopRecording(int instanceId) { Integer sessionId = getActiveRecordingSession(instanceId); if (sessionId == null) { mFrames.remove(instanceId); + mScreenDensities.remove(instanceId); + mExecutor.execute(() -> mTreeStates.remove(instanceId)); return; } mStoppingSessions.put(instanceId, sessionId); @@ -140,19 +171,19 @@ public void stopRecording(int instanceId) { mInitialTreePendingSessions.remove(instanceId, sessionId); if (!mRecordingSessions.containsKey(instanceId)) { mFrames.remove(instanceId); + mScreenDensities.remove(instanceId); + mTreeStates.remove(instanceId); } }); } public void recordCreateNode(int instanceId, int sign, String tagName, PropBundle bundle, - ReadableMapBuffer initialStyles, boolean isFlatten, int nodeIndex) { + ReadableMapBuffer initialStyles) { long opTimeMillis = SystemClock.uptimeMillis(); Map bundleSnapshot = snapshotPropBundle(bundle); Map initialStyleSnapshot = snapshotStyleMapBuffer(initialStyles); recordUIOperation(instanceId, EVENT_LYNX_UI_CREATE_NODE, - new CreateNodePayload( - sign, tagName, bundleSnapshot, initialStyleSnapshot, isFlatten, nodeIndex), - opTimeMillis); + new CreateNodePayload(sign, tagName, bundleSnapshot, initialStyleSnapshot), opTimeMillis); } public void recordNodeRelation( @@ -173,39 +204,57 @@ public void recordUpdateProps(int instanceId, int sign, boolean tendToFlatten, P public void recordUpdateLayout(int instanceId, int sign, float x, float y, float width, float height, float[] paddings, float[] margins, float[] borders, float[] bounds, - float[] sticky, float maxHeight, int nodeIndex) { + float[] sticky, float maxHeight) { long opTimeMillis = SystemClock.uptimeMillis(); recordUIOperation(instanceId, EVENT_LYNX_UI_UPDATE_LAYOUT, new UpdateLayoutPayload(sign, x, y, width, height, copyFloatArray(paddings), copyFloatArray(margins), copyFloatArray(borders), copyFloatArray(bounds), - copyFloatArray(sticky), maxHeight, nodeIndex), + copyFloatArray(sticky), maxHeight), opTimeMillis); } - public void recordInvoke( - int instanceId, int sign, String method, ReadableMap params, long context, int callback) { + public void recordInvoke(int instanceId, int sign, String method, ReadableMap params) { long opTimeMillis = SystemClock.uptimeMillis(); Map paramsSnapshot = snapshotReadableMap(params); recordUIOperation(instanceId, EVENT_LYNX_UI_INVOKE, - new InvokePayload(sign, method, paramsSnapshot, context, callback), opTimeMillis); + new InvokePayload(sign, method, paramsSnapshot), opTimeMillis); + } + + public void recordUpdateExtraData(int instanceId, int sign, String type, String text) { + if (type == null || text == null) { + return; + } + long opTimeMillis = SystemClock.uptimeMillis(); + recordUIOperation(instanceId, EVENT_LYNX_UI_UPDATE_EXTRA_DATA, + new UpdateExtraDataPayload(sign, type, text), opTimeMillis); + } + + public void recordInputValue( + int instanceId, int sign, String value, int selectionStart, int selectionEnd) { + if (value == null) { + return; + } + long opTimeMillis = SystemClock.uptimeMillis(); + recordUIOperation(instanceId, EVENT_LYNX_UI_UPDATE_INPUT_VALUE, + new UpdateInputValuePayload(sign, value, selectionStart, selectionEnd), opTimeMillis); } public void recordInitialTree(int instanceId, int[] signs, String[] tagNames, Object[] bundles, - Object[] initialStyles, boolean[] isFlattens, int[] nodeIndexes, int[] parentSigns, - int[] childIndexes, float[] layouts, boolean[] hasBounds, boolean[] hasSticky) { + Object[] initialStyles, int[] parentSigns, int[] childIndexes, float[] layouts, + boolean[] hasBounds, boolean[] hasSticky, String[] textContents) { Integer sessionId = mRecordingSessions.get(instanceId); if (sessionId == null) { return; } - if (!isValidInitialTree(signs, tagNames, bundles, initialStyles, isFlattens, nodeIndexes, - parentSigns, childIndexes, layouts, hasBounds, hasSticky)) { + if (!isValidInitialTree(signs, tagNames, bundles, initialStyles, parentSigns, childIndexes, + layouts, hasBounds, hasSticky)) { finishInitialTreeRecording(instanceId, sessionId); return; } int count = signs.length; long opTimeMillis = SystemClock.uptimeMillis(); - List operations = new ArrayList<>(count * 4); + List nodeSnapshots = new ArrayList<>(count); for (int index = 0; index < count; index++) { PropBundle bundle = bundles[index] instanceof PropBundle ? (PropBundle) bundles[index] : null; ReadableMapBuffer initialStyle = initialStyles[index] instanceof ReadableMapBuffer @@ -213,44 +262,36 @@ public void recordInitialTree(int instanceId, int[] signs, String[] tagNames, Ob : null; Map bundleSnapshot = snapshotPropBundle(bundle); Map initialStyleSnapshot = snapshotStyleMapBuffer(initialStyle); - operations.add(new UIOperationRecord(EVENT_LYNX_UI_CREATE_NODE, - new CreateNodePayload(signs[index], tagNames[index], bundleSnapshot, initialStyleSnapshot, - isFlattens[index], nodeIndexes[index]), - opTimeMillis)); - if (parentSigns[index] >= 0) { - operations.add(new UIOperationRecord(EVENT_LYNX_UI_INSERT_NODE, - new NodeRelationPayload(parentSigns[index], signs[index], childIndexes[index]), - opTimeMillis)); - } - operations.add(new UIOperationRecord(EVENT_LYNX_UI_UPDATE_PROPS, - new UpdatePropsPayload( - signs[index], isFlattens[index], bundleSnapshot, initialStyleSnapshot), - opTimeMillis)); - int layoutOffset = index * INITIAL_TREE_LAYOUT_STRIDE; - operations.add(new UIOperationRecord(EVENT_LYNX_UI_UPDATE_LAYOUT, - new UpdateLayoutPayload(signs[index], layouts[layoutOffset + INITIAL_TREE_X_INDEX], - layouts[layoutOffset + INITIAL_TREE_Y_INDEX], - layouts[layoutOffset + INITIAL_TREE_WIDTH_INDEX], - layouts[layoutOffset + INITIAL_TREE_HEIGHT_INDEX], - copyFourFloats(layouts, layoutOffset + INITIAL_TREE_PADDING_INDEX), - copyFourFloats(layouts, layoutOffset + INITIAL_TREE_MARGIN_INDEX), - copyFourFloats(layouts, layoutOffset + INITIAL_TREE_BORDER_INDEX), - hasBounds[index] ? copyFourFloats(layouts, layoutOffset + INITIAL_TREE_BOUNDS_INDEX) - : null, - hasSticky[index] ? copyFourFloats(layouts, layoutOffset + INITIAL_TREE_STICKY_INDEX) - : null, - layouts[layoutOffset + INITIAL_TREE_MAX_HEIGHT_INDEX], nodeIndexes[index]), - opTimeMillis)); - } - final List initialTreeOperations = Collections.unmodifiableList(operations); + String textContent = + textContents != null && textContents.length == count ? textContents[index] : null; + nodeSnapshots.add(new InitialNodeSnapshot(signs[index], tagNames[index], bundleSnapshot, + initialStyleSnapshot, parentSigns[index], childIndexes[index], + layouts[layoutOffset + INITIAL_TREE_X_INDEX], + layouts[layoutOffset + INITIAL_TREE_Y_INDEX], + layouts[layoutOffset + INITIAL_TREE_WIDTH_INDEX], + layouts[layoutOffset + INITIAL_TREE_HEIGHT_INDEX], + copyFourFloats(layouts, layoutOffset + INITIAL_TREE_PADDING_INDEX), + copyFourFloats(layouts, layoutOffset + INITIAL_TREE_MARGIN_INDEX), + copyFourFloats(layouts, layoutOffset + INITIAL_TREE_BORDER_INDEX), + hasBounds[index] ? copyFourFloats(layouts, layoutOffset + INITIAL_TREE_BOUNDS_INDEX) + : null, + hasSticky[index] ? copyFourFloats(layouts, layoutOffset + INITIAL_TREE_STICKY_INDEX) + : null, + layouts[layoutOffset + INITIAL_TREE_MAX_HEIGHT_INDEX], textContent)); + } + final List initialNodes = Collections.unmodifiableList(nodeSnapshots); mExecutor.execute(() -> { if (!isRecordingSession(instanceId, sessionId)) { return; } + RecorderTreeState treeState = new RecorderTreeState(sessionId); + List initialTreeOperations = + treeState.buildInitialOperations(initialNodes, opTimeMillis); + mTreeStates.put(instanceId, treeState); if (!initialTreeOperations.isEmpty()) { emitFrame(instanceId, TRACK_TYPE_LYNX_UI_OP, EVENT_LYNX_UI_FRAME, - new UIFramePayload(initialTreeOperations), opTimeMillis); + new UIFramePayload(initialTreeOperations, getScreenDensity(instanceId)), opTimeMillis); } mInitialTreePendingSessions.remove(instanceId, sessionId); }); @@ -260,8 +301,18 @@ public void recordTouch( int instanceId, int eventType, List points, long timestampMillis) { List pointSnapshot = points == null ? null : Collections.unmodifiableList(new ArrayList<>(points)); - recordFrame(instanceId, TRACK_TYPE_LYNX_TOUCH, eventType, new TouchPayload(pointSnapshot), - timestampMillis); + Integer sessionId = getActiveRecordingSession(instanceId); + if (sessionId == null) { + return; + } + mExecutor.execute(() -> { + if (!isRecordingSession(instanceId, sessionId)) { + return; + } + RecorderTreeState treeState = getOrCreateTreeStateOnExecutor(instanceId, sessionId); + emitFrame(instanceId, TRACK_TYPE_LYNX_TOUCH, eventType, + new TouchPayload(treeState.sanitizeTouchPoints(pointSnapshot)), timestampMillis); + }); } private void recordUIOperation(int instanceId, int eventType, Object data, long opTimeMillis) { @@ -274,12 +325,17 @@ private void recordUIOperation(int instanceId, int eventType, Object data, long || isInitialTreeRecordingPending(instanceId, sessionId)) { return; } + RecorderTreeState treeState = getOrCreateTreeStateOnExecutor(instanceId, sessionId); + UIOperationRecord operation = treeState.applyOperation(eventType, data, opTimeMillis); + if (operation == null) { + return; + } PendingUIFrame pendingFrame = mPendingUIFrames.get(instanceId); if (pendingFrame == null || pendingFrame.sessionId != sessionId) { pendingFrame = new PendingUIFrame(sessionId); mPendingUIFrames.put(instanceId, pendingFrame); } - pendingFrame.operations.add(new UIOperationRecord(eventType, data, opTimeMillis)); + pendingFrame.operations.add(operation); if (pendingFrame.flushScheduled) { return; } @@ -324,6 +380,15 @@ public void clearFrames(int instanceId) { }); } + private RecorderTreeState getOrCreateTreeStateOnExecutor(int instanceId, int sessionId) { + RecorderTreeState state = mTreeStates.get(instanceId); + if (state == null || state.sessionId != sessionId) { + state = new RecorderTreeState(sessionId); + mTreeStates.put(instanceId, state); + } + return state; + } + private void requestVSyncFlush(int instanceId, int sessionId, PendingUIFrame pendingFrame) { RequestFlushFrameCallback frameCallback = new RequestFlushFrameCallback(instanceId, sessionId); pendingFrame.frameCallback = frameCallback; @@ -348,7 +413,8 @@ private void flushPendingUIFrame(int instanceId, int sessionId, long frameTimeNa pendingFrame.flushScheduled = false; pendingFrame.frameCallback = null; emitFrame(instanceId, TRACK_TYPE_LYNX_UI_OP, EVENT_LYNX_UI_FRAME, - new UIFramePayload(operations), toUptimeMillis(frameTimeNanos)); + new UIFramePayload(operations, getScreenDensity(instanceId)), + toUptimeMillis(frameTimeNanos)); } private static long toUptimeMillis(long frameTimeNanos) { @@ -586,18 +652,259 @@ private static float[] copyFourFloats(float[] values, int offset) { } private static boolean isValidInitialTree(int[] signs, String[] tagNames, Object[] bundles, - Object[] initialStyles, boolean[] isFlattens, int[] nodeIndexes, int[] parentSigns, - int[] childIndexes, float[] layouts, boolean[] hasBounds, boolean[] hasSticky) { + Object[] initialStyles, int[] parentSigns, int[] childIndexes, float[] layouts, + boolean[] hasBounds, boolean[] hasSticky) { if (signs == null || tagNames == null || bundles == null || initialStyles == null - || isFlattens == null || nodeIndexes == null || parentSigns == null || childIndexes == null - || layouts == null || hasBounds == null || hasSticky == null) { + || parentSigns == null || childIndexes == null || layouts == null || hasBounds == null + || hasSticky == null) { return false; } int count = signs.length; return tagNames.length == count && bundles.length == count && initialStyles.length == count - && isFlattens.length == count && nodeIndexes.length == count && parentSigns.length == count - && childIndexes.length == count && hasBounds.length == count && hasSticky.length == count - && layouts.length >= count * INITIAL_TREE_LAYOUT_STRIDE; + && parentSigns.length == count && childIndexes.length == count && hasBounds.length == count + && hasSticky.length == count && layouts.length >= count * INITIAL_TREE_LAYOUT_STRIDE; + } + + private static Map sanitizeBundleForNode( + Map bundle, NodeRecord node, boolean maskInputContent) { + if (bundle == null) { + return null; + } + Map result = new LinkedHashMap<>(bundle); + Map props = getPropsFromBundle(bundle); + if (props != null) { + result.put("props", sanitizePropsForNode(props, node, maskInputContent)); + } + return Collections.unmodifiableMap(result); + } + + private static Map sanitizeBoxBundleForNode( + Map bundle, NodeRecord node) { + if (bundle == null) { + return createMaskBundle(node == null ? ByteReplayMask.NONE : node.ownMask); + } + Map result = new LinkedHashMap<>(bundle); + Map props = getPropsFromBundle(bundle); + if (props != null) { + result.put("props", sanitizeBoxPropsForNode(props, node)); + } + return Collections.unmodifiableMap(result); + } + + private static Map sanitizePropsForNode( + Map props, NodeRecord node, boolean maskInputContent) { + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : props.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + if ((maskInputContent || (node != null && node.isPasswordInput)) && isInputContentKey(key)) { + result.put(key, maskInputContentValue(value)); + continue; + } + result.put(key, value); + } + return Collections.unmodifiableMap(result); + } + + private static Map sanitizeBoxPropsForNode( + Map props, NodeRecord node) { + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : props.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + if (isBoxSensitivePropKey(key)) { + result.put(key, maskInputContentValue(value)); + continue; + } + result.put(key, value); + } + String maskValue = getMaskValue(node == null ? ByteReplayMask.NONE : node.ownMask); + if (maskValue != null && !result.containsKey(LYNX_BYTE_REPLAY_MASK_PROP)) { + result.put(LYNX_BYTE_REPLAY_MASK_PROP, maskValue); + } + return Collections.unmodifiableMap(result); + } + + private static Map sanitizeInputContentMap(Map values) { + if (values == null) { + return null; + } + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : values.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + if (isInputContentKey(key)) { + result.put(key, maskInputContentValue(value)); + } else { + result.put(key, sanitizeInputContentValue(value)); + } + } + return Collections.unmodifiableMap(result); + } + + private static Object maskInputContentValue(Object value) { + if (value instanceof CharSequence) { + return maskText(String.valueOf(value)); + } + return MASKED_TEXT; + } + + private static String maskText(String text) { + if (text == null) { + return null; + } + int length = text.codePointCount(0, text.length()); + StringBuilder result = new StringBuilder(length); + for (int index = 0; index < length; ++index) { + result.append(MASKED_TEXT); + } + return result.toString(); + } + + private static Map createMaskBundle(ByteReplayMask mask) { + String maskValue = getMaskValue(mask); + if (maskValue == null) { + return null; + } + Map props = new LinkedHashMap<>(); + props.put(LYNX_BYTE_REPLAY_MASK_PROP, maskValue); + Map bundle = new LinkedHashMap<>(); + bundle.put("props", Collections.unmodifiableMap(props)); + return Collections.unmodifiableMap(bundle); + } + + private static String getMaskValue(ByteReplayMask mask) { + switch (mask) { + case TRUE: + return MASK_VALUE_TRUE; + case IGNORE: + return MASK_VALUE_IGNORE; + case NAME: + return MASK_VALUE_NAME; + case STRICT: + return MASK_VALUE_STRICT; + case NONE: + default: + return null; + } + } + + private static Object sanitizeInputContentValue(Object value) { + Map map = toStringObjectMap(value); + if (map != null) { + return sanitizeInputContentMap(map); + } + if (value instanceof Iterable) { + List result = new ArrayList<>(); + for (Object item : (Iterable) value) { + result.add(sanitizeInputContentValue(item)); + } + return Collections.unmodifiableList(result); + } + if (value != null && value.getClass().isArray()) { + List result = new ArrayList<>(); + int length = Array.getLength(value); + for (int i = 0; i < length; i++) { + result.add(sanitizeInputContentValue(Array.get(value, i))); + } + return Collections.unmodifiableList(result); + } + return value; + } + + private static Map getPropsFromBundle(Map bundle) { + return bundle == null ? null : toStringObjectMap(bundle.get("props")); + } + + private static boolean hasMaskKey(Map bundle) { + Map props = getPropsFromBundle(bundle); + return props != null && props.containsKey(LYNX_BYTE_REPLAY_MASK_PROP); + } + + private static ByteReplayMask parseMask(Map bundle) { + Map props = getPropsFromBundle(bundle); + if (props == null) { + return ByteReplayMask.NONE; + } + Object value = props.get(LYNX_BYTE_REPLAY_MASK_PROP); + return parseMaskValue(value); + } + + private static ByteReplayMask parseMaskValue(Object value) { + if (value == null) { + return ByteReplayMask.NONE; + } + String mask = String.valueOf(value); + if (MASK_VALUE_TRUE.equals(mask)) { + return ByteReplayMask.TRUE; + } + if (MASK_VALUE_IGNORE.equals(mask)) { + return ByteReplayMask.IGNORE; + } + if (MASK_VALUE_NAME.equals(mask)) { + return ByteReplayMask.NAME; + } + if (MASK_VALUE_STRICT.equals(mask)) { + return ByteReplayMask.STRICT; + } + return ByteReplayMask.NONE; + } + + private static boolean hasInputTypeKey(Map bundle) { + Map props = getPropsFromBundle(bundle); + return props != null && props.containsKey(INPUT_TYPE_PROP); + } + + private static boolean isPasswordInputType(Map bundle) { + Map props = getPropsFromBundle(bundle); + if (props == null) { + return false; + } + Object value = props.get(INPUT_TYPE_PROP); + return INPUT_PASSWORD_TYPE.equals(value); + } + + private static boolean isInputTag(String tagName) { + return INPUT_TAG.equals(tagName) || X_INPUT_TAG.equals(tagName); + } + + private static boolean isTextInputTag(String tagName) { + return isInputTag(tagName) || TEXTAREA_TAG.equals(tagName) || X_TEXTAREA_TAG.equals(tagName); + } + + private static boolean isInputContentKey(String key) { + return "value".equals(key) || "text".equals(key) || "default-value".equals(key) + || "defaultValue".equals(key); + } + + private static boolean isBoxSensitivePropKey(String key) { + return isInputContentKey(key) || isMediaSourceKey(key); + } + + private static boolean isMediaSourceKey(String key) { + return "src".equals(key) || "source".equals(key) || "url".equals(key) || "uri".equals(key) + || "poster".equals(key); + } + + private static boolean isMediaTag(String tagName) { + return "image".equals(tagName) || "inline-image".equals(tagName) || "video".equals(tagName) + || "audio".equals(tagName) || "x-video-engine".equals(tagName); + } + + @SuppressWarnings("unchecked") + private static Map toStringObjectMap(Object value) { + if (!(value instanceof Map)) { + return null; + } + Map source = (Map) value; + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : source.entrySet()) { + Object key = entry.getKey(); + if (key != null) { + result.put(String.valueOf(key), entry.getValue()); + } + } + return result; } public static void put(JSONObject target, String key, Object value) { @@ -629,6 +936,9 @@ private static Object toJsonValue(Object value) { UIFramePayload payload = (UIFramePayload) value; JSONObject result = new JSONObject(); put(result, "operations", payload.operations); + if (payload.screenDensity != null) { + put(result, "screenDensity", payload.screenDensity); + } return result; } if (value instanceof CreateNodePayload) { @@ -636,8 +946,6 @@ private static Object toJsonValue(Object value) { JSONObject result = new JSONObject(); put(result, "sign", payload.sign); put(result, "tagName", payload.tagName); - put(result, "isFlatten", payload.isFlatten); - put(result, "nodeIndex", payload.nodeIndex); put(result, "bundle", payload.bundle); put(result, "initialStyles", payload.initialStyles); return result; @@ -654,7 +962,9 @@ private static Object toJsonValue(Object value) { UpdatePropsPayload payload = (UpdatePropsPayload) value; JSONObject result = new JSONObject(); put(result, "sign", payload.sign); - put(result, "tendToFlatten", payload.tendToFlatten); + if (payload.tendToFlatten != null) { + put(result, "tendToFlatten", payload.tendToFlatten); + } put(result, "bundle", payload.bundle); put(result, "styles", payload.styles); return result; @@ -673,7 +983,23 @@ private static Object toJsonValue(Object value) { put(result, "bounds", payload.bounds); put(result, "sticky", payload.sticky); put(result, "maxHeight", payload.maxHeight); - put(result, "nodeIndex", payload.nodeIndex); + return result; + } + if (value instanceof UpdateExtraDataPayload) { + UpdateExtraDataPayload payload = (UpdateExtraDataPayload) value; + JSONObject result = new JSONObject(); + put(result, "sign", payload.sign); + put(result, "type", payload.type); + put(result, "text", payload.text); + return result; + } + if (value instanceof UpdateInputValuePayload) { + UpdateInputValuePayload payload = (UpdateInputValuePayload) value; + JSONObject result = new JSONObject(); + put(result, "sign", payload.sign); + put(result, "value", payload.value); + put(result, "selectionStart", payload.selectionStart); + put(result, "selectionEnd", payload.selectionEnd); return result; } if (value instanceof InvokePayload) { @@ -682,8 +1008,6 @@ private static Object toJsonValue(Object value) { put(result, "sign", payload.sign); put(result, "method", payload.method); put(result, "params", payload.params); - put(result, "context", payload.context); - put(result, "callback", payload.callback); return result; } if (value instanceof TouchPayload) { @@ -751,6 +1075,422 @@ private static Object toJsonValue(Object value) { return String.valueOf(value); } + private enum ByteReplayMask { NONE, TRUE, IGNORE, NAME, STRICT } + + private static final class RecorderTreeState { + private final int sessionId; + private final Map nodes = new HashMap<>(); + + private RecorderTreeState(int sessionId) { + this.sessionId = sessionId; + } + + private List buildInitialOperations( + List initialNodes, long opTime) { + nodes.clear(); + for (InitialNodeSnapshot node : initialNodes) { + NodeRecord record = getOrCreateNode(node.sign); + record.tagName = node.tagName; + updateNodeFromBundle(record, node.bundle, true); + } + for (InitialNodeSnapshot node : initialNodes) { + if (node.parentSign >= 0) { + setParent(node.sign, node.parentSign); + } + } + + List operations = new ArrayList<>(initialNodes.size() * 4); + for (InitialNodeSnapshot node : initialNodes) { + if (shouldDropNode(node.sign)) { + continue; + } + boolean boxNode = isBoxNode(node.sign); + operations.add(new UIOperationRecord(EVENT_LYNX_UI_CREATE_NODE, + new CreateNodePayload(node.sign, boxNode ? VIEW_TAG : node.tagName, + boxNode ? getBoxNodeBundle(node.sign, node.bundle) + : sanitizeBundleForNode( + node.bundle, nodes.get(node.sign), shouldMaskText(node.sign)), + node.initialStyles), + opTime)); + Integer parentSign = getRecordedParentSign(node.sign); + if (parentSign != null) { + operations.add(new UIOperationRecord(EVENT_LYNX_UI_INSERT_NODE, + new NodeRelationPayload(parentSign, node.sign, node.childIndex), opTime)); + } + if (!boxNode) { + operations.add(new UIOperationRecord(EVENT_LYNX_UI_UPDATE_PROPS, + new UpdatePropsPayload(node.sign, null, + sanitizeBundleForNode( + node.bundle, nodes.get(node.sign), shouldMaskText(node.sign)), + node.initialStyles), + opTime)); + } + operations.add(new UIOperationRecord(EVENT_LYNX_UI_UPDATE_LAYOUT, + new UpdateLayoutPayload(node.sign, node.x, node.y, node.width, node.height, + copyFloatArray(node.paddings), copyFloatArray(node.margins), + copyFloatArray(node.borders), copyFloatArray(node.bounds), + copyFloatArray(node.sticky), node.maxHeight), + opTime)); + if (!boxNode && node.text != null) { + operations.add(new UIOperationRecord(EVENT_LYNX_UI_UPDATE_EXTRA_DATA, + new UpdateExtraDataPayload(node.sign, TEXT_EXTRA_DATA_TYPE, + shouldMaskText(node.sign) ? maskText(node.text) : node.text), + opTime)); + } + } + return Collections.unmodifiableList(operations); + } + + private UIOperationRecord applyOperation(int eventType, Object payload, long opTime) { + switch (eventType) { + case EVENT_LYNX_UI_CREATE_NODE: + return applyCreateNode((CreateNodePayload) payload, opTime); + case EVENT_LYNX_UI_INSERT_NODE: + return applyInsertNode((NodeRelationPayload) payload, opTime); + case EVENT_LYNX_UI_REMOVE_NODE: + return applyRemoveNode((NodeRelationPayload) payload, opTime); + case EVENT_LYNX_UI_DESTROY_NODE: + return applyDestroyNode((NodeRelationPayload) payload, opTime); + case EVENT_LYNX_UI_UPDATE_PROPS: + return applyUpdateProps((UpdatePropsPayload) payload, opTime); + case EVENT_LYNX_UI_UPDATE_LAYOUT: + return applyUpdateLayout((UpdateLayoutPayload) payload, opTime); + case EVENT_LYNX_UI_INVOKE: + return applyInvoke((InvokePayload) payload, opTime); + case EVENT_LYNX_UI_UPDATE_EXTRA_DATA: + return applyUpdateExtraData((UpdateExtraDataPayload) payload, opTime); + case EVENT_LYNX_UI_UPDATE_INPUT_VALUE: + return applyUpdateInputValue((UpdateInputValuePayload) payload, opTime); + default: + return new UIOperationRecord(eventType, payload, opTime); + } + } + + private List sanitizeTouchPoints(List points) { + if (points == null) { + return null; + } + List result = new ArrayList<>(points.size()); + for (TouchPointPayload point : points) { + Integer targetId = point.targetId == null ? null : getVisibleTargetSign(point.targetId); + result.add(new TouchPointPayload(point.x, point.y, point.touchId, targetId)); + } + return Collections.unmodifiableList(result); + } + + private UIOperationRecord applyCreateNode(CreateNodePayload payload, long opTime) { + NodeRecord node = getOrCreateNode(payload.sign); + node.tagName = payload.tagName; + updateNodeFromBundle(node, payload.bundle, true); + if (shouldDropNode(payload.sign)) { + return null; + } + if (isBoxNode(payload.sign)) { + return new UIOperationRecord(EVENT_LYNX_UI_CREATE_NODE, + new CreateNodePayload(payload.sign, VIEW_TAG, + getBoxNodeBundle(payload.sign, payload.bundle), payload.initialStyles), + opTime); + } + return new UIOperationRecord(EVENT_LYNX_UI_CREATE_NODE, + new CreateNodePayload(payload.sign, payload.tagName, + sanitizeBundleForNode(payload.bundle, node, shouldMaskText(payload.sign)), + payload.initialStyles), + opTime); + } + + private UIOperationRecord applyInsertNode(NodeRelationPayload payload, long opTime) { + setParent(payload.childSign, payload.parentSign); + if (shouldDropNode(payload.childSign)) { + return null; + } + Integer parentSign = getRecordedParentSign(payload.childSign); + if (parentSign == null) { + return null; + } + return new UIOperationRecord(EVENT_LYNX_UI_INSERT_NODE, + new NodeRelationPayload(parentSign, payload.childSign, payload.index), opTime); + } + + private UIOperationRecord applyRemoveNode(NodeRelationPayload payload, long opTime) { + UIOperationRecord operation = null; + if (!shouldDropNode(payload.childSign)) { + Integer parentSign = getRecordedParentSign(payload.childSign); + if (parentSign != null) { + operation = new UIOperationRecord(EVENT_LYNX_UI_REMOVE_NODE, + new NodeRelationPayload(parentSign, payload.childSign, null), opTime); + } + } + detachFromParent(payload.childSign); + return operation; + } + + private UIOperationRecord applyDestroyNode(NodeRelationPayload payload, long opTime) { + UIOperationRecord operation = null; + if (!shouldDropNode(payload.childSign)) { + Integer parentSign = getRecordedParentSign(payload.childSign); + operation = new UIOperationRecord(EVENT_LYNX_UI_DESTROY_NODE, + new NodeRelationPayload( + parentSign == null ? payload.parentSign : parentSign, payload.childSign, null), + opTime); + } + destroySubtree(payload.childSign); + return operation; + } + + private UIOperationRecord applyUpdateProps(UpdatePropsPayload payload, long opTime) { + NodeRecord node = getOrCreateNode(payload.sign); + updateNodeFromBundle(node, payload.bundle, false); + if (shouldDropNode(payload.sign)) { + return null; + } + boolean boxNode = isBoxNode(payload.sign); + return new UIOperationRecord(EVENT_LYNX_UI_UPDATE_PROPS, + new UpdatePropsPayload(payload.sign, payload.tendToFlatten, + boxNode ? getBoxNodeBundle(payload.sign, payload.bundle) + : sanitizeBundleForNode(payload.bundle, node, shouldMaskText(payload.sign)), + payload.styles), + opTime); + } + + private UIOperationRecord applyUpdateLayout(UpdateLayoutPayload payload, long opTime) { + if (shouldDropNode(payload.sign)) { + return null; + } + return new UIOperationRecord(EVENT_LYNX_UI_UPDATE_LAYOUT, payload, opTime); + } + + private UIOperationRecord applyInvoke(InvokePayload payload, long opTime) { + if (shouldDropNode(payload.sign) || isBoxNode(payload.sign)) { + return null; + } + NodeRecord node = nodes.get(payload.sign); + Map params = + (node != null && node.isPasswordInput) || shouldMaskText(payload.sign) + ? sanitizeInputContentMap(payload.params) + : payload.params; + return new UIOperationRecord( + EVENT_LYNX_UI_INVOKE, new InvokePayload(payload.sign, payload.method, params), opTime); + } + + private UIOperationRecord applyUpdateExtraData(UpdateExtraDataPayload payload, long opTime) { + if (shouldDropNode(payload.sign) || isBoxNode(payload.sign)) { + return null; + } + String text = payload.text; + if (TEXT_EXTRA_DATA_TYPE.equals(payload.type) && shouldMaskText(payload.sign)) { + text = maskText(payload.text); + } + return new UIOperationRecord(EVENT_LYNX_UI_UPDATE_EXTRA_DATA, + new UpdateExtraDataPayload(payload.sign, payload.type, text), opTime); + } + + private UIOperationRecord applyUpdateInputValue(UpdateInputValuePayload payload, long opTime) { + if (shouldDropNode(payload.sign) || isBoxNode(payload.sign)) { + return null; + } + NodeRecord node = nodes.get(payload.sign); + if (node == null || !isTextInputTag(node.tagName)) { + return null; + } + String value = payload.value; + if (node.isPasswordInput || shouldMaskText(payload.sign)) { + value = maskText(payload.value); + } + return new UIOperationRecord(EVENT_LYNX_UI_UPDATE_INPUT_VALUE, + new UpdateInputValuePayload( + payload.sign, value, payload.selectionStart, payload.selectionEnd), + opTime); + } + + private NodeRecord getOrCreateNode(int sign) { + NodeRecord node = nodes.get(sign); + if (node == null) { + node = new NodeRecord(sign); + nodes.put(sign, node); + } + return node; + } + + private Map getBoxNodeBundle(int sign, Map bundle) { + NodeRecord node = nodes.get(sign); + return sanitizeBoxBundleForNode(bundle, node); + } + + private void updateNodeFromBundle( + NodeRecord node, Map bundle, boolean resetMissingMask) { + if (resetMissingMask || hasMaskKey(bundle)) { + node.ownMask = parseMask(bundle); + } + if (isInputTag(node.tagName)) { + if (resetMissingMask || hasInputTypeKey(bundle)) { + node.isPasswordInput = isPasswordInputType(bundle); + } + } else { + node.isPasswordInput = false; + } + } + + private void setParent(int childSign, int parentSign) { + NodeRecord child = getOrCreateNode(childSign); + detachFromParent(childSign); + child.parentSign = parentSign; + if (parentSign >= 0) { + NodeRecord parent = getOrCreateNode(parentSign); + if (!parent.children.contains(childSign)) { + parent.children.add(childSign); + } + } + } + + private void detachFromParent(int sign) { + NodeRecord child = nodes.get(sign); + if (child == null || child.parentSign < 0) { + return; + } + NodeRecord parent = nodes.get(child.parentSign); + if (parent != null) { + parent.children.remove(Integer.valueOf(sign)); + } + child.parentSign = -1; + } + + private void destroySubtree(int sign) { + NodeRecord node = nodes.get(sign); + if (node == null) { + return; + } + List children = new ArrayList<>(node.children); + for (Integer childSign : children) { + destroySubtree(childSign); + } + detachFromParent(sign); + nodes.remove(sign); + } + + private boolean shouldDropNode(int sign) { + return isHiddenByBoxAncestor(sign); + } + + private boolean isBoxNode(int sign) { + return isIgnoredRoot(sign) || isFilteredMedia(sign); + } + + private boolean isIgnoredRoot(int sign) { + NodeRecord node = nodes.get(sign); + return node != null && node.ownMask == ByteReplayMask.IGNORE && !isHiddenByBoxAncestor(sign); + } + + private boolean isFilteredMedia(int sign) { + NodeRecord node = nodes.get(sign); + return node != null && isMediaTag(node.tagName) && hasStrictMaskOnPath(sign) + && !isHiddenByBoxAncestor(sign); + } + + private boolean shouldMaskText(int sign) { + NodeRecord node = nodes.get(sign); + if (node != null && node.isPasswordInput) { + return true; + } + if (node != null && node.ownMask == ByteReplayMask.NAME) { + return true; + } + return hasSubtreeTextMaskOnPath(sign); + } + + private boolean hasSubtreeTextMaskOnPath(int sign) { + Integer current = sign; + int guard = nodes.size() + 1; + while (current != null && guard-- > 0) { + NodeRecord node = nodes.get(current); + if (node == null) { + return false; + } + if (node.ownMask == ByteReplayMask.TRUE || node.ownMask == ByteReplayMask.STRICT) { + return true; + } + current = node.parentSign >= 0 ? node.parentSign : null; + } + return false; + } + + private boolean hasStrictMaskOnPath(int sign) { + Integer current = sign; + int guard = nodes.size() + 1; + while (current != null && guard-- > 0) { + NodeRecord node = nodes.get(current); + if (node == null) { + return false; + } + if (node.ownMask == ByteReplayMask.STRICT) { + return true; + } + current = node.parentSign >= 0 ? node.parentSign : null; + } + return false; + } + + private boolean isHiddenByBoxAncestor(int sign) { + NodeRecord node = nodes.get(sign); + if (node == null) { + return false; + } + Integer current = node.parentSign >= 0 ? node.parentSign : null; + int guard = nodes.size() + 1; + while (current != null && guard-- > 0) { + if (isBoxNode(current)) { + return true; + } + NodeRecord parent = nodes.get(current); + if (parent == null) { + return false; + } + current = parent.parentSign >= 0 ? parent.parentSign : null; + } + return false; + } + + private Integer getRecordedParentSign(int sign) { + NodeRecord node = nodes.get(sign); + if (node == null || node.parentSign < 0 || shouldDropNode(node.parentSign)) { + return null; + } + return node.parentSign; + } + + private Integer getVisibleTargetSign(int sign) { + NodeRecord node = nodes.get(sign); + if (node == null || !shouldDropNode(sign)) { + return sign; + } + Integer current = sign; + int guard = nodes.size() + 1; + while (current != null && guard-- > 0) { + if (isBoxNode(current)) { + return current; + } + NodeRecord currentNode = nodes.get(current); + if (currentNode == null) { + return null; + } + current = currentNode.parentSign >= 0 ? currentNode.parentSign : null; + } + return null; + } + } + + private static final class NodeRecord { + private final int sign; + private String tagName; + private int parentSign = -1; + private final List children = new ArrayList<>(); + private ByteReplayMask ownMask = ByteReplayMask.NONE; + private boolean isPasswordInput = false; + + private NodeRecord(int sign) { + this.sign = sign; + } + } + private static final class RecorderThreadFactory implements ThreadFactory { private final AtomicInteger mThreadNumber = new AtomicInteger(1); @@ -843,10 +1583,15 @@ private void flushPendingUIFrameOnExecutor(int instanceId, int sessionId, long t } if (!operations.isEmpty()) { emitFrame(instanceId, TRACK_TYPE_LYNX_UI_OP, EVENT_LYNX_UI_FRAME, - new UIFramePayload(operations), timestampMillis); + new UIFramePayload(operations, getScreenDensity(instanceId)), timestampMillis); } } + private Float getScreenDensity(int instanceId) { + Float screenDensity = mScreenDensities.get(instanceId); + return screenDensity != null && screenDensity > 0.f ? screenDensity : null; + } + private static final class FrameCache { private final int mCapacity; private final Deque mFrames = new ArrayDeque<>(); @@ -919,9 +1664,54 @@ private UIOperationRecord(int eventType, Object payload, long opTime) { private static final class UIFramePayload { private final List operations; + private final Float screenDensity; - private UIFramePayload(List operations) { + private UIFramePayload(List operations, Float screenDensity) { this.operations = operations; + this.screenDensity = screenDensity; + } + } + + private static final class InitialNodeSnapshot { + private final int sign; + private final String tagName; + private final Map bundle; + private final Map initialStyles; + private final int parentSign; + private final int childIndex; + private final float x; + private final float y; + private final float width; + private final float height; + private final float[] paddings; + private final float[] margins; + private final float[] borders; + private final float[] bounds; + private final float[] sticky; + private final float maxHeight; + private final String text; + + private InitialNodeSnapshot(int sign, String tagName, Map bundle, + Map initialStyles, int parentSign, int childIndex, float x, float y, + float width, float height, float[] paddings, float[] margins, float[] borders, + float[] bounds, float[] sticky, float maxHeight, String text) { + this.sign = sign; + this.tagName = tagName; + this.bundle = bundle; + this.initialStyles = initialStyles; + this.parentSign = parentSign; + this.childIndex = childIndex; + this.x = x; + this.y = y; + this.width = width; + this.height = height; + this.paddings = paddings; + this.margins = margins; + this.borders = borders; + this.bounds = bounds; + this.sticky = sticky; + this.maxHeight = maxHeight; + this.text = text; } } @@ -930,17 +1720,13 @@ private static final class CreateNodePayload { private final String tagName; private final Map bundle; private final Map initialStyles; - private final boolean isFlatten; - private final int nodeIndex; - private CreateNodePayload(int sign, String tagName, Map bundle, - Map initialStyles, boolean isFlatten, int nodeIndex) { + private CreateNodePayload( + int sign, String tagName, Map bundle, Map initialStyles) { this.sign = sign; this.tagName = tagName; this.bundle = bundle; this.initialStyles = initialStyles; - this.isFlatten = isFlatten; - this.nodeIndex = nodeIndex; } } @@ -958,12 +1744,12 @@ private NodeRelationPayload(int parentSign, int childSign, Integer index) { private static final class UpdatePropsPayload { private final int sign; - private final boolean tendToFlatten; + private final Boolean tendToFlatten; private final Map bundle; private final Map styles; private UpdatePropsPayload( - int sign, boolean tendToFlatten, Map bundle, Map styles) { + int sign, Boolean tendToFlatten, Map bundle, Map styles) { this.sign = sign; this.tendToFlatten = tendToFlatten; this.bundle = bundle; @@ -983,11 +1769,10 @@ private static final class UpdateLayoutPayload { private final float[] bounds; private final float[] sticky; private final float maxHeight; - private final int nodeIndex; private UpdateLayoutPayload(int sign, float x, float y, float width, float height, float[] paddings, float[] margins, float[] borders, float[] bounds, float[] sticky, - float maxHeight, int nodeIndex) { + float maxHeight) { this.sign = sign; this.x = x; this.y = y; @@ -999,7 +1784,32 @@ private UpdateLayoutPayload(int sign, float x, float y, float width, float heigh this.bounds = bounds; this.sticky = sticky; this.maxHeight = maxHeight; - this.nodeIndex = nodeIndex; + } + } + + private static final class UpdateExtraDataPayload { + private final int sign; + private final String type; + private final String text; + + private UpdateExtraDataPayload(int sign, String type, String text) { + this.sign = sign; + this.type = type; + this.text = text; + } + } + + private static final class UpdateInputValuePayload { + private final int sign; + private final String value; + private final int selectionStart; + private final int selectionEnd; + + private UpdateInputValuePayload(int sign, String value, int selectionStart, int selectionEnd) { + this.sign = sign; + this.value = value; + this.selectionStart = selectionStart; + this.selectionEnd = selectionEnd; } } @@ -1007,16 +1817,11 @@ private static final class InvokePayload { private final int sign; private final String method; private final Map params; - private final long context; - private final int callback; - private InvokePayload( - int sign, String method, Map params, long context, int callback) { + private InvokePayload(int sign, String method, Map params) { this.sign = sign; this.method = method; this.params = params; - this.context = context; - this.callback = callback; } } diff --git a/platform/android/lynx_xelement/lynx_xelement_input/src/main/java/com/lynx/xelement/input/LynxUIBaseInput.kt b/platform/android/lynx_xelement/lynx_xelement_input/src/main/java/com/lynx/xelement/input/LynxUIBaseInput.kt index 46ced86984..4b6b1ee511 100644 --- a/platform/android/lynx_xelement/lynx_xelement_input/src/main/java/com/lynx/xelement/input/LynxUIBaseInput.kt +++ b/platform/android/lynx_xelement/lynx_xelement_input/src/main/java/com/lynx/xelement/input/LynxUIBaseInput.kt @@ -44,6 +44,7 @@ import com.lynx.tasm.behavior.ui.LynxBaseUI import com.lynx.tasm.behavior.ui.LynxUI import com.lynx.tasm.event.LynxDetailEvent import com.lynx.tasm.fontface.FontFaceManager +import com.lynx.tasm.recording.LynxFrameRecorder import com.lynx.tasm.utils.ColorUtils import com.lynx.tasm.utils.ContextUtils import com.lynx.tasm.utils.PixelUtils @@ -191,19 +192,14 @@ open class LynxUIBaseInput(context: LynxContext, params: Any?) : LynxUI