Merge remote-tracking branch 'upstream/trunk' into OAK-11438

Author: nfsantos

oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/progress/IndexingProgressReporter.java

@@ -91,7 +91,7 @@ public void setMessagePrefix(String messagePrefix) {
     }
 
     public void traversedNode(PathSource pathSource) throws CommitFailedException {
-        if (++traversalCount % 100_000 == 0) {
+        if (++traversalCount % 10_000 == 0) {
             double rate = traversalRateEstimator.getNodesTraversedPerSecond();
             String formattedRate = String.format("%1.2f nodes/s, %1.2f nodes/hr", rate, rate * 3600);
             String estimate = estimatePendingTraversal(rate);

oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java

@@ -216,6 +216,7 @@ private boolean updateSuggester(Analyzer analyzer, Calendar currentTime) throws
      * Checks if last suggestion build time was done sufficiently in the past AND that there were non-zero indexedNodes
      * stored in the last run. Note, if index is updated only to rebuild suggestions, even then we update indexedNodes,
      * which would be zero in case it was a forced update of suggestions.
+     *
      * @return is suggest dict should be updated
      */
     private boolean shouldUpdateSuggestions(Calendar currentTime) {

oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/LuceneIndexWriterConfig.java

@@ -39,10 +39,10 @@ public class LuceneIndexWriterConfig {
     public final static String RAM_PER_THREAD_HARD_LIMIT_MB_KEY = "oak.index.lucene.ramPerThreadHardLimitMB";
 
     private final int maxBufferedDeleteTerms = SystemPropertySupplier.create(
-            MAX_BUFFERED_DELETE_TERMS_KEY, IndexWriterConfig.DISABLE_AUTO_FLUSH)
+                    MAX_BUFFERED_DELETE_TERMS_KEY, IndexWriterConfig.DISABLE_AUTO_FLUSH)
             .loggingTo(LOG).get();
     private final int ramPerThreadHardLimitMB = SystemPropertySupplier.create(
-            RAM_PER_THREAD_HARD_LIMIT_MB_KEY, IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB)
+                    RAM_PER_THREAD_HARD_LIMIT_MB_KEY, IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB)
             .loggingTo(LOG).get();
 
     private final double ramBufferSizeMB;

oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java

@@ -34,12 +34,17 @@
 import java.util.concurrent.TimeUnit;
 
 import org.apache.jackrabbit.guava.common.base.Stopwatch;
+import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
+import org.apache.jackrabbit.oak.commons.json.JsopReader;
+import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader;
 import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedSortBatchTask.Result;
 import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore;
 import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession;
 import org.apache.jackrabbit.oak.plugins.index.IndexingReporter;
 import org.apache.jackrabbit.oak.plugins.index.MetricsFormatter;
 import org.apache.jackrabbit.oak.plugins.index.MetricsUtils;
+import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
 import org.apache.jackrabbit.oak.stats.StatisticsProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -207,7 +212,9 @@ private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception {
                 int valueLength = buffer.getInt();
                 String value = new String(buffer.array(), buffer.arrayOffset() + buffer.position(), valueLength, StandardCharsets.UTF_8);
                 textSize += entry.getPath().length() + value.length() + 2;
-                treeStore.putNode(entry.getPath(), value);
+                String path = entry.getPath();
+                value = removePropertiesOfBundledNodes(path, value);
+                treeStore.putNode(path, value);
             }
             session.checkpoint();
             unmergedRoots++;
@@ -230,4 +237,68 @@ private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception {
         }
     }
 
+    /**
+     * If there are any, remove properties of bundled nodes (jcr:content/...) from the JSON-encoded node.
+     *
+     * @param path the path
+     * @param value the JSON-encoded node
+     * @return the cleaned JSON
+     */
+    public static String removePropertiesOfBundledNodes(String path, String value) {
+        if (value.indexOf("\"jcr:content/") < 0) {
+            return value;
+        }
+        // possibly the node contains a bundled property, but we are not sure
+        // try to de-serialize
+        NodeStateEntryReader nodeReader = new NodeStateEntryReader(new MemoryBlobStore());
+        try {
+            // the following line will throw an exception if de-serialization fails
+            nodeReader.read(path + "|" + value);
+            // ok it did not: it was a false positive
+            return value;
+        } catch (Exception e) {
+            LOG.warn("Unable to de-serialize due to presence of bundled properties: {} = {}", path, value);
+            JsopReader reader = new JsopTokenizer(value);
+            JsopBuilder writer = new JsopBuilder();
+            reader.read('{');
+            writer.object();
+            if (!reader.matches('}')) {
+                do {
+                    String key = reader.readString();
+                    reader.read(':');
+                    // skip properties that contain "/"
+                    boolean skip = key.indexOf('/') >= 0;
+                    if (!skip) {
+                        writer.key(key);
+                    }
+                    if (reader.matches('[')) {
+                        if (!skip) {
+                            writer.array();
+                        }
+                        do {
+                            String raw = reader.readRawValue();
+                            if (!skip) {
+                                writer.encodedValue(raw);
+                            }
+                        } while (reader.matches(','));
+                        reader.read(']');
+                        if (!skip) {
+                            writer.endArray();
+                        }
+                    } else {
+                        String raw = reader.readRawValue();
+                        if (!skip) {
+                            writer.encodedValue(raw);
+                        }
+                    }
+                } while (reader.matches(','));
+            }
+            reader.read('}');
+            writer.endObject();
+            String result = writer.toString();
+            LOG.warn("Cleaned bundled properties: {} = {}", path, result);
+            return result;
+        }
+    }
+
 }

oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RemovePropertiesOfBundledNodesTest.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.indexer.document.tree.store;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedTreeStoreTask;
+import org.junit.Test;
+
+public class RemovePropertiesOfBundledNodesTest {
+
+    @Test
+    public void cleanUp() {
+        // this is similar to the real-world case
+        verify("{\"jcr:created\":\"dat:2020-05-06T17:15:13.971Z\",\"jcr:primaryType\":\"nam:nt:file\",\"jcr:createdBy\":\"admin\",\"jcr:content/jcr:lastModified\":\"dat:2025-01-21T03:37:42.095Z\",\"jcr:content/jcr:lastModifiedBy\":\"test\"}",
+                "{\"jcr:created\":\"dat:2020-05-06T17:15:13.971Z\",\"jcr:primaryType\":\"nam:nt:file\",\"jcr:createdBy\":\"admin\"}");
+
+        // generic entries
+        verify("{}", "{}");
+        verify("{\"c\":null,\"b\":\"x\",\"a\":123,\"d\":[1,2,null,\"x\"]}",
+                "{\"c\":null,\"b\":\"x\",\"a\":123,\"d\":[1,2,null,\"x\"]}");
+
+        // false positive
+        verify("{\"c\":\"jcr:content/that\"}",
+                "{\"c\":\"jcr:content/that\"}");
+
+        // generic entries that need cleaning
+        verify("{\"c\":null,\"jcr:content/this\":null,\"a\":123,\"jcr:content/that\":[1,2,null,\"x\"]}",
+                "{\"c\":null,\"a\":123}");
+        verify("{\"c\":null,\"jcr:content/this\":null,\"a\":123,\"array\":[1,2,null,\"x\"]}",
+                "{\"c\":null,\"a\":123,\"array\":[1,2,null,\"x\"]}");
+
+    }
+
+    static void verify(String input, String expected) {
+        String v2 = PipelinedTreeStoreTask.removePropertiesOfBundledNodes("/test", input);
+        assertEquals(expected, v2);
+
+    }
+}