[#9631] Add API to retrieve CLP replaced tokens with count and ordering

Author: intr3p1d

exceptiontrace/exceptiontrace-common/src/main/java/com/navercorp/pinpoint/exceptiontrace/common/pinot/PinotFunctions.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2024 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.navercorp.pinpoint.exceptiontrace.common.pinot;
+
+/**
+ * @author intr3p1d
+ */
+public enum PinotFunctions {
+    ARRAY_SLICE_INT("arraySliceInt"),
+    ARRAY_SLICE_STRING("arraySliceString");
+
+    private final String name;
+
+    PinotFunctions(String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+}

exceptiontrace/exceptiontrace-web/pom.xml

@@ -29,6 +29,15 @@
             <groupId>com.navercorp.pinpoint</groupId>
             <artifactId>pinpoint-exceptiontrace-common</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-text</artifactId>
+            <version>1.10.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.mapstruct</groupId>
             <artifactId>mapstruct</artifactId>

exceptiontrace/exceptiontrace-web/src/main/java/com/navercorp/pinpoint/exceptiontrace/web/config/ExceptionTraceRegistryHandler.java

@@ -15,11 +15,13 @@
  */
 package com.navercorp.pinpoint.exceptiontrace.web.config;
 
+import com.navercorp.pinpoint.exceptiontrace.web.entity.ClpConvertedEntity;
 import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionMetaDataEntity;
 import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionTraceSummaryEntity;
 import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionTraceValueViewEntity;
 import com.navercorp.pinpoint.exceptiontrace.web.entity.GroupedFieldNameEntity;
-import com.navercorp.pinpoint.exceptiontrace.web.util.ExceptionTraceQueryParameter;
+import com.navercorp.pinpoint.exceptiontrace.web.query.ClpQueryParameter;
+import com.navercorp.pinpoint.exceptiontrace.web.query.ExceptionTraceQueryParameter;
 import com.navercorp.pinpoint.mybatis.MyBatisRegistryHandler;
 import org.apache.ibatis.type.TypeAliasRegistry;
 import org.apache.ibatis.type.TypeHandlerRegistry;
@@ -34,7 +36,9 @@ public void registerTypeAlias(TypeAliasRegistry typeAliasRegistry) {
         typeAliasRegistry.registerAlias(GroupedFieldNameEntity.class);
         typeAliasRegistry.registerAlias(ExceptionTraceSummaryEntity.class);
         typeAliasRegistry.registerAlias(ExceptionTraceValueViewEntity.class);
+        typeAliasRegistry.registerAlias(ClpConvertedEntity.class);
         typeAliasRegistry.registerAlias(ExceptionTraceQueryParameter.class);
+        typeAliasRegistry.registerAlias(ClpQueryParameter.class);
     }
 
     @Override

exceptiontrace/exceptiontrace-web/src/main/java/com/navercorp/pinpoint/exceptiontrace/web/controller/ExceptionTraceController.java

@@ -18,10 +18,12 @@
 
 import com.navercorp.pinpoint.common.server.util.time.Range;
 import com.navercorp.pinpoint.common.server.util.time.RangeValidator;
+import com.navercorp.pinpoint.exceptiontrace.web.model.ClpConverted;
 import com.navercorp.pinpoint.exceptiontrace.web.model.ExceptionTraceSummary;
 import com.navercorp.pinpoint.exceptiontrace.web.model.ExceptionTraceValueView;
+import com.navercorp.pinpoint.exceptiontrace.web.query.ClpQueryParameter;
 import com.navercorp.pinpoint.exceptiontrace.web.service.ExceptionTraceService;
-import com.navercorp.pinpoint.exceptiontrace.web.util.ExceptionTraceQueryParameter;
+import com.navercorp.pinpoint.exceptiontrace.web.query.ExceptionTraceQueryParameter;
 import com.navercorp.pinpoint.exceptiontrace.web.util.GroupByAttributes;
 import com.navercorp.pinpoint.exceptiontrace.web.view.ExceptionMetaDataView;
 import com.navercorp.pinpoint.exceptiontrace.web.view.ExceptionTraceView;
@@ -159,6 +161,37 @@ public List<ExceptionTraceSummary> getListOfExceptionMetaDataWithDynamicGroupBy(
         );
     }
 
+    @GetMapping("/replacedTokens")
+    public List<ClpConverted> getReplacedVariables(
+            @RequestParam("applicationName") @NotBlank String applicationName,
+            @RequestParam(value = "agentId", required = false) String agentId,
+            @RequestParam("from") @PositiveOrZero long from,
+            @RequestParam("to") @PositiveOrZero long to,
+
+            @RequestParam("logType") String logType,
+            @RequestParam("targetColumn") String targetColumn,
+            @RequestParam("targetIndex") int targetIndex
+    ) {
+        Range range = Range.between(from, to);
+        rangeValidator.validate(range);
+
+        ClpQueryParameter queryParameter = new ClpQueryParameter.Builder()
+                .setTableName(tableName)
+                .setTenantId(tenantProvider.getTenantId())
+                .setApplicationName(applicationName)
+                .setAgentId(agentId)
+                .setRange(Range.between(from, to))
+                .setTimePrecision(DETAILED_TIME_PRECISION)
+                .setLogType(logType)
+                .setTargetColumn(targetColumn)
+                .setTargetIndex(targetIndex)
+                .build();
+
+        return exceptionTraceService.getReplacedVariables(
+                queryParameter
+        );
+    }
+
     @GetMapping("/chart")
     public ExceptionTraceView getCollectedExceptionMetaDataByGivenRange(
             @RequestParam("applicationName") @NotBlank String applicationName,

exceptiontrace/exceptiontrace-web/src/main/java/com/navercorp/pinpoint/exceptiontrace/web/dao/ExceptionTraceDao.java

@@ -18,9 +18,11 @@
 
 
 import com.navercorp.pinpoint.exceptiontrace.common.model.ExceptionMetaData;
+import com.navercorp.pinpoint.exceptiontrace.web.model.ClpConverted;
 import com.navercorp.pinpoint.exceptiontrace.web.model.ExceptionTraceSummary;
 import com.navercorp.pinpoint.exceptiontrace.web.model.ExceptionTraceValueView;
-import com.navercorp.pinpoint.exceptiontrace.web.util.ExceptionTraceQueryParameter;
+import com.navercorp.pinpoint.exceptiontrace.web.query.ClpQueryParameter;
+import com.navercorp.pinpoint.exceptiontrace.web.query.ExceptionTraceQueryParameter;
 import com.navercorp.pinpoint.exceptiontrace.web.view.ExceptionMetaDataView;
 
 import java.util.List;
@@ -34,4 +36,5 @@ public interface ExceptionTraceDao {
     ExceptionMetaData getException(ExceptionTraceQueryParameter exceptionTraceQueryParameter);
     List<ExceptionTraceSummary> getSummaries(ExceptionTraceQueryParameter exceptionTraceQueryParameter);
     List<ExceptionTraceValueView> getValueViews(ExceptionTraceQueryParameter exceptionTraceQueryParameter);
+    List<ClpConverted> getReplacedVariables(ClpQueryParameter queryParameter);
 }

exceptiontrace/exceptiontrace-web/src/main/java/com/navercorp/pinpoint/exceptiontrace/web/dao/PinotExceptionTraceDao.java

@@ -17,13 +17,16 @@
 package com.navercorp.pinpoint.exceptiontrace.web.dao;
 
 import com.navercorp.pinpoint.exceptiontrace.common.model.ExceptionMetaData;
+import com.navercorp.pinpoint.exceptiontrace.web.entity.ClpConvertedEntity;
 import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionMetaDataEntity;
 import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionTraceSummaryEntity;
 import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionTraceValueViewEntity;
 import com.navercorp.pinpoint.exceptiontrace.web.mapper.ExceptionMetaDataEntityMapper;
+import com.navercorp.pinpoint.exceptiontrace.web.model.ClpConverted;
 import com.navercorp.pinpoint.exceptiontrace.web.model.ExceptionTraceSummary;
 import com.navercorp.pinpoint.exceptiontrace.web.model.ExceptionTraceValueView;
-import com.navercorp.pinpoint.exceptiontrace.web.util.ExceptionTraceQueryParameter;
+import com.navercorp.pinpoint.exceptiontrace.web.query.ClpQueryParameter;
+import com.navercorp.pinpoint.exceptiontrace.web.query.ExceptionTraceQueryParameter;
 import com.navercorp.pinpoint.exceptiontrace.web.view.ExceptionMetaDataView;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -49,6 +52,7 @@ public class PinotExceptionTraceDao implements ExceptionTraceDao {
     private static final String SELECT_EXACT_QUERY = "selectExactException";
     private static final String SELECT_SUMMARIES_QUERY = "selectSummaries";
     private static final String SELECT_VALUEVIEWS_QUERY = "selectValueViews";
+    private static final String SELECT_CLP_VARIABLES_QUERY = "selectClpVariables";
 
     private final SqlSessionTemplate sqlPinotSessionTemplate;
 
@@ -105,4 +109,13 @@ public List<ExceptionTraceValueView> getValueViews(ExceptionTraceQueryParameter
                         )
                 ).collect(Collectors.toList());
     }
+
+    @Override
+    public List<ClpConverted> getReplacedVariables(ClpQueryParameter queryParameter) {
+        List<ClpConvertedEntity> clpConvertedEntities = this.sqlPinotSessionTemplate.selectList(NAMESPACE + SELECT_CLP_VARIABLES_QUERY, queryParameter);
+
+        return clpConvertedEntities.stream()
+                .map(mapper::toClpConverted)
+                .collect(Collectors.toList());
+    }
 }

exceptiontrace/exceptiontrace-web/src/main/java/com/navercorp/pinpoint/exceptiontrace/web/entity/ClpConvertedEntity.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2024 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.navercorp.pinpoint.exceptiontrace.web.entity;
+
+/**
+ * @author intr3p1d
+ */
+public class ClpConvertedEntity {
+
+    private String replacedToken;
+    private int count;
+
+    public ClpConvertedEntity() {
+    }
+
+    public String getReplacedToken() {
+        return replacedToken;
+    }
+
+    public void setReplacedToken(String replacedToken) {
+        this.replacedToken = replacedToken;
+    }
+
+    public int getCount() {
+        return count;
+    }
+
+    public void setCount(int count) {
+        this.count = count;
+    }
+}

exceptiontrace/exceptiontrace-web/src/main/java/com/navercorp/pinpoint/exceptiontrace/web/mapper/CLPMapper.java

@@ -15,7 +15,13 @@
  */
 package com.navercorp.pinpoint.exceptiontrace.web.mapper;
 
+
+import org.apache.commons.text.StringEscapeUtils;
+
 import java.nio.charset.StandardCharsets;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * @author intr3p1d
@@ -26,17 +32,76 @@ public class CLPMapper {
     public final static char DICTIONARY_VARIABLE_VALUE = '\u0011';
     public final static char NON_DICTIONARY_VALUE = '\u0012';
 
-    public final static String DICTIONARY_REPLACEMENT = "▨▨▨";
-    public final static String NON_DICTIONARY_REPLACEMENT = "▧▧▧";
+    public final static String DICTIONARY_REPLACEMENT = "<ClpPlaceHolder dict %d/>";
+    public final static String NON_DICTIONARY_REPLACEMENT = "<ClpPlaceHolder non-dict %d/>";
+
+    public final static String SIMPLE_REPLACEMENT = "▨▨▨";
+
+    static String fixAndEscapeLogType(String encodedLogType) {
+        return replaceDictPlaceHolder(replaceNonDictPlaceHolder(
+                escapeXml(correctEncoding(
+                        encodedLogType
+                ))
+        ));
+    }
+
+    static String processEncodedLogType(String encodedLogType) {
+        return replaceSimple(correctEncoding(
+                encodedLogType
+        ));
+    }
 
-    static String makeReadableString(String encodedLogType) {
-        byte[] encodedLogTypeBytes = encodedLogType.getBytes(StandardCharsets.ISO_8859_1);
+    static String correctEncoding(String isoString) {
+        byte[] encodedLogTypeBytes = isoString.getBytes(StandardCharsets.ISO_8859_1);
         return new String(encodedLogTypeBytes, StandardCharsets.UTF_8);
     }
 
-    static String replacePlaceHolders(String encodedLogType) {
-        return encodedLogType
-                .replaceAll(String.valueOf(DICTIONARY_VARIABLE_VALUE), DICTIONARY_REPLACEMENT)
-                .replaceAll(String.valueOf(NON_DICTIONARY_VALUE), NON_DICTIONARY_REPLACEMENT);
+    static String escapeXml(String rawString) {
+        return StringEscapeUtils.escapeHtml4(rawString);
+    }
+
+    static String replaceNonDictPlaceHolder(String encodedLogType) {
+        return replaceHolder(
+                encodedLogType,
+                String.valueOf(NON_DICTIONARY_VALUE),
+                x -> stringFunction(NON_DICTIONARY_REPLACEMENT, x)
+        );
+    }
+
+    static String replaceDictPlaceHolder(String encodedLogType) {
+        return replaceHolder(
+                encodedLogType,
+                String.valueOf(DICTIONARY_VARIABLE_VALUE),
+                x -> stringFunction(DICTIONARY_REPLACEMENT, x)
+        );
+    }
+
+    static String replaceSimple(String encodedLogType) {
+        return replaceHolder(replaceHolder(
+                        encodedLogType,
+                        String.valueOf((DICTIONARY_VARIABLE_VALUE)),
+                        x -> stringFunction(SIMPLE_REPLACEMENT, x)
+                ),
+                String.valueOf(NON_DICTIONARY_VALUE),
+                x -> stringFunction(SIMPLE_REPLACEMENT, x)
+        );
+    }
+
+    static String stringFunction(String format, int index) {
+        return String.format(format, index);
+    }
+
+    static String replaceHolder(String encodedLogType, String replacedCharacter, Function<Integer, String> replacement) {
+        Pattern pattern = Pattern.compile(replacedCharacter);
+        Matcher matcher = pattern.matcher(encodedLogType);
+        StringBuilder result = new StringBuilder();
+        int wordCount = 0;
+
+        while (matcher.find()) {
+            matcher.appendReplacement(result, replacement.apply(wordCount++));
+        }
+        matcher.appendTail(result);
+
+        return result.toString();
     }
 }