Merge pull request #536 from HXSecurity/beta

Release v1.11.0

Author: lostsnow

dongtai-agent/src/main/java/io/dongtai/iast/agent/Agent.java

@@ -39,6 +39,7 @@ private static String[] parseAgentArgs(String[] args) throws ParseException {
         attachOptions.addOption(build("log_path", "log_path", "optional: DongTai agent log print path."));
         attachOptions.addOption(build("log_disable_collector", "log_disable_collector", "optional: DongTai agent disable log collector."));
         attachOptions.addOption(build("disabled_plugins", "disabled_plugins", "optional: DongTai agent disable plugins."));
+        attachOptions.addOption(build("disabled_features", "disabled_features", "optional: DongTai agent disable features."));
 
         CommandLineParser parser = new DefaultParser();
         HelpFormatter formatter = new HelpFormatter();

dongtai-agent/src/main/java/io/dongtai/iast/agent/IastProperties.java

@@ -34,6 +34,7 @@ public class IastProperties {
         put("log_disable_collector", PropertyConstant.PROPERTY_LOG_DISABLE_COLLECTOR);
         put("uuid_path", PropertyConstant.PROPERTY_UUID_PATH);
         put("disabled_plugins", PropertyConstant.PROPERTY_DISABLED_PLUGINS);
+        put("disabled_features", PropertyConstant.PROPERTY_DISABLED_FEATURES);
     }};
 
     private static IastProperties instance;

dongtai-agent/src/main/java/io/dongtai/iast/agent/LogCollector.java

@@ -22,6 +22,15 @@ public static void extractFluent() {
         }
         try {
             if (!isMacOs() && !isWindows()) {
+                String agentId = String.valueOf(AgentRegisterReport.getAgentId());
+                FLUENT_FILE_CONF = IastProperties.getInstance().getTmpDir() + "fluent-" + agentId + ".conf";
+                FileUtils.getResourceToFile("bin/fluent.conf", FLUENT_FILE_CONF);
+                FileUtils.confReplace(FLUENT_FILE_CONF);
+
+                String multiParserFile = IastProperties.getInstance().getTmpDir() + "parsers_multiline.conf";
+                FileUtils.getResourceToFile("bin/parsers_multiline.conf", multiParserFile);
+                FileUtils.confReplace(multiParserFile);
+
                 FLUENT_FILE = IastProperties.getInstance().getTmpDir() + "fluent";
                 File f = new File(FLUENT_FILE);
                 if (f.exists()) {
@@ -34,10 +43,6 @@ public static void extractFluent() {
                     FileUtils.getResourceToFile("bin/fluent", FLUENT_FILE);
                 }
 
-                String agentId = String.valueOf(AgentRegisterReport.getAgentId());
-                FLUENT_FILE_CONF = IastProperties.getInstance().getTmpDir() + "fluent-" + agentId + ".conf";
-                FileUtils.getResourceToFile("bin/fluent.conf", FLUENT_FILE_CONF);
-                FileUtils.confReplace(FLUENT_FILE_CONF);
                 if (!(new File(FLUENT_FILE)).setExecutable(true)) {
                     DongTaiLog.warn(ErrorCode.FLUENT_SET_EXECUTABLE_FAILED, FLUENT_FILE);
                 }

dongtai-agent/src/main/java/io/dongtai/iast/agent/middlewarerecognition/dubbo/DubboService.java

@@ -13,7 +13,7 @@ public boolean isMatch(RuntimeMXBean paramRuntimeMXBean, ClassLoader loader) {
         } catch (Throwable ignored) {
         }
         try {
-            loader.loadClass("com.alibaba.dubbo.monitor.support.MonitorFilter");
+            loader.loadClass(" com.alibaba.dubbo.monitor.support.MonitorFilter".substring(1));
             return true;
         } catch (Throwable ignored) {
         }

dongtai-agent/src/main/resources/bin/fluent.conf

@@ -3,6 +3,7 @@
   Daemon       OFF
   Log_Level    error
   HTTP_Server  Off
+  parsers_file parsers_multiline.conf
 [INPUT]
   Name         tail
   Path         ${LOG_PATH}
@@ -13,6 +14,7 @@
   Buffer_Max_Size   16MB
   Skip_Long_Lines   On
   Read_from_Head    true
+  multiline.parser multiline-regex-test
 [FILTER]
   Name         record_modifier
   Match        *

dongtai-agent/src/main/resources/bin/parsers_multiline.conf

@@ -0,0 +1,6 @@
+[MULTILINE_PARSER]
+    name          multiline-regex-test
+    type          regex
+    flush_timeout 1000
+    rule      "start_state"   "/(\d+\d+\-\d+\-\d+ \d+\:\d+\:\d+)(.*)/"  "cont"
+    rule      "cont"          "/(^\s+at.*|^Caused.*|^\s+\.\.\..*)/"     "cont"

dongtai-common/src/main/java/io/dongtai/iast/common/constants/AgentConstant.java

@@ -1,7 +1,7 @@
 package io.dongtai.iast.common.constants;
 
 public class AgentConstant {
-    public static final String VERSION_VALUE = "v1.10.0";
+    public static final String VERSION_VALUE = "v1.11.0";
     public static final String LANGUAGE = "JAVA";
     public static final String THREAD_NAME_PREFIX = "DongTai-IAST-";
     public static final String THREAD_NAME_PREFIX_CORE = "DongTai-IAST-Core-";

dongtai-common/src/main/java/io/dongtai/iast/common/constants/PropertyConstant.java

@@ -25,12 +25,12 @@ public class PropertyConstant {
     public static final String PROPERTY_JAR_API_URL = "iast.jar.api.url";
     public static final String PROPERTY_LOG_ADDRESS = "dongtai.log.address";
     public static final String PROPERTY_LOG_PORT = "dongtai.log.port";
-    public static final String PROPERTY_FALLBACK_VERSION = "dongtai.fallback.version";
     public static final String PROPERTY_DUMP_CLASS_PATH = "iast.dump.class.path";
     public static final String PROPERTY_DUMP_CLASS_ENABLE = "iast.dump.class.enable";
     public static final String PROPERTY_SERVICE_HEARTBEAT_INTERVAL = "iast.service.heartbeat.interval";
     public static final String PROPERTY_RESPONSE_LENGTH = "dongtai.response.length";
     public static final String PROPERTY_POLICY_PATH = "dongtai.policy.path";
     public static final String PROPERTY_UUID_PATH = "dongtai.uuid.path";
     public static final String PROPERTY_DISABLED_PLUGINS = "dongtai.disabled.plugins";
+    public static final String PROPERTY_DISABLED_FEATURES = "dongtai.disabled_features";
 }

dongtai-core/src/main/java/io/dongtai/iast/core/EngineManager.java

@@ -136,16 +136,16 @@ public static void enterHttpEntry(Map<String, Object> requestMeta) {
         }
         REQUEST_CONTEXT.set(requestMeta);
         TRACK_MAP.set(new HashMap<Integer, MethodEvent>(1024));
-        TAINT_HASH_CODES.set(new HashSet<Integer>());
-        TAINT_RANGES_POOL.set(new HashMap<Integer, TaintRanges>());
+        TAINT_HASH_CODES.set(new HashSet<Long>());
+        TAINT_RANGES_POOL.set(new HashMap<Long, TaintRanges>());
         ScopeManager.SCOPE_TRACKER.getScope(Scope.HTTP_ENTRY).enter();
     }
 
     public static void enterDubboEntry(Map<String, Object> requestMeta) {
         REQUEST_CONTEXT.set(requestMeta);
         TRACK_MAP.set(new HashMap<Integer, MethodEvent>(1024));
-        TAINT_HASH_CODES.set(new HashSet<Integer>());
-        TAINT_RANGES_POOL.set(new HashMap<Integer, TaintRanges>());
+        TAINT_HASH_CODES.set(new HashSet<Long>());
+        TAINT_RANGES_POOL.set(new HashMap<Long, TaintRanges>());
         ScopeManager.SCOPE_TRACKER.getScope(Scope.DUBBO_ENTRY).enter();
     }
 }

dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/PluginRegister.java

@@ -12,6 +12,7 @@
 import io.dongtai.iast.core.bytecode.enhance.plugin.spring.DispatchApiCollector;
 import io.dongtai.iast.core.handler.hookpoint.models.policy.Policy;
 import io.dongtai.iast.core.handler.hookpoint.models.policy.PolicyManager;
+import io.dongtai.iast.core.utils.PropertyUtils;
 import org.objectweb.asm.ClassVisitor;
 
 import java.util.*;
@@ -28,7 +29,7 @@ public class PluginRegister {
 
     public PluginRegister() {
         this.plugins = new ArrayList<>();
-        List<String> disabledPlugins = getdisabledPlugins();
+        List<String> disabledPlugins = PropertyUtils.getDisabledPlugins();
         List<DispatchPlugin> allPlugins = new ArrayList<>(Arrays.asList(
                 new DispatchApiCollector(),
                 new DispatchJ2ee(),
@@ -43,12 +44,6 @@ public PluginRegister() {
         this.plugins.add(new DispatchClassPlugin());
     }
 
-    private List<String> getdisabledPlugins() {
-        return Optional.ofNullable(System.getProperty("dongtai.disabled.plugins"))
-                .map(s -> Arrays.asList(s.split(",")))
-                .orElse(null);
-    }
-
     public ClassVisitor initial(ClassVisitor classVisitor, ClassContext context, PolicyManager policyManager) {
         Policy policy = policyManager.getPolicy();
         if (policy == null) {