Remove the Arcsight module and the modules framework (#16794)

Remove all module related code
- remove arcsight module
- remove module framework
- remove module tests
- remove module configs

Author: kaisecheng

build.gradle

@@ -334,7 +334,6 @@ tasks.register("assembleTarDistribution") {
   inputs.files fileTree("${projectDir}/bin")
   inputs.files fileTree("${projectDir}/config")
   inputs.files fileTree("${projectDir}/lib")
-  inputs.files fileTree("${projectDir}/modules")
   inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
   inputs.files fileTree("${projectDir}/logstash-core/lib")
   inputs.files fileTree("${projectDir}/logstash-core/src")
@@ -351,7 +350,6 @@ tasks.register("assembleOssTarDistribution") {
   inputs.files fileTree("${projectDir}/bin")
   inputs.files fileTree("${projectDir}/config")
   inputs.files fileTree("${projectDir}/lib")
-  inputs.files fileTree("${projectDir}/modules")
   inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
   inputs.files fileTree("${projectDir}/logstash-core/lib")
   inputs.files fileTree("${projectDir}/logstash-core/src")
@@ -366,7 +364,6 @@ tasks.register("assembleZipDistribution") {
   inputs.files fileTree("${projectDir}/bin")
   inputs.files fileTree("${projectDir}/config")
   inputs.files fileTree("${projectDir}/lib")
-  inputs.files fileTree("${projectDir}/modules")
   inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
   inputs.files fileTree("${projectDir}/logstash-core/lib")
   inputs.files fileTree("${projectDir}/logstash-core/src")
@@ -383,7 +380,6 @@ tasks.register("assembleOssZipDistribution") {
   inputs.files fileTree("${projectDir}/bin")
   inputs.files fileTree("${projectDir}/config")
   inputs.files fileTree("${projectDir}/lib")
-  inputs.files fileTree("${projectDir}/modules")
   inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
   inputs.files fileTree("${projectDir}/logstash-core/lib")
   inputs.files fileTree("${projectDir}/logstash-core/src")

config/logstash.yml

@@ -181,38 +181,6 @@
 #
 # api.auth.basic.password_policy.mode: WARN
 #
-# ------------ Module Settings ---------------
-# Define modules here.  Modules definitions must be defined as an array.
-# The simple way to see this is to prepend each `name` with a `-`, and keep
-# all associated variables under the `name` they are associated with, and
-# above the next, like this:
-#
-# modules:
-#   - name: MODULE_NAME
-#     var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE
-#     var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE
-#     var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE
-#     var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE
-#
-# Module variable names must be in the format of
-#
-# var.PLUGIN_TYPE.PLUGIN_NAME.KEY
-#
-# modules:
-#
-# ------------ Cloud Settings ---------------
-# Define Elastic Cloud settings here.
-# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy
-# and it may have an label prefix e.g. staging:dXMtZ...
-# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host'
-# cloud.id: <identifier>
-#
-# Format of cloud.auth is: <user>:<pass>
-# This is optional
-# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password'
-# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password'
-# cloud.auth: elastic:<password>
-#
 # ------------ Queuing Settings --------------
 #
 # Internal queuing model, "memory" for legacy in-memory based queuing and

docker/data/logstash/env2yaml/env2yaml.go

@@ -17,11 +17,12 @@ package main
 import (
 	"errors"
 	"fmt"
-	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"log"
 	"os"
 	"strings"
+
+	"gopkg.in/yaml.v2"
 )
 
 var validSettings = []string{
@@ -67,7 +68,6 @@ var validSettings = []string{
 	"log.level",
 	"log.format",
 	"log.format.json.fix_duplicate_message_fields",
-	"modules",
 	"metric.collect",
 	"path.logs",
 	"path.plugins",
@@ -126,8 +126,6 @@ var validSettings = []string{
 	"xpack.management.elasticsearch.ssl.cipher_suites",
 	"xpack.geoip.download.endpoint",
 	"xpack.geoip.downloader.enabled",
-	"cloud.id",
-	"cloud.auth",
 }
 
 // Given a setting name, return a downcased version with delimiters removed.

logstash-core/lib/logstash/config/modules_common.rb

@@ -71,37 +71,5 @@ def config_path
     def config_path?
       !(config_path.nil? || config_path.empty?)
     end
-
-    def modules_cli_setting
-      @settings.get_setting("modules.cli")
-    end
-
-    def modules_cli
-      modules_cli_setting.value
-    end
-
-    def modules_cli?
-      !(modules_cli.nil? || modules_cli.empty?)
-    end
-
-    def modules_setting
-      @settings.get_setting("modules")
-    end
-
-    def modules
-      modules_setting.value
-    end
-
-    def modules?
-      !(modules.nil? || modules.empty?)
-    end
-
-    def both_module_configs?
-      modules_cli? && modules?
-    end
-
-    def modules_defined?
-      modules_cli? || modules?
-    end
   end
 end end end

logstash-core/lib/logstash/config/source/base.rb

@@ -179,7 +179,7 @@ def pipeline_configs
 
     def match?
       # see basic settings predicates and getters defined in the base class
-      (config_string? || config_path?) && !(modules_cli? || modules?) && !automatic_reload_with_config_string?
+      (config_string? || config_path?) && !automatic_reload_with_config_string?
     end
 
     def config_conflict?

logstash-core/lib/logstash/config/source/local.rb

@@ -48,7 +48,7 @@ def pipeline_configs
     end
 
     def match?
-      if modules_cli? || modules? || config_string? || config_path?
+      if config_string? || config_path?
         return false
       end
       detect_pipelines if !@detect_pipelines_called
@@ -62,7 +62,7 @@ def invalid_pipelines_detected?
     def config_conflict?
       @conflict_messages.clear
       # are there any auto-reload conflicts?
-      if !(modules_cli? || modules? || config_string? || config_path?)
+      if !(config_string? || config_path?)
         detect_pipelines if !@detect_pipelines_called
         if @detected_marker.nil?
           @conflict_messages << I18n.t("logstash.runner.config-pipelines-failed-read", :path => pipelines_yaml_location)
@@ -74,7 +74,7 @@ def config_conflict?
           @conflict_messages << @detected_marker.message
         end
       else
-        do_warning? && logger.warn("Ignoring the 'pipelines.yml' file because modules or command line options are specified")
+        do_warning? && logger.warn("Ignoring the 'pipelines.yml' file because command line options are specified")
       end
       @conflict_messages.any?
     end

logstash-core/lib/logstash/config/source/modules.rb

@@ -16,7 +16,6 @@
 # under the License.
 
 require "logstash/config/source/local"
-require "logstash/config/source/modules"
 require "logstash/config/source/multi_local"
 require "thread"
 require "set"