update path.log to path.logs directory, and respect it in log4j2

Fixes #5876

Author: talevy

config/log4j2.properties

@@ -2,37 +2,26 @@ status = error
 name = LogstashPropertiesConfig
 
 appender.console.type = Console
-appender.console.name = STDOUT
-
-# Pattern Logging
-#
+appender.console.name = console
 appender.console.layout.type = PatternLayout
 appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
-
 # JSON Logging
-#
 # appender.console.layout.type = JSONLayout
 # appender.console.layout.compact = true
 # appender.console.layout.eventEol = true
 
-# Rolling File Appender
-#
-#property.filename = /tmp/logstash/logstash.log
-#
-# appender.rolling.type = RollingFile
-# appender.rolling.name = RollingFile
-# appender.rolling.fileName = ${filename}
-# appender.rolling.filePattern = /tmp/logstash/logstash-%d{MM-dd-yy-HH-mm-ss}-%i.log.gz
-# appender.rolling.policies.type = Policies
-# appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
-# appender.rolling.policies.time.interval = 2
-# appender.rolling.policies.time.modulate = true
-# appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
-# appender.rolling.policies.size.size=100MB
-# appender.rolling.strategy.type = DefaultRolloverStrategy
-# appender.rolling.strategy.max = 5
-# appender.rolling.layout.type = PatternLayout
-# appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+appender.rolling.type = RollingFile
+appender.rolling.name = rolling
+appender.rolling.fileName = ${sys:ls.logs}/logstash.log
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
+appender.rolling.filePattern = ${sys:ls.logs}/logstash-%d{yyyy-MM-dd}.log
+appender.rolling.policies.type = Policies
+appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval = 1
+appender.rolling.policies.time.modulate = true
 
 rootLogger.level = error
-rootLogger.appenderRef.stdout.ref = STDOUT
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.rolling.ref = rolling
+

config/logstash.yml

@@ -107,7 +107,7 @@
 #   * trace
 #
 # log.level: warn
-# path.log:
+# path.logs: LOGSTASH_HOME/logs
 #
 # ------------ Other Settings --------------
 #

docs/static/command-line-flags.asciidoc

@@ -65,8 +65,8 @@ added[5.0.0-alpha3, Command-line flags have dots instead of dashes in their name
   `PATH/logstash/TYPE/NAME.rb` where `TYPE` is `inputs`, `filters`, `outputs`, or `codecs`,
   and `NAME` is the name of the plugin.
 
-*`-l, --path.log FILE`*::
-  Write Logstash internal logs to the given file. Without this flag, Logstash will emit logs to standard output.
+*`-l, --path.logs PATH`*::
+  Directory to Write Logstash internal logs to.
 
 *`--log.level LEVEL`*::
  Set the log level for Logstash. Possible values are:

logstash-core/lib/logstash/environment.rb

@@ -36,7 +36,6 @@ module Environment
             Setting::String.new("log.level", "warn", true, ["fatal", "error", "warn", "debug", "info", "trace"]),
            Setting::Boolean.new("version", false),
            Setting::Boolean.new("help", false),
-            Setting::String.new("path.log", nil, false),
             Setting::String.new("log.format", "plain", true, ["json", "plain"]),
             Setting::String.new("http.host", "127.0.0.1"),
             Setting::PortRange.new("http.port", 9600..9700),

logstash-core/lib/logstash/runner.rb

@@ -19,11 +19,12 @@
 
 class LogStash::Runner < Clamp::StrictCommand
   include LogStash::Util::Loggable
-  # The `path.settings` need to be defined in the runner instead of the `logstash-core/lib/logstash/environment.rb`
+  # The `path.settings` and `path.logs` need to be defined in the runner instead of the `logstash-core/lib/logstash/environment.rb`
   # because the `Environment::LOGSTASH_HOME` doesn't exist in the context of the `logstash-core` gem.
   #
   # See issue https://github.com/elastic/logstash/issues/5361
   LogStash::SETTINGS.register(LogStash::Setting::String.new("path.settings", ::File.join(LogStash::Environment::LOGSTASH_HOME, "config")))
+  LogStash::SETTINGS.register(LogStash::Setting::String.new("path.logs", ::File.join(LogStash::Environment::LOGSTASH_HOME, "logs")))
 
   # Node Settings
   option ["-n", "--node.name"], "NAME",
@@ -77,9 +78,10 @@ class LogStash::Runner < Clamp::StrictCommand
     :default => LogStash::SETTINGS.get_default("path.plugins")
 
   # Logging Settings
-  option ["-l", "--path.log"], "FILE",
+  option ["-l", "--path.logs"], "PATH",
     I18n.t("logstash.runner.flag.log"),
-    :attribute_name => "path.log"
+    :attribute_name => "path.logs",
+    :default => LogStash::SETTINGS.get_default("path.logs")
 
   option "--log.level", "LEVEL", I18n.t("logstash.runner.flag.log_level"),
     :default => LogStash::SETTINGS.get_default("log.level"),
@@ -172,8 +174,9 @@ def run(args)
     # Configure Logstash logging facility, this need to be done before everything else to
     # make sure the logger has the correct settings and the log level is correctly defined.
     # TODO(talevy): cleanly support `path.logs` setting in log4j
+    java.lang.System.setProperty("ls.logs", setting("path.logs"))
     unless java.lang.System.getProperty("log4j.configurationFile")
-      log4j_config_location = setting("path.settings") + "/log4j2.properties"
+      log4j_config_location = ::File.join(setting("path.settings"), "log4j2.properties")
       LogStash::Logging::Logger::initialize(log4j_config_location)
     end
 

logstash-core/locales/en.yml

@@ -194,6 +194,8 @@ en:
         path_settings: |+
           Directory containing logstash.yml file. This can also be
           set through the LS_SETTINGS_DIR environment variable.
+        path_logs: |+
+            Directory to Write Logstash internal logs to.
         auto_reload: |+
           Monitor configuration changes and reload
           whenever it is changed.

pkg/centos/after-install.sh

@@ -4,7 +4,7 @@ chown logstash:logstash /var/lib/logstash
 chmod 0644 /etc/logrotate.d/logstash
 sed -i \
   -e 's|# path.config:|path.config: /etc/logstash/conf.d|' \
-  -e 's|# path.log:|path.log: /var/log/logstash/logstash.log|' \
+  -e 's|# path.logs:|path.logs: /var/log/logstash|' \
   -e 's|# path.data:|path.data: /var/lib/logstash|' \
   /etc/logstash/logstash.yml
 /usr/share/logstash/bin/system-install /etc/logstash/startup.options

pkg/debian/after-install.sh

@@ -7,7 +7,7 @@ chmod 755 /etc/logstash
 chmod 0644 /etc/logrotate.d/logstash
 sed -i \
   -e 's|# path.config:|path.config: /etc/logstash/conf.d|' \
-  -e 's|# path.log:|path.log: /var/log/logstash/logstash.log|' \
+  -e 's|# path.logs:|path.logs: /var/log/logstash|' \
   -e 's|# path.data:|path.data: /var/lib/logstash|' \
   /etc/logstash/logstash.yml
 /usr/share/logstash/bin/system-install /etc/logstash/startup.options

pkg/ubuntu/after-install.sh

@@ -6,7 +6,7 @@ chown logstash:logstash /var/lib/logstash
 chmod 0644 /etc/logrotate.d/logstash
 sed -i \
   -e 's|# path.config:|path.config: /etc/logstash/conf.d|' \
-  -e 's|# path.log:|path.log: /var/log/logstash/logstash.log|' \
+  -e 's|# path.logs:|path.logs: /var/log/logstash|' \
   -e 's|# path.data:|path.data: /var/lib/logstash|' \
   /etc/logstash/logstash.yml
 /usr/share/logstash/bin/system-install /etc/logstash/startup.options