diff --git a/pom.xml b/pom.xml index 3f352af3..0a216283 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,7 @@ org.apache.httpcomponents httpclient - 4.4 + 4.5.3 redis.clients @@ -109,6 +109,16 @@ 2.15 true + + org.jenkins-ci.plugins + conditional-buildstep + 1.3.6 + + + org.jenkins-ci.plugins + flexible-publish + 0.15.2 + org.mockito mockito-core @@ -258,6 +268,40 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + display-info + + display-info + enforce + + validate + + + + + org.codehaus.plexus:plexus-classworlds + org.jenkins-ci.plugins:conditional-buildstep + org.codehaus.plexus:plexus-utils + com.google.guava:guava + commons-logging:commons-logging + com.google.code.findbugs:jsr305 + net.java.dev.jna:jna + org.apache.ant:ant + org.apache.maven:maven-embedder + org.apache.maven:maven-core + org.apache.maven:maven-aether-provider + org.jenkins-ci.plugins:matrix-project + + + + + + + diff --git a/src/main/java/jenkins/plugins/logstash/GloballyEnabledMode.java b/src/main/java/jenkins/plugins/logstash/GloballyEnabledMode.java new file mode 100644 index 00000000..1ff5eb70 --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/GloballyEnabledMode.java @@ -0,0 +1,22 @@ +package jenkins.plugins.logstash; + +public enum GloballyEnabledMode +{ + + // TODO: Introduce a DATAONLY mode, that allows to send a start event and an end event only without sending the log itself. + OFF("Do not send logs globally."), + LINEMODE("Send log line by line."), + NOTIFIERMODE("Send log at the end of the build."); + + private String description; + + private GloballyEnabledMode(String description) + { + this.description = description; + } + + public String getDescrition() + { + return description; + } +} diff --git a/src/main/java/jenkins/plugins/logstash/LogstashConfiguration.java b/src/main/java/jenkins/plugins/logstash/LogstashConfiguration.java index 589322c4..2294c465 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashConfiguration.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashConfiguration.java @@ -39,9 +39,11 @@ public class LogstashConfiguration extends GlobalConfiguration private LogstashIndexer logstashIndexer; private Boolean enabled; private boolean dataMigrated = false; - private boolean enableGlobally = false; + private transient boolean enableGlobally = false; private boolean milliSecondTimestamps = true; private transient LogstashIndexer activeIndexer; + private GloballyEnabledMode globalMode; + private int maxLines = 1000; // a flag indicating if we're currently in the configure method. private transient boolean configuring = false; @@ -63,6 +65,26 @@ public LogstashConfiguration() activeIndexer = logstashIndexer; } + public int getMaxLines() + { + return maxLines; + } + + public void setMaxLines(int maxLines) + { + this.maxLines = maxLines; + } + + public GloballyEnabledMode getGlobalMode() + { + return globalMode; + } + + public void setGlobalMode(GloballyEnabledMode globalMode) + { + this.globalMode = globalMode; + } + public boolean isEnabled() { return enabled == null ? false: enabled; @@ -73,14 +95,23 @@ public void setEnabled(boolean enabled) this.enabled = enabled; } + @Deprecated public boolean isEnableGlobally() { - return enableGlobally; + return Objects.equals(globalMode, GloballyEnabledMode.LINEMODE); } + @Deprecated public void setEnableGlobally(boolean enableGlobally) { - this.enableGlobally = enableGlobally; + if (enableGlobally) + { + globalMode = GloballyEnabledMode.LINEMODE; + } + else + { + globalMode = GloballyEnabledMode.OFF; + } } public boolean isMilliSecondTimestamps() @@ -230,6 +261,17 @@ public void migrateData() dataMigrated = true; save(); } + if (globalMode == null) + { + if (enableGlobally == true) + { + globalMode = GloballyEnabledMode.LINEMODE; + } + else + { + globalMode = GloballyEnabledMode.OFF; + } + } } @Override @@ -246,7 +288,7 @@ public boolean configure(StaplerRequest staplerRequest, JSONObject json) throws save(); return true; } - + configuring = true; // when we bind the stapler request we get a new instance of logstashIndexer. diff --git a/src/main/java/jenkins/plugins/logstash/LogstashConsoleLogFilter.java b/src/main/java/jenkins/plugins/logstash/LogstashConsoleLogFilter.java index d5995163..e24f9e68 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashConsoleLogFilter.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashConsoleLogFilter.java @@ -8,9 +8,9 @@ import hudson.Extension; import hudson.console.ConsoleLogFilter; -import hudson.model.AbstractBuild; import hudson.model.AbstractProject; import hudson.model.Run; +import hudson.tasks.Publisher; @Extension(ordinal = 1000) public class LogstashConsoleLogFilter extends ConsoleLogFilter implements Serializable @@ -19,17 +19,24 @@ public class LogstashConsoleLogFilter extends ConsoleLogFilter implements Serial private static final Logger LOGGER = Logger.getLogger(LogstashConsoleLogFilter.class.getName()); private transient Run run; + public LogstashConsoleLogFilter() {} public LogstashConsoleLogFilter(Run run) { this.run = run; } + private static final long serialVersionUID = 1L; + /** + * {@inheritDoc} + * + */ @Override public OutputStream decorateLogger(Run build, OutputStream logger) throws IOException, InterruptedException { + LogstashConfiguration configuration = LogstashConfiguration.getInstance(); if (!configuration.isEnabled()) { @@ -37,27 +44,36 @@ public OutputStream decorateLogger(Run build, OutputStream logger) throws IOExce return logger; } - if (build != null && build instanceof AbstractBuild) + + // A pipeline step uses the constructor which sets run. + if (run != null) { - if (isLogstashEnabled(build)) - { - LogstashWriter logstash = getLogStashWriter(build, logger); - return new LogstashOutputStream(logger, logstash); - } - else + if (run.getAction(LogstashMarkerAction.class) != null) { + LOGGER.log(Level.FINEST, "Logstash is enabled globally. No need to decorate the logger another time for {0}", + run.toString()); return logger; } + return getLogstashOutputStream(run, logger); } - if (run != null) - { - LogstashWriter logstash = getLogStashWriter(run, logger); - return new LogstashOutputStream(logger, logstash); - } - else + + LogstashMarkerAction markerAction = new LogstashMarkerAction(); + build.addAction(markerAction); + + // Not pipeline step so @{code build} should be set. + if (isLogstashEnabled(build)) { - return logger; + markerAction.setLineModeEnabled(true); + return getLogstashOutputStream(build, logger); } + + return logger; + } + + private LogstashOutputStream getLogstashOutputStream(Run run, OutputStream logger) + { + LogstashWriter logstash = getLogStashWriter(run, logger); + return new LogstashOutputStream(logger, logstash); } LogstashWriter getLogStashWriter(Run build, OutputStream errorStream) @@ -65,23 +81,42 @@ LogstashWriter getLogStashWriter(Run build, OutputStream errorStream) return new LogstashWriter(build, errorStream, null, build.getCharset()); } - private boolean isLogstashEnabled(Run build) + private boolean isLogstashEnabledGlobally() { LogstashConfiguration configuration = LogstashConfiguration.getInstance(); - if (configuration.isEnableGlobally()) + if (configuration.getGlobalMode() == GloballyEnabledMode.LINEMODE) { + LOGGER.log(Level.FINEST, "Line mode is enabled globally."); return true; } + return false; + } + + private boolean isLogstashEnabled(Run build) + { + if (build == null) + { + return false; + } if (build.getParent() instanceof AbstractProject) { AbstractProject project = (AbstractProject)build.getParent(); - if (project.getProperty(LogstashJobProperty.class) != null) + LogstashJobProperty property = project.getProperty(LogstashJobProperty.class); + if (property != null) { - return true; + LOGGER.log(Level.FINEST, "Property is set and disableGlobal is: " + property.isDisableGlobal()); + return !property.isDisableGlobal(); + } + else + { + if (PluginImpl.getLogstashNotifier(project) != null) + { + return false; + } + return isLogstashEnabledGlobally(); } } return false; } - } diff --git a/src/main/java/jenkins/plugins/logstash/LogstashJobProperty.java b/src/main/java/jenkins/plugins/logstash/LogstashJobProperty.java index e595d928..b44c6c08 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashJobProperty.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashJobProperty.java @@ -1,38 +1,40 @@ package jenkins.plugins.logstash; import org.kohsuke.stapler.DataBoundConstructor; -import org.kohsuke.stapler.StaplerRequest; +import org.kohsuke.stapler.DataBoundSetter; import hudson.Extension; import hudson.model.AbstractProject; import hudson.model.Job; -import hudson.model.JobProperty; -import hudson.model.JobPropertyDescriptor; -import net.sf.json.JSONObject; +import jenkins.model.OptionalJobProperty; /** * This JobProperty is a marker to decide if logs should be sent to an indexer. * */ -public class LogstashJobProperty extends JobProperty> +public class LogstashJobProperty extends OptionalJobProperty> { + private boolean disableGlobal; + @DataBoundConstructor public LogstashJobProperty() {} + public boolean isDisableGlobal() + { + return disableGlobal; + } + + @DataBoundSetter + public void setDisableGlobal(boolean disableGlobal) + { + this.disableGlobal = disableGlobal; + } + @Extension - public static class DescriptorImpl extends JobPropertyDescriptor + public static class DescriptorImpl extends OptionalJobPropertyDescriptor { - @Override - public JobProperty newInstance(StaplerRequest req, JSONObject formData) throws FormException - { - if (formData.containsKey("enable")) - { - return new LogstashJobProperty(); - } - return null; - } @Override public String getDisplayName() diff --git a/src/main/java/jenkins/plugins/logstash/LogstashMarkerAction.java b/src/main/java/jenkins/plugins/logstash/LogstashMarkerAction.java new file mode 100644 index 00000000..476d4b6d --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/LogstashMarkerAction.java @@ -0,0 +1,52 @@ +package jenkins.plugins.logstash; + +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; + +import hudson.Extension; +import hudson.model.InvisibleAction; + +/* + * A marker for Runs containing information about Logstash configuration of the job + * When enabled per JobProperty it doesn't make sense to send the data another time in the notifier + * And when the notifier decides to send at the end, we need to know the number of lines to send. + * Note: When wrapped in a conditional build step or flexible publish, the standard detection will not catch it + */ +@Extension +@Restricted(NoExternalUse.class) +public class LogstashMarkerAction extends InvisibleAction +{ + private boolean lineModeEnabled; + private boolean runNotifierAtEnd; + private int maxLines; + + public boolean isRunNotifierAtEnd() + { + return runNotifierAtEnd; + } + + public void setRunNotifierAtEnd(boolean runNotifierAtEnd) + { + this.runNotifierAtEnd = runNotifierAtEnd; + } + + public int getMaxLines() + { + return maxLines; + } + + public void setMaxLines(int maxLines) + { + this.maxLines = maxLines; + } + + public boolean isLineModeEnabled() + { + return lineModeEnabled; + } + + public void setLineModeEnabled(boolean lineModeEnabled) + { + this.lineModeEnabled = lineModeEnabled; + } +} \ No newline at end of file diff --git a/src/main/java/jenkins/plugins/logstash/LogstashNotifier.java b/src/main/java/jenkins/plugins/logstash/LogstashNotifier.java index b5221d35..78461a4f 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashNotifier.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashNotifier.java @@ -46,6 +46,7 @@ import java.io.IOException; import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.DataBoundSetter; import org.kohsuke.stapler.QueryParameter; import org.jenkinsci.Symbol; @@ -61,11 +62,24 @@ public class LogstashNotifier extends Notifier implements SimpleBuildStep { private final int maxLines; private final boolean failBuild; + private boolean runNotifierAtEnd; @DataBoundConstructor public LogstashNotifier(int maxLines, boolean failBuild) { this.maxLines = maxLines; this.failBuild = failBuild; + + } + + public boolean isRunNotifierAtEnd() + { + return runNotifierAtEnd; + } + + @DataBoundSetter + public void setRunNotifierAtEnd(boolean runNotifierAtEnd) + { + this.runNotifierAtEnd = runNotifierAtEnd; } public int getMaxLines() @@ -99,12 +113,43 @@ private boolean perform(Run run, TaskListener listener) { return true; } + // globally enabled and active. No need to send the data another time. + LogstashMarkerAction markerAction = getLogstashMarkerAction(run); + if (markerAction != null && markerAction.isLineModeEnabled()) + { + listener.getLogger().println("LogstashNotifier ignored. The data is already sent line by line."); + return true; + } + + if (runNotifierAtEnd) + { + markerAction.setRunNotifierAtEnd(true); + markerAction.setMaxLines(maxLines); + return true; + } + PrintStream errorPrintStream = listener.getLogger(); LogstashWriter logstash = getLogStashWriter(run, errorPrintStream, listener); logstash.writeBuildLog(maxLines); return !(failBuild && logstash.isConnectionBroken()); } + /** + * Returns the LogstashMarkerAction of this build. + * @param run The {@link Run} to get the action from. + * @return LogstashMarkerAction, never {@code null} + */ + private synchronized LogstashMarkerAction getLogstashMarkerAction(Run run) + { + LogstashMarkerAction markerAction = run.getAction(LogstashMarkerAction.class); + if (markerAction == null) + { + markerAction = new LogstashMarkerAction(); + run.addAction(markerAction); + } + + return markerAction; + } // Method to encapsulate calls for unit-testing LogstashWriter getLogStashWriter(Run run, OutputStream errorStream, TaskListener listener) { return new LogstashWriter(run, errorStream, listener, run.getCharset()); diff --git a/src/main/java/jenkins/plugins/logstash/LogstashRunListener.java b/src/main/java/jenkins/plugins/logstash/LogstashRunListener.java new file mode 100644 index 00000000..5ba2460a --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/LogstashRunListener.java @@ -0,0 +1,60 @@ +package jenkins.plugins.logstash; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.acegisecurity.ui.AbstractProcessingFilter; + +import hudson.Extension; +import hudson.model.AbstractProject; +import hudson.model.Run; +import hudson.model.TaskListener; +import hudson.model.listeners.RunListener; +import hudson.tasks.Publisher; + +@Extension +public class LogstashRunListener extends RunListener> +{ + + private static final Logger LOGGER = Logger.getLogger(LogstashRunListener.class.getName()); + + @Override + public void onFinalized(Run run) + { + LogstashConfiguration config = LogstashConfiguration.getInstance(); + if (!config.isEnabled()) + { + return; + } + LogstashMarkerAction markerAction = run.getAction(LogstashMarkerAction.class); + if ((markerAction != null && markerAction.isRunNotifierAtEnd()) || config.getGlobalMode() == GloballyEnabledMode.NOTIFIERMODE) + { + int maxLines = config.getMaxLines(); + if (run.getParent() instanceof AbstractProject) + { + AbstractProject project = (AbstractProject) run.getParent(); + //don't run if project has a the wrapper explicitly enabled or disabled global settings + if (project.getProperty(LogstashJobProperty.class) != null) + { + LOGGER.log(Level.FINEST, "Job Property is set. Disabling global notifier mode."); + return; + } + //don't run if project has the notifier explicitly enabled + if (PluginImpl.getLogstashNotifier(project) != null) + { + if (markerAction == null || !markerAction.isRunNotifierAtEnd()) + { + LOGGER.log(Level.FINEST, "Job has explicit Notifier not running at end."); + return; + } + } + } + if (markerAction != null && markerAction.isRunNotifierAtEnd()) + { + maxLines = markerAction.getMaxLines(); + } + LogstashWriter logstash = new LogstashWriter(run, null, null, run.getCharset()); + logstash.writeBuildLog(maxLines); + } + } +} diff --git a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java index 2523814a..810760c6 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java @@ -39,6 +39,7 @@ import java.io.OutputStream; import java.nio.charset.Charset; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.List; @@ -122,6 +123,9 @@ public void writeBuildLog(int maxLines) { try { if (maxLines < 0) { logLines = build.getLog(Integer.MAX_VALUE); + } else if (maxLines == 0) + { + logLines = Collections.emptyList(); } else { logLines = build.getLog(maxLines); } diff --git a/src/main/java/jenkins/plugins/logstash/PluginImpl.java b/src/main/java/jenkins/plugins/logstash/PluginImpl.java index bf946ed8..840f47b6 100644 --- a/src/main/java/jenkins/plugins/logstash/PluginImpl.java +++ b/src/main/java/jenkins/plugins/logstash/PluginImpl.java @@ -25,11 +25,21 @@ import hudson.DescriptorExtensionList; import hudson.Plugin; +import hudson.model.AbstractProject; import hudson.model.Descriptor; +import hudson.model.Project; +import hudson.tasks.BuildStep; +import hudson.tasks.Builder; +import hudson.tasks.Publisher; import jenkins.plugins.logstash.configuration.LogstashIndexer; import java.util.logging.Logger; +import org.jenkins_ci.plugins.flexible_publish.ConditionalPublisher; +import org.jenkins_ci.plugins.flexible_publish.FlexiblePublisher; +import org.jenkinsci.plugins.conditionalbuildstep.ConditionalBuilder; +import org.jenkinsci.plugins.conditionalbuildstep.singlestep.SingleConditionalBuilder; + public class PluginImpl extends Plugin { private final static Logger LOG = Logger.getLogger(PluginImpl.class.getName()); @@ -46,5 +56,63 @@ public DescriptorExtensionList, Descriptor { return LogstashIndexer.all(); } + + /** + * Gets the {@link LogstashNotifier} that is configured in this project. + * + * @param project The project to look for a {@link LogstashNotifier}. + * @return The {@link LogstashNotifier} or null, if no {@link LogstashNotifier} is configured. + */ + public static LogstashNotifier getLogstashNotifier(AbstractProject project) + { + if (project instanceof Project) + { + Project p = (Project) project; + for (Builder builder: p.getBuilders()) + { + if (builder instanceof SingleConditionalBuilder) + { + SingleConditionalBuilder scb = (SingleConditionalBuilder) builder; + if (scb.getBuildStep() instanceof LogstashNotifier) + { + return (LogstashNotifier) scb.getBuildStep(); + } + } + if (builder instanceof ConditionalBuilder) + { + ConditionalBuilder cb = (ConditionalBuilder) builder; + for (BuildStep bs: cb.getConditionalbuilders()) + { + if (bs instanceof LogstashNotifier) + { + return (LogstashNotifier) bs; + } + } + } + } + } + for (Publisher publisher: project.getPublishersList()) + { + if (publisher instanceof LogstashNotifier) + { + return (LogstashNotifier) publisher; + } + if (publisher instanceof FlexiblePublisher) + { + FlexiblePublisher fp = (FlexiblePublisher) publisher; + for (ConditionalPublisher cp: fp.getPublishers()) + { + for (BuildStep bs: cp.getPublisherList()) + { + if (bs instanceof LogstashNotifier) + { + return (LogstashNotifier) bs; + } + } + } + } + } + return null; + } } diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashConfiguration/config.jelly b/src/main/resources/jenkins/plugins/logstash/LogstashConfiguration/config.jelly index 1ad826e2..9014aa62 100644 --- a/src/main/resources/jenkins/plugins/logstash/LogstashConfiguration/config.jelly +++ b/src/main/resources/jenkins/plugins/logstash/LogstashConfiguration/config.jelly @@ -3,13 +3,26 @@ - - - - + + + + + + + + + + + + + +
+ + + - + diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashConfiguration/help-enableGlobally.html b/src/main/resources/jenkins/plugins/logstash/LogstashConfiguration/help-enableGlobally.html deleted file mode 100644 index 4034f221..00000000 --- a/src/main/resources/jenkins/plugins/logstash/LogstashConfiguration/help-enableGlobally.html +++ /dev/null @@ -1 +0,0 @@ -Checking will make all non pipeline builds forward their log to the above indexer. \ No newline at end of file diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashConfiguration/help-globalMode.html b/src/main/resources/jenkins/plugins/logstash/LogstashConfiguration/help-globalMode.html new file mode 100644 index 00000000..341e9943 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/LogstashConfiguration/help-globalMode.html @@ -0,0 +1,3 @@ +OFF: Do not send log for each build. +Line Mode: Enable sending an event for each log line. This will not enable it for pipeline jobs. +Notifier Mode: Enable sending the log at the end of the build in a single event \ No newline at end of file diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashJobProperty/config.jelly b/src/main/resources/jenkins/plugins/logstash/LogstashJobProperty/config-details.jelly similarity index 54% rename from src/main/resources/jenkins/plugins/logstash/LogstashJobProperty/config.jelly rename to src/main/resources/jenkins/plugins/logstash/LogstashJobProperty/config-details.jelly index 00a6dd27..d1ddc988 100644 --- a/src/main/resources/jenkins/plugins/logstash/LogstashJobProperty/config.jelly +++ b/src/main/resources/jenkins/plugins/logstash/LogstashJobProperty/config-details.jelly @@ -1,5 +1,6 @@ - - + + + diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashJobProperty/help-disableGlobal.html b/src/main/resources/jenkins/plugins/logstash/LogstashJobProperty/help-disableGlobal.html new file mode 100644 index 00000000..699994b9 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/LogstashJobProperty/help-disableGlobal.html @@ -0,0 +1 @@ +If a global logstash forwarder is set, disable it and do not send logs to an indexer, neither per line nor as a notifier. An explicitly configured notifier might still send logs to an indexer. \ No newline at end of file diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashNotifier/config.jelly b/src/main/resources/jenkins/plugins/logstash/LogstashNotifier/config.jelly index e3097eef..41662561 100644 --- a/src/main/resources/jenkins/plugins/logstash/LogstashNotifier/config.jelly +++ b/src/main/resources/jenkins/plugins/logstash/LogstashNotifier/config.jelly @@ -1,11 +1,14 @@ - + - + + + + diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashNotifier/help.html b/src/main/resources/jenkins/plugins/logstash/LogstashNotifier/help.html index 29230e51..f65ed372 100644 --- a/src/main/resources/jenkins/plugins/logstash/LogstashNotifier/help.html +++ b/src/main/resources/jenkins/plugins/logstash/LogstashNotifier/help.html @@ -1,3 +1,6 @@
-

Send the tail of the log to Logstash.

+

Send the tail of the log to Logstash.
+ Any changes to the result, that happen after the notifier is run are not reflected in the upload. + If the a project enables uploading the log via a JobProperty in line mode as well, the Notifier gets disabled. +

diff --git a/src/main/webapp/help/help-enableGlobally.html b/src/main/webapp/help/help-enableGlobally.html new file mode 100644 index 00000000..639f7348 --- /dev/null +++ b/src/main/webapp/help/help-enableGlobally.html @@ -0,0 +1,16 @@ +
+Allows to enable the upload globally for all jobs. If jobs explicitly configure uploading this will overwrite the settings defined here.
+E.g. you enable the notifier mode globally, a project defines the line mode, the logs will be uploaded in line mode only. +
    +
  • OFF:
    + Do not send log for each build.
    • +
  • Line Mode:
    + Enable sending an event for each log line. This will not enable it for pipeline jobs. +
    • +
  • + Notifier Mode:
    + Enable sending the log at the end of the build in a single event.
    + This will never fail the build even when uploading fails. +
    • +
+
\ No newline at end of file diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashNotifier/help-maxLines.html b/src/main/webapp/help/help-maxLines.html similarity index 62% rename from src/main/resources/jenkins/plugins/logstash/LogstashNotifier/help-maxLines.html rename to src/main/webapp/help/help-maxLines.html index 8c04962a..2beac01e 100644 --- a/src/main/resources/jenkins/plugins/logstash/LogstashNotifier/help-maxLines.html +++ b/src/main/webapp/help/help-maxLines.html @@ -1,5 +1,7 @@

The maximum number of log lines to send to Logstash.
If the log is bigger than this, only the most recent lines are sent.
- -1 indicates there is no maximum (for very large logs, consider using the Logstash Job Property instead).

+ 0 indicates to send no log, basically just the job data is sent.
+ -1 indicates there is no maximum (for very large logs, consider using the Logstash Job Property instead).
+

diff --git a/src/main/webapp/help/help-runNotifierAtEnd.html b/src/main/webapp/help/help-runNotifierAtEnd.html new file mode 100644 index 00000000..1c01e67e --- /dev/null +++ b/src/main/webapp/help/help-runNotifierAtEnd.html @@ -0,0 +1,5 @@ +
+ Runs the upload of the tail of the log after the build has completely finished. + The difference to the regular notifier is that the result is final (also for pipeline builds) and the log is complete.
+ The failBuild option is ignored in this case. +
diff --git a/src/test/java/jenkins/plugins/logstash/LogstashConsoleLogFilterTest.java b/src/test/java/jenkins/plugins/logstash/LogstashConsoleLogFilterTest.java index 869cf147..4ae20bf2 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashConsoleLogFilterTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashConsoleLogFilterTest.java @@ -6,12 +6,16 @@ import static org.mockito.Mockito.verify; import hudson.model.AbstractBuild; +import hudson.model.Descriptor; import hudson.model.Project; import hudson.model.Run; +import hudson.tasks.Publisher; +import hudson.util.DescribableList; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.Collections; import jenkins.plugins.logstash.persistence.BuildData; @@ -61,19 +65,26 @@ LogstashWriter getLogStashWriter(Run build, OutputStream errorStream) { @Mock Project mockProject; @Mock BuildData mockBuildData; @Mock LogstashWriter mockWriter; + @Mock LogstashJobProperty mockProperty; + private DescribableList> publishers; + private MockLogstashConsoleLogFilter consoleLogFilter; @Before public void before() throws Exception { + publishers = new DescribableList<>(mockProject); + buffer = new ByteArrayOutputStream(); + consoleLogFilter = new MockLogstashConsoleLogFilter(mockWriter); + PowerMockito.mockStatic(LogstashConfiguration.class); when(LogstashConfiguration.getInstance()).thenReturn(logstashConfiguration); - when(logstashConfiguration.isEnableGlobally()).thenReturn(false); + when(logstashConfiguration.getGlobalMode()).thenReturn(GloballyEnabledMode.OFF); when(logstashConfiguration.isEnabled()).thenReturn(true); when(mockWriter.isConnectionBroken()).thenReturn(false); when(mockBuild.getParent()).thenReturn(mockProject); - when(mockProject.getProperty(LogstashJobProperty.class)).thenReturn(new LogstashJobProperty()); + when(mockProject.getProperty(LogstashJobProperty.class)).thenReturn(null); + when(mockProject.getPublishersList()).thenReturn(publishers); - buffer = new ByteArrayOutputStream(); } @After @@ -83,9 +94,27 @@ public void after() throws Exception { buffer.close(); } + private void assertIsOriginalOutputStream(OutputStream result) + { + // Verify results + assertNotNull("Result was null", result); + assertTrue("Result is not the right type", result == buffer); + assertEquals("Results don't match", "", buffer.toString()); + } + + private void assertIsLogstashOutputStream(OutputStream result) + { + // Verify results + assertNotNull("Result was null", result); + assertTrue("Result is not the right type", result instanceof LogstashOutputStream); + assertSame("Result has wrong writer", mockWriter, ((LogstashOutputStream) result).getLogstashWriter()); + assertEquals("Results don't match", "", buffer.toString()); + } + @Test - public void decorateLoggerSuccess() throws Exception { - MockLogstashConsoleLogFilter consoleLogFilter = new MockLogstashConsoleLogFilter(mockWriter); + public void successBadWriter() throws Exception { + when(mockProject.getProperty(LogstashJobProperty.class)).thenReturn(mockProperty); + when(mockWriter.isConnectionBroken()).thenReturn(true); // Unit under test OutputStream result = consoleLogFilter.decorateLogger(mockBuild, buffer); @@ -94,56 +123,97 @@ public void decorateLoggerSuccess() throws Exception { assertNotNull("Result was null", result); assertTrue("Result is not the right type", result instanceof LogstashOutputStream); assertSame("Result has wrong writer", mockWriter, ((LogstashOutputStream) result).getLogstashWriter()); - assertEquals("Results don't match", "", buffer.toString()); + assertEquals("Error was not written", "Mocked Constructor failure", buffer.toString()); verify(mockWriter).isConnectionBroken(); } @Test - public void decorateLoggerSuccessLogstashNotEnabled() throws Exception { - when(mockProject.getProperty(LogstashJobProperty.class)).thenReturn(null); + public void successJobPropertyEnabled() throws Exception { + when(mockProject.getProperty(LogstashJobProperty.class)).thenReturn(mockProperty); + + // Unit under test + OutputStream result = consoleLogFilter.decorateLogger(mockBuild, buffer); + + assertIsLogstashOutputStream(result); + verify(mockWriter).isConnectionBroken(); + } - MockLogstashConsoleLogFilter consoleLogFilter = new MockLogstashConsoleLogFilter(mockWriter); + @Test + public void successJobPropertySetDisable() throws Exception { + when(mockProject.getProperty(LogstashJobProperty.class)).thenReturn(mockProperty); + when(mockProperty.isDisableGlobal()).thenReturn(true); // Unit under test OutputStream result = consoleLogFilter.decorateLogger(mockBuild, buffer); - // Verify results - assertNotNull("Result was null", result); - assertTrue("Result is not the right type", result == buffer); - assertEquals("Results don't match", "", buffer.toString()); + assertIsOriginalOutputStream(result); } @Test - public void decorateLoggerSuccessBadWriter() throws Exception { - when(mockWriter.isConnectionBroken()).thenReturn(true); + public void successGlobalOffJobPropertyNull() throws Exception { + // Unit under test + OutputStream result = consoleLogFilter.decorateLogger(mockBuild, buffer); - MockLogstashConsoleLogFilter consoleLogFilter = new MockLogstashConsoleLogFilter(mockWriter); + assertIsOriginalOutputStream(result); + } + + @Test + public void successGlobalLineMode() throws IOException, InterruptedException + { + when(logstashConfiguration.getGlobalMode()).thenReturn(GloballyEnabledMode.LINEMODE); // Unit under test OutputStream result = consoleLogFilter.decorateLogger(mockBuild, buffer); - // Verify results - assertNotNull("Result was null", result); - assertTrue("Result is not the right type", result instanceof LogstashOutputStream); - assertSame("Result has wrong writer", mockWriter, ((LogstashOutputStream) result).getLogstashWriter()); - assertEquals("Error was not written", "Mocked Constructor failure", buffer.toString()); + assertIsLogstashOutputStream(result); verify(mockWriter).isConnectionBroken(); } @Test - public void decorateLoggerSuccessEnabledGlobally() throws IOException, InterruptedException + public void successGlobalLineModeWithNotifier() throws Exception { + when(logstashConfiguration.getGlobalMode()).thenReturn(GloballyEnabledMode.LINEMODE); + publishers.add(new LogstashNotifier(10, false)); + + // Unit under test + OutputStream result = consoleLogFilter.decorateLogger(mockBuild, buffer); + + assertIsOriginalOutputStream(result); + } + + @Test + public void successGlobalLineModeWithJobPropertyDisabled() throws Exception { + when(mockProperty.isDisableGlobal()).thenReturn(true); + when(logstashConfiguration.getGlobalMode()).thenReturn(GloballyEnabledMode.LINEMODE); + publishers.add(new LogstashNotifier(10, false)); + + // Unit under test + OutputStream result = consoleLogFilter.decorateLogger(mockBuild, buffer); + + assertIsOriginalOutputStream(result); + } + + @Test + public void successGlobalNotifierModeWithJobProperty() throws IOException, InterruptedException { - when(logstashConfiguration.isEnableGlobally()).thenReturn(true); - MockLogstashConsoleLogFilter buildWrapper = new MockLogstashConsoleLogFilter(mockWriter); + when(mockProject.getProperty(LogstashJobProperty.class)).thenReturn(mockProperty); + when(logstashConfiguration.getGlobalMode()).thenReturn(GloballyEnabledMode.NOTIFIERMODE); // Unit under test - OutputStream result = buildWrapper.decorateLogger(mockBuild, buffer); + OutputStream result = consoleLogFilter.decorateLogger(mockBuild, buffer); - // Verify results - assertNotNull("Result was null", result); - assertTrue("Result is not the right type", result instanceof LogstashOutputStream); - assertSame("Result has wrong writer", mockWriter, ((LogstashOutputStream) result).getLogstashWriter()); - assertEquals("Results don't match", "", buffer.toString()); + assertIsLogstashOutputStream(result); verify(mockWriter).isConnectionBroken(); } + + @Test + public void successGlobalNotifierMode() throws Exception { + when(mockProject.getProperty(LogstashJobProperty.class)).thenReturn(null); + when(logstashConfiguration.getGlobalMode()).thenReturn(GloballyEnabledMode.NOTIFIERMODE); + publishers.add(new LogstashNotifier(10, false)); + + // Unit under test + OutputStream result = consoleLogFilter.decorateLogger(mockBuild, buffer); + + assertIsOriginalOutputStream(result); + } } diff --git a/src/test/java/jenkins/plugins/logstash/LogstashIntegrationTest.java b/src/test/java/jenkins/plugins/logstash/LogstashIntegrationTest.java index b2e7ff7c..cfbaec79 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashIntegrationTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashIntegrationTest.java @@ -17,6 +17,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.jvnet.hudson.test.FailureBuilder; import org.jvnet.hudson.test.JenkinsRule; import com.michelin.cio.hudson.plugins.maskpasswords.MaskPasswordsBuildWrapper; @@ -30,6 +31,7 @@ import hudson.model.Slave; import hudson.model.queue.QueueTaskFuture; import hudson.plugins.ansicolor.AnsiColorBuildWrapper; +import hudson.tasks.BuildStep; import net.sf.json.JSONArray; import jenkins.plugins.logstash.configuration.MemoryIndexer; import jenkins.plugins.logstash.persistence.MemoryDao; @@ -58,6 +60,7 @@ public void setup() throws Exception MemoryIndexer indexer = new MemoryIndexer(memoryDao); config.setLogstashIndexer(indexer); config.setEnabled(true); + config.setMaxLines(1000); slave = jenkins.createSlave(); slave.setLabelString("myLabel"); @@ -203,21 +206,34 @@ public void passwordsAreMaskedWithGlobalMaskPasswordsConfiguration() throws Exce } @Test - public void enableGlobally() throws Exception + public void enableGlobalLineMode() throws Exception { - LogstashConfiguration.getInstance().setEnableGlobally(true); + LogstashConfiguration.getInstance().setGlobalMode(GloballyEnabledMode.LINEMODE); + QueueTaskFuture f = project.scheduleBuild2(0); + project.getBuildersList().add(new FailureBuilder()); + + FreeStyleBuild build = f.get(); + assertThat(build.getResult(), equalTo(Result.FAILURE)); + List dataLines = memoryDao.getOutput(); + assertThat(dataLines.size(), is(6)); + JSONObject lastLine = dataLines.get(dataLines.size()-1); + assertThat(lastLine.getJSONArray("message").get(0).toString(),equalTo("Finished: FAILURE")); + } + + @Test + public void enableGlobalNotifierMode() throws Exception + { + LogstashConfiguration.getInstance().setGlobalMode(GloballyEnabledMode.NOTIFIERMODE); QueueTaskFuture f = project.scheduleBuild2(0); FreeStyleBuild build = f.get(); assertThat(build.getResult(), equalTo(Result.SUCCESS)); List dataLines = memoryDao.getOutput(); - assertThat(dataLines.size(), is(4)); + assertThat(dataLines.size(), is(1)); JSONObject firstLine = dataLines.get(0); - JSONObject lastLine = dataLines.get(dataLines.size()-1); - JSONObject data = firstLine.getJSONObject("data"); - assertThat(data.getString("buildHost"),equalTo("Jenkins")); - assertThat(data.getString("buildLabel"),equalTo("master")); - assertThat(lastLine.getJSONArray("message").get(0).toString(),equalTo("Finished: SUCCESS")); + int messageSize = firstLine.getJSONArray("message").size(); + System.out.println("Message SIze: " + messageSize); + assertThat(firstLine.getJSONArray("message").get(messageSize-1).toString(),equalTo("Finished: SUCCESS")); } @Test @@ -284,4 +300,75 @@ public void disabledWillNotWrite() throws Exception List dataLines = memoryDao.getOutput(); assertThat(dataLines.size(), equalTo(0)); } + + @Test + public void enableGlobalLineModeNotifierWillSend() throws Exception + { + LogstashConfiguration.getInstance().setGlobalMode(GloballyEnabledMode.LINEMODE); + project.getPublishersList().add(new LogstashNotifier(10, false)); + + QueueTaskFuture f = project.scheduleBuild2(0); + + FreeStyleBuild build = f.get(); + assertThat(build.getResult(), equalTo(Result.SUCCESS)); + List dataLines = memoryDao.getOutput(); + assertThat(dataLines.size(), is(1)); + JSONObject firstLine = dataLines.get(0); + JSONObject data = firstLine.getJSONObject("data"); + assertThat(firstLine.getJSONArray("message").size(),not(equalTo(1))); + assertThat(data.getString("buildHost"),equalTo("Jenkins")); + assertThat(data.getString("buildLabel"),equalTo("master")); + assertThat(data.getString("result"),equalTo("SUCCESS")); + } + + private void assertNotifierNotCalled(List dataLines) + { + boolean foundNotifierMessage = false; + for (JSONObject line: dataLines) + { + assertThat(line.getJSONArray("message").size(), equalTo(1)); + if (line.getJSONArray("message").get(0).toString().equals("LogstashNotifier ignored. The data is already sent line by line.")) + { + foundNotifierMessage = true; + break; + } + } + assertThat(foundNotifierMessage, equalTo(true)); + } + + @Test + public void enabledJobPropertyNotifierWillNotSend() throws Exception + { + project.addProperty(new LogstashJobProperty()); + project.getPublishersList().add(new LogstashNotifier(10, false)); + + QueueTaskFuture f = project.scheduleBuild2(0); + + FreeStyleBuild build = f.get(); + assertThat(build.getResult(), equalTo(Result.SUCCESS)); + List dataLines = memoryDao.getOutput(); + assertThat(dataLines.size(), is(5)); + JSONObject lastLine = dataLines.get(dataLines.size()-1); + assertThat(lastLine.getJSONArray("message").get(0).toString(),equalTo("Finished: SUCCESS")); + assertNotifierNotCalled(dataLines); + } + + @Test + public void notifierAtEnd() throws Exception + { + QueueTaskFuture f = project.scheduleBuild2(0); + LogstashNotifier notifier = new LogstashNotifier(2, false); + notifier.setRunNotifierAtEnd(true); + project.getPublishersList().add(notifier); + + FreeStyleBuild build = f.get(); + assertThat(build.getResult(), equalTo(Result.SUCCESS)); + List dataLines = memoryDao.getOutput(); + assertThat(dataLines.size(), is(1)); + JSONObject firstLine = dataLines.get(0); + int messageSize = firstLine.getJSONArray("message").size(); + assertThat(messageSize,equalTo(2)); + assertThat(firstLine.getJSONArray("message").get(messageSize-1).toString(),equalTo("Finished: SUCCESS")); + } + } diff --git a/src/test/java/jenkins/plugins/logstash/LogstashNotifierTest.java b/src/test/java/jenkins/plugins/logstash/LogstashNotifierTest.java index e2fe4ea7..b176d38b 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashNotifierTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashNotifierTest.java @@ -9,6 +9,7 @@ import hudson.model.BuildListener; import hudson.model.AbstractBuild; import hudson.model.AbstractProject; +import hudson.model.Action; import hudson.model.TaskListener; import hudson.model.Run; import hudson.model.Result; @@ -62,6 +63,7 @@ LogstashWriter getLogStashWriter(Run run, OutputStream errorStream, TaskLi } static class MockRun

, R extends AbstractBuild> extends Run { + Result result; MockRun(P job) throws IOException { @@ -136,6 +138,8 @@ public void performSuccess() throws Exception { verify(mockListener).getLogger(); verify(mockWriter).writeBuildLog(3); verify(mockWriter).isConnectionBroken(); + verify(mockBuild).getAction(LogstashMarkerAction.class); + verify(mockBuild).addAction(any(LogstashMarkerAction.class)); assertEquals("Errors were written", "", errorBuffer.toString()); @@ -172,6 +176,8 @@ public void performBadWriterDoNotFailBuild() throws Exception { verify(mockListener).getLogger(); verify(mockWriter).writeBuildLog(3); verify(mockWriter).isConnectionBroken(); + verify(mockBuild).getAction(LogstashMarkerAction.class); + verify(mockBuild).addAction(any(LogstashMarkerAction.class)); assertEquals("Error was not written", "Mocked Constructor failure", errorBuffer.toString()); } @@ -213,6 +219,8 @@ public void performBadWriterDoFailBuild() throws Exception { verify(mockListener).getLogger(); verify(mockWriter).writeBuildLog(3); verify(mockWriter, times(2)).isConnectionBroken(); + verify(mockBuild).getAction(LogstashMarkerAction.class); + verify(mockBuild).addAction(any(LogstashMarkerAction.class)); assertEquals("Error was not written", "Mocked Constructor failure", errorBuffer.toString()); } @@ -264,6 +272,8 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { verify(mockListener).getLogger(); verify(mockWriter).writeBuildLog(3); verify(mockWriter, times(3)).isConnectionBroken(); + verify(mockBuild).getAction(LogstashMarkerAction.class); + verify(mockBuild).addAction(any(LogstashMarkerAction.class)); assertThat("Wrong error message", errorBuffer.toString(), containsString(errorMsg)); } @@ -286,6 +296,8 @@ public void performAllLines() throws Exception { verify(mockListener).getLogger(); verify(mockWriter).writeBuildLog(-1); verify(mockWriter, times(2)).isConnectionBroken(); + verify(mockBuild).getAction(LogstashMarkerAction.class); + verify(mockBuild).addAction(any(LogstashMarkerAction.class)); assertEquals("Errors were written", "", errorBuffer.toString()); } @@ -308,6 +320,8 @@ public void performZeroLines() throws Exception { verify(mockListener).getLogger(); verify(mockWriter).writeBuildLog(0); verify(mockWriter, times(2)).isConnectionBroken(); + verify(mockBuild).getAction(LogstashMarkerAction.class); + verify(mockBuild).addAction(any(LogstashMarkerAction.class)); assertEquals("Errors were written", "", errorBuffer.toString()); } diff --git a/src/test/java/jenkins/plugins/logstash/PipelineTest.java b/src/test/java/jenkins/plugins/logstash/PipelineTest.java index 83f45724..fa0b2bce 100644 --- a/src/test/java/jenkins/plugins/logstash/PipelineTest.java +++ b/src/test/java/jenkins/plugins/logstash/PipelineTest.java @@ -1,6 +1,7 @@ package jenkins.plugins.logstash; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; import java.util.List; @@ -70,6 +71,24 @@ public void logstashSendNotifier() throws Exception assertThat(data.getString("result"),equalTo("SUCCESS")); } + @Test + public void logstashSendNotifierAtEnd() throws Exception + { + WorkflowJob p = j.jenkins.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("node {" + + "echo 'Message'\n" + + "currentBuild.result = 'SUCCESS'\n" + + "step([$class: 'LogstashNotifier', failBuild: true, maxLines: 5, runNotifierAtEnd: true])" + + "}", true)); + j.assertBuildStatusSuccess(p.scheduleBuild2(0).get()); + List dataLines = memoryDao.getOutput(); + assertThat(dataLines.size(), is(1)); + JSONObject firstLine = dataLines.get(0); + int messageSize = firstLine.getJSONArray("message").size(); + assertThat(messageSize,equalTo(5)); + assertThat(firstLine.getJSONArray("message").get(messageSize-1).toString(),equalTo("Finished: SUCCESS")); + +} @Test public void logstashSend() throws Exception {