Skip to content

kafka-plugin segfaults #138

@ChristianKniep

Description

@ChristianKniep

Hey there,

I just stumbled upon the kafka plugin and I am thrilled. Sadly syslog segaults if I use it.
The error and how to recreate it could be found here: https://github.com/qnib/docker-kafka
syslog-ng was installed like this: https://github.com/qnib/docker-syslog/blob/master/Dockerfile

RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
    curl -Ls -o /etc/yum.repos.d/czanik-syslog-ng36-epel-7.repo https://copr.fedoraproject.org/coprs/czanik/syslog-ng36/repo/epel-7/czanik-syslog-ng36-epel-7.repo
RUN yum install -y unzip syslog-ng nmap syslog-ng-incubator-kafka yum install syslog-ng-incubator-grok

The config...

[root@653b8af98c85 conf.d]# cat /etc/syslog-ng/syslog-ng.conf
@version:3.6

# syslog-ng configuration file.
#
# This should behave pretty much like the original syslog on RedHat. But
# it could be configured a lot smarter.
#
# See syslog-ng(8) and syslog-ng.conf(5) for more information.
#
# Note: it also sources additional configuration files (*.conf)
#       located in /etc/syslog-ng/conf.d/

options {
    flush_lines (0);
    time_reopen (10);
    log_fifo_size (1000);
    chain_hostnames (off);
    use_dns (no);
    use_fqdn (no);
    create_dirs (no);
    keep_hostname (yes);
    stats_freq(0);
};

source s_sys {
    file ("/proc/kmsg" program_override("kernel") flags(kernel));
    unix-dgram ("/dev/log");
    internal();
    udp(ip(0.0.0.0) port(514));
    tcp(ip(0.0.0.0) port(514));
};

# Source additional configuration files (.conf extension only)
@include "/etc/syslog-ng/conf.d/*.conf"


# vim:ft=syslog-ng:ai:si:ts=4:sw=4:et:

[root@653b8af98c85 conf.d]# cat /etc/syslog-ng/conf.d/
kafka.conf              kafka.conf.disabled     logstash.conf.disabled
[root@653b8af98c85 conf.d]# cat /etc/syslog-ng/conf.d/kafka.conf
destination d_kafka {
  channel {
    rewrite {
      set("${HOST}"    value(".eventv1.host"));
      set("1"          value(".eventv1.@version"));
      set("${ISODATE}" value(".eventv1.@timestamp") condition("${.eventv1.@timestamp}" eq ""));
      set("${MESSAGE}" value(".eventv1.message")    condition("${.eventv1.message}" eq ""));
      set("${MSG}"     value(".eventv1.message")    condition("${.eventv1.message}" eq ""));
      set("generic"    value(".eventv1.type")       condition("${.eventv1.type}" eq ""));
    };
    destination {
      kafka(properties(metadata.broker.list("localhost:9092")
                       queue.buffering.max.ms("1"))
            topic("syslog")
            payload("$(format-json --key .eventv1.* --rekey .eventv1.* --shift 9)"));
    };
  };
};

log {
    source(s_sys);
    destination(d_kafka);
};
[root@653b8af98c85 conf.d]#

I would love to use it with kafka... Hope there is a fix...

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions