master-all.conf

input {
  file {
    type => "bluecoat-proxy"
    start_position => "beginning"
    path => "/Bluecoat/*.log"
  }
  file {
    type => "mcafee-ips"
    start_position => "beginning"
    path => "/McAfee_IPS/*.csv"
  }
  file {
    type => "checkpoint-firewall"
    start_position => "beginning"
    path => "/Checkpoint/*.txt"
  }
  file {
    type => "iis-weblogs"
    start_position => "beginning"
    path => "/W3SVC1/u_ex*.log"
  }
}
filter {
  if ([message] =~ /^#/) {
    drop { }
    }
  ##########################
  #  Blue Coat Proxy Logs  #
  ##########################
  if [type] == "bluecoat-proxy" {
    csv {
      columns => ["date","time","time-taken","c-ip","sc-status","s-action","sc-bytes","cs-bytes","cs-method","cs-uri-scheme","cs-host","cs-uri-port","cs-uri-path","cs-uri-query","cs-username","cs-auth-group","s-hierarchy","s-supplier-name","rs-content-type","cs-referer","cs-user-agent","sc-filter-result","cs-categories","x-virus-id","s-ip"]
    separator => " "
    remove_field => [ "message" ]
    remove_field => [ "time-taken" ]
    remove_field => [ "x-virus-id" ]
    }
  mutate {
    replace => [ "date", "%{date} %{time}" ]
    remove_field => [ "time" ]
    }
  date {
    match => [ "date", "YYYY-MM-dd HH:mm:ss" ]
    target => [ "datetime"]
    remove_field => [ "date" ]
    }
  geoip {
      source => "s-ip"
    }
  }
  #########################
  #    Checkpoint Logs    #
  #########################
  if [type] == "checkpoint-firewall" {
    csv {
      columns => ["number","date","time","interface","orgin","fw-type","action","service","s-port","s-ip","d-ip","proto","rule","rule-name","rule-number","user","information","product","src-machine","src-user"]
      separator => " "
      remove_field => [ "message" ]
      remove_field => [ "rule" ]
      remove_field => [ "rule-name" ]
      remove_field => [ "rule-number" ]
      remove_field => [ "user" ]
      remove_field => [ "product" ]
      remove_field => [ "number" ]
      remove_field => [ "interface" ]
      remove_field => [ "src-machine" ]
      remove_field => [ "src-user" ]
    }
    mutate {
      replace => [ "date", "%{date} %{time}" ]
      remove_field => [ "time" ]
    }
    date {
      match => [ "date", "ddMMMYYYY HH:mm:ss" ]
      target => [ "datetime"]
      remove_field => [ "date" ]
    }
    geoip {
      source => "s-ip"
    }
  }
  #########################
  #    McAfee IPS Logs    #
  #########################
  if [type] == "mcafee-ips" {
    csv {
      columns => ["number","date","severity","attack-name","s-ip","d-ip","dst-port","result","attack-count","direction","layer","src-c","dst-c","app-name","protection","assignment","exe-name"]
      separator => ","
      remove_field => [ "message" ]
    }
    date {
      match => [ "date", "EEE MMM dd HH:mm:ss 'SGT' yyyy" ]
      timezone => "Asia/Singapore"
      target => [ "datetime"]
      remove_field => [ "date" ]
    }
    geoip {
      source => "s-ip"
    }
  }
  #########################
  #      IIS LOGS         #
  #########################
  if [type] == "iis-weblogs" {
    csv {
      columns => ["date","time","s-ip","cs-method","cs-uri-stem","cs-uri-query","s-port","cs-username","c-ip","cs(User-Agent)","sc-status","sc-substatus","sc-win32-status","time-taken"]
      separator => " "
      remove_field => [ "sc-win32-status" ]
      remove_field => [ "sc-substatus" ]
      remove_field => [ "message" ]
    }
    mutate {
      replace => [ "date", "%{date} %{time}" ]
      remove_field => [ "time" ]
    }
    date {
      match => [ "date", "YYYY-MM-DD HH:mm:ss" ]
      target => [ "datetime"]
      remove_field => [ "date" ]
    }
    geoip {
      source => "s-ip"
    } 
  } 
}
output {
  elasticsearch {
    embedded => true
    index => "general-logs"
  }
}