harbor.yaml

# bypass-global-no-service-config-checks
# bypass-global-facets-path-checks
# bypass-global-grok-parser-count-per-pipeline-checks
id: harbor
metric_id: harbor
backend_only: false
facets:
  - groups:
      - Database
    name: Database
    path: db.instance
    source: log
  - groups:
      - Database
    name: Operation Type
    path: db.operation
    source: log
  - groups:
      - Database
    name: User
    path: db.user
    source: log
  - groups:
      - Web Access
    name: Method
    path: http.method
    source: log
  - groups:
      - Web Access
    name: Referer
    path: http.referer
    source: log
  - groups:
      - Web Access
    name: Status Code
    path: http.status_code
    source: log
  - groups:
      - Web Access
    name: URL Path
    path: http.url
    source: log
  - groups:
      - Web Access
    name: URL Host
    path: http.url_details.host
    source: log
  - groups:
      - Web Access
    name: URL Path
    path: http.url_details.path
    source: log
  - groups:
      - Web Access
    name: URL Port
    path: http.url_details.port
    source: log
  - groups:
      - Web Access
    name: URL scheme
    path: http.url_details.scheme
    source: log
  - groups:
      - Web Access
    name: User-Agent
    path: http.useragent
    source: log
  - groups:
      - Web Access
    name: Browser
    path: http.useragent_details.browser.family
    source: log
  - groups:
      - Web Access
    name: Device
    path: http.useragent_details.device.family
    source: log
  - groups:
      - Web Access
    name: OS
    path: http.useragent_details.os.family
    source: log
  - groups:
      - Source Code
    name: Logger Name
    path: logger.name
    source: log
  - groups:
      - Web Access
    name: Client IP
    path: network.client.ip
    source: log
  - groups:
      - Database
    name: Redis PID
    path: pid
    source: log
  - groups:
      - Database
    name: Session ID
    path: postgres.session_id
    source: log
pipeline:
  type: pipeline
  name: Harbor
  enabled: true
  filter:
    query: source:harbor
  processors:
    - type: grok-parser
      name: Extract prefix
      enabled: true
      source: message
      samples:
        - 'May 24 15:14:58 172.30.0.1 jobservice[1304]: 2019-05-24T15:14:58Z [INFO] [/common/config/store/driver/rest.go:24]: get configuration from url: http://core:8080/api/internal/configurations'
        - 'Jun  3 19:49:04 172.19.0.1 proxy[1297]: 172.19.0.1 - "GET /harbor/sign-in?redirect_url=%2Fharbor%2Fprojects HTTP/1.1" 200 693 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" 0.001 0.000 .'
        - 10.60.2.10 - - [02/Dec/2021:13:50:26 +0000] "GET / HTTP/1.1" 200 532 "-" "Go-http-client/1.1"
      grok:
        supportRules: |
          _date (%{date("MMM  d HH:mm:ss"):syslog.timestamp}|%{date("MMM dd HH:mm:ss"):syslog.timestamp})
          _log_prefix %{_date} %{ip} %{word:syslog.appname}\[\d+\]:\s+

        matchRules: |
          harbor_common (%{_log_prefix})?%{data:msg}
    - type: grok-parser
      name: Harbor nginx rules
      enabled: true
      source: msg
      samples:
        - 172.19.0.1 - "GET /harbor/sign-in?redirect_url=%2Fharbor%2Fprojects HTTP/1.1" 200 693 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" 0.001 0.000 .
        - 172.19.0.1 - "GET /api/health HTTP/1.1" 308 172 "-" "insomnia/6.5.3" 0.000 - .
        - 172.19.0.7 - - [04/Jun/2019:13:12:09 +0000] "GET /api/health HTTP/1.1" 200 9
        - 172.19.0.7 - - [04/Jun/2019:13:20:16 +0000] "GET / HTTP/1.1" 200 693 "-" "Go-http-client/1.1"
      grok:
        supportRules: |
          # nginx
          # See nginx pipeline
          _nginx_auth %{notSpace:http.auth:nullIf("-")}
          _nginx_bytes_written %{integer:network.bytes_written}
          _nginx_client_ip %{ipOrHost:network.client.ip}
          _nginx_version HTTP\/%{regex("\\d+\\.\\d+"):http.version}
          _nginx_url %{notSpace:http.url}
          _nginx_ident %{notSpace:http.ident:nullIf("-")}
          _nginx_user_agent %{regex("[^\\\"]*"):http.useragent}
          _nginx_referer %{notSpace:http.referer:nullIf("-")}
          _nginx_status_code %{integer:http.status_code}
          _nginx_method %{word:http.method}
          _nginx_date_access %{date("dd/MMM/yyyy:HH:mm:ss Z")}
          _nginx_x_forwarded_for %{regex("[^\\\"]*"):http._x_forwarded_for:nullIf("-")}

          nginx.common %{_nginx_client_ip} %{_nginx_ident} "(?>%{_nginx_method} |)%{_nginx_url}(?> %{_nginx_version}|)" %{_nginx_status_code} (?>%{_nginx_bytes_written}|-)
          nginx.common2 %{_nginx_client_ip} %{_nginx_ident} %{_nginx_auth} \[%{_nginx_date_access}\] "(?>%{_nginx_method} |)%{_nginx_url}(?> %{_nginx_version}|)" %{_nginx_status_code} (?>%{_nginx_bytes_written}|-)
          nginx.combined %{nginx.common} "%{_nginx_referer}?" "%{_nginx_user_agent}"( "%{_nginx_x_forwarded_for}")?.*
          nginx.combined2 %{nginx.common2} "%{_nginx_referer}?" "%{_nginx_user_agent}"( "%{_nginx_x_forwarded_for}")?.*
        matchRules: |
          nginx_proxy %{nginx.combined}

          nginx_registryctl %{nginx.common2}

          nginx_portal_registry %{nginx.combined2}
    - type: grok-parser
      name: Harbor core/jobservice
      enabled: true
      source: msg
      samples:
        - '2019/06/04 13:20:57 #033[1;44m[D] [server.go:2619] |      127.0.0.1|#033[42m 200 #033[0m|     9.9379ms|   match|#033[44m GET     #033[0m /api/ping   r:/api/ping#033[0m'
        - '2019-06-03T19:43:27Z [INFO] [/common/dao/base.go:84]: Registering database: type-PostgreSQL host-postgresql port-5432 databse-registry sslmode-"disable"'
        - '2019-05-24T15:08:41Z [ERROR] [/common/config/store/driver/rest.go:27]: Failed on load rest config err:Get http://core:8080/api/internal/configurations: dial tcp 172.28.0.7:8080: getsockopt: connection refused, url:http://core:8080/api/internal/configurations'
        - '2019-06-04T13:07:21Z [ERROR] [/replication/adapter/image_registry.go:145]: failed to ping registry https://demo.goharbor.io: http error: code 401, message'
      grok:
        supportRules: |
          _core_date %{date("yyyy/MM/dd HH:mm:ss")}
          _jobservice_date %{date("yyyy-MM-dd'T'HH:mm:ssz"):}
          _colorcode %{regex("(#033\\[((0|1);)?\\d+m)")}
          _common_severity %{notSpace:severity}
          _logger_name %{regex("[^ :]*"):logger.name}\:\d+
          _client_ip %{ipOrHost:network.client.ip}
          _status_code %{integer:http.status_code}
          _duration %{number:duration:scale(1000000)}
          _http_method %{word:http.method}
          _http_url %{notSpace:http.url}
        matchRules: |
          core %{_core_date} %{_colorcode}?\[%{_common_severity}\]\s+\[%{_logger_name}\]\s+\|\s+%{_client_ip}\s*\|%{_colorcode}?\s+%{_status_code}\s+%{_colorcode}?\s*\|\s+%{_duration}ms\s*\|\s+%{notSpace}\|%{_colorcode}?\s+%{_http_method}\s+%{_colorcode}\s+%{_http_url}.*

          jobservice_or_simple_core %{_jobservice_date}\s+\[%{_common_severity}\]\s+\[%{_logger_name}\]:\s+%{data}
    - type: grok-parser
      name: Harbor postgres db
      enabled: true
      source: msg
      samples:
        - "LOG:  stats_timestamp 2019-05-24 19:48:20.586618+00 is later than collector's time 2019-05-24 19:48:20.564753+00 for database 16388"
        - 'LOG:  statement: SELECT * FROM playground;'
      grok:
        supportRules: |
          # postgres
          # See postgres pipeline
          _pg_raw_query %{data:db.statement}
          _pg_severity %{?>notSpace:severity}
        matchRules: |
          postgres %{_pg_severity}:\s+(%{regex("statement:")}\s+%{_pg_raw_query}|%{data})
    - type: grok-parser
      name: Harbor redis db
      enabled: true
      source: msg
      samples:
        - '8:M 03 Jun 19:43:16.664 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128.'
        - 8:M 03 Jun 19:53:18.077 * 10 changes in 300 seconds. Saving...
      grok:
        supportRules: |
          # redis
          # See redis pipeline
          _redis_date (%{date("dd MMM HH:mm:ss.SSS")}|%{date("dd MMM yyyy HH:mm:ss.SSS")})
          _redis_pid %{integer:pid}
          _redis_severity %{notSpace:redis.severity}
          _redis_role %{word:role}
        matchRules: |
          redis %{_redis_pid}:%{_redis_role} %{_redis_date} %{_redis_severity} %{data}
    - type: grok-parser
      name: Harbor chartmuseum
      enabled: true
      source: msg
      samples:
        - '{"L":"INFO","T":"2019-06-03T19:49:08.093Z","M":"[47] Request served","path":"/admin_private/index.yaml","comment":"","latency":"38.750171ms","clientIP":"172.20.0.3","method":"GET","statusCode":200,"reqID":"9e60fdcc-25a1-49da-9b26-4ca3deee6714"}'
        - '{"L":"INFO","T":"2019-05-24T15:16:55.523Z","M":"Starting ChartMuseum","port":9999}'
      grok:
        matchRules: |
          chartmuseum_full %{data:properties:json}
        supportRules: ''
    - type: grok-parser
      name: Chartmuseum parse duration
      enabled: true
      source: properties.latency
      samples:
        - 38.750171ms
        - 0.038750171s
      grok:
        matchRules: |
          latency (%{number:duration:scale(1000000)}ms|%{number:duration:scale(1000000000)}s)
        supportRules: ''
    - type: grok-parser
      name: Harbor parser
      enabled: true
      source: msg
      samples:
        - 'time="2019-06-04T13:26:57.2139804Z" level=warning msg="error authorizing context: authorization token required" go.version=go1.11.8 http.request.host="registry:5000" http.request.id=f87f0124-2631-4518-b0ed-3b741fe2d9e3 http.request.method=GET http.request.referer="http://registry:5000/v2" http.request.remoteaddr="172.19.0.7:41842" http.request.uri="/v2/" http.request.useragent="Go-http-client/1.1"'
      grok:
        matchRules: |
          registry %{data:properties:keyvalue}
        supportRules: ''
    - type: date-remapper
      name: Define `syslog.timestamp` as the official date of the log
      enabled: true
      sources:
        - syslog.timestamp
    - type: grok-parser
      name: Extract db.operation from statement
      enabled: true
      source: db.statement
      grok:
        supportRules: |
        matchRules: |
          extract_operation %{word:db.operation} .*
      samples:
        - SELECT * FROM playground;
    - type: attribute-remapper
      name: Map `properties.http.request.referer` to `http.referer`
      enabled: true
      sources:
        - properties.http.request.referer
      target: http.referer
      preserveSource: false
      overrideOnConflict: true
      sourceType: attribute
      targetType: attribute
    - type: attribute-remapper
      name: Map `properties.method`, `properties.http.request.method` to `http.method`
      enabled: true
      sources:
        - properties.method
        - properties.http.request.method
      target: http.method
      preserveSource: false
      overrideOnConflict: true
      sourceType: attribute
      targetType: attribute
    - type: attribute-remapper
      name: Map `properties.http.request.useragent` to `http.useragent`
      enabled: true
      sources:
        - properties.http.request.useragent
      target: http.useragent
      preserveSource: false
      overrideOnConflict: true
      sourceType: attribute
      targetType: attribute
    - type: attribute-remapper
      name: Map `properties.path`, `properties.http.request.uri` to `http.url`
      enabled: true
      sources:
        - properties.path
        - properties.http.request.uri
      target: http.url
      preserveSource: false
      overrideOnConflict: true
      sourceType: attribute
      targetType: attribute
    - type: attribute-remapper
      name: Map `properties.clientIP` to `network.client.ip`
      enabled: true
      sources:
        - properties.clientIP
      target: network.client.ip
      preserveSource: false
      overrideOnConflict: true
      sourceType: attribute
      targetType: attribute
    - type: attribute-remapper
      name: Map `properties.statusCode` to `http.status_code`
      enabled: true
      sources:
        - properties.statusCode
      target: http.status_code
      preserveSource: false
      overrideOnConflict: true
      sourceType: attribute
      targetType: attribute
    - type: category-processor
      name: Severity from status code
      enabled: true
      categories:
        - filter:
            query: '@http.status_code:[200 TO 299]'
          name: ok
        - filter:
            query: '@http.status_code:[300 TO 399]'
          name: notice
        - filter:
            query: '@http.status_code:[400 TO 499]'
          name: warning
        - filter:
            query: '@http.status_code:[500 TO 599]'
          name: error
      target: severity
    - type: category-processor
      name: Severity from redis severity symbols
      enabled: true
      categories:
        - filter:
            query: '@redis.severity: "."'
          name: debug
        - filter:
            query: '@redis.severity: "-"'
          name: verbose
        - filter:
            query: '@redis.severity: "*"'
          name: notice
        - filter:
            query: '@redis.severity: "#"'
          name: warning
      target: severity
    - type: url-parser
      name: Parse http.url
      enabled: true
      sources:
        - http.url
      target: http.url_details
    - type: user-agent-parser
      name: Parse http.useragent
      enabled: true
      sources:
        - http.useragent
      target: http.useragent_details
      encoded: false
    - type: status-remapper
      name: Define `severity`, `properties.level`, `properties.L` as the official status of the log
      enabled: true
      sources:
        - severity
        - properties.level
        - properties.L
    - type: message-remapper
      name: Define `properties.M`, `msg` as the official message of the log
      enabled: true
      sources:
        - properties.M
        - msg
    - type: service-remapper
      name: Define `syslog.appname` as the official service of the log
      enabled: true
      sources:
        - syslog.appname