Logstash 管道配置示例

Logstash 管道配置示例

钉钉推送管道

if [logtype] in ["brtc-vconsole", "brtc-vrtm", "brtc-vrest", "brtc-vrectifier"] {
    if ([docs][alert][type]) {
        aggregate {
            task_id => "%{[logtype]}:%{[docs][alert][type]}"
            code => "
                map['types'] = event.get('[docs][alert][type]').downcase
                map['alerts'] ||= 0
                map['alerts'] += 1
                map['alert_list'] ||= []
                map['alert_list'] << [event.get('[logs][trace]').to_s, event.get('[logs][msg]').to_s].reject(&:empty?).join(':')
            "
            push_map_as_event_on_timeout => true
            timeout => 60
            timeout_task_id_field => "logs.channel"
            timeout_tags => ['_aggregate_timeout_tags']
            timeout_code => "
                event.set('logtype', 'aggregate_alert')
                event.set('logs_type', event.get('types'))
                event.set('should_alert', event.get('alerts') >= 1)
                event.set('message', event.get('alert_list').join('\n'))
            "
        }
    }
}

if [should_alert] {
    http {
        http_method => "post"
        #url => "https://oapi.dingtalk.com/robot/send?access_token=cc707a187337af8802504f4cbfeeb14fc00197540b6a8ffab6aecdab042b6a74"
        url => "https://brtc-apitest.baijiayun.com/vcs/admin/stats/rt/bill/apps?app_id=777777777777777777777"
        format => "json"
        mapping => {
            "msgtype" => "text"
            "text" => {
                "content" => "[告警服务][ENV:TEST][错误日志统计:%{logs.channel}]\n最近一分钟共发生%{alerts}次%{logs_level}级别错误.\n错误信息:\n%{message}"
            }
        }
    }
}

input {
    pipeline {
        address => "test-brtc-dirty-log-handler"
    }
}

filter {
    if "json" in [tags] {
        json {
            source => "message"
            target => "docs"
        }
        mutate {
            add_field => {
                "[logs][logger]" => "%{[docs][logger]}"
            }
            add_field => {
                "[logs][caller]" => "%{[docs][caller]}"
            }
            add_field => {
                "[logs][level]" => "%{[docs][level]}"
            }
            add_field => {
                "[logs][msg]" => "%{[docs][msg]}"
            }
            add_field => {
                "[logs][trace]" => "%{[docs][trace]}"
            }
            add_field => {
                "[logs][time]" => "%{[docs][time]}"
            }
            remove_field => [
                "source", "beat", "prospector", "input", "docs", "tags", "@version"
            ]
        }
    }

    #if [logtype] in ["brtc-vconsole", "brtc-vrtm", "brtc-vrest", "brtc-vrectifier"] {
    #    if [logs][level] == "WARN" {
    #        mutate {
    #            add_field => {
    #                "[logs][tag]" => "WARNS..."
    #            }
    #        }
    #    }
    #}

    if [logtype] in ["brtc-vconsole", "brtc-vrtm", "brtc-vrest", "brtc-vrectifier"] {
        if [logs][level] == "ERROR" {
            aggregate {
                task_id => "%{[logtype]}:%{[logs][level]}"
                code => "
                    map['levels'] = event.get('[logs][level]').downcase
                    map['alerts'] ||= 0
                    map['alerts'] += 1
                    map['alert_list'] ||= []
                    map['alert_list'] << [event.get('[logs][trace]').to_s, event.get('[logs][msg]').to_s].reject(&:empty?).join(':')
                "
                push_map_as_event_on_timeout => true
                timeout => 60
                timeout_task_id_field => "logs.channel"
                timeout_tags => ['_aggregate_timeout_tags']
                timeout_code => "
                    event.set('logtype', 'aggregate_alert')
                    event.set('logs_level', event.get('levels'))
                    event.set('should_alert', event.get('alerts') > 1)
                    event.set('message', event.get('alert_list').join('\n'))
                "
            }
        }
    }
}

output {
    #stdout {
    #    codec => rubydebug {
    #        metadata => true
    #    }
    #}

    if [logtype] == "aggregate_alert" {
        stdout {
            codec => rubydebug {
                metadata => true
            }
        }

        if [should_alert] {
            http {
                http_method => "post"
                url => "https://oapi.dingtalk.com/robot/send?access_token=cc707a187337af8802504f4cbfeeb14fc00197540b6a8ffab6aecdab042b6a74"
                #url => "https://brtc-apitest.baijiayun.com/vcs/admin/stats/rt/bill/apps?app_id=777777777777777777777"
                format => "json"
                mapping => {
                    "msgtype" => "text"
                    "text" => {
                        "content" => "[告警服务][ENV:TEST][错误日志统计:%{logs.channel}]\n最近一分钟共发生%{alerts}次%{logs_level}级别错误.\n错误信息:\n%{message}"
                    }
                }
            }
        }
    }

    elasticsearch {
        hosts => ["es-internet.baijiayun.com:9200"]
        user => "elastic"
        password => "4321bjsl"
        index => "test-%{logtype}-v1-%{+YYYY.MM.dd}"
    }

}

• 3 Awesome Ways To Use Ruby’s Gsub Method