Logstash深入收集syslog收集日志 - 栀雾のblog

Single

Logstash深入收集syslog收集日志

Post on 2023-08-15 48 0

安装配置rsyslog
安装配置haproxy
连接远端机器的nginx集群
修改haproxy配置文件
logstash通过rsyslog收集haproxy

在centos 6及之前的版本叫做syslog，centos 7开始叫做rsyslog。官方网址：http://www.rsyslog.com/

安装配置rsyslog

## 安装rsyslog
[root@logstash ~]# yum install -y rsyslog

## 编辑rsyslog配置文件
[root@logstash ~]# vim /etc/rsyslog.conf
 15行 $ModLoad imudp
 16行 $UDPServerRun 514
 19行 $ModLoad imtcp 
 20行 $InputTCPServerRun 514
最后一行加入
local7.*     @@10.0.0.91:2222

## 启动服务
[root@logstash ~]# systemctl start rsyslog

# 检查进程
[root@logstash ~]# ps -ef | grep rsyslog
root        839      1  0 10:37 ?        00:00:01 /usr/sbin/rsyslogd -n
root       3688   3449  0 20:07 pts/0    00:00:00 grep --color=auto rsyslog

安装配置haproxy

# 安装haproxy
[root@logstash ~]# yum install -y haproxy

# 编辑haproxy配置文件
[root@logstash ~]# vim /etc/haproxy/haproxy.cfg 
global
maxconn 100000
chroot /var/lib/haproxy
uid 99
gid 99
daemon
nbproc 1
pidfile /var/run/haproxy.pid
log 127.0.0.1 local6 info

defaults
option http-keep-alive
option  forwardfor
maxconn 100000
mode http
timeout connect 300000ms
timeout client  300000ms
timeout server  300000ms

listen stats
 mode http
 bind 0.0.0.0:9999
 stats enable
 log global
 stats uri     /haproxy-status
 stats auth    haadmin:123456

#frontend web_port
frontend web_port
        bind 0.0.0.0:80
        mode http
        option httplog
        log global
        option  forwardfor
###################ACL Setting##########################
        acl pc          hdr_dom(host) -i www.elk.com
        acl mobile      hdr_dom(host) -i m.elk.com
###################USE ACL##############################
        use_backend     pc_host        if  pc
        use_backend     mobile_host    if  mobile
########################################################

backend pc_host
        mode    http
        option  httplog
        balance source
        server phl  10.0.0.51:8091 check inter 2000 rise 3 fall 2 weight 1

backend mobile_host
        mode    http
        option  httplog
        balance source
        server jng  10.0.0.51:8090 check inter 2000 rise 3 fall 2 weight 1

# 启动haproxy
[root@logstash ~]# systemctl start haproxy

# 检查端口
[root@logstash ~]# netstat -lntup
tcp        0      0 0.0.0.0:9999            0.0.0.0:*               LISTEN      3727/haproxy        
tcp        0      0 0.0.0.0:80              0.0.0.0:*               LISTEN      3727/haproxy 

# 检查进程
[root@logstash ~]# ps -ef | grep haproxy
root       3725      1  0 20:13 ?        00:00:00 /usr/sbin/haproxy-systemd-wrapper -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid
nobody     3726   3725  0 20:13 ?        00:00:00 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -Ds
nobody     3727   3726  0 20:13 ?        00:00:00 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -Ds
root       3730   3449  0 20:16 pts/0    00:00:00 grep --color=auto haproxy

# 修改nginx配置文件
[root@logstash ~]# vim /etc/nginx/conf.d/jng.conf
server{
        listen 8090;
        server_name _;
        root /jng;
        index index.html;
        access_log /var/log/nginx/blog.jng.com_access_json.log json;
}

[root@logstash ~]# vim /etc/nginx/conf.d/phl.conf
server{
        listen 8091;
        server_name _;
        root /phl;
        index index.html;
        access_log /var/log/nginx/blog.phl.com_access_json.log json;
}

# 修改nginx主配置文件
[root@logstash ~]# vim /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

http {
#    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
#                      '$status $body_bytes_sent "$http_referer" '
#                      '"$http_user_agent" "$http_x_forwarded_for"';
log_format json '{"@timestamp":"$time_iso8601",'
                '"host":"$server_addr",'
                '"clientip":"$remote_addr",'
                '"size":$body_bytes_sent,'
                '"responsetime":$request_time,'
                '"upstreamtime":"$upstream_response_time",'
                '"upstreamhost":"$upstream_addr",'
                '"http_host":"$host",'
                '"url":"$uri",'
                '"domain":"$host",'
                '"xff":"$http_x_forwarded_for",'
                '"referer":"$http_referer",'
                '"status":"$status"}';
#    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 4096;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    include /etc/nginx/conf.d/*.conf;

}

# 创建站点目录
[root@logstash ~]# mkdir /phl
[root@logstash ~]# mkdir /jng

# 创建index文件
[root@logstash ~]# vim /phl/index.html
phl
[root@logstash ~]# vim /jng/index.html
jng

# 启动nginx
[root@logstash ~]# systemctl start nginx

# 浏览器访问10.0.0.91:9999/haproxy-status

连接远端机器的nginx集群

# db01
# 安装nginx
[root@db01 ~]# yum install -y nginx

# 推送配置文件
[root@logstash ~]# scp /etc/nginx/conf.d/* 10.0.0.51:/etc/nginx/conf.d/
jng.conf          100%  168   207.7KB/s   00:00    
phl.conf          100%  168   252.5KB/s   00:00 

# 推送站点目录
[root@logstash ~]# scp -r /phl/ 10.0.0.51:/
root@10.0.0.51's password: 
index.html        100%    4     5.9KB/s   00:00
[root@logstash ~]# scp -r /jng/ 10.0.0.51:/
root@10.0.0.51's password: 
index.html        100%    4     5.9KB/s   00:00 

# 推送nginx主配置文件
[root@logstash ~]# scp /etc/nginx/nginx.conf 10.0.0.51:/etc/nginx/
root@10.0.0.51's password: 
nginx.conf        100% 1052     1.7MB/s   00:00

# 启动nginx
[root@db01 ~]# systemctl start nginx

# 浏览器访问
10.0.0.51：8090
10.0.0.51：8091

修改haproxy配置文件

# 修改配置haproxy文件
[root@logstash ~]# vim /etc/haproxy/haproxy.cfg
chroot /var/lib/haproxy
uid 99
gid 99
daemon
nbproc 1
pidfile /var/run/haproxy.pid
log 127.0.0.1 local7 info

defaults
option http-keep-alive
option  forwardfor
maxconn 100000
mode http
timeout connect 300000ms
timeout client  300000ms
timeout server  300000ms

listen stats
 mode http
 bind 0.0.0.0:9999
 stats enable
 log global
 stats uri     /haproxy-status
 stats auth    haadmin:123456

#frontend web_port
frontend web_port
        bind 0.0.0.0:80
        mode http
        option httplog
        log global
        option  forwardfor
###################ACL Setting##########################
        acl pc          hdr_dom(host) -i www.elk.com
        acl mobile      hdr_dom(host) -i m.elk.com
###################USE ACL##############################
        use_backend     pc_host        if  pc
        use_backend     mobile_host    if  mobile
########################################################

backend pc_host
        mode    http
        option  httplog
        balance static-rr
        server jng_10.0.0.51  10.0.0.51:8091 check inter 2000 rise 3 fall 2 weight 1
        server jng2_10.0.0.91  10.0.0.91:8091 check inter 2000 rise 3 fall 2 weight 1

backend mobile_host
        mode    http
        option  httplog
        balance static-rr
        server phl_10.0.0.51  10.0.0.51:8090 check inter 2000 rise 3 fall 2 weight 1
        server phl2_10.0.0.91  10.0.0.91:8090 check inter 2000 rise 3 fall 2 weight 1
# 重启haproxy
[root@elk03 conf.d]# systemctl restart haproxy

# 本地域名解析
10.0.0.91 www.elk.com
10.0.0.91 m.elk.com

logstash通过rsyslog收集haproxy

# 日志输出到屏幕
[root@logstash ~]# vim /etc/logstash/conf.d/haproxy.conf
input{
        syslog{
                type => 'rsyslog_haproxy_log'
                port => 2222
        }
}
output{
        stdout{
                codec => rubydebug
        }
        elasticsearch{
                hosts => ['10.0.0.82:9200']
                index => '%{type}-%{+yyyy.MM.dd}'

# 启动
[root@logstash ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/haproxy.conf

上传到es

logstash监听了2222端口
rsyslog是接收到了haproxy的日志然后转发给10.0.0.83:2222
logstash启动2222端口实时接受rsyslog传来的日志
如果logstash停掉 2222消失，2222是logstash起的

暂无评论

发表评论取消回复

表情

ELK结合kafka

ELK结合kafka

Kibana入门

Elasticsearch内部分片及分片处…

X