cd /usr/local

tar -zxvf elasticsearch-6.3.2.tar.gz

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.2.tar.gz

rm -rf /usr/local/elasticsearch-6.3.2.tar.gz

#创建日志文件夹

mkdir -p /usr/local/elasticsearch-6.3.2/log

#修改elasticsearch.yml配置文件 ----------------------------------------------------------------------

cat > /usr/local/elasticsearch-6.3.2/config/elasticsearch.yml <<EOF

#cluster.name 集群的cluster.name 必须一致

cluster.name: elasticsearch

#node.name  集群的node.name 不一致

node.name: node-8

#path to data

path.data: /usr/local/elasticsearch-6.3.2/data

#path  to log

path.logs:  /usr/local/elasticsearch-6.3.2/log

#设置内存不会到交换分区上 需要安装插件 在5.0之后没有该配置了

#bootstrap.mlockall: true

node.master: true

#监听的主机

network.host: 0.0.0.0

#集群设置

discovery.zen.ping.unicast.hosts: ["10.29.112.172", "10.29.113.121"]

#监听的端口

http.port: 9200

transport.tcp.port: 9300

bootstrap.memory_lock: false

bootstrap.system_call_filter: false

#便于监控插件走http协议获取相关数据

http.cors.enabled: true

http.cors.allow-origin: "*"

#回收缓存  避免es服务器内存一直增大 

#缓存数据+当前查询需要缓存的数据量到达断路器限制时，会返回Data too large错误

indices.fielddata.cache.size:  60%

EOF

discovery.zen.ping.unicast.hosts: ["10.6.11.176", "10.6.11.177"]

建议使用ip 避免dns 解析出问题导致es 故障

修改/etc/security/limits.conf 参数

yum 安装的es默认是普通用户 需要修改limit

yum 安装权限是默认设置好的 会创建用户 用户组

修改/etc/sysctl.conf

cat > /etc/sysctl.conf  <<EOF

vm.max_map_count=655360

EOF

sysctl -p

修改/etc/security/limits.d/90-nproc.conf

cat > /etc/security/limits.d/90-nproc.conf <<EOF

*          soft    nproc     2048

root       soft    nproc     unlimited

EOF

如果需要 也可以改大  elastic用户 正常情况够用

ulimit -a

查看opne file 值

如果过小 需要改大  建议65536

useradd elastic

chown -R elastic:elastic  elasticsearch-6.3.2

su - elastic -c "sh /usr/local/elasticsearch-6.3.2/bin/elasticsearch -d"

log_format log_json '{ "@timestamp": "$time_local", '

'"remote_addr": "$remote_addr", '

'"referer": "$http_referer", '

'"request": "$request", '

'"status": $status, '

'"bytes": $body_bytes_sent, '

'"agent": "$http_user_agent", '

'"x_forwarded": "$http_x_forwarded_for", '

'"up_addr": "$upstream_addr",'

'"up_host": "$upstream_http_host",'

'"up_resp_time": "$upstream_response_time",'

'"request_time": "$request_time"'

' }';

input {

    syslog {

       type => "system-syslog"

       host => "10.6.11.176"

       port => "514"

    }

    file {

        path => "/var/log/messages"

        type => "system"

        start_position => "beginning"

    }

    file {

        path => "/usr/local/elasticsearch-6.3.2/logs/elasticsearch.log"

        type => "es-log"

        start_position => "beginning"

        codec => multiline {

            pattern => "^\["

            negate => true

            what => "previous"

        }

    }

    file{

       path => "/var/log/nginx/access_json.log"

       codec => json

       start_position => "beginning"

       type => "nginx-log"

    }

    file {

        path => "/tmp/slow.log"

        type => "mysql-slow-log"

        start_position => "beginning"

        codec => multiline {

            pattern => "^# [email protected]:"

            negate => true

                what => "previous"

        }

     }

}

output{

    if [type] == "system-syslog" {

           redis {

                  host => "10.6.11.176"

                     port => "6379"

                     password => "[email protected]"

                     db => "6"

                     data_type => "list"

                     key => "system-syslog"

              }

       }

    if [type] == "system" {

           redis {

                  host => "10.6.11.176"

                     port => "6379"

                     password => "[email protected]"

                     db => "6"

                     data_type => "list"

                     key => "system"

              }

       }

    if [type] == "nginx-log" {

           redis {

                  host => "10.6.11.176"

                     port => "6379"

                     password => "[email protected]"

                     db => "6"

                     data_type => "list"

                     key => "nginx-log"

              }

       }

}

#从redis读 往es写

input {

    redis {

           type => "system"

              host => "10.6.11.176"

              port => "6379"

              password => "[email protected]"

              db => "6"

              data_type => "list"

              key => "system"

       }

    redis {

           type => "nginx-log"

              host => "10.6.11.176"

              port => "6379"

              password => "[email protected]"

              db => "6"

              data_type => "list"

              key => "nginx-log"

       }

    redis {

           type => "system-syslog"

              host => "10.6.11.176"

              port => "6379"

              password => "[email protected]"

              db => "6"

              data_type => "list"

              key => "system-syslog"

       }            

}

output {

    if [type] == "system-syslog" {

        elasticsearch {

            hosts => ["10.6.11.176:9200"]

            index => "system-syslog-%{+YYYY.MM.dd}"

                      }

    }

    if [type] == "system" {

        elasticsearch {

            hosts => ["10.6.11.176:9200"]

            index => "system-%{+YYYY.MM.dd}"

        }

    }    

    if [type] == "nginx-log" {

        elasticsearch {

            hosts => ["10.6.11.176:9200"]

            index => "nginx-log-%{+YYYY.MM.dd}"

        }

    }       

    if [type] == "mysql-slow-log" {

        elasticsearch {

            hosts => ["10.6.11.176:9200"]

            index => "mysql-slow-log-%{+YYYY.MM.dd}"

        }

    }

}

#从本地redis 写到远程redis

input {

    redis {

           host => "10.6.11.176"

              port => "6379"

              password => "[email protected]"

              db => "6"

              data_type => "list"

              key => "demo"

       }

}

output {

    redis {

           host => "10.6.11.177"

              port => "6379"

              password => "[email protected]"

              db => "6"

              data_type => "list"

              key => "demo"

       }

}

执行之前

执行完之后