Graylog 3.3.8安装笔记（yum，docker，docker-compose）
前记：
大多时候，公司系统并不大，使用ELK的成本太高，可以使用一些替代方案，除了ELK还有很多日志管理工具，这里就介绍其中的一个很不错的日志方案：Graylog，Graylog是一个可以跟ELK相提并论的日志管理的后起之秀，一个开源的 log 收容器，背后的储存是搭配 mongodb，而搜寻引擎则由 elasticsearch 提供，自身集成web端，不需要单独部署，目前最新为3.3版本。
由于种种原因，个人尝试了Graylog 3.3的三种安装方式，记录如下：
CentOS 7.2 + Graylog 3.3
1，关闭SELinux
#set enforce 0
#sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
#reboot
2，安装pwgen
#yum install pwgen
3，安装jdk
#yum install java-1.8.0-openjdk.x86_64
#查看，添加java_home
[[email protected] /]# java -version
openjdk version "1.8.0_262"
OpenJDK Runtime Environment (build 1.8.0_262-b10)
OpenJDK 64-Bit Server VM (build 25.262-b10, mixed mode)
[[email protected] /]#
[[email protected] ~]# echo $JAVA_HOME
/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.262.b10-0.el7_8.x86_64
[[email protected] ~]# whereis java
java: /usr/bin/java /usr/lib/java /etc/java /usr/share/java /usr/share/man/man1/java.1.gz
[[email protected] ~]#
[[email protected] ~]#
[[email protected] ~]# ll /usr/bin/java
lrwxrwxrwx. 1 root root 22 10月 26 11:27 /usr/bin/java -> /etc/alternatives/java
[[email protected] ~]#
[[email protected] ~]#
[[email protected] ~]# ll /etc/alternatives/java
lrwxrwxrwx. 1 root root 73 10月 26 11:27 /etc/alternatives/java -> /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.262.b10-0.el7_8.x86_64/jre/bin/java
[[email protected] ~]#
[[email protected] /]# vi /etc/profile
JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.262.b10-0.el7_8.x86_64
JRE_HOME=$JAVA_HOME/jre
PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
CLASSPATH=:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib/dt.jar
export JAVA_HOME JRE_HOME PATH CLASS_PATH
[[email protected] /]# source /etc/profile
方法一：docker setup
#安装需要的软件包， yum-util 提供yum-config-manager功能，另外两个是devicemapper驱动依赖的
#yum install -y yum-utils device-mapper-persistent-data lvm2
#设置yum源
#yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
#查看所有仓库中所有docker版本，选择合适的版本
#yum list docker-ce --showduplicates | sort -r
#安装Docker
#yum install docker-ce-17.12.1.ce
#启动Docker，加入开机启动
#systemctl start docker
#systemctl enable docker
#验证安装是否成功
#docker version
#搜索jdk安装包并下载安装jdk1.8
# yum search java|grep jdk
# yum install java-1.8.0-openjdk
[[email protected] ~]#docker pull mongo
[[email protected] ~]#docker pull elasticsearch/elasticsearch:6.3.2
[[email protected] ~]#docker pull graylog/graylog:3.2
[[email protected] ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
0be0d2b348a5 graylog/graylog:3.2 "tini -- /docker-ent鈥 5 minutes ago Up 5 minutes (healthy) 0.0.0.0:9000->9000/tcp, 0.0.0.0:12201->12201/udp graylog
b63acee0f472 docker.elastic.co/elasticsearch/elasticsearch:6.3.2 "/usr/local/bin/dock鈥 6 minutes ago Up 6 minutes 0.0.0.0:9200->9200/tcp, 0.0.0.0:9300->9300/tcp es
2da50aefbe5e mongo:latest "docker-entrypoint.s鈥 8 minutes ago Up 8 minutes 0.0.0.0:27017->27017/tcp mongo
[[email protected] ~]#
[[email protected] ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
mongo latest ba0c2ff8d362 4 weeks ago 492MB
graylog/graylog 3.2 5131daabda91 4 months ago 484MB
docker.elastic.co/elasticsearch/elasticsearch 6.3.2 96dd1575de0f 2 years ago 826MB
[[email protected] ~]#
[[email protected] ~]#docker pull elasticsearch/elasticsearch:6.8.13 #想安装es6.8.13的版本,卸载旧的，安装新版本
[[email protected] ~]#
[[email protected] ~]# docker stop es
es
[[email protected] ~]# docker rmi es
Error: No such image: es
[[email protected] ~]# docker rm es
es
[[email protected] ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
elasticsearch 6.8.13 a7e1d4b5ee81 9 days ago 827MB
mongo latest ba0c2ff8d362 4 weeks ago 492MB
graylog/graylog 3.2 5131daabda91 4 months ago 484MB
docker.elastic.co/elasticsearch/elasticsearch 6.3.2 96dd1575de0f 2 years ago 826MB
[[email protected] ~]# docker rmi 96dd1575de0f
Untagged: docker.elastic.co/elasticsearch/elasticsearch:6.3.2
Untagged: docker.elastic.co/elasticsearch/[email protected]:8f06aecf7227dbc67ee62d8d05db680f8a29d0296ecd74c60d21f1fe665e04b0
Deleted: sha256:96dd1575de0ff2d2759f85216f4e3e36313873e00e69e6611e28a86d1c0482af
Deleted: sha256:ad08c25cce3f77b278b941551a316206a9fdfe3eb594bdf7ddacc59635387ef3
Deleted: sha256:ef49e95d6ffd579e5f1abc86e64fd849d4515fd6775e2d207a96cd11124cb1a5
Deleted: sha256:0b2e448546f14b324520263af11e4ebf9883d48274ccdf666a35b58d04d249a3
Deleted: sha256:325c4b940b6740999af5346822e9a87e35a20a7e201994b13f70106ef3f53049
Deleted: sha256:32544a489f0a0b4479102f44caf5a1eef3901f0d7805b68bab2ab32a1ca894d3
Deleted: sha256:cee94d9c38c3257b621eb6ebd915fe7c8b873ee18698b773f4819e2dce51a488
Deleted: sha256:f9ff6c381b67d861193bc30018f39ed9728a0cbcb37be7b38ef63b126fce6634
Deleted: sha256:bcc97fbfc9e1a709f0eb78c1da59caeb65f43dc32cd5deeb12b8c1784e5b8237
[[email protected] ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
elasticsearch 6.8.13 a7e1d4b5ee81 9 days ago 827MB
mongo latest ba0c2ff8d362 4 weeks ago 492MB
graylog/graylog 3.2 5131daabda91 4 months ago 484MB
[[email protected] ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
0be0d2b348a5 graylog/graylog:3.2 "tini -- /docker-ent鈥 19 minutes ago Up 19 minutes (healthy) 0.0.0.0:9000->9000/tcp, 0.0.0.0:12201->12201/udp graylog
2da50aefbe5e mongo:latest "docker-entrypoint.s鈥 22 minutes ago Up 22 minutes 0.0.0.0:27017->27017/tcp mongo
docker run \
--name mongo \
-p 27017:27017 \
-v /data/mongodb/configdb:/var/lib/mongodb/configdb/ \
-v /data/mongodb/db/:/var/lib/mongodb/ \
-d mongo:latest
docker run \
--name elasticsearch \
-p 9200:9200 -p 9300:9300 \
-e "discovery.type=single-node" \
-e http.cors.allow-origin="*" \
-e http.cors.enabled=true \
-e network.host=0.0.0.0 \
-d elasticsearch:6.8.13
docker run \
--link mongo:mongo \
--link elasticsearch:elasticsearch \
--name graylog \
-p 9000:9000 \
-p 12201:12201/udp \
-e GRAYLOG_HTTP_EXTERNAL_URI=http://172.16.86.107:9000/ \
-e GRAYLOG_ROOT_TIMEZONE=Asia/Shanghai \
-e GRAYLOG_WEB_ENDPOINT_URI="http://172.16.86.107:9000/:9000/api" \
-e GRAYLOG_PASSWORD_SECRET=somepasswordpepper \
-e GRAYLOG_ROOT_PASSWORD_SHA2=8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918 \
-d graylog/graylog:3.3
[[email protected] ~]# docker run \
>--name mongo \
>-p 27017:27017 \
>-v /data/mongodb/configdb:/var/lib/mongodb/configdb/ \
>-v /data/mongodb/db/:/var/lib/mongodb/ \
>-d mongo:latest
[[email protected] ~]#
[[email protected] ~]# docker run \
> --name elasticsearch \
> -p 9200:9200 -p 9300:9300 \
> -e "discovery.type=single-node" \
> -e http.cors.allow-origin="*" \
> -e http.cors.enabled=true \
> -e network.host=0.0.0.0 \
> -d elasticsearch:6.8.13
0ad458b12a3c1a5a69d7f81a6c2959d00519f5d6f0890287028b3361890404e6
[[email protected] ~]#
[[email protected] ~]# docker run \
> --link mongo:mongo \
> --link elasticsearch:elasticsearch \
> --name graylog \
> -p 9000:9000 \
> -p 12201:12201/udp \
> -e GRAYLOG_HTTP_EXTERNAL_URI=http://172.16.86.107:9000/ \
> -e GRAYLOG_ROOT_TIMEZONE=Asia/Shanghai \
> -e GRAYLOG_WEB_ENDPOINT_URI="http://172.16.86.107:9000/:9000/api" \
> -e GRAYLOG_PASSWORD_SECRET=somepasswordpepper \
> -e GRAYLOG_ROOT_PASSWORD_SHA2=8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918 \
> -d graylog/graylog:3.3
3ad423c72f71bc6eb3cdf7ea3e427bdcf684d216f534d6f2436d3f7c62b89e91
docker: Error response from daemon: Cannot link to a non running container: /mongo AS /graylog/mongo.
[[email protected] ~]#
[[email protected] ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
3ad423c72f71 graylog/graylog:3.3 "tini -- /docker-ent…" 19 seconds ago Created 0.0.0.0:9000->9000/tcp, 0.0.0.0:12201->12201/udp graylog
0ad458b12a3c elasticsearch:6.8.13 "/usr/local/bin/dock…" 27 seconds ago Up 26 seconds 0.0.0.0:9200->9200/tcp, 0.0.0.0:9300->9300/tcp elasticsearch
2da50aefbe5e mongo:latest "docker-entrypoint.s…" 2 hours ago Exited (0) About an hour ago mongo
[[email protected] ~]#
[[email protected] ~]#
[[email protected] ~]#
[[email protected] ~]# docker start mongo
mongo
[[email protected] ~]#
[[email protected] ~]# docker start graylog
graylog
[[email protected] ~]#
[[email protected] ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
3ad423c72f71 graylog/graylog:3.3 "tini -- /docker-ent…" About a minute ago Up 11 seconds (health: starting) 0.0.0.0:9000->9000/tcp, 0.0.0.0:12201->12201/udp graylog
0ad458b12a3c elasticsearch:6.8.13 "/usr/local/bin/dock…" About a minute ago Up About a minute 0.0.0.0:9200->9200/tcp, 0.0.0.0:9300->9300/tcp elasticsearch
2da50aefbe5e mongo:latest "docker-entrypoint.s…" 2 hours ago Up 40 seconds 0.0.0.0:27017->27017/tcp mongo
[[email protected] ~]#
方法二：docker-compose setup
1、安装python-pip
#yum -y install epel-release
#yum -y install python-pip
2、安装docker-compose
#pip install docker-compose
3.创建配置目录
# mkdir -p ./graylog/config
# chmod -R 777 graylog/
# cd ./graylog/config
# wget https://raw.githubusercontent.com/Graylog2/graylog-docker/3.3/config/graylog.conf
# wget https://raw.githubusercontent.com/Graylog2/graylog-docker/3.3/config/log4j2.xml
4.编辑docker-compose.yml文件
#vi ./graylog/config/docker-compose.yml
version: '2'
services:
# MongoDB: https://hub.docker.com/_/mongo/
mongodb:
container_name: mongo
image: mongo:3.6.20
# volumes:
# - mongo_data:/data/db
# Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/6.x/docker.html
elasticsearch:
container_name: es
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.10
# volumes:
# - es_data:/usr/share/elasticsearch/data
environment:
- TZ=Asia/Shanghai
- http.host=0.0.0.0
- transport.host=localhost
- network.host=0.0.0.0
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
mem_limit: 1g
# Graylog: https://hub.docker.com/r/graylog/graylog/
graylog:
container_name: graylog
image: graylog/graylog:3.3
# volumes:
# - graylog_journal:/usr/share/graylog/data/journal
# - /usr/local/graylog:/usr/share/graylog/data/config
environment:
# CHANGE ME (must be at least 16 characters)!
- GRAYLOG_PASSWORD_SECRET=somepasswordpepper
# Password: admin
- GRAYLOG_ROOT_PASSWORD_SHA2=8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
- GRAYLOG_HTTP_EXTERNAL_URI=http://172.16.86.107:9000/
- GRAYLOG_ROOT_TIMEZONE=Asia/Shanghai
links:
- mongodb:mongo
- elasticsearch
depends_on:
- mongodb
- elasticsearch
ports:
# Graylog web interface and REST API
- 9000:9000
# Syslog TCP
- 1514:1514
# Syslog UDP
- 1514:1514/udp
# GELF TCP
- 12201:12201
# GELF UDP
- 12201:12201/udp
# Volumes for persisting data, see https://docs.docker.com/engine/admin/volumes/volumes/
#volumes:
# mongo_data:
# driver: local
# es_data:
# driver: local
# graylog_journal:
# driver: local
5.启动
# cd graylog/config
# docker-compose up -d
#######################################
# docker-compose help
build Build or rebuild services
config Validate and view the Compose file
create Create services
down Stop and remove containers, networks, images, and volumes
events Receive real time events from containers
exec Execute a command in a running container
help Get help on a command
images List images
kill Kill containers
logs View output from containers
pause Pause services
port Print the public port for a port binding
ps List containers
pull Pull service images
push Push service images
restart Restart services
rm Remove stopped containers
run Run a one-off command
scale Set number of containers for a service
start Start services
stop Stop services
top Display the running processes
unpause Unpause services
up Create and start containers
version Show the Docker-Compose version information
# docker logs --since 150m graylogy
#######################################
方法3：yum setup
#安装mongodb
[[email protected] ~]# vi /etc/yum.repos.d/mongodb-org.repo
[mongodb-org-3.6]
name=MongoDB Repository
#由于MongoDB 官方镜像源在国外，yum安装较慢，这里使用阿里云MongoDB源
https://mirrors.aliyun.com/mongodb/yum/redhat/$releasever/mongodb-org/3.6/x86_64/
#baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/3.6/x86_64/
gpgcheck=1
enabled=1
gpgkey=https://www.mongodb.org/static/pgp/server-3.6.asc
[[email protected] ~]# yum install mongodb-org
[[email protected] ~]# systemctl daemon-reload
[[email protected] ~]# systemctl enable mongod.service
[[email protected] ~]# systemctl start mongod.service
#安装Elasticsearch
[[email protected] ~]# vi /etc/yum.repos.d/elasticsearch.repo
[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/oss-6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
[[email protected] ~]# yum install elasticsearch-oss
[[email protected] ~]# vi /etc/elasticsearch/elasticsearch.yml
cluster.name: graylog
action.destructive_requires_name: false
[[email protected] ~]# vi /etc/sysconfig/elasticsearch
JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.262.b10-0.el7_8.x86_64
[[email protected] ~]# systemctl daemon-reload
[[email protected] ~]# systemctl enable elasticsearch.service
Created symlink from /etc/systemd/system/multi-user.target.wants/elasticsearch.service to /usr/lib/systemd/system/elasticsearch.service.
[[email protected] ~]# systemctl start elasticsearch.service
#安装Graylog
[[email protected] ~]# rpm -Uvh https://packages.graylog2.org/repo/packages/graylog-3.3-repository_latest.rpm
[[email protected] ~]# yum install graylog-server -y
[[email protected] config]# pwgen -N 1 -s 96 //设置password_secret
btbWDsHUt3mtgJ5FolZ45JxaLfRuHOX6LVlbun0UFgRKzzzmhcljSWphjLk96v6dgHFGvU4QuR9PsTWE6bswmhrLvs
[[email protected] config]#
[[email protected] config]# echo -n "Enter Password: " && head -1 </dev/stdin | tr -d '\n' | sha256sum | cut -d" " -f1
Enter Password: admin //输入密码，该密码为web登录的密码
8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
[[email protected] config]#
[[email protected] ~]# vi /etc/graylog/server/server.conf
password_secret = btbWDsHUt3mtgJ5FolZ45JxaLfRuHOX6LVlbun0UFgRKzzzmhcljSWphjLk96v6dgHFGvU4QuR9PsTWE6bswmhrLvs
root_password_sha2 = 8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
root_timezone = Asia/Shanghai
http_bind_address = 0.0.0.0:9000 #http绑定的IP与端口
allow_highlighting = true #查询结果高亮
#elasticsearch是单点单源
elasticsearch_shards = 1
elasticsearch_replicas = 0
# Email transport
transport_email_enabled = true
transport_email_hostname = smtp_host_server
transport_email_port = 25
transport_email_subject_prefix = [graylog]
transport_email_from_email = [email protected]
[[email protected] server]# systemctl daemon-reload
[[email protected] server]# systemctl enable graylog-server
Created symlink from /etc/systemd/system/multi-user.target.wants/graylog-server.service to /usr/lib/systemd/system/graylog-server.service.
[[email protected] server]# systemctl start graylog-server
[[email protected] server]#
#配置防火墙
[[email protected] server]# firewall-cmd --list-all
FirewallD is not running
[[email protected] server]# systemctl start firewalld
[[email protected] server]# firewall-cmd --list-all
public (active)
target: default
icmp-block-inversion: no
interfaces: ens192
sources:
services: dhcpv6-client ssh
ports:
protocols:
masquerade: no
forward-ports:
sourceports:
icmp-blocks:
rich rules:
[[email protected] server]# firewall-cmd --state
running
[[email protected] server]#
[[email protected] server]# firewall-cmd --add-port=9000/tcp --permanent
success
[[email protected] server]# firewall-cmd --add-forward-port=port=514:proto=tcp:toport=1514 --permanent
success
[[email protected] server]# firewall-cmd --add-forward-port=port=514:proto=udp:toport=1514 --permanent
success
[[email protected] server]# firewall-cmd --reload
success
[[email protected] server]# firewall-cmd --list-all
public (active)
target: default
icmp-block-inversion: no
interfaces: ens192
sources:
services: dhcpv6-client ssh
ports: 9000/tcp
protocols:
masquerade: no
forward-ports: port=514:proto=tcp:toport=1514:toaddr=
port=514:proto=udp:toport=1514:toaddr=
sourceports:
icmp-blocks:
rich rules:
[[email protected] server]#
/* CentOS6使用
iptables -t nat -A PREROUTING -p tcp --dport 514 -j REDIRECT --to 1514
iptables -t nat -A PREROUTING -p udp --dport 514 -j REDIRECT --to 1514
service iptables save
*/
#安装企业版插件试用
[[email protected] ~]# yum install graylog-enterprise-plugins
#正式使用Graylog
http://172.16.86.107:9000/
默认账号密码为 admin/admin
测试一：
添加一个gelf http 的INPUT，TCP默认
#发送一个测试
#curl -XPOST http://localhost:12201/gelf -p0 -d '{"message":"hello这是一条消息", "host":"127.0.0.1", "facility":"test", "topic": "meme"}'
看到测试信息上传到Graylog主机
测试二：
添加一个syslog INPUTS，UDP端口用1514，确保input running起来
#默认syslog 514会出现低于1024的端口号会无法启动的情况，具体原因可以参考官方说明文档
#cisco 交换机端添加log server，指定Graylog主机
#conf t
logging source-interface Loopback1
logging host 172.16.86.107
#wr
###增加这一行后，过了许久，交换机日志才最终进来
[[email protected] ~]# vi /etc/rsyslog.conf
*.* @172.16.86.107:1514
[[email protected] ~]# systemctl restart rsyslog.service
#发送一个测试
[[email protected] ~]# logger -p mail.info "test mail