curl-L-O//artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.7.0-linux-x86_64.tar.gz
tar -xzvf filebeat-7.7.0-linux-x86_64.tar.gz

tar -zxvf logstash-7.7.0.tar.gz

./bin/logstash -e 'input { stdin { } } output { stdout {} }'

1、設置系統配置
ulimit #暫時修改，切換到該用戶es，ulimit -n 65535 
/etc/security/limits.conf #永久修改 es -  nofile  65535
ulimit -a #查看當前用戶的資源限制

2、禁用sawpping
方式一：
swapoff -a #臨時禁用所有的swap文件
vim /etc/fstab #注釋掉所有的swap相關的行，永久禁用

方式二：
cat /proc/sys/vm/swappiness #查看該值
sysctl vm.swappiness=1 #臨時修改該值為1
vim /etc/sysctl.conf #修改文件 永久生效
vm.swappiness = 1 #如果有該值，則修改該值，若沒有，則追加該選項，sysctl -p生效命令

方式三：
配置elasticsearch.yml文件，添加如下配置：
bootstrap.memory_lock: true
GET _nodes?filter_path=**.mlockall  #檢查如上配置是否成功
注意：如果試圖分配比可用記憶體更多的記憶體，mlockall可能會導致JVM或shell會話退出!

3、配置文件描述符
ulimit -n 65535  #臨時修改
vim /etc/security/limits.conf #永久修改
es         soft    nproc     65535
es         hard    nproc     65535

4、配置虛擬記憶體
sysctl -w vm.max_map_count=262144 #臨時修改該值
vim /etc/sysctl.conf #永久修改
vm.max_map_count=262144

5、配置執行緒數
ulimit -u 4096 #臨時修改
vim /etc/security/limits.conf #永久修改

groupadd  elastic
useradd elk -d /data/hd05/elk -g elastic
echo '2edseoir@' | passwd elk --stdin

解壓：tar -zxvf elasticsearch-7.7.0-linux-x86_64.tar.gz
建立軟鏈接：ln –s elasticsearch-7.7.0  es

$ES_HOME：/data/hd05/elk/elasticsearch-7.7.0
bin: $ES_HOME/bin  #es啟動命令和插件安裝命令
conf：$ES_HOME/conf #elasticsearch.yml配置文件目錄
data：$ES_HOME/data  #對應的參數path.data，用於存放索引分片數據文件
logs：$ES_HOME/logs  #對應的參數path.logs，用於存放日誌
jdk：$ES_HOME/jdk  #自帶支援該es版本的jdk
plugins： $ES_HOME/jplugins #插件存放目錄
lib： $ES_HOME/lib #存放依賴包，比如java類庫
modules： $ES_HOME/modules #包含所有的es模組

Vim ~/.bashrc
############往後面添加如下內容######################
export JAVA_HOME=/data/hd05/elk/es/jdk
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar\:/lib/tools.jar

配置java參數
一種是通過修改/data/hd05/elk/elasticsearch-7.7.0/config/jvm.options文件修改jvm參數，一個使用過一個變數ES_JAVA_OPTS來聲明jvm參數
/data/hd05/elk/elasticsearch-7.7.0/config/jvm.options介紹：
8:-Xmx2g  #表示只適合java8
8-:-Xmx2g  #表示適合高於java8的版本
8-9:-Xmx2g #表示適合java8，和java9
其他配置，都是jvm的相關參數，如果要想明白，得去看java虛擬機

通過變數ES_JAVA_OPTS來聲明jvm參數：
例如：export ES_JAVA_OPTS="$ES_JAVA_OPTS -Djava.io.tmpdir=/path/to/temp/dir"
./bin/elasticsearch

[elk@lgh config]$ cat  jvm.options  | egrep -v '^$|#'                 
-Xms2g
-Xmx2g
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
14-:-XX:G1ReservePercent=25
14-:-XX:InitiatingHeapOccupancyPercent=30
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m

./bin/elasticsearch-certutil ca -out config/elastic-certificates.p12 -pass "password"

./bin/elasticsearch-certutil ca  #創建集群認證機構，需要交互輸入密碼
./bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12  #為節點頒發證書，與上面密碼一樣
執行./bin/elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password 並輸入第一步輸入的密碼 
執行./bin/elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password 並輸入第一步輸入的密碼 
將生成的elastic-certificates.p12、elastic-stack-ca.p12文件移動到config目錄下

[elk@lgh config]$ cat  elasticsearch.yml  | egrep -v '^$|#'
cluster.name: my_cluster
node.name: lgh01
node.data: true
node.master: true
path.data: /data/hd05/elk/elasticsearch-7.7.0/data
path.logs: /data/hd05/elk/elasticsearch-7.7.0/logs
network.host: 192.168.110.130
http.port: 9200
transport.tcp.port: 9300
discovery.seed_hosts: ["192.168.110.130","192.168.110.131","192.168.110.132","192.168.110.133"]
cluster.initial_master_nodes: ["lgh01","lgh02","lgh03"]
cluster.routing.allocation.cluster_concurrent_rebalance: 32
cluster.routing.allocation.node_concurrent_recoveries: 32
cluster.routing.allocation.node_initial_primaries_recoveries: 32
http.cors.enabled: true
http.cors.allow-origin: '*'

#下面的是配置x-pack和tsl/ssl加密通訊的
xpack.security.enabled: true
xpack.license.self_generated.type: basic
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: elastic-certificates.p12

bootstrap.memory_lock: false   #centos6需要配置
bootstrap.system_call_filter: false #centos6需要配置

[elk@lgh elasticsearch-7.7.0]$ ./bin/elasticsearch-setup-passwords interactive
Enter password for the elasticsearch keystore : 
Initiating the setup of passwords for reserved users elastic,apm_system,kibana,logstash_system,beats_system,remote_monitoring_user.
You will be prompted to enter passwords as the process progresses.
Please confirm that you would like to continue [y/N]y


Enter password for [elastic]: 
Reenter password for [elastic]: 
Enter password for [apm_system]: 
Reenter password for [apm_system]: 
Enter password for [kibana]: 
Reenter password for [kibana]: 
Enter password for [logstash_system]: 
Reenter password for [logstash_system]: 
Enter password for [beats_system]: 
Reenter password for [beats_system]: 
Enter password for [remote_monitoring_user]: 
Reenter password for [remote_monitoring_user]: 
1qaz@WSXChanged password for user [apm_system]
Changed password for user [kibana]
Changed password for user [logstash_system]
Changed password for user [beats_system]
Changed password for user [remote_monitoring_user]

#Running with built in server
git clone git://github.com/mobz/elasticsearch-head.git
cd elasticsearch-head
npm install
npm run start
open http://localhost:9100/

[elk@lgh config]$ cat kibana.yml  | egrep -v "^$|#"
server.port: 5601
server.host: "0.0.0.0"
server.name: "my-kibana"
elasticsearch.hosts: ["//192.168.110.130:9200","//192.168.110.131:9200","//192.168.110.132:9200"]
elasticsearch.preserveHost: true
kibana.index: ".kibana"
elasticsearch.username: "elastic"
elasticsearch.password: "password"   #或者使用keystore的保存的密碼"${ES_PWD}"

[elk@lgh ~]$ tail -3 logstash-tutorial.log 
86.1.76.62 - - [04/Jan/2015:05:30:37 +0000] "GET /projects/xdotool/ HTTP/1.1" 200 12292 "//www.haskell.org/haskellwiki/Xmonad/Frequently_asked_questions" "Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20140205 Firefox/24.0 Iceweasel/24.3.0"
86.1.76.62 - - [04/Jan/2015:05:30:37 +0000] "GET /reset.css HTTP/1.1" 200 1015 "//www.semicomplete.com/projects/xdotool/" "Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20140205 Firefox/24.0 Iceweasel/24.3.0"
86.1.76.62 - - [04/Jan/2015:05:30:37 +0000] "GET /style2.css HTTP/1.1" 200 4877 "//www.semicomplete.com/projects/xdotool/" "Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20140205 Firefox/24.0 Iceweasel/24.3.0"

#=========================== Filebeat inputs =============================

filebeat.inputs:

# Each - is an input. Most options can be set at the input level, so
# you can use different inputs for various configurations.
# Below are the input specific configurations.

- type: log

  # Change to true to enable this input configuration.
  enabled: true

  # Paths that should be crawled and fetched. Glob based paths.
  paths:
    - /data/elk/logstash-tutorial.log  #這裡使用的是apache的日誌格式
    #- c:\programdata\elasticsearch\logs\*

  # Exclude lines. A list of regular expressions to match. It drops the lines that are
  # matching any regular expression from the list.
  #exclude_lines: ['^DBG']

  # Include lines. A list of regular expressions to match. It exports the lines that are
  # matching any regular expression from the list.
  #include_lines: ['^ERR', '^WARN']

  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
  # are matching any regular expression from the list. By default, no files are dropped.
  #exclude_files: ['.gz$']

  # Optional additional fields. These fields can be freely picked
  # to add additional information to the crawled log files for filtering
  #fields:
  #  level: debug
  #  review: 1

  ### Multiline options

  # Multiline can be used for log messages spanning multiple lines. This is common
  # for Java Stack Traces or C-Line Continuation

  # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
  #multiline.pattern: ^\[

  # Defines if the pattern set under pattern should be negated or not. Default is false.
  #multiline.negate: false

  # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
  # that was (not) matched before or after or as long as a pattern is not matched based on negate.
  # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
  #multiline.match: after
#================================ Outputs =====================================
output.kafka:
  hosts: ["192.168.110.130:9092"]   #配置kafka的broker
  topic: 'filebeat_test'   #配置topic 名字
  partition.round_robin:
    reachable_only: false
  required_acks: 1
  compression: gzip
  max_message_bytes: 1000000

cd filebeat-7.7.0-linux-x86_64 && nohup ./filebeat -e &

cd logstash-7.7.0/ && mkidr conf.d
cd conf.d
vim apache.conf 
################apache.conf文件中填入如下內容##############################
input {
        kafka{
                bootstrap_servers => "192.168.110.130:9092"
                topics => ["filebeat_test"]
                group_id => "test123"
                auto_offset_reset => "earliest"
        }
    }
filter {
json
        {
                source => "message"
        }
        grok {
        match => { "message" => "%{COMBINEDAPACHELOG}"}
        remove_field => "message"
    }
}
output {
    stdout { codec => rubydebug }
    elasticsearch {
                    hosts => ["192.168.110.130:9200","192.168.110.131:9200","10.18.126.224:9200","192.168.110.132:9200"]
                    index => "test_kakfa"
                    user => "elastic"
                    password => "${ES_PWD}"
                }
}

cd logstash-7.7.0/ && nohup ./bin/logstash -f conf.d/apache.conf &