ceph 004 糾刪碼池 修改參數 cephx認證
- 2022 年 8 月 12 日
- 筆記
- ceph 分散式存儲
複習ceph003
存儲池為邏輯概念，存儲池可以佔用整個集群的所有空間
[root@ceph01 ~]# ceph osd pool create pool1
pool 'pool1' created
[root@ceph01 ~]# ceph osd pool application enable pool1 rgw
enabled application 'rgw' on pool 'pool1'
[root@ceph01 ~]# ceph osd pool ls detail
pool 1 'device_health_metrics' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 300 flags hashpspool stripe_width 0 pg_num_min 1 application mgr_devicehealth
pool 4 'pool1' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 309 flags hashpspool stripe_width 0 application rgw
[root@ceph01 ~]# ceph osd pool ls detail
pool 1 'device_health_metrics' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 300 flags hashpspool stripe_width 0 pg_num_min 1 application mgr_devicehealth
pool 4 'pool1' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 309 flags hashpspool stripe_width 0 application rgw
[root@ceph01 ~]# ceph pg dump pgs_brief | grep ^4
dumped pgs_brief
4.1f active+clean [6,5,1] 6 [6,5,1] 6
4.1e active+clean [0,4,7] 0 [0,4,7] 0
4.1d active+clean [7,3,1] 7 [7,3,1] 7
4.1c active+clean [6,1,4] 6 [6,1,4] 6
4.1b active+clean [4,1,8] 4 [4,1,8] 4
4.1a active+clean [4,0,7] 4 [4,0,7] 4
4.19 active+clean [6,0,5] 6 [6,0,5] 6
4.18 active+clean [3,1,6] 3 [3,1,6] 3
4.17 active+clean [0,7,3] 0 [0,7,3] 0
4.16 active+clean [0,4,8] 0 [0,4,8] 0
4.15 active+clean [8,0,5] 8 [8,0,5] 8
4.14 active+clean [1,5,7] 1 [1,5,7] 1
4.13 active+clean [4,6,1] 4 [4,6,1] 4
4.12 active+clean [1,8,4] 1 [1,8,4] 1
4.11 active+clean [4,0,7] 4 [4,0,7] 4
4.10 active+clean [1,6,4] 1 [1,6,4] 1
4.f active+clean [7,1,5] 7 [7,1,5] 7
4.4 active+clean [1,7,3] 1 [1,7,3] 1
4.3 active+clean [6,5,1] 6 [6,5,1] 6
4.2 active+clean [5,6,2] 5 [5,6,2] 5
4.1 active+clean [6,5,0] 6 [6,5,0] 6
4.5 active+clean [3,6,0] 3 [3,6,0] 3
4.0 active+clean [0,4,6] 0 [0,4,6] 0
4.6 active+clean [1,4,6] 1 [1,4,6] 1
4.7 active+clean [1,6,3] 1 [1,6,3] 1
4.8 active+clean [8,3,0] 8 [8,3,0] 8
4.9 active+clean [6,1,5] 6 [6,1,5] 6
4.a active+clean [5,1,7] 5 [5,1,7] 5
4.b active+clean [8,1,4] 8 [8,1,4] 8
4.c active+clean [4,1,8] 4 [4,1,8] 4
4.d active+clean [7,1,4] 7 [7,1,4] 7
4.e active+clean [4,6,1] 4 [4,6,1] 4
先寫主後寫備。三副本有自我恢復機制，在主osd壞掉後，備暫時成為主。然後再找一個osd成為第三個副本
老版不會自動擴
pg自動擴大 從 32開始
pg計算器
100osd 每個osd承載pg數量有限（100-200 pg/osd）
100osd * 200 = 20000pg
規劃創建2個存儲池
pool1 10000pg
pool2 10000pg
這個池osd承載數量可以改，但最好不要
在這個20000pg平均下來，性能會好一些
rados為集群內部的操作命令，排錯調試
mom_allow_pool_delete = true全局影像化修改此參數允許刪除存儲池
ceph osd pool set pool1 nodelete false 單獨設置存儲池可不可刪除
存儲池命名空間
分隔存儲池中數據，允許訪問哪一個存儲池裡面的哪一個命名空間
[root@ceph01 ~]# cp /etc/passwd .
[root@ceph01 ~]#
[root@ceph01 ~]# rados -p pool1 put password passwd
[root@ceph01 ~]# rados -p pool1 ls
password
[root@ceph01 ~]# rados -p pool1 -N sys put password01 passwd
[root@ceph01 ~]# rados -p pool1 ls
password
[root@ceph01 ~]# rados -p pool1 -N sys ls
password01
[root@ceph01 ~]# rados -p pool1 --all ls
password
sys password01
允許用戶訪問存儲池，就可以訪問所有的命名空間
只限制用戶只能訪問 sys這個命名空間，其他命名空間就無法訪問
糾刪碼池
10M 複製池 30M
10M 糾刪碼 < 30M 糾刪碼池可以節省空間
n = k + m
4M
k為數據塊 2 每個數據塊大小2M 如果為3數據塊則為1.3 1.3*3
m為編碼塊 2 編碼塊2M 1.3*2 1.3*5=6.5M
兩個數據塊基於演算法，會得到兩個校驗塊 （raid6）
10M佔了20M
pool 5 'pool2' erasure profile default size 4 min_size 3 crush_rule 1 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 313 flags hashpspool stripe_width 8192
[root@ceph01 ~]# ceph osd erasure-code-profile get default
k=2
m=2
plugin=jerasure
technique=reed_sol_van
[root@ceph01 ~]#
四個副本最多只能壞一個副本
壞2校驗塊數據不丟，但誰能保證壞的兩個都是校驗快呢
[root@ceph01 ~]# ceph osd pool application enable pool2 rgw
enabled application 'rgw' on pool 'pool2'
[root@ceph01 ~]# rados -p pool2 put passwd02 passwd
[root@ceph01 ~]# ceph pg dump pgs_brief | grep ^5
dumped pgs_brief
5.f active+undersized [5,2147483647,7,0] 5 [5,2147483647,7,0] 5
5.c active+undersized [1,3,6,2147483647] 1 [1,3,6,2147483647] 1
5.d active+undersized [7,1,3,2147483647] 7 [7,1,3,2147483647] 7
5.a active+undersized [2,6,2147483647,5] 2 [2,6,2147483647,5] 2
5.b active+undersized [7,2,4,2147483647] 7 [7,2,4,2147483647] 7
5.8 active+undersized [6,1,2147483647,4] 6 [6,1,2147483647,4] 6
5.9 active+undersized [7,5,2147483647,2] 7 [7,5,2147483647,2] 7
5.6 active+undersized [2,4,8,2147483647] 2 [2,4,8,2147483647] 2
5.7 active+undersized [5,0,7,2147483647] 5 [5,0,7,2147483647] 5
5.1 active+undersized [3,0,2147483647,7] 3 [3,0,2147483647,7] 3
5.4 active+undersized [7,2,4,2147483647] 7 [7,2,4,2147483647] 7
5.0 active+undersized [6,2,3,2147483647] 6 [6,2,3,2147483647] 6
5.3 active+undersized [0,2147483647,4,6] 0 [0,2147483647,4,6] 0
5.2 active+undersized [5,2,7,2147483647] 5 [5,2,7,2147483647] 5
5.5 active+undersized [0,7,4,2147483647] 0 [0,7,4,2147483647] 0
5.e active+undersized [6,4,1,2147483647] 6 [6,4,1,2147483647] 6
5.11 active+undersized [6,3,2147483647,1] 6 [6,3,2147483647,1] 6
5.10 active+undersized [4,6,2,2147483647] 4 [4,6,2,2147483647] 4
5.13 active+undersized [6,1,2147483647,4] 6 [6,1,2147483647,4] 6
5.12 active+undersized [6,3,2,2147483647] 6 [6,3,2,2147483647] 6
5.15 active+undersized [5,0,8,2147483647] 5 [5,0,8,2147483647] 5
5.14 active+undersized [0,7,4,2147483647] 0 [0,7,4,2147483647] 0
5.17 active+undersized [2,8,2147483647,4] 2 [2,8,2147483647,4] 2
5.16 active+undersized [3,1,2147483647,8] 3 [3,1,2147483647,8] 3
5.19 active+undersized [1,5,2147483647,7] 1 [1,5,2147483647,7] 1
5.18 active+undersized [4,7,1,2147483647] 4 [4,7,1,2147483647] 4
5.1b active+undersized [5,1,2147483647,7] 5 [5,1,2147483647,7] 5
5.1a active+undersized [8,4,0,2147483647] 8 [8,4,0,2147483647] 8
5.1d active+undersized [1,6,5,2147483647] 1 [1,6,5,2147483647] 1
5.1c active+undersized [4,6,0,2147483647] 4 [4,6,0,2147483647] 4
5.1f active+undersized [3,2147483647,0,8] 3 [3,2147483647,0,8] 3
5.1e active+undersized [0,7,3,2147483647] 0 [0,7,3,2147483647] 0
[root@ceph01 ~]# ceph osd map pool2 passwd02
osdmap e314 pool 'pool2' (5) object 'passwd02' -> pg 5.e793df76 (5.16) -> up ([3,1,NONE,8], p3) acting ([3,1,NONE,8], p3)
[root@ceph01 ~]#
更改故障域host 為 osd
[root@ceph01 ~]# ceph osd erasure-code-profile set ec01 crush-failure-domain=osd k=3 m=2
[root@ceph01 ~]# ceph osd erasure-code-profile get ec01
crush-device-class=
crush-failure-domain=osd
crush-root=default
jerasure-per-chunk-alignment=false
k=3
m=2
plugin=jerasure
technique=reed_sol_van
w=8
[root@ceph01 ~]# ceph osd pool create pool3 erasure ec01
pool 'pool3' created
[root@ceph01 ~]# ceph pg dump pgs_brief | grep ^6
dumped pgs_brief
6.1d active+clean [1,8,6,2,5] 1 [1,8,6,2,5] 1
6.1c active+clean [7,1,3,4,2] 7 [7,1,3,4,2] 7
6.1f active+clean [2,6,8,7,3] 2 [2,6,8,7,3] 2
6.1e active+clean [8,6,5,2,4] 8 [8,6,5,2,4] 8
6.19 active+clean [5,1,8,6,0] 5 [5,1,8,6,0] 5
6.18 active+clean [2,1,0,6,3] 2 [2,1,0,6,3] 2
6.1b active+clean [6,8,7,5,3] 6 [6,8,7,5,3] 6
6.1a active+clean [4,3,1,8,6] 4 [4,3,1,8,6] 4
6.15 active+clean [3,0,8,2,7] 3 [3,0,8,2,7] 3
6.14 active+clean [2,4,3,5,8] 2 [2,4,3,5,8] 2
6.17 active+clean [6,2,8,7,0] 6 [6,2,8,7,0] 6
6.16 active+clean [0,4,2,3,5] 0 [0,4,2,3,5] 0
6.11 active+clean [2,0,8,6,7] 2 [2,0,8,6,7] 2
6.10 active+clean [0,8,1,5,7] 0 [0,8,1,5,7] 0
6.13 active+clean [2,1,8,3,4] 2 [2,1,8,3,4] 2
6.12 active+clean [8,7,6,2,3] 8 [8,7,6,2,3] 8
6.d active+clean [5,1,8,0,7] 5 [5,1,8,0,7] 5
6.6 active+clean [1,4,3,8,0] 1 [1,4,3,8,0] 1
6.1 active+clean [6,4,1,3,5] 6 [6,4,1,3,5] 6
6.0 active+clean [0,7,6,5,3] 0 [0,7,6,5,3] 0
6.3 active+clean [4,5,7,3,0] 4 [4,5,7,3,0] 4
6.7 active+clean [5,2,0,4,1] 5 [5,2,0,4,1] 5
6.2 active+clean [3,0,2,8,5] 3 [3,0,2,8,5] 3
6.4 active+clean [1,5,6,2,3] 1 [1,5,6,2,3] 1
6.5 active+clean [5,4,1,2,3] 5 [5,4,1,2,3] 5
6.a active+clean [5,4,0,7,2] 5 [5,4,0,7,2] 5
6.b active+clean [1,3,4,6,7] 1 [1,3,4,6,7] 1
6.8 active+clean [3,6,2,8,4] 3 [3,6,2,8,4] 3
6.9 active+clean [0,5,7,2,6] 0 [0,5,7,2,6] 0
6.e active+clean [4,8,1,2,5] 4 [4,8,1,2,5] 4
6.f active+clean [2,6,7,3,4] 2 [2,6,7,3,4] 2
6.c active+clean [1,4,5,8,3] 1 [1,4,5,8,3] 1
4M的對象你切分成3個數據塊，
數據發生變化也要重新計算
k=3 m=2 就最多壞2個 3+2好一些 但是最多讓你壞1個保險
創建完ec01規則不能修改 如果有存儲池在應用這個規則
性能比複製池差，因為時通過校驗塊，來保證可靠性，所以會消耗cpu計算
一個文件被切成對象，如果用的是糾刪碼池，對象還得被切塊，就是/k的數量，然後計算出編碼塊來保證可靠性。
一些相關的參數
k m rule(crush-failure-domain)
crush-device-class 僅將某一類設備支援的osd用於池 ssd hdd nvme
crush-root 設置crush規則集的根節點
ceph的配置
/etc/ceph
ceph.conf 集群配置文件的入口
[root@ceph01 ceph]# cat ceph.conf
# minimal ceph.conf for cb8f4abe-14a7-11ed-a76d-000c2939fb75
[global]
fsid = cb8f4abe-14a7-11ed-a76d-000c2939fb75
mon_host = [v2:192.168.92.11:3300/0,v1:192.168.92.11:6789/0]
mon其實有三個，可以手動更新一下
更新ceph配置
集群網路（心跳，重平衡之類），和客戶端訪問的網路得分開好一些
[root@ceph01 ceph]# ceph config get mon public_network
192.168.92.0/24
[root@ceph01 ceph]# ceph config get mon cluster_network
xxxxxx
我沒設置
可以通過set改
[root@ceph01 ceph]# ceph -s --cluster ceph
cluster:
id: cb8f4abe-14a7-11ed-a76d-000c2939fb75
health: HEALTH_WARN
1 osds exist in the crush map but not in the osdmap
Degraded data redundancy: 1/10 objects degraded (10.000%), 1 pg degraded, 32 pgs undersized
services:
mon: 3 daemons, quorum ceph01.example.com,ceph02,ceph03 (age 22h)
mgr: ceph01.example.com.wvuoii(active, since 22h), standbys: ceph02.alqzfq
osd: 9 osds: 9 up (since 22h), 9 in (since 2d)
data:
pools: 4 pools, 97 pgs
objects: 3 objects, 2.9 KiB
usage: 9.4 GiB used, 171 GiB / 180 GiB avail
pgs: 1/10 objects degraded (10.000%)
65 active+clean
31 active+undersized
1 active+undersized+degraded
[root@ceph01 ceph]# ls
COPYING sample.ceph.conf
[root@ceph01 ceph]# pwd
/usr/share/doc/ceph
[root@ceph01 ceph]# cd /etc/ceph/
[root@ceph01 ceph]# ls
ceph.client.admin.keyring ceph.conf ceph.pub rbdmap
[root@ceph01 ceph]#
ceph.conf .conf前面的就是集群名
區分集群名
ceph -s
health: HEALTH_WARN
1 osds exist in the crush map but not in the osdmap
Degraded data redundancy: 1/10 objects degraded (10.000%), 1 pg degraded, 32 pgs undersized
糾刪碼那個池引發的報錯
[root@ceph01 ceph]# ceph osd pool delete pool2 pool2 --yes-i-really-really-mean-it
Error EPERM: pool deletion is disabled; you must first set the mon_allow_pool_delete config option to true before you can destroy a pool
[root@ceph01 ceph]# ceph config set mon mon_allow_pool_delete true
[root@ceph01 ceph]# ceph osd pool delete pool2 pool2 --yes-i-really-really-mean-it
pool 'pool2' removed
[root@ceph01 ceph]#
[root@ceph01 ceph]# ceph -s
cluster:
id: cb8f4abe-14a7-11ed-a76d-000c2939fb75
health: HEALTH_WARN
1 osds exist in the crush map but not in the osdmap
services:
mon: 3 daemons, quorum ceph01.example.com,ceph02,ceph03 (age 23h)
mgr: ceph01.example.com.wvuoii(active, since 23h), standbys: ceph02.alqzfq
osd: 9 osds: 9 up (since 23h), 9 in (since 2d)
data:
health_warn一般問題不大
報錯資訊就會少一條
ceph -s --name client.admin
ceph -s --id admin
元數據用來區分
在客戶端區分做區分集群名之類
配置參數
以前改配置文件，現在通過命令行改
圖形介面也可以看
[root@ceph01 ceph]# ceph config ls | grep mon | grep delete
mon_allow_pool_delete
mon_fake_pool_delete
[root@ceph01 ceph]# ceph config get mon mon_allow_pool_delete
true
對存儲池操作由mon
對osd讀寫由osd決定
寫在資料庫，立即生效
ceph orch restart mon 重啟集群所有mon
ceph orch daemon restart ods.1 針對性重啟
ceph config dump
查看資料庫的當前值
[root@ceph01 ~]# ceph config ls | grep mon | grep delete
mon_allow_pool_delete
mon_fake_pool_delete
過濾有用的設置值
[root@ceph01 ~]# ceph config set mon mon_allow_pool_delete false
[root@ceph01 ~]# ceph config get mon mon_allow_pool_delete
false
更改值並查看
或者
寫一個配置文件
[root@ceph01 ~]# cat ceph.conf
[global]
mon_allow_pool_delete = false
使用此配置文件修改值
使用這個文件生效
ceph config assimilate-conf -i ceph.conf
可以知道自己寫了哪些內容
ceph 認證授權
ceph auth ls 查看所有用戶
[root@serverc ~]# ceph config ls | grep auth | grep required
auth_cluster_required
auth_service_required
auth_client_required
[root@serverc ~]# ceph config get mon auth_cluster_required
cephx
創建密鑰
key來認證，然後訪問集群
訪問
創建用戶
[root@ceph01 ~]# ceph auth get-or-create client.user1 --id admin
[client.user1]
key = AQB+PPdiAgZZBxAAWKZ3s0Cu3y7o7yEVZqgKKQ==
使用id admin創建的
[root@ceph01 ~]# ceph auth get-or-create client.admin > /etc/ceph/ceph.client.user1.keyring
[root@ceph01 ~]# ceph -s --name client.user1
2022-08-13T13:56:00.302+0800 7f6eaaffd700 -1 monclient(hunting): handle_auth_bad_method server allowed_methods [2] but i only support [2,1]
[errno 13] RADOS permission denied (error connecting to the cluster)
[root@ceph01 ~]#
無許可權
許可權：
r：mon 獲取集群資訊
osd 獲取對象資訊
w：mon 修改集群資訊，mon參數，創建和修改存儲池
osd 上傳對象 刪除對象
授權
[root@serverc ~]# ceph auth caps client.user1 mon 'allow r'
updated caps for client.user1
[root@serverc ~]# ceph osd pool create pool1 --id user1
Error EACCES: access denied
[root@serverc ~]# ceph auth caps client.user1 mon 'allow rw'
updated caps for client.user1
[root@serverc ~]# ceph osd pool create pool1 --id user1
pool 'pool1' created
[root@serverc ~]#
[root@serverc ceph]# ceph auth caps client.user1 mon 'allow rw' osd 'allow r'
updated caps for client.user1
[root@serverc ceph]# rados -p pool1 ls --id user1
[root@serverc ceph]#
[root@serverc ceph]# ceph auth caps client.user1 mon 'allow rw' osd 'allow rw'
updated caps for client.user1
[root@serverc ceph]# rados -p pool1 ls --id user1
[root@serverc ceph]# cp /etc/passwd .
[root@serverc ceph]# rados -p pool1 put file2 passwd --id user1
[root@serverc ceph]# rados -p pool1 ls --id user1
file2
查看許可權
[root@ceph01 ceph]# ceph auth get client.user1
exported keyring for client.user1
[client.user1]
key = AQB+PPdiAgZZBxAAWKZ3s0Cu3y7o7yEVZqgKKQ==
caps mon = "allow r"
[root@ceph01 ceph]#
