Home > Menu > Plateformes > 000 (Pkgdata) > Installer Ceph Backup

Installer Ceph Backup

Infra

3 serveurs OVH Advance-Stor

CPU : AMD EPYC 4344P - 8c/16t - 3.8GHz/5.3GHz
RAM : 32GB DDR5 5200MHz
Disques : 2×960 GB SSD SATA + 4×22 TB HDD SAS

Installer les serveurs

OS

Partitionnement :

Partitionning

Ansible

Si besoin de debug ajouter : “-vvv -e ‘debops__no_log=false’”

ansible-playbook -i inventory/000_hosts playbooks/debops_bootstrap.yml --vault-id @prompt --flush-cache --user debian --limit ceph_backup
ansible-playbook -i inventory/000_hosts playbooks/debops_site.yml --vault-id @prompt --flush-cache --limit ceph_backup
reboot

Tuning Performance Disque SAS

Mettre à jour les firmwares

openSeaChest_Firmware -d all --modelMatch ST22000NM004E --fwdlInfo | grep Revision
openSeaChest_Firmware -d all --modelMatch ST22000NM004E --downloadFW LongsPeakBPExosX22SAS-SED-512E-ET02.LOD
openSeaChest_Firmware -d all --modelMatch ST22000NM004E --downloadFW LongsPeakBPExosX22SAS-SED-512E-ET03.LOD

Désactiver Write Cache (passe de 130MB/s à 200MB/s en écriture)

openSeaChest_Configure -d all --modelMatch ST22000NM004E --writeCache info
openSeaChest_Configure -d all --modelMatch ST22000NM004E --writeCache disable
for i in $(seq 0 3); do echo "write through" > /sys/class/scsi_disk/0\:0\:$i\:0/cache_type; done

Ceph

CEPH_RELEASE=18.2.2 ; curl --silent --remote-name --location https://download.ceph.com/rpm-${CEPH_RELEASE}/el9/noarch/cephadm
chmod +x cephadm
sudo ./cephadm add-repo --release reef
sudo chmod 644 /etc/apt/trusted.gpg.d/ceph.release.gpg
sudo ./cephadm add-repo --release reef
sudo ./cephadm install cephadm ceph ceph-volume ceph-fuse
rm cephadm

Configurer Ceph

cephadm bootstrap --mon-id on004.cephbackup --mon-ip 192.168.1.1 --cluster-network 192.168.1.0/24

Copier le contenu de /etc/ceph/ceph.pub dans ~/work/infras/inventory/group_vars/ceph_backup/debops_users.yml puis lancer :

ansible-playbook -i inventory/000_hosts playbooks/debops_site.yml --vault-id @prompt --flush-cache --limit ceph_backup --tags role::root_account

ceph orch host add ov005 192.168.1.2 --labels _admin
ceph orch host add ov006 192.168.1.3 --labels _admin

Sur ov004 :

for i in $(seq 0 3); do lvcreate -L 50GB -n db-$i vg; done

ceph orch daemon add osd ov004:data_devices=/dev/sda,db_devices=/dev/vg/db-0,osds_per_device=1
ceph orch daemon add osd ov004:data_devices=/dev/sdb,db_devices=/dev/vg/db-1,osds_per_device=1
ceph orch daemon add osd ov004:data_devices=/dev/sdc,db_devices=/dev/vg/db-2,osds_per_device=1
ceph orch daemon add osd ov004:data_devices=/dev/sdd,db_devices=/dev/vg/db-3,osds_per_device=1

Sur ov005 :

for i in $(seq 0 3); do lvcreate -L 50GB -n db-$i vg; done

ceph orch daemon add osd ov005:data_devices=/dev/sda,db_devices=/dev/vg/db-0,osds_per_device=1
ceph orch daemon add osd ov005:data_devices=/dev/sdb,db_devices=/dev/vg/db-1,osds_per_device=1
ceph orch daemon add osd ov005:data_devices=/dev/sdc,db_devices=/dev/vg/db-2,osds_per_device=1
ceph orch daemon add osd ov005:data_devices=/dev/sdd,db_devices=/dev/vg/db-3,osds_per_device=1

Sur ov006 :

for i in $(seq 0 3); do lvcreate -L 50GB -n db-$i vg; done

ceph orch daemon add osd ov006:data_devices=/dev/sda,db_devices=/dev/vg/db-0,osds_per_device=1
ceph orch daemon add osd ov006:data_devices=/dev/sdb,db_devices=/dev/vg/db-1,osds_per_device=1
ceph orch daemon add osd ov006:data_devices=/dev/sdc,db_devices=/dev/vg/db-2,osds_per_device=1
ceph orch daemon add osd ov006:data_devices=/dev/sdd,db_devices=/dev/vg/db-3,osds_per_device=1

Service RGW

Activation :

ceph orch apply rgw cephbackup --realm=eu-west --zonegroup=ovh --zone=backups
ceph dashboard set-rgw-credentials

Configuration HTTPS :

ceph config-key set rgw/cert/rgw.cephbackup -i /etc/pki/realms/pkgdata.backup/private/key_chain.pem

et vérifier :

ceph orch ls --service-type rgw --export
ceph config-key get rgw/cert/rgw.cephbackup

Ignorer la vérification du certificat depuis le dashboard

SSLCertVerificationError hostname ov005 doesn't match either of *.pkgdata.com, pkgdata.com

ceph dashboard set-rgw-api-ssl-verify False

# radosgw-admin realm list
radosgw-admin realm create --rgw-realm=eu-west --default
radosgw-admin realm default --rgw-realm=eu-west

radosgw-admin zonegroup create --rgw-zonegroup=ovh --rgw-realm=eu-west --master --default
radosgw-admin zonegroup default --rgw-zonegroup=ovh

radosgw-admin zone create --rgw-zone=backups --rgw-zonegroup=ovh --master --default
radosgw-admin zone default --rgw-zone=backups

radosgw-admin period update --commit

# ceph df
# ceph osd erasure-code-profile get raid5
# ceph osd lspools
ceph osd erasure-code-profile set raid5 k=2 m=1
ceph osd pool create backup-data-ec01 erasure raid5
ceph osd pool application enable backup-data-ec01 rgw

radosgw-admin zonegroup get | jq '.placement_targets += [{"name": "'"ec_placement"'", "tags": [], "storage_classes": ["STANDARD"]}]' > zonegroup.json
radosgw-admin zonegroup set --infile zonegroup.json

radosgw-admin zone get | jq '.placement_pools += [{"key": "'"ec_placement"'", "val": {"index_pool": "backups.rgw.buckets.index", "storage_classes": {"STANDARD": {"data_pool": "'"backup-data-ec01"'"}}, "data_extra_pool": "backups.rgw.buckets.non-ec", "index_type": 0}}]' > zone.json
radosgw-admin zone set --infile zone.json

ceph orch ps
ceph orch daemon restart rgw.cephbackup.ov004.rydmwh
ceph orch daemon restart rgw.cephbackup.ov005.hpfpvf

radosgw-admin user create --uid='user1' --display-name='First User' --access-key='S3user1' --secret-key='S3user1key'
radosgw-admin subuser create --uid='user1' --subuser='user1:swift' --secret-key='Swiftuser1key' --access=full

Truster le certificat SSL autosigné

cp secret/pki/authorities/root/subject/cert.pem /usr/local/share/ca-certificates/root-ca.pkgdata.backup.crt
sudo update-ca-certificates -f

Tuning perf

ceph config set osd osd_mclock_override_recovery_settings true
ceph config set osd osd_mclock_profile high_recovery_ops

Upgrade Debian Bookworm > Trixie

ceph osd add-noout osd.3
ceph osd add-noout osd.5
ceph osd add-noout osd.6
ceph osd add-noout osd.7
apt update
apt upgrade -y
apt full-upgrade -y
apt autoremove -y
apt clean

reboot

sed -i 's/bookworm/trixie/g' /etc/apt/sources.list
sed -i 's/bookworm/trixie/g' /etc/apt/sources.list.d/*.list
rm /etc/apt/sources.list.d/ceph.list

apt update
apt upgrade --without-new-pkgs -y
apt full-upgrade -y
apt autoremove -y
apt clean

apt install docker-cli
apt install --reinstall grub-efi-amd64
update-grub

mv /etc/systemd/network/enp9s0f0.network /etc/systemd/network/enp9s0f0np0.network
mv /etc/systemd/network/enp9s0f1.network /etc/systemd/network/enp9s0f1np1.network
sed -i 's/enp9s0f0/enp9s0f0np0/' /etc/systemd/network/enp9s0f0np0.network
sed -i 's/enp9s0f1/enp9s0f1np1/' /etc/systemd/network/enp9s0f1np1.network

cat > /etc/udev/rules.d/95-ceph-osd.rules <<EOF
# Règles pour les volumes LVM générés par Ceph (données OSD)
ACTION=="add|change", SUBSYSTEM=="block", ENV{DM_VG_NAME}=="ceph-*", OWNER="167", GROUP="167", MODE="0660"

# Règle pour ton volume LVM de base de données (RocksDB) personnalisé (vg0)
ACTION=="add|change", SUBSYSTEM=="block", ENV{DM_VG_NAME}=="vg*", ENV{DM_LV_NAME}=="db-*", OWNER="167", GROUP="167", MODE="0660"
EOF

dpkg -l | grep bookworm | awk '{print $2}' | xargs apt-get remove --purge -y
apt install cephadm ceph

reboot

Upgrade Ceph 19.2.4 > 20.2.1

ceph osd set noout
ceph osd set noscrub
ceph osd set nodeep-scrub
ceph osd set noautoscale
ceph osd set norebalance

root@ov004:/home/pkgdata # ceph health detail
HEALTH_WARN noout,norebalance,noscrub,nodeep-scrub flag(s) set
[WRN] OSDMAP_FLAGS: noout,norebalance,noscrub,nodeep-scrub flag(s) set

root@ov004:/home/pkgdata # ceph orch upgrade start --ceph-version 20.2.1 --daemon-types mgr --limit 1
Initiating upgrade to quay.io/ceph/ceph:v20.2.1

root@ov004:/home/pkgdata # ceph orch ps --daemon-type mgr
NAME              HOST   PORTS             STATUS         REFRESHED  AGE  MEM USE  MEM LIM  VERSION  IMAGE ID      CONTAINER ID  
mgr.ov004.oovptt  ov004  *:8443,9283,8765  running (5d)      8m ago  22M     469M        -  19.2.4   9d4dd962f80d  12748a29a937  
mgr.ov005.mulmvd  ov005  *:8443,9283,8765  running (78s)    73s ago  22M     378M        -  20.2.1   fb63cba66eea  d5db426eef07

root@ov004:/home/pkgdata # ceph orch upgrade start --ceph-version 20.2.1 --daemon-types mgr --limit 1
Initiating upgrade to quay.io/ceph/ceph:v20.2.1

root@ov004:/home/pkgdata # ceph orch ps --daemon-type mgr
NAME              HOST   PORTS             STATUS        REFRESHED  AGE  MEM USE  MEM LIM  VERSION  IMAGE ID      CONTAINER ID  
mgr.ov004.oovptt  ov004  *:8443,9283,8765  running (3m)    26s ago  22M     594M        -  20.2.1   fb63cba66eea  789b15d2e803  
mgr.ov005.mulmvd  ov005  *:8443,9283,8765  running (3m)     2m ago  22M     462M        -  20.2.1   fb63cba66eea  6d4af56a36a3

ceph orch upgrade start --ceph-version 20.2.1 --daemon-types mon

root@ov004:/home/pkgdata # ceph orch ps --daemon-type mon
NAME                  HOST   PORTS  STATUS        REFRESHED  AGE  MEM USE  MEM LIM  VERSION  IMAGE ID      CONTAINER ID  
mon.on004.cephbackup  ov004         running (5m)     4m ago  22M    32.0M    2048M  20.2.1   fb63cba66eea  c71b841e8312  
mon.ov005             ov005         running (4m)     3m ago  22M    22.2M    2048M  20.2.1   fb63cba66eea  64cae672b836  
mon.ov006             ov006         running (3m)     3m ago  22M    22.2M    2048M  20.2.1   fb63cba66eea  677acc642af8  
mon.ov007             ov007         running (3m)     3m ago   2w    33.2M    2048M  20.2.1   fb63cba66eea  40e8f184b612

ceph orch upgrade start --ceph-version 20.2.1 --daemon-types crash
ceph orch upgrade start --ceph-version 20.2.1 --daemon-types osd

ceph osd require-osd-release tentacle

ceph orch upgrade start --ceph-version 20.2.1

ceph osd unset noout
ceph osd unset norebalance
ceph osd unset noscrub
ceph osd unset nodeep-scrub
ceph osd unset noautoscale

Upgrade v20.2.2

ceph orch upgrade start --image quay.io/ceph/ceph:v20.2.2
ceph orch upgrade status
ceph config set global container_image quay.io/ceph/ceph:v20.2.2

ceph dashboard set-prometheus-api-host https://192.168.1.1:9095
ceph dashboard set-prometheus-api-ssl-verify false

ceph dashboard set-alertmanager-api-host https://192.168.1.1:9093
ceph dashboard set-alertmanager-api-ssl-verify false

ceph mgr module disable dashboard
ceph mgr module enable dashboard

Configuration de MGMT-Gateway

cat <<EOF | ceph orch apply -i -
service_type: mgmt-gateway
service_id: gateway
placement:
  hosts:
    - ov004
    - ov005
networks:
  - 192.168.1.0/24
spec:
  port: 8444
  ssl: true
  ssl_cert: |
$(sed 's/^/    /' /etc/pki/realms/pkgdata.net/public/chain.pem)
  ssl_key: |
$(sed 's/^/    /' /etc/pki/realms/pkgdata.net/private/key.pem)
  extra_container_args:
    - "--add-host=ov004.pkgdata.net:192.168.1.1"
    - "--add-host=ov005.pkgdata.net:192.168.1.2"
EOF


ceph dashboard set-alertmanager-api-ssl-verify false
ceph dashboard set-prometheus-api-ssl-verify false
ceph config-key set mgr/dashboard/prometheus_api_ssl_verify false
ceph config-key set mgr/dashboard/alertmanager_api_ssl_verify false
ceph config set mgr mgr/cephadm/secure_monitoring_stack true
ceph orch prometheus set-credentials admin XXXXXXXX
ceph orch alertmanager set-credentials admin XXXXXX