1# 禁用selinux
2setenforce 0
3sed -i s'/SELINUX=enforcing/SELINUX=disabled'/g /etc/selinux/config
4
5# 关闭防火墙
6systemctl disable --now firewalld

 1cat > /etc/hosts <<EOF
 2127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
 3::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
 410.10.10.47 ceph-node1
 510.10.10.48 ceph-node2
 610.10.10.49 ceph-node3
 7EOF
 8
 9# 分发到各节点
10scp /etc/hosts ceph-node2:/etc
11scp /etc/hosts ceph-node3:/etc

1# 生成私钥
2ssh-keygen -t rsa -b 2048 -P '' -f ~/.ssh/id_rsa
3
4# 分发到各节点
5ssh-copy-id ceph-node1
6ssh-copy-id ceph-node2
7ssh-copy-id ceph-node3

1# 设置时区
2timedatectl set-timezone Asia/Shanghai
3# 配置时间同步
4yum -y install chrony
5systemctl enable --now chronyd

 1cat > /etc/yum.repos.d/ceph.repo <<EOF
 2[ceph-luminous-noarch]
 3name = ceph-luminous-noarch
 4baseurl = https://mirrors.tuna.tsinghua.edu.cn/ceph/rpm-luminous/el7/noarch/
 5enabled = 1
 6gpgcheck = 1
 7gkgkey = http://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc
 8[ceph-luminous-x64]
 9name = ceph-luminous-x64
10baseurl = https://mirrors.tuna.tsinghua.edu.cn/ceph/rpm-luminous/el7/x86_64/
11enabled = 1
12gpgcheck = 1
13gkgkey = http://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc
14EOF

1export CEPH_DEPLOY_REPO_URL=http://mirrors.tuna.tsinghua.edu.cn/ceph/rpm-luminous/el7
2export CEPH_DEPLOY_GPG_URL=http://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc

1yum -y install epel-release
2# 使用国内epel源
3sed -e 's!^metalink=!#metalink=!g' \
4    -e 's!^#baseurl=!baseurl=!g' \
5    -e 's!//download\.fedoraproject\.org/pub!//mirrors.tuna.tsinghua.edu.cn!g' \
6    -e 's!http://mirrors\.tuna!https://mirrors.tuna!g' \
7    -i /etc/yum.repos.d/epel.repo /etc/yum.repos.d/epel-testing.repo
8yum -y install ceph-deploy

1mkdir my-cluster
2cd my-cluster

1ceph-deploy new ceph-node1 ceph-node2 ceph-node3

1vim ceph.conf
2...
3public_network = 10.10.10.0/24
4cluster_network = 10.10.10.0/24

1ceph-deploy install --release luminous ceph-node1 ceph-node2 ceph-node3

1ceph --version
2ceph version 12.2.13 (584a20eb0237c657dc0567da126be145106aa47e) luminous (stable)

1ceph-deploy mon create-initial

1ceph-deploy admin ceph-node1 ceph-node2 ceph-node3

1ceph-deploy osd create ceph-node1 --data /dev/vdb
2ceph-deploy osd create ceph-node2 --data /dev/vdb
3ceph-deploy osd create ceph-node3 --data /dev/vdb

1ceph osd tree
2ID CLASS WEIGHT  TYPE NAME           STATUS REWEIGHT PRI-AFF
3-1       0.58589 root default
4-3       0.19530     host ceph-node1
5 0   hdd 0.19530         osd.0           up  1.00000 1.00000
6-5       0.19530     host ceph-node2
7 1   hdd 0.19530         osd.1           up  1.00000 1.00000
8-7       0.19530     host ceph-node3
9 2   hdd 0.19530         osd.2           up  1.00000 1.00000

1chmod +r /etc/ceph/ceph.client.admin.keyring

1ceph-deploy mgr create ceph-node1 ceph-node2 ceph-node3

1ceph health
2HEALTH_OK

1rados mkpool rbd
2successfully created pool rbd

1rbd create rbd1 --size 1024

 1rbd list
 2rbd1
 3
 4# 查看rbd细节
 5rbd --image rbd1 info
 6rbd image 'rbd1':
 7	size 1GiB in 256 objects
 8	order 22 (4MiB objects)
 9	block_name_prefix: rbd_data.103d6b8b4567
10	format: 2
11	features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
12	flags:
13	create_timestamp: Sat Jun 20 21:17:50 2020

1rbd feature disable rbd1 object-map fast-diff deep-flatten
2rbd map rbd/rbd1
3/dev/rbd0
4
5# 格式化磁盘
6mkfs.xfs /dev/rbd0
7# 挂载
8mount /dev/rbd0 /opt

1rbd rm rbd1
2Removing image: 100% complete...done.

1rados rmpool rbd rbd --yes-i-really-really-mean-it
2Check your monitor configuration - `mon allow pool delete` is set to false by default, change it to true to allow deletion of pools
3# 修改ceph.conf
4vim /etc/ceph/ceph.conf
5...
6mon_allow_pool_delete = true
7
8# 重启ceph-mon.target
9systemctl restart ceph-mon.target

1rados rmpool rbd rbd --yes-i-really-really-mean-it
2successfully deleted pool rbd

1ceph-deploy rgw create ceph-node1 ceph-node2 ceph-node3

 1radosgw-admin user create --uid=admin --display-name=admin --email=admin@example.com
 2{
 3    "user_id": "admin",
 4    "display_name": "admin",
 5    "email": "admin@example.com",
 6    "suspended": 0,
 7    "max_buckets": 1000,
 8    "auid": 0,
 9    "subusers": [],
10    "keys": [
11        {
12            "user": "admin",
13            "access_key": "837H72BJ7KJ4ZO7Q7PJL",
14            "secret_key": "GYgDMcqxFI68A5K10sWlA2GF9cknohFPqUb6499b"
15        }
16    ],
17    "swift_keys": [],
18    "caps": [],
19    "op_mask": "read, write, delete",
20    "default_placement": "",
21    "placement_tags": [],
22    "bucket_quota": {
23        "enabled": false,
24        "check_on_raw": false,
25        "max_size": -1,
26        "max_size_kb": 0,
27        "max_objects": -1
28    },
29    "user_quota": {
30        "enabled": false,
31        "check_on_raw": false,
32        "max_size": -1,
33        "max_size_kb": 0,
34        "max_objects": -1
35    },
36    "temp_url_keys": [],
37    "type": "rgw"
38}

1radosgw-admin user rm --uid=admin

1yum -y install s3cmd

 1s3cmd --configure \
 2        --access_key=837H72BJ7KJ4ZO7Q7PJL \
 3        --secret_key=GYgDMcqxFI68A5K10sWlA2GF9cknohFPqUb6499b \
 4        --host=10.10.10.47:7480 \
 5        --host-bucket=test-bucket \
 6        --no-ssl
 7
 8Enter new values or accept defaults in brackets with Enter.
 9Refer to user manual for detailed description of all options.
10
11Access key and Secret key are your identifiers for Amazon S3. Leave them empty for using the env variables.
12Access Key [837H72BJ7KJ4ZO7Q7PJL]:
13Secret Key [GYgDMcqxFI68A5K10sWlA2GF9cknohFPqUb6499b]:
14Default Region [US]:
15
16Use "s3.amazonaws.com" for S3 Endpoint and not modify it to the target Amazon S3.
17S3 Endpoint [10.10.10.47:7480]:
18
19Use "%(bucket)s.s3.amazonaws.com" to the target Amazon S3. "%(bucket)s" and "%(location)s" vars can be used
20if the target S3 system supports dns based buckets.
21DNS-style bucket+hostname:port template for accessing a bucket [test-bucket]:
22
23Encryption password is used to protect your files from reading
24by unauthorized persons while in transfer to S3
25Encryption password:
26Path to GPG program [/usr/bin/gpg]:
27
28When using secure HTTPS protocol all communication with Amazon S3
29servers is protected from 3rd party eavesdropping. This method is
30slower than plain HTTP, and can only be proxied with Python 2.7 or newer
31Use HTTPS protocol [No]:
32
33On some networks all internet access must go through a HTTP proxy.
34Try setting it here if you can't connect to S3 directly
35HTTP Proxy server name:
36
37New settings:
38  Access Key: 837H72BJ7KJ4ZO7Q7PJL
39  Secret Key: GYgDMcqxFI68A5K10sWlA2GF9cknohFPqUb6499b
40  Default Region: US
41  S3 Endpoint: 10.10.10.47:7480
42  DNS-style bucket+hostname:port template for accessing a bucket: test-bucket
43  Encryption password:
44  Path to GPG program: /usr/bin/gpg
45  Use HTTPS protocol: False
46  HTTP Proxy server name:
47  HTTP Proxy server port: 0
48
49Test access with supplied credentials? [Y/n] y
50Please wait, attempting to list all buckets...
51Success. Your access key and secret key worked fine :-)
52
53Now verifying that encryption works...
54Not configured. Never mind.
55
56Save settings? [y/N] y
57Configuration saved to '/root/.s3cfg'

1s3cmd mb s3://test-bucket
2Bucket 's3://test-bucket/' created

1s3cmd ls
22020-06-20 14:54  s3://test-bucket

1s3cmd put ceph.conf s3://test-bucket
2upload: 'ceph.conf' -> 's3://test-bucket/ceph.conf'  [1 of 1]
3 340 of 340   100% in    2s   163.23 B/s  done

1s3cmd ls s3://test-bucket
22020-06-20 14:55          340  s3://test-bucket/ceph.conf

1# 开启mgr功能
2ceph mgr module enable dashboard
3
4# 生成并安装自签名的证书
5ceph dashboard create-self-signed-cert  
6
7# 创建一个dashboard登录用户名密码
8ceph dashboard ac-user-create guest 1q2w3e4r administrator 

1# 指定集群dashboard的访问端口
2ceph config-key set mgr/dashboard/server_port 7000
3
4# 指定集群 dashboard的访问IP
5ceph config-key set mgr/dashboard/server_addr $IP 

1# 创建rgw用户
2radosgw-admin user info --uid=admin
3
4# 提供Dashboard证书
5ceph dashboard set-rgw-api-access-key $access_key
6ceph dashboard set-rgw-api-secret-key $secret_key
7
8# 配置rgw主机名和端口
9ceph dashboard set-rgw-api-host 192.168.0.251