## 安装glusterfs(所有节点)
yum -y install centos-release-gluster
yum -y install glusterfs-server
## 启动并加入开机自启
systemctl start glusterd
systemctl enable glusterd
## 每个节点进行时间同步
yum install ntpdate
ntpdate cn.ntp.org.cn
## 查看磁盘和分区信息
[root@backup ~]# fdisk -l
Disk /dev/sda: 21.5 GB, 21474836480 bytes, 41943040 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0x000c155d
Device Boot Start End Blocks Id System
/dev/sda1 * 2048 1026047 512000 83 Linux
/dev/sda2 1026048 3123199 1048576 82 Linux swap / Solaris
/dev/sda3 3123200 41943039 19409920 83 Linux
## 在所有gfs-node节点新增一块硬盘(docker01 docker02 backup)
- 关机
- 添加磁盘
## 查看磁盘和分区信息
[root@docker01 ~]# fdisk -l
Disk /dev/sda: 21.5 GB, 21474836480 bytes, 41943040 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0x000c155d
Device Boot Start End Blocks Id System
/dev/sda1 * 2048 1026047 512000 83 Linux
/dev/sda2 1026048 3123199 1048576 82 Linux swap / Solaris
/dev/sda3 3123200 41943039 19409920 83 Linux
Disk /dev/sdb: 21.5 GB, 21474836480 bytes, 41943040 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
## 格式化硬盘(docker01 docker02 backup)
fdisk /dev/sdb
Welcome to fdisk (util-linux 2.23.2).
Changes will remain in memory only, until you decide to write them.
Be careful before using the write command.
Device does not contain a recognized partition table
Building a new DOS disklabel with disk identifier 0x6a5b5325.
Command (m for help): n
Partition type:
p primary (0 primary, 0 extended, 4 free)
e extended
Select (default p): p
Partition number (1-4, default 1):
First sector (2048-41943039, default 2048):
Using default value 2048
Last sector, +sectors or +size{K,M,G} (2048-41943039, default 41943039):
Using default value 41943039
Partition 1 of type Linux and of size 20 GiB is set
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
## 格式化分区(docker01 docker02 backup)
mkfs.ext4 /dev/sdb1
## 执行挂载命令并查看挂载情况(永久挂载)(docker01 docker02 backup)
mkdir /sdb1
mount /dev/sdb1 /sdb1
[root@backup ~]# df -Th
Filesystem Type Size Used Avail Use% Mounted on
devtmpfs devtmpfs 476M 0 476M 0% /dev
tmpfs tmpfs 487M 0 487M 0% /dev/shm
tmpfs tmpfs 487M 7.7M 479M 2% /run
tmpfs tmpfs 487M 0 487M 0% /sys/fs/cgroup
/dev/sda3 xfs 19G 1.7G 17G 10% /
/dev/sda1 xfs 497M 125M 373M 26% /boot
tmpfs tmpfs 98M 0 98M 0% /run/user/0
/dev/sdb1 ext4 20G 45M 19G 1% /sdb1
## 启动glusterfsd并添加hosts解析
[root@docker01 ~]# systemctl start glusterfsd
[root@docker02 ~]# systemctl start glusterfsd
[root@backup ~]# systemctl start glusterfsd
[root@harbor ~]# echo "10.0.0.101 gfs-node1" >> /etc/hosts
[root@harbor ~]# echo "10.0.0.102 gfs-node2" >> /etc/hosts
[root@harbor ~]# echo "10.0.0.41 gfs-node3" >> /etc/hosts
[root@backup ~]# echo "10.0.0.101 gfs-node1" >> /etc/hosts
[root@backup ~]# echo "10.0.0.102 gfs-node2" >> /etc/hosts
[root@backup ~]# echo "10.0.0.41 gfs-node3" >> /etc/hosts
[root@docker01 ~]# echo "10.0.0.101 gfs-node1" >> /etc/hosts
[root@docker01 ~]# echo "10.0.0.102 gfs-node2" >> /etc/hosts
[root@docker01 ~]# echo "10.0.0.41 gfs-node3" >> /etc/hosts
[root@docker02 ~]# echo "10.0.0.101 gfs-node1" >> /etc/hosts
[root@docker02 ~]# echo "10.0.0.102 gfs-node2" >> /etc/hosts
[root@docker02 ~]# echo "10.0.0.41 gfs-node3" >> /etc/hosts
## 在任一节点上操作添加另外两个节点(在gfs-node1上操作)
[root@docker01 ~]# gluster peer probe gfs-node1
peer probe: Probe on localhost not needed
[root@docker01 ~]# gluster peer probe gfs-node2
peer probe: success
[root@docker01 ~]# gluster peer probe gfs-node3
peer probe: success
[root@docker01 ~]# netstat -antp|grep gluster
tcp 0 0 0.0.0.0:24007 0.0.0.0:* LISTEN 853/glusterd
tcp 0 0 10.0.0.101:49151 10.0.0.102:24007 ESTABLISHED 853/glusterd
tcp 0 0 10.0.0.101:24007 10.0.0.41:49151 ESTABLISHED 853/glusterd
tcp 0 0 10.0.0.101:49150 10.0.0.41:24007 ESTABLISHED 853/glusterd
tcp 0 0 10.0.0.101:24007 10.0.0.102:49151 ESTABLISHED 853/glusterd
## 检查gluster集群状态,在其中任意一个节点中可以看到另外两个节点信息
[root@docker01 ~]# gluster peer status
Number of Peers: 2
Hostname: gfs-node2
Uuid: 34c6133f-a4f4-4c20-b154-0bed6acd4acb
State: Peer in Cluster (Connected)
Hostname: gfs-node3
Uuid: 49c04608-fcee-42ef-b399-819abb5cd657
State: Peer in Cluster (Connected)
[root@docker02 ~]# gluster peer status
Number of Peers: 2
Hostname: gfs-node1
Uuid: 7097bd00-c28d-4b2b-acae-4611c4541c12
State: Peer in Cluster (Connected)
Hostname: gfs-node3
Uuid: 49c04608-fcee-42ef-b399-819abb5cd657
State: Peer in Cluster (Connected)
[root@backup ~]# gluster peer status
Number of Peers: 2
Hostname: gfs-node1
Uuid: 7097bd00-c28d-4b2b-acae-4611c4541c12
State: Peer in Cluster (Connected)
Hostname: gfs-node2
Uuid: 34c6133f-a4f4-4c20-b154-0bed6acd4acb
State: Peer in Cluster (Connected)
创建分布式卷
## 创建分布式卷
[root@docker01 ~]# gluster volume create distribute-volume gfs-node1:/sdb1 gfs-node2:/sdb1 gfs-node3:/sdb1 force
volume create: distribute-volume: success: please start the volume to access data
[root@docker01 ~]# gluster volume list
distribute-volume
[root@docker01 ~]# gluster volume info distribute-volume
Volume Name: distribute-volume
Type: Distribute #可以看到这个类型为分布式卷
Volume ID: 54e6c08c-1cd6-4599-a9f0-be86d37ed80c
Status: Created
Snapshot Count: 0
Number of Bricks: 3
Transport-type: tcp
Bricks: #共有三个brick存储服务器
Brick1: gfs-node1:/sdb1
Brick2: gfs-node2:/sdb1
Brick3: gfs-node3:/sdb1
Options Reconfigured:
storage.fips-mode-rchecksum: on
transport.address-family: inet
nfs.disable: on
## 创建之后的卷需要要启用后才能使用,启动分布式卷
[root@docker01 ~]# gluster volume start distribute-volume
volume start: distribute-volume: success
[root@docker01 ~]# gluster volume status
Status of volume: distribute-volume
Gluster process TCP Port RDMA Port Online Pid
------------------------------------------------------------------------------
Brick gfs-node1:/sdb1 49152 0 Y 8314
Brick gfs-node2:/sdb1 49152 0 Y 3162
Brick gfs-node3:/sdb1 49152 0 Y 2550
Task Status of Volume distribute-volume
------------------------------------------------------------------------------
There are no active volume tasks
## 安装客户端
[root@harbor ~]# yum -y install glusterfs glusterfs-fuse
## 客户端执行目录挂载
[root@harbor ~]# mkdir -p /data/distribute-volume
[root@harbor ~]# mount.glusterfs gfs-node1:distribute-volume /data/distribute-volume
[root@harbor ~]# df -Th
gfs-node1:distribute-volume fuse.glusterfs 59G 737M 56G 2% /data/distribute-volume
## 在/data目录中产生一些测试文件
[root@harbor ~]# cd /data/
[root@harbor data]# dd if=/dev/zero of=test1.log bs=1M count=10
10+0 records in
10+0 records out
10485760 bytes (10 MB) copied, 0.00499322 s, 2.1 GB/s
[root@harbor data]# dd if=/dev/zero of=test2.log bs=1M count=10
10+0 records in
10+0 records out
10485760 bytes (10 MB) copied, 0.00368149 s, 2.8 GB/s
[root@harbor data]# dd if=/dev/zero of=test3.log bs=1M count=10
10+0 records in
10+0 records out
10485760 bytes (10 MB) copied, 0.00541283 s, 1.9 GB/s
## 将所有test文件移动到distribute-volume中,检查存储情况,因为创建的是分布式卷,文件会分别被存放在不同的节点中
[root@harbor data]# mv test1.log distribute-volume/
[root@harbor data]# mv test2.log distribute-volume/
[root@harbor data]# mv test3.log distribute-volume/
[root@docker01 ~]# ll /sdb1/
total 10260
drwx------ 2 root root 16384 Oct 9 18:39 lost+found
-rw-r--r-- 2 root root 10485760 Oct 9 19:01 test3.log
[root@docker02 ~]# ll /sdb1/
total 20504
drwx------ 2 root root 16384 Oct 9 18:39 lost+found
-rw-r--r-- 2 root root 10485760 Oct 9 19:00 test1.log
-rw-r--r-- 2 root root 10485760 Oct 9 19:00 test2.log
[root@backup ~]# ll /sdb1/
total 16
drwx------ 2 root root 16384 Oct 9 18:39 lost+found
创建复制式存储卷
## 创建磁盘(方法同上)
## 格式化
[root@docker01 ~]# fdisk /dev/sdc
Welcome to fdisk (util-linux 2.23.2).
Changes will remain in memory only, until you decide to write them.
Be careful before using the write command.
Device does not contain a recognized partition table
Building a new DOS disklabel with disk identifier 0xdc055445.
Command (m for help): n
Partition type:
p primary (0 primary, 0 extended, 4 free)
e extended
Select (default p): p
Partition number (1-4, default 1):
First sector (2048-41943039, default 2048):
Using default value 2048
Last sector, +sectors or +size{K,M,G} (2048-41943039, default 41943039):
Using default value 41943039
Partition 1 of type Linux and of size 20 GiB is set
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
[root@docker01 ~]# mkfs.ext4 /dev/sdc1
[root@docker02 ~]# fdisk /dev/sdc
Welcome to fdisk (util-linux 2.23.2).
Changes will remain in memory only, until you decide to write them.
Be careful before using the write command.
Device does not contain a recognized partition table
Building a new DOS disklabel with disk identifier 0xdc055445.
Command (m for help): n
Partition type:
p primary (0 primary, 0 extended, 4 free)
e extended
Select (default p): p
Partition number (1-4, default 1):
First sector (2048-41943039, default 2048):
Using default value 2048
Last sector, +sectors or +size{K,M,G} (2048-41943039, default 41943039):
Using default value 41943039
Partition 1 of type Linux and of size 20 GiB is set
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
[root@docker02 ~]# mkfs.ext4 /dev/sdc1
[root@backup ~]# fdisk /dev/sdc
Welcome to fdisk (util-linux 2.23.2).
Changes will remain in memory only, until you decide to write them.
Be careful before using the write command.
Device does not contain a recognized partition table
Building a new DOS disklabel with disk identifier 0x3c6c395b.
Command (m for help): n
Partition type:
p primary (0 primary, 0 extended, 4 free)
e extended
Select (default p): p
Partition number (1-4, default 1):
First sector (2048-41943039, default 2048):
Using default value 2048
Last sector, +sectors or +size{K,M,G} (2048-41943039, default 41943039):
Using default value 41943039
Partition 1 of type Linux and of size 20 GiB is set
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
[root@backup ~]# mkfs.ext4 /dev/sdc1
[root@docker01 ~]# mkdir /sdc1
[root@docker01 ~]# mount -a
[root@docker02 ~]# mkdir /sdc1
[root@docker02 ~]# mount -a
[root@backup ~]# mkdir /sdc1
[root@backup ~]# mount -a
## 永久挂载
[root@docker01 ~]# echo "/dev/sdc1 /sdc1 ext4 defaults 0 0" >> /etc/fstab
[root@docker02 ~]# echo "/dev/sdc1 /sdc1 ext4 defaults 0 0" >> /etc/fstab
[root@backup ~]# echo "/dev/sdc1 /sdc1 ext4 defaults 0 0" >> /etc/fstab
## 创建复制式存储卷
[root@docker01 ~]# gluster volume create replica-volume replica 3 gfs-node1:/sdc1 gfs-node2:/sdc1 gfs-node3:/sdc1 force
volume create: replica-volume: success: please start the volume to access data
[root@docker01 ~]# gluster volume start replica-volume
volume start: replica-volume: success
[root@docker01 ~]# gluster volume status
Status of volume: distribute-volume
Gluster process TCP Port RDMA Port Online Pid
------------------------------------------------------------------------------
Brick gfs-node1:/sdb1 49152 0 Y 8314
Brick gfs-node2:/sdb1 49152 0 Y 3162
Brick gfs-node3:/sdb1 49152 0 Y 2550
Task Status of Volume distribute-volume
------------------------------------------------------------------------------
There are no active volume tasks
Status of volume: replica-volume
Gluster process TCP Port RDMA Port Online Pid
------------------------------------------------------------------------------
Brick gfs-node1:/sdc1 49153 0 Y 13740
Brick gfs-node2:/sdc1 49153 0 Y 4971
Brick gfs-node3:/sdc1 49153 0 Y 4163
Self-heal Daemon on localhost N/A N/A Y 13757
Self-heal Daemon on gfs-node3 N/A N/A Y 4180
Self-heal Daemon on gfs-node2 N/A N/A Y 4988
Task Status of Volume replica-volume
------------------------------------------------------------------------------
There are no active volume tasks
## 客户端挂载测试
[root@harbor data]# mkdir -p /data/replica-volume
[root@harbor data]# echo "gfs-node1:replica-volume /data/replica-volume glusterfs defaults 0 0" >> /etc/fstab
[root@harbor data]# mount -a
[root@harbor data]# cd /data/replica-volume/
[root@harbor replica-volume]# ll
total 0
[root@harbor replica-volume]# dd if=/dev/zero of=test-replica1.log bs=1M count=10
10+0 records in
10+0 records out
10485760 bytes (10 MB) copied, 0.149423 s, 70.2 MB/s
## 前往各个节点进行查看,因为时复制卷,每个节点都有挂载点的内容
[root@docker01 ~]# ll /sdc1/
total 10240
-rw-r--r-- 2 root root 10485760 Oct 9 19:33 test-replica1.log
[root@docker02 ~]# ll /sdc1/
total 10240
-rw-r--r-- 2 root root 10485760 Oct 9 19:33 test-replica1.log
[root@backup ~]# ll /sdc1/
total 10240
-rw-r--r-- 2 root root 10485760 Oct 9 19:33 test-replica1.log
Comments | NOTHING