CentOS8下FastdDFS安装

Fastdfs简介

FastDFS 是一个开源的高性能分布式文件系统（DFS）。 它的主要功能包括：文件存储，文件同步和文件访问，以及高容量和负载平衡。主要解决了海量数据存储问题，特别适合以中小文件（建议范围：4KB < file_size <500MB）为载体的在线服务。

FastDFS 系统有三个角色：跟踪服务器(Tracker Server)、存储服务器(Storage Server)和客户端(Client)。

  • Tracker Server：跟踪服务器，主要做调度工作，起到均衡的作用；负责管理所有的 storage server和 group，每个 storage 在启动后会连接 Tracker，告知自己所属 group 等信息，并保持周期性心跳。

  • Storage Server：存储服务器，主要提供容量和备份服务；以 group 为单位，每个 group 内可以有多台 storage server，数据互为备份。

  • Client：客户端，上传下载数据的服务器，也就是我们自己的项目所部署在的服务器。

安装Fastdfs

安装依赖包

1sudo dnf -y install wget \
2     gcc \
3     make \
4     perl-interpreter \
5     pcre-devel \
6     zlib-devel \
7     openssl-devel

安装libfastcommon

1wget https://github.com/happyfish100/libfastcommon/archive/V1.0.43.tar.gz
2tar xf V1.0.43.tar.gz
3cd libfastcommon-1.0.43
4./make.sh && sudo make install

安装Fastdfs

1wget https://github.com/happyfish100/fastdfs/archive/V6.06.tar.gz
2tar xf V6.06.tar.gz
3cd fastdfs-6.06
4export LANG=en_US
5export LANGUAGE=en_US.UTF-8
6export LC_ALL=ex_US.UTF-8
7./make.sh
8sudo ./make.sh install

生成配置文件

1sudo ./setup.sh /etc/fdfs
2ls /etc/fdfs/
3client.conf         http.conf   storage.conf      storage_ids.conf.sample  tracker.conf.sample
4client.conf.sample  mime.types  storage_ids.conf  tracker.conf

准备存储目录

1sudo mkdir -pv /data/fastdfs/{tracker,client,storage,storage_data}
2mkdir: created directory '/data'
3mkdir: created directory '/data/fastdfs'
4mkdir: created directory '/data/fastdfs/tracker'
5mkdir: created directory '/data/fastdfs/client'
6mkdir: created directory '/data/fastdfs/storage'
7mkdir: created directory '/data/fastdfs/storage_data'

修改配置文件

tracker.conf

1sudo vim /etc/fdfs/tracker.conf
2base_path = /data/fastdfs/tracker

storage.conf

1sudo /etc/fdfs/storage.conf
2bind_addr = 172.16.10.10
3base_path = /data/fastdfs/storage
4store_path0 = = /data/fastdfs/storage_data
5tracker_server = 172.16.10.10:22122

client.conf

1sudo vim /etc/fdfs/client.conf
2base_path = /data/fastdfs/client
3tracker_server = 172.16.10.10:22122

运行Fastdfs服务

运行tracker服务

1sudo /usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf restart

运行storage服务

1sudo /usr/bin/fdfs_storaged /etc/fdfs/storage.conf restart

作为服务启动

1sudo service fdfs_trackerd restart
2sudo service fdfs_storaged restart

运行监控程序

1/usr/bin/fdfs_monitor /etc/fdfs/client.conf

运行测试程序

1/usr/bin/fdfs_test <client_conf_filename> <operation>
2/usr/bin/fdfs_test1 <client_conf_filename> <operation>

测试上传文件

 1/usr/bin/fdfs_test /etc/fdfs/client.conf upload /usr/include/stdlib.h
 2This is FastDFS client test program v6.06
 3
 4Copyright (C) 2008, Happy Fish / YuQing
 5
 6FastDFS may be copied only under the terms of the GNU General
 7Public License V3, which may be found in the FastDFS source kit.
 8Please visit the FastDFS Home Page http://www.fastken.com/
 9for more detail.
10
11[2020-03-16 15:38:27] DEBUG - base_path=/data/fastdfs/client, connect_timeout=5, network_timeout=60, tracker_server_count=1, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0
12
13tracker_query_storage_store_list_without_group:
14	server 1. group_name=, ip_addr=172.16.10.10, port=23000
15
16group_name=group1, ip_addr=172.16.10.10, port=23000
17storage_upload_by_filename
18group_name=group1, remote_filename=M00/00/00/rBAKCl5vLPOAYbhqAACLRdtk8t842096.h
19source ip address: 172.16.10.10
20file timestamp=2020-03-16 15:38:27
21file size=35653
22file crc32=3680826079
23example file url: http://172.16.10.10/group1/M00/00/00/rBAKCl5vLPOAYbhqAACLRdtk8t842096.h
24storage_upload_slave_by_filename
25group_name=group1, remote_filename=M00/00/00/rBAKCl5vLPOAYbhqAACLRdtk8t842096_big.h
26source ip address: 172.16.10.10
27file timestamp=2020-03-16 15:38:27
28file size=35653
29file crc32=3680826079
30example file url: http://172.16.10.10/group1/M00/00/00/rBAKCl5vLPOAYbhqAACLRdtk8t842096_big.h

查看刚上传的文件

1ls -lh /data/fastdfs/storage_data/data/00/00/rBAKCl5vLPOAYbhqAACLRdtk8t842096*
2-rw-r--r-- 1 root root 35K Mar 16 15:38 /data/fastdfs/storage_data/data/00/00/rBAKCl5vLPOAYbhqAACLRdtk8t842096.h
3-rw-r--r-- 1 root root  49 Mar 16 15:38 /data/fastdfs/storage_data/data/00/00/rBAKCl5vLPOAYbhqAACLRdtk8t842096.h-m
4-rw-r--r-- 1 root root 35K Mar 16 15:38 /data/fastdfs/storage_data/data/00/00/rBAKCl5vLPOAYbhqAACLRdtk8t842096_big.h
5-rw-r--r-- 1 root root  49 Mar 16 15:38 /data/fastdfs/storage_data/data/00/00/rBAKCl5vLPOAYbhqAACLRdtk8t842096_big.h-m

与nginx进行整合

安装nginx

nginx服务为fastdfs文件服务提供web浏览服务，需要安装fastdfs-nginx-module模块

下载地址：fastdfs-nginx-module

 1# 下载nginx最新稳定版本
 2wget http://nginx.org/download/nginx-1.16.1.tar.gz
 3tar xf nginx-1.16.1.tar.gz
 4
 5# 下载fastdfs-nginx-module模块
 6wget https://github.com/happyfish100/fastdfs-nginx-module/archive/V1.22.tar.gz
 7tar xf V1.22.tar.gz
 8
 9# 修改头文件路径
10sed -i  '/CORE_INCS=/s@local/@@' fastdfs-nginx-module-1.22/src/config
11
12# 拷贝模块配置文件并修改配置的路径
13sudo cp fastdfs-nginx-module-1.22/src/mod_fastdfs.conf /etc/fdfs
14sudo sed -i '/^base_path/s@/tmp@/data/fastdfs/storage@' /etc/fdfs/mod_fastdfs.conf
15sudo sed -i '/^tracker_server/s@tracker:22122@172.16.10.10:22122@' /etc/fdfs/mod_fastdfs.conf
16sudo sed -i '/^url_have_group_name/s@false@true@' /etc/fdfs/mod_fastdfs.conf
17sudo sed -i '/^store_path0/s@/home/yuqing/fastdfs@/data/fastdfs/storage_data@' /etc/fdfs/mod_fastdfs.conf
18
19# 编译nginx
20cd nginx-1.16.1
21
22# 需要依赖pcre-devel zlib-devel openssl-devel
23./configure --add-module=../fastdfs-nginx-module-1.22/src \
24  --with-http_ssl_module
25make -j4 && sudo make install

配置nginx

 1user  root;
 2worker_processes  4;
 3
 4#error_log  logs/error.log;
 5#error_log  logs/error.log  notice;
 6#error_log  logs/error.log  info;
 7
 8#pid        logs/nginx.pid;
 9
10
11events {
12    worker_connections  1024;
13}
14
15
16http {
17    include       mime.types;
18    default_type  application/octet-stream;
19
20    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
21    #                  '$status $body_bytes_sent "$http_referer" '
22    #                  '"$http_user_agent" "$http_x_forwarded_for"';
23
24    #access_log  logs/access.log  main;
25
26    sendfile        on;
27    #tcp_nopush     on;
28
29    #keepalive_timeout  0;
30    keepalive_timeout  65;
31
32    #gzip  on;
33
34    server {
35        listen       80;
36        server_name  localhost;
37
38        #charset koi8-r;
39
40        #access_log  logs/host.access.log  main;
41
42        location / {
43            root   html;
44            index  index.html index.htm;
45        }
46
47        #location /group1/M00/ {
48        location ~ /group([0-9])/M00 {
49            alias   /data/fastdfs/storage_data/data;
50            ngx_fastdfs_module;
51        }
52
53        #error_page  404              /404.html;
54        error_page   500 502 503 504  /50x.html;
55        location = /50x.html {
56            root   html;
57        }
58    }
59}

运行nginx

1cd /usr/local/nginx && sudo sbin/nginx -t && sudo sbin/nginx

测试上传和下载

 1wget http://172.16.10.10/group1/M00/00/00/rBAKCl5vLPOAYbhqAACLRdtk8t842096_big.h
 2--2020-03-16 16:59:50--  http://172.16.10.10/group1/M00/00/00/rBAKCl5vLPOAYbhqAACLRdtk8t842096_big.h
 3Connecting to 172.16.10.10:80... connected.
 4HTTP request sent, awaiting response... 200 OK
 5Length: 35653 (35K) [application/octet-stream]
 6Saving to: 'rBAKCl5vLPOAYbhqAACLRdtk8t842096_big.h'
 7
 8rBAKCl5vLPOAYbhqAACLRdtk8t8 100%[===========================================>]  34.82K  --.-KB/s    in 0.001s
 9
102020-03-16 16:59:50 (47.3 MB/s) - 'rBAKCl5vLPOAYbhqAACLRdtk8t842096_big.h' saved [35653/35653]

1fdfs_test /etc/fdfs/client.conf upload beego_flow.png
2...
3example file url: http://172.16.10.10/group1/M00/00/00/rBAKCl5vQHKABVdeAACkAPmoVhM645_big.png

打开浏览器查看效果

fastdfs

由于fastdfs是分布式的文件服务器，可以很方便的扩展，后面启动多个的时候吧tracker_server的地址添加上去就行。

后续会介绍go使用第三方库对fastdfs进行管理操作。