[root@localhost ~]# cd keepalived–1.2。2
[root@localhost keepalived–1.2。2]# 。/configure —prefix=/
Keepalived configuration
————————
Keepalived version       : 1.2。2
Compiler                 : gcc
Compiler flags           : –g –O2 –DETHERTYPE_IPV6=0x86dd
Extra Lib                : –lpopt –lssl –lcrypto
Use IPVS Framework       : No
IPVS sync daemon support : No
Use VRRP Framework       : Yes
Use Debug flags          : No

這里只需要啟用VRRP就可以

[root@localhost keepalived-1.2.2]# make && make install

安裝后，配置為隨機(jī)啟動(dòng)服務(wù)

chmod +x /etc/rc.d/init.d/keepalived

chkconfig –add keepalived

chkconfig –level 21 keepalived on

使用的配置文件是/etc/keepalived/keepalived.conf。

主控服務(wù)器10.1.1.105上的配置文件是，每?jī)擅腌娛褂媚_本檢測(cè)mfsmaster運(yùn)行情況，發(fā)現(xiàn)運(yùn)行失敗，就停止keepaled服務(wù)

! Configuration File for keepalived
global_defs {
router_id LVS_STTD
}
vrrp_script check_run {
script "/Data/apps/mfs/keepalived_check_mfsmaster.sh"
interval 2
}
vrrp_sync_group VG1 {
group {
VI_1
}
}
vrrp_instance VI_1 {
state MASTER
interface eth1
virtual_router_id 88
priority 100
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
track_script {
check_run
}
virtual_ipaddress {
10.1。1.103
}
}

/Data/apps/mfs/keepalived_check_mfsmaster.sh腳本，如mfsmaster未運(yùn)行，則停止keepalived服務(wù)
#!/bin/sh
CHECK_TIME=2
mfspath="/Data/apps/mfs/sbin/mfsmaster"
function check_mfsmaster () {
ps –ef | grep mfsmaster | grep "/Data/apps/mfs/sbin/mfsmaster" | grep –v "grep"
if [ $? = 0 ] ;then
MFS_OK=1
else
MFS_OK=0
fi
return $MFS_OK
}
while [ $CHECK_TIME –ne 0 ]
do
let "CHECK_TIME -= 1"
check_mfsmaster
if [ $MFS_OK = 1 ] ; then
CHECK_TIME=0
exit 0
fi

if [ $MFS_OK –eq 0 ] &&  [ $CHECK_TIME –eq 0 ] ;then
/etc/init.d/keepalived stop
exit 1
fi
done

元數(shù)據(jù)日志服務(wù)器上keepalved配置

! Configuration File for keepalived
global_defs {
router_id LVS_STTD
}

vrrp_sync_group VG1 {
group {
VI_1
}
notify_master "/Data/apps/mfs/keepalived_notify.sh master"
notify_backup "/Data/apps/mfs/keepalived_notify.sh backup"
}
vrrp_instance VI_1 {
state BACKUP
interface eth1
virtual_router_id 88
priority 80
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}

virtual_ipaddress {
10.1。1.103
}
}

/Data/apps/mfs/keepalived_notify.sh腳本
#!/bin/bash

MFS_HOME=/Data/apps/mfs
MFSMARSTER=${MFS_HOME} /sbin/mfsmaster
MFSMETARESTORE=${MFS_HOME} /sbin/mfsmetarestore
MFS_DATA_PATH=${MFS_HOME} /var/mfs

function backup2master(){
$MFSMETARESTORE –m ${MFS_DATA_PATH} /metadata.mfs.back –o ${MFS_DATA_PATH} /metadata.mfs $MFS_DATA_PATH/changelog_ml*.mfs
$MFSMARSTER start
}

function master2backup(){
$MFSMARSTER stop
/Data/apps/mfs/sbin/mfsmetalogger start
}

function ERROR(){
echo "USAGE: keepalived_notify.sh master|backup "
}

case $1 in
master)
backup2master
;;
backup)
master2backup
;;
*)
ERROR
;;
esac

五、故障切換

在10.1.1.105上停止mfsmaster服務(wù)，查看日志

[root@localhost init.d]# tail –f /var/log/messages
May  2 12:18:40 localhost snmpd[28923]: Connection from UDP: [211.157。110.180]:64027->[114.113。149.105]
May  2 12:18:40 localhost snmpd[28923]: Connection from UDP: [211.157。110.180]:64027->[114.113。149.105]
May  2 12:18:40 localhost snmpd[28923]: Connection from UDP: [211.157。110.180]:19812->[114.113。149.105]
May  2 12:18:40 localhost snmpd[28923]: Connection from UDP: [211.157。110.180]:19812->[114.113。149.105]
May  2 12:18:41 localhost snmpd[28923]: Connection from UDP: [211.157。110.180]:9005->[114.113。149.105]
May  2 12:18:41 localhost snmpd[28923]: Connection from UDP: [211.157。110.180]:9005->[114.113。149.105]
May  2 12:18:41 localhost snmpd[28923]: Connection from UDP: [211.157。110.180]:4508->[114.113。149.105]
May  2 12:18:41 localhost snmpd[28923]: Connection from UDP: [211.157。110.180]:36607->[114.113。149.105]
May  2 12:18:41 localhost snmpd[28923]: Connection from UDP: [211.157。110.180]:31566->[114.113。149.105]
May  2 12:18:41 localhost snmpd[28923]: Connection from UDP: [211.157。110.180]:12040->[114.113。149.105]
May  2 12:19:17 localhost mfsmaster[30383]: set gid to 501
May  2 12:19:17 localhost mfsmaster[30383]: set uid to 501
May  2 12:19:17 localhost mfsmaster[28772]: matocu: closing *:9421
May  2 12:19:17 localhost mfsmaster[28772]: matocs: closing *:9420
May  2 12:19:17 localhost mfsmaster[28772]: matoml: closing *:9419
May  2 12:19:20 localhost Keepalived: Terminating on signal
May  2 12:19:20 localhost Keepalived: Stopping Keepalived v1.2.2 (04/23,2012)
May  2 12:19:20 localhost Keepalived_vrrp: Terminating VRRP child process on signal
May  2 12:19:20 localhost Keepalived_vrrp: VRRP_Instance(VI_1) removing protocol VIPs。

查看10.1.1.104的日志

[root@localhost log]# tail –f messages
May  2 12:19:17 localhost mfschunkserver[17620]: connection reset by Master
May  2 12:19:17 localhost mfsmetalogger[6105]: connection was reset by Master
May  2 12:19:20 localhost mfschunkserver[17620]: connecting .。.
May  2 12:19:20 localhost mfsmetalogger[6105]: connecting .。.
May  2 12:19:20 localhost mfschunkserver[17620]: connection failed, error: ECONNREFUSED (Connection refused)
May  2 12:19:20 localhost mfsmetalogger[6105]: connection failed, error: ECONNREFUSED (Connection refused)
May  2 12:19:23 localhost Keepalived_vrrp: VRRP_Instance(VI_1) Transition to MASTER STATE
May  2 12:19:23 localhost Keepalived_vrrp: VRRP_Group(VG1) Syncing instances to MASTER state
May  2 12:19:23 localhost mfsmaster[17690]: set gid to 504
May  2 12:19:23 localhost mfsmaster[17690]: set uid to 504
May  2 12:19:23 localhost mfsmaster[17690]: sessions have been loaded
May  2 12:19:23 localhost mfsmaster[17690]: exports file has been loaded
May  2 12:19:23 localhost mfsmaster[17690]: mfstopology configuration file (/Data/apps/mfs/etc/mfstopology.cfg) not found – network topology not defined
May  2 12:19:23 localhost mfsmaster[17690]: stats file has been loaded
May  2 12:19:23 localhost mfsmaster[17690]: master <-> metaloggers module: listen on *:9419
May  2 12:19:23 localhost mfsmaster[17690]: master <-> chunkservers module: listen on *:9420
May  2 12:19:23 localhost mfsmaster[17690]: main master server module: listen on *:9421
May  2 12:19:23 localhost mfsmaster[17690]: open files limit: 5000
May  2 12:19:23 localhost mfschunkserver[17620]: testing chunk: /Data/testshared/03/chunk_0000000000000003_00000001.mfs
May  2 12:19:24 localhost Keepalived_vrrp: VRRP_Instance(VI_1) Entering MASTER STATE
May  2 12:19:24 localhost Keepalived_vrrp: VRRP_Instance(VI_1) setting protocol VIPs。
May  2 12:19:24 localhost avahi–daemon[7367]: Registering new address record for 10.1。1.103 on eth1.
May  2 12:19:24 localhost Keepalived_vrrp: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth1 for 10.1。1.103
May  2 12:19:24 localhost mfschunkserver[17620]: connecting .。.
May  2 12:19:24 localhost mfsmetalogger[6105]: connecting .。.
May  2 12:19:25 localhost mfsmaster[17690]: chunkserver register begin (packet version: 5) – ip: 10.1。1.116, port: 9422
May  2 12:19:25 localhost mfsmaster[17690]: chunkserver register end (packet version: 5) – ip: 10.1。1.116, port: 9422, usedspace: 2801012736 (2.61 GiB)， totalspace: 169845575680 (158.18 GiB)
May  2 12:19:27 localhost mfschunkserver[17620]: connected to Master
May  2 12:19:27 localhost mfsmetalogger[6105]: connected to Master
May  2 12:19:27 localhost mfsmaster[17690]: chunkserver register begin (packet version: 5) – ip: 10.1。1.104, port: 9422
May  2 12:19:27 localhost mfsmaster[17690]: chunkserver register end (packet version: 5) – ip: 10.1。1.104, port: 9422, usedspace: 1069522944 (1.00 GiB)， totalspace: 275084394496 (256.19 GiB)
May  2 12:19:28 localhost mfsmetalogger[6105]: metadata downloaded 1490B/0.000284s (5.246 MB/s)
May  2 12:19:28 localhost mfsmetalogger[6105]: changelog_0 downloaded 0B/0.000001s (0.000 MB/s)
May  2 12:19:28 localhost mfsmetalogger[6105]: changelog_1 downloaded 0B/0.000001s (0.000 MB/s)
May  2 12:19:28 localhost mfsmetalogger[6105]: sessions downloaded 205B/0.000096s (2.135 MB/s)
May  2 12:19:29 localhost Keepalived_vrrp: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth1 for 10.1。1.103
May  2 12:19:33 localhost mfschunkserver[17620]: testing chunk: /Data/testshared/0E/chunk_000000000000000E_00000001.mfs
May  2 12:19:43 localhost mfschunkserver[17620]: testing chunk: /Data/testshared/0F/chunk_000000000000000F_00000001.mfs
May  2 12:19:53 localhost mfschunkserver[17620]: testing chunk: /Data/testshared/06/chunk_0000000000000006_00000001.mfs

已經(jīng)自動(dòng)切換了

六、簡(jiǎn)單性能測(cè)試

小文件

[root@localhost f1]# dd if=/dev/zero of=1.img bs=100K count=5000

5000+0 records in

5000+0 records out

512000000 bytes (512 MB) copied, 6.26102 s, 81.8 MB/s

大文件

[root@localhost f1]# dd if=/dev/zero of=1.img bs=1M count=5000

5000+0 records in

5000+0 records out

5242880000 bytes (5.2 GB) copied, 61.5205 s, 85.2 MB/s

[root@localhost f1]# dd if=/dev/zero of=1.img bs=50K count=5000

dd if=/dev/zero of=1.img bs=10K count=5000

5000+0 records in

5000+0 records out

256000000 bytes (256 MB) copied, 3.16866 s, 80.8 MB/s

[root@localhost f1]# dd if=/dev/zero of=1.img bs=10K count=5000

5000+0 records in

5000+0 records out

51200000 bytes (51 MB) copied, 0.582408 s, 87.9 MB/s

創(chuàng)建1000X1000個(gè)小文件

[root@localhost test]# time ./1000.sh

real 177m8.487s

user 6m36.276s

sys 32m4.413s

本機(jī)測(cè)試

小文件

[root@hadoop03 test]# dd if=/dev/zero of=1.img bs=100K count=5000

5000+0 records in

5000+0 records out

512000000 bytes (512 MB) copied, 0.871519 s, 587 MB/s

大文件

[root@hadoop03 test]# dd if=/dev/zero of=1.img bs=1M count=5000

5000+0 records in

5000+0 records out

5242880000 bytes (5.2 GB) copied, 23.7836 s, 220 MB/s

[root@hadoop03 test]# dd if=/dev/zero of=1.img bs=50K count=5000

5000+0 records in

5000+0 records out

256000000 bytes (256 MB) copied, 2.0681 s, 124 MB/s

1000*1000個(gè)小文件

[root@hadoop03 test]# time ./1000.sh

real 32m1.278s

user 5m19.947s

sys 28m54.985s

1000.sh腳本內(nèi)容

#!/bin/bash

for ((i=0;i<1000;i++))

do

mkdir ${i}

cd ${i}

for ((j=0;j<1000;j++))

do

cp /Data/webapps/img.muzhiwan.com/mfs/test/1.img ${j}

done

cd 。.

done

七、注意

注意/etc/hosts里面mfsmaster指向虛擬ip，否則切換到從服務(wù)器時(shí)候報(bào)錯(cuò)

Apr 24 16:19:15 localhost mfschunkserver[5833]: connecting .。.
Apr 24 16:19:15 localhost mfsmetalogger[5829]: connecting .。.
Apr 24 16:19:15 localhost mfschunkserver[5833]: connection failed, error: ECONNREFUSED (Connection refused)
Apr 24 16:19:15 localhost mfsmetalogger[5829]: connection failed, error: ECONNREFUSED (Connection refused)

對(duì)文件夾goal的設(shè)置，會(huì)影響新增加的文件，但是不會(huì)影響已有的文件

可以使用/Data/apps/mfs/sbin/mfscgiserv啟動(dòng)web gui，監(jiān)控MooseFS運(yùn)行情況

也可以使用nagios監(jiān)控MooseFS運(yùn)行情況

八、參考資料

http://www.moosefs.org/reference-guide.html  官方手冊(cè)

http://bbs.chinaunix.net/thread-1644309-1-1.html  shinelian總結(jié)的mfs權(quán)威指南

http://sery.blog.51cto.com/10037/263515  田逸的分布式文件系統(tǒng)MFS(moosefs)實(shí)現(xiàn)存儲(chǔ)共享(第二版)

http://blog.csdn.net/liuyunfengheda/article/details/5260278  流云隨風(fēng)的MFS總結(jié)

http://blog.csdn.net/pc620/article/details/6327956  常見(jiàn)問(wèn)題

分享到

zhaohang

相關(guān)推薦