将公司的gitlab服务器进行备份，并且将备份rsync到异地机房，因为备份文件太大，所以这里做了切割成多个文件，进行多线程传输。
#!/usr/bin/env perl ################ #filename: gitback.pl #version : 1.0 #author : zzq #date : 20160705 #function: The transmission of gitlab backup files to the remote host ######################### use strict; use threads; use Thread::Queue; use File::Rsync; use threads::shared; use File::Find; use File::Basename; use POSIX qw(strftime); use Log::Log4perl qw(:easy); use Mail::Sender; my ($logger,$Msg,$Subject,$filename,$srcFile,$fileName,$filePath,$fileSize,$backFilename); my $fileCount; my $filemsg; my $message; ######### remote host rsync config ################## my $remotehost = 'rsync@10.xx.xx.xx::gitbackup'; ############# back file dir############## my $backFiledir = '/data/gitbackups/'; ######## max threads count############## my $thread_max = 10; ########## current time ############# my $currentTime = strftime("%Y%m%d%H%M", localtime(time)); $Msg = "$currentTime\n"; ########## log file ############## my $logFile = "gitback.log"; my $splitFileQueue = Thread::Queue->new(); my $remotefileQueue = Thread::Queue->new(); ############## define smtp ###################### #Sender my $send_user='monitor@.cc.cn'; #passwd for Sender #my $send_user_pw='123'; #smtp address my $smtp_server='10.168.xx.xx'; #define mail subject $Subject = "gitback-$currentTime"; #address my @address=qw(xxx.cn); ################################################# ################### define log conf ######################### my $log_logger = get_logger(); $log_logger->level($INFO); my $file_appender = Log::Log4perl::Appender->new( "Log::Dispatch::File", filename => "$logFile", mode => "append", ); $log_logger->add_appender($file_appender); my $layout = Log::Log4perl::Layout::PatternLayout->new( "%d %p > %F{1} %M -- %m%n" ); $file_appender->layout($layout); ################################################### main(); sub main { Usage() unless ( scalar(@ARGV) == 0); $log_logger->info("Begin:$currentTime"); $srcFile = backup(); print "srcFile is $srcFile\n"; ($fileName,$filePath) = fileparse($srcFile) if (-f $srcFile); $fileSize=(stat $srcFile)[7]; srcFileMd5($srcFile); splitFile($srcFile); find(\&findSplitFile,$filePath); $fileCount = $splitFileQueue->pending(); $Msg = "$Msg"."FileCount: $fileCount\n"."$filemsg"; splitFileMd5Thread(); rsyncThread(); for my $addr (@address){ sendMail($addr); } $currentTime=strftime("%Y%m%d%H%M", localtime(time)); $log_logger->info("End:$currentTime"); } ###################Access to the backup name for gitlab ################ sub backup { my $result=`gitlab-rake gitlab:backup:create`; my $flag=`echo $?`; if ($flag == 0){ $log_logger->info("gitlab-backup-create : success "); my @result=split(/\n/,$result); for my $cc (@result){ if ($cc =~ /(? \d+_gitlab_backup.tar)/){ $backFilename=$+{'filename'}; $log_logger->info("gitlab-backup-filename : $backFilename"); } } }else{ $log_logger->info("gitlab-backup-create : fail "); } return "$backFiledir$backFilename"; } ############# Search the file after segmentation ################ sub findSplitFile { if ( $_ =~ /$fileName\.\d{3}$/) { $log_logger->info("splitFileQueue enqueue : $_"); $filemsg = "$filemsg"."$_\n"; $splitFileQueue->enqueue($_); $log_logger->info("remotefileQueue enqueue : $_"); $remotefileQueue->enqueue($_); # $log_logger->info("print to logfile : $_"); } } ############## Search the MD5 file after segmentation ############ sub splitFileMd5Thread { while($splitFileQueue->pending()) { my $Queue = $splitFileQueue->dequeue(); $log_logger->info("splitfileQueueMD5 dequeue : $Queue "); threads->create(\&splitFileMd5,$Queue); } foreach my $thread (threads->list(threads::all)){ if ($thread->is_joinable()){ $thread->join(); } } foreach my $thread ( threads->list(threads::all) ) { $thread->join(); } } ################## Search for the source file md5 ################# sub srcFileMd5 { my $file=shift; my $md5_command=`md5sum $file`; my $flag=`echo $?`; if ($flag == 0) { my ($file_md5,$filename) = split(/\s+/,$md5_command); my $log = "srcfile:$file filesize:$fileSize file_md5sum:$file_md5"; $Msg = "$Msg"."srcfile:$file filesize:$fileSize file_md5sum:$file_md5\n"; $log_logger->info($log); }else { my $file_md5 = "system command maybe exec fail, command result code is $md5_command"; my $log = "srcfile:$file filesize:$fileSize file_md5sum:$file_md5"; $Msg = "$Msg"."srcfile:$file filesize:$fileSize file_md5sum:$file_md5\n"; $log_logger->warn($log); } } ####### split File MD5 ############### sub splitFileMd5 { my $file=shift; my $md5_command=`md5sum $filePath$file`; my $flag=`echo $?`; if ($flag == 0) { my ($file_md5,$filename) = split(/\s+/,$md5_command); my $log = "file:$file file_md5sum:$file_md5"; $log_logger->info($log); } } ################# Will be cut by the source file ################ sub splitFile { my $srcFile = shift; my $split_command =`split -b 50M $srcFile -d -a 3 $srcFile.`; $split_command = `echo $? `; if ($split_command == 0 ){ my $splitFileCode = 'success'; $log_logger->info(" srcFile:$srcFile desc:$splitFileCode splitFileResult:$split_command"); }else{ my $splitFileCode = 'fail'; $log_logger->warn(" srcFile:$srcFile desc:$splitFileCode splitFileResult:$split_command"); } } sub rsyncThread { while ( $remotefileQueue->pending() ){ if (scalar threads->list() < $thread_max) { my $readQueue = $remotefileQueue->dequeue(); $log_logger->info("remotefileQueue dequeue : $readQueue "); threads->create(\&rsync,$readQueue); } foreach my $thread (threads->list(threads::all)){ if ($thread->is_joinable()){ $thread->join(); } } } foreach my $thread ( threads->list(threads::all) ) { $thread->join(); } } sub rsync { my $file = shift; my $obj = File::Rsync->new( { archive => 1, compress => 1, checksum => 1, recursive => 1, owner => 1, devices => 1, group => 1, perms => 1, times => 1, verbose => 1, progress => 1, stats => 1, links => 1, 'hard-links' => 1, 'ignore-times' => 1, 'password-file' => './rsync.pass', } ); $obj->exec( { src => "$filePath$file", dest => $remotehost } ); #or warn ($log_logger->warn("rsync: $file to $remotehost fail!" )); my $rval = $obj->realstatus; if ($rval == 0 ) { $log_logger->info("rsync: $file to $remotehost Success!\n"); }else{ my $rsyncError = sprintf ("%s", $obj->err); $message = "$message"."Transfer Failed:\nfile:$file reason:$rsyncError"; $log_logger->info("rsync: $file to $remotehost fail reason:$rsyncError"); } } sub sendMail { my $CONTACTEMAIL = shift; my $sender = new Mail::Sender{ ctype => 'text/plain; charset=utf-8', encoding => 'utf-8', } ; die "Error in mailing : $Mail::Sender::Error\n" unless ref $sender; if ($sender->MailMsg({ smtp => $smtp_server, from => $send_user, to => $CONTACTEMAIL, subject => $Subject, msg => $Msg, # file => "$file", # auth => 'LOGIN', # authid => $send_user, # authpwd => $send_user_pw, charset=>'utf-8' }) < 0) { die $log_logger->error("senermail fail -- $Mail::Sender::Error\n"); } $sender->Close(); } sub Usage { print <<"END"; perl gitback.pl END exit 2; }