OBServer 运行日志压缩、归档、删除脚本

本文分享 OBServer 运行日志压缩、归档、删除的相关脚本代码。

需求说明

日志保留更长时间、并定期删除。

设计思路

• 不与 OBServer 本身的功能冲突。

• 不对 OBServer 的日志清理策略产生影响。

• 日志保留的时间更长。

• 考虑异常场景，同时只有一个进程在跑。

• 压缩对磁盘负载要小。

• 支持手动修改策略。

实现步骤

1. 切换到 admin 用户。

   # su - admin
   

2. 创建脚本存放文件夹 delete_ob_log。

   # mkdir /home/admin/delete_ob_log
   

3. 切换到 delete_ob_log 文件夹。

   # cd /home/admin/delete_ob_log
   

4. 创建脚本。

   # vi compress_and_delete_archive_log_file.sh
   

   #!/bin/bash
   #compress log files before a few minutes, eg. 10*60 is 10 hours
   keep_min=$((10*60))
   # keep days of compressed log files
   compressed_log_keep_day=7
   compressed_audit_log_keep_day=7
   archive_dir=/home/admin/ob_log_archive
   log_dir=/home/admin/oceanbase/log
   scripts_dir=/home/admin/delete_ob_log
   audit_log_dir=/home/admin/oceanbase/audit
   audit_archive_dir=/home/admin/audit_log_archive
   
   
   current_year=`date +%Y`
   free_space=`df -m | grep home | awk '{print $4}'`
   used_space_percent=`df -m | grep home | awk '{print $5}'`
   ip_prefix=`ifconfig  | grep inet | grep -v inet6 | grep -v 127.0.0.1 | grep -v 172.xx.x.x | awk '{print $2}' | awk -F '.' '{print $4}'`
   dir_name=`hostname`"-$ip_prefix"
   
   
   if [ ! -d $archive_dir/$dir_name/`date +%Y%m%d` ] ; then
     echo "archive log dir is not exists, now creating..."
     mkdir -p $archive_dir/$dir_name/`date +%Y%m%d`
   fi
   
   
   if [ ! -d $audit_archive_dir/$dir_name/`date +%Y%m%d` ] ; then
     echo "audit log dir is not exists, now creating..."
     mkdir -p $audit_archive_dir/$dir_name/`date +%Y%m%d`
   fi
   
   function archive_compress_log_file(){
     index=0
     begin_id=0
   
     if [ ! -f "$scripts_dir/compress_file_$file_type.txt" ]; then
       touch $scripts_dir/compress_file_$file_type.txt
     fi
   
     line_count=`wc -l $scripts_dir/compress_file_$file_type.txt | awk '{print $1}'`
     if [ "$line_count" != 1 ]; then 
       >$scripts_dir/compress_file_$file_type.txt
     fi
   
     begin_file_name=`cat $scripts_dir/compress_file_$file_type.txt`
   
     file_list=`find $log_dir -mmin +"$keep_min" -type f -name "$file_type.$current_year*" | sort`
     file_arr=($file_list)
   
     length=${#file_arr[@]}
     echo "$file_type file count: $length"
     for f in ${file_arr[*]}
     do
       if [ x"$begin_file_name" == x"$f" ]; then
         begin_id=$(($index+1))
         break
       fi
       index=$(($index+1))
     done
   
     echo "$file_type file begin index: $begin_id"
   
     for((i="$begin_id";i<length;i++))
     do
       echo "**********following $file_type log will be compressed......"
       file_path=${file_arr[$i]}
       file_name=${file_path##*/}
       tar zcvf $archive_dir/$dir_name/`date +%Y%m%d`/$file_name.tar.gz $file_path
       echo ${file_arr[$i]} >$scripts_dir/compress_file_$file_type.txt
     done
   
   }
   
   function archived_log_file_delete(){
     echo "**********begin $file_type compressed file delete......"
     find $archive_dir/$dir_name -mtime +"$compressed_log_keep_day" -type f -name "$file_type.$current_year*" | sort
     res=`find $archive_dir/$dir_name -mtime +"$compressed_log_keep_day" -type f -name "$file_type.$current_year*" | sort`
     if [ ! "$res" ]; then
       echo "********no $file_type compressed log will be deleted......"
     else
       echo "********$file_type compressed log will be deleted......"
       find $archive_dir/$dir_name -mtime +"$compressed_log_keep_day" -type f -name "$file_type.$current_year*" -exec rm -rf {} \;
     fi
     echo "**********end $file_type  compressed file delete......"
   }
   
   
   function archive_compress_audit_file(){
     local audit_index=0
     local audit_begin_id=0
   
     if [ ! -f "$scripts_dir/compress_file_audit.txt" ]; then
       touch $scripts_dir/compress_file_audit.txt
     fi
   
     audit_line_count=`wc -l $scripts_dir/compress_file_audit.txt | awk '{print $1}'`
     if [ "$audit_line_count" != 1 ]; then 
       >$scripts_dir/compress_file_audit.txt
     fi
   
     begin_audit_file_name=`cat $scripts_dir/compress_file_audit.txt`
   
     audit_file_list=`find $audit_log_dir -mmin +"$keep_min" -type f -name "observer_*.aud" | sort`
     audit_file_arr=($audit_file_list)
   
     audit_length=${#audit_file_arr[@]}
     echo "audit file count: $audit_length"
     for f in ${audit_file_arr[*]}
     do
       if [ x"$begin_audit_file_name" == x"$f" ]; then
         audit_begin_id=$(($audit_index+1))
         break
       fi
       audit_index=$(($audit_index+1))
     done
   
     echo "audit file begin index: $audit_begin_id"
   
     for((i="$audit_begin_id";i<audit_length;i++))
     do
       echo "**********following $file_type log will be compressed......"
       file_path=${audit_file_arr[$i]}
       file_name=${file_path##*/}
       tar zcvf $audit_archive_dir/$dir_name/`date +%Y%m%d`/$file_name.tar.gz $file_path
       if [ $? == 0 ]; then 
         rm -rf $file_path
         echo "$file_path deleted success."
       fi
       echo ${audit_file_arr[$i]} >$scripts_dir/compress_file_audit.txt
     done
   
   }
   
   
   function archived_audit_file_delete(){
     echo "**********begin audit compressed file delete......"
     find $audit_archive_dir/$dir_name -mtime +"$compressed_audit_log_keep_day" -type f -name "observer_*.aud*" | sort
     res=`find $audit_archive_dir/$dir_name -mtime +"$compressed_audit_log_keep_day" -type f -name "observer_*.aud*" | sort`
     if [ ! "$res" ]; then
       echo "********no $file_type compressed log will be deleted......"
     else
       echo "********$file_type compressed log will be deleted......"
       find $archive_dir/$dir_name -mtime +"$compressed_log_keep_day" -type f -name "observer_*.aud*" -exec rm -rf {} \;
     fi
     echo "**********end $file_type  compressed file delete......"
   }
   
   
   echo "============================================================================================"
   file_type_list="rootservice.log rootservice.log.wf observer.log trace.log election.log election.log.wf"
   for file_type in $file_type_list
   do
   archive_compress_log_file $file_type
   archived_log_file_delete $file_type
   done
   echo "============================================================================================"
   
   archive_compress_audit_file
   archived_audit_file_delete
   echo "============================================================================================"
   rm -rf $scripts_dir/lock.file
   echo "done:"`date`
   echo "============================================================================================"
   

5. 赋予可执行 compress_and_delete_archive_log_file.sh 的权限。

   # chmod +x compress_and_delete_archive_log_file.sh
   

6. 设置调用脚本。

   # vi compress_and_delete_archive_log_file_daemon.sh
   

   #!/bin/bash
   scripts_dir=/home/admin/delete_ob_log
   
   if [ ! -f $scripts_dir/lock.file ];then
     touch $scripts_dir/lock.file
     nohup $scripts_dir/compress_and_delete_archive_log_file.sh >> $scripts_dir/compress_and_delete_archive_log_file.log 2>&1 &
   else 
     echo "the compress script is running, or lock file is exists."
     ps -ef | grep compress_and_delete_archive_log_file | grep -v compress_and_delete_archive_log_file_damon 
   fi
   
   echo "done:"`date`
   

7. 赋予可执行 compress_and_delete_archive_log_file_daemon.sh 的权限。

   # chmod +x compress_and_delete_archive_log_file_daemon.sh
   

8. 切换到 root 用户。

   # su - root
   

9. 把脚本放入计划任务里定期执行。

   # vi /etc/cron.d/ob_log_compress.cron
   

   SHELL=/bin/bash
   PATH=/sbin:/bin:/usr/sbin:/usr/bin
   MAILTO=""
   00 * * * * admin /home/admin/delete_ob_log/compress_and_delete_archive_log_file_daemon.sh >> /home/admin/delete_ob_log/compress_and_delete_archive_log_file_daemon.log 2>&1
   

10. 赋予可执行 ob_log_compress.cron 的权限。

    # chmod 644 /etc/cron.d/ob_log_compress.cron
    

适用版本

OceanBase 数据库所有版本。