菜单开关

周梦康 发表于 2017-09-14 1700 次浏览 标签 : shelloss

免费领取阿里云优惠券 我的直播 - 《PHP 进阶之路》

原文

工具下载:https://help.aliyun.com/document_detail/57053.html
文档说明:https://help.aliyun.com/document_detail/56990.html
下面的笔记写的比较简单,自己备份。

需求

迁移账号bucket目录
原始Azhoumengkangwww
目的地Bmengkangwww

配置

$ cat /Users/zhoumengkang/Downloads/ossimport-2.2.2/conf/local_job.cfg |grep -v "^#"|awk '{if (length($0) > 0){print $0}}'
isIncremental=false
incrementalModeInterval=86400
srcType=local
srcAccessKey=
srcSecretKey=
srcDomain=
srcBucket=
srcPrefix=
destAccessKey=
destSecretKey=
destDomain=http://oss-cn-hangzhou.aliyuncs.com
destBucket=
destPrefix=
appId=0
httpListFilePath=c:/example/http.list
httpPrefixColumn=1
relativePathColumn=2
jobName=local_test
jobType=import
importSince=0
lastModify=0
isSkipExistFile=false
taskObjectCountLimit=10000
taskObjectSizeLimit=1000000000
scanThreadCount=1
maxMultiThreadScanDepth=1

那么需要修改的就是

isIncremental=false
incrementalModeInterval=86400
- srcType=local
+ srcType=oss
- srcAccessKey=
+ srcAccessKey=xxx
- srcSecretKey=
+ srcSecretKey=xxx
- srcDomain=
+ srcDomain=http://oss-cn-hangzhou-internal.aliyuncs.com
- srcBucket=
+ srcBucket=zhoumengkang
- srcPrefix=
+ srcPrefix=www
- destAccessKey=
+ destAccessKey=xxx
- destSecretKey=
+ destSecretKey=xxxx
- destDomain=http://oss-cn-hangzhou.aliyuncs.com
+ destDomain=http://oss-cn-hangzhou-internal.aliyuncs.com
- destBucket=
+ destBucket=mengkang
- destPrefix=
+ destPrefix=www
appId=0
httpListFilePath=c:/example/http.list
httpPrefixColumn=1
relativePathColumn=2
jobName=local_test
jobType=import
importSince=0
lastModify=0
isSkipExistFile=false
taskObjectCountLimit=10000
taskObjectSizeLimit=1000000000
scanThreadCount=1
maxMultiThreadScanDepth=1

实操记录

机器:2核4G
设置了conf/sys.properties

#表示同时执行迁移的工作线程数
workerTaskThreadNum=40
#表示分发任务的并行线程数
dispatcherThreadNum=10
top - 09:38:31 up 15:29,  2 users,  load average: 6.98, 3.10, 1.16
Tasks:  78 total,   2 running,  76 sleeping,   0 stopped,   0 zombie
Cpu0  : 43.7%us,  3.7%sy,  0.0%ni, 12.3%id,  0.0%wa,  0.0%hi, 40.3%si,  0.0%st
Cpu1  : 56.0%us,  5.7%sy,  0.0%ni, 37.0%id,  0.3%wa,  0.0%hi,  1.0%si,  0.0%st
Mem:   4043712k total,  1354176k used,  2689536k free,   156796k buffers
Swap:        0k total,        0k used,        0k free,   323304k cached

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
14045 XXX       18   0 2600m 529m  12m S 129.9 13.4   3:09.99 java

sh import.sh 执行服务启动之后,开始输出job stats比较耗费性能,直接终止脚本,而迁移交易已经在后台运行了。终止该脚本,负载会下降2左右,我的实践。

统计下来,每小时 50G 的传输量。

脚本不错,可以参考

#!/bin/bash
# One Key Import for Linux

#######################################################
# Define function
#######################################################

function yes_or_no()
{
    while [ true ]               
    do
       echo -n "$*, Yes or No: "
       read x
       case "$x" in
         y | Y | yes | Yes | YES ) return 0;;
         n | N | no | No | NO ) return 1;;
         * ) echo "Answer yes or no"
       esac
    done
}

function start_import_service()
{
    nohup java -Dskip_exist_file=false -jar ${work_dir}/bin/ossimport2.jar -c ${work_dir}/conf/sys.properties start > ${work_dir}/logs/ossimport2.log 2>&1 &
    sleep 1s
    echo "Start import service completed."
}

function stop_import_service()
{
    ps axu | grep "ossimport2.jar.* start" | grep -v grep | awk '{print "kill -9 "$2}' | bash > /dev/null 2>&1
    sleep 1s
    echo "Stop import service completed."
}

function submit_job()
{
    java -jar $work_dir/bin/ossimport2.jar -c $work_dir/conf/sys.properties submit $work_dir/conf/local_job.cfg > ${work_dir}/logs/submit.log 2>&1
    submit_result=$(grep "Error:" ${work_dir}/logs/submit.log)
}

function clean_job()
{
    java -jar $work_dir/bin/ossimport2.jar -c $work_dir/conf/sys.properties clean ${job_name}
    echo "Clean job:${job_name} completed."
}

function stat_job()
{
    java -jar $work_dir/bin/ossimport2.jar -c $work_dir/conf/sys.properties stat detail > ${work_dir}/logs/job_stat.log 2>&1
    cat ${work_dir}/logs/job_stat.log
}

function retry_failed_tasks()
{
    java -jar $work_dir/bin/ossimport2.jar -c $work_dir/conf/sys.properties retry ${job_name} > ${work_dir}/logs/retry.log 2>&1
    retry_result=$(cat ${work_dir}/logs/retry.log)
    if [ -z "$retry_result" ]; then
        echo "None failed tasks for job:${job_name}."
    else
        cat ${work_dir}/logs/retry_result.log
        echo "Retry has been submitted."
    fi
}

#######################################################
# Start of main
#######################################################

work_dir=
job_name="local_test"
submit_result=

# get work dir
osname=$(uname)
if [ "$osname" = "Linux" ]; then
    work_dir=$( dirname $(readlink -f $0) )
else
    work_dir=$( cd "$( dirname "$0" )" && pwd )
fi

# submit job
while [ 0 ]; do
    submit_job
    if [ -z "$submit_result" ]; then
        # successful
        cat ${work_dir}/logs/submit.log
        break
    else
        # failed
        yes_or_no "Clean the previous job"
        is_clean_job=$?
        if [ $is_clean_job == 0 ]; then
            stop_import_service
            clean_job
        else
            exit 1
        fi
    fi
done


# start service
start_import_service

# stat
while [ 0 ]; do
    sleep 5s
    stat_job 

    # job failed
    is_job_failed=$(grep "JobState:Failed" ${work_dir}/logs/job_stat.log)
    if [ -n "$is_job_failed" ]; then
        yes_or_no "Retry the failed tasks"
        is_retry=$?
        if [ $is_retry == 0 ]; then
            retry_failed_tasks
        else
            exit 2
        fi
    fi
    
    # job successful
    is_job_completed=$(grep "JobState:Succeed" ${work_dir}/logs/job_stat.log)
    if [ -n "$is_job_completed" ]; then
        break
    fi
done
echo "Import to oss completed."

# stop service

yes_or_no "Stop import service"
is_stop_service=$?
if [ $is_stop_service == 0 ]; then
    stop_import_service
fi

exit 0

嗨,老铁,欢迎来到我的博客!

如果觉得我的内容还不错的话,可以关注下我在 segmentfault.com 上的直播。我主要从事 PHP 和 Java 方面的开发,《深入 PHP 内核》作者之一。

[视频直播] PHP 进阶之路 - 亿级 pv 网站架构的技术细节与套路 直播中我将毫无保留的分享我这六年的全部工作经验和踩坑的故事,以及会穿插着一些面试中的 考点难点加分点

评论列表