基于 Graviton2处理器构建容器化基因分析工作负载

{
    "jobDefinitionName": "jd-mapping-graviton",
    "type": "container",
    "parameters": {
        "r2": "source/SRR11676645_2.fastq.gz",
        "dbtable": "reads_graviton",
        "sample": "SRR11676645",
        "script": "mapping.sh",
        "r1": "source/SRR11676645_1.fastq.gz"
    },
    "containerProperties": {
        "image": "123456789012.dkr.ecr.cn-northwest-1.amazonaws.com.cn/mapping-graviton:latest",
        "command": [
            "sh",
            "Ref::script",
            "Ref::sample",
            "Ref::r1",
            "Ref::r2",
            "Ref::dbtable"
        ],
        "jobRoleArn": "arn:aws-cn:iam::123456789012:role/BatchJobRole",
        "volumes": [
            {
                "name": "efs",
                "efsVolumeConfiguration": {
                    "fileSystemId": "fs-0a8685ad57ce63a3f",
                    "rootDirectory": "mapping"
                }
            }
        ],
        "environment": [
            {
                "name": "S3_MOUNT_POINT",
                "value": "/s3"
            }
        ],
        "mountPoints": [
            {
                "containerPath": "/data",
                "sourceVolume": "efs"
            }
        ],
        "privileged": true,
        "resourceRequirements": [
            {
                "value": "64",
                "type": "VCPU"
            },
            {
                "value": "500000",
                "type": "MEMORY"
            }
        ]
    },
    "platformCapabilities": [
        "EC2"
    ]
}

aws batch register-job-definition --cli-input-json file://jd.json

sample=$1
r1=$2
r2=$3
dbtable=$4
echo 'sample: '$sample', r1: '$r1', r2: '$r2', dbtable: '$dbtable

echo 'mount s3 bucket'
mkdir -p $S3_MOUNT_POINT
goofys --region cn-northwest-1 sun-test $S3_MOUNT_POINT
base_dir=$S3_MOUNT_POINT
ls -lh $base_dir

echo 'job done set status to 1'
aws dynamodb execute-statement --statement "UPDATE $dbtable SET status=1 WHERE sample='$sample'"

echo 'bwa start'
mkdir /result
bwa mem -t 64 $base_dir/ref_data/derep_all.fa $base_dir/$r1 $base_dir/$r2 > /result/$sample.sam
echo 'samtools start'
samtools sort /result/$sample.sam -@ 64 -o /result/$sample.bam
echo 'rm sample'
rm /result/$sample.sam

echo 'job done set status to 2'
aws dynamodb execute-statement --statement "UPDATE $dbtable SET status=2 WHERE sample='$sample'"

echo 'coverm filter start'
coverm filter --min-read-percent-identity 0.95 --min-read-aligned-percent 0.75 -b /result/$sample.bam -o /result/${sample}_filter.bam -t 64
echo 'copy '${sample}_filter.bam' to s3'
mkdir -p $base_dir/results/$dbtable
cp /result/${sample}_filter.bam $base_dir/results/$dbtable/

echo 'job done set status to 3'
aws dynamodb execute-statement --statement "UPDATE $dbtable SET status=3 WHERE sample='$sample'"

echo 'rm result '$sample.bam' from local disk'
rm /result/$sample.bam

echo 'coverm contig start'
coverm contig --trim-max 90 --trim-min 10 --min-read-aligned-percent 70 -t 64 --bam-files /result/${sample}_filter.bam > /result/${sample}_coverage.csv
echo 'copy to s3'
cp /result/${sample}_coverage.csv $base_dir/results/$dbtable

echo 'job done set status to 4'
aws dynamodb execute-statement --statement "UPDATE $dbtable SET status=4 WHERE sample='$sample'"

echo 'rm result '${sample}_filter.bam' from local disk'
rm /result/${sample}_filter.bam

dbtable='reads_graviton'
item=`aws ddb select $dbtable --filter 'status = 0' `
count=`echo $item | awk '{print $2}'`
echo 'count: '$count
if [ $count -eq 0 ]
then
        echo 'end'
        break
fi
info=`echo $item | awk '{for(i=10;i<=NF;i=i+9){print $i}}'`
fastq_1=`echo $item | awk '{for(i=6;i<=NF;i=i+9){print $i}}'`
fastq_2=`echo $item | awk '{for(i=8;i<=NF;i=i+9){print $i}}'`

for ((i=1;i<=$count;i++))
do
        sample=`echo $info|awk '{print $'$i'}'`
        echo 'sample= '$sample
        r1=`echo $fastq_1|awk '{print $'$i'}'`
        echo 'r1= '$r1
        r2=`echo $fastq_2|awk '{print $'$i'}'`
        echo 'r2= '$r2      
        jobname=${dbtable}_${sample%.*}
        echo 'jobnane= '$jobname
        aws batch submit-job --job-name $jobname --job-queue q-mapping-graviton --job-definition jd-mapping-graviton:1 --parameters script=mapping.sh,sample=$sample,r1=$r1,r2=$r2,dbtable=$dbtable
done