#!/bin/sh
#$ -S /bin/sh

# sh chipatlas/sh/checkCoreDump.sh

if [ "$1" = "q" ]; then  ## qsub モード
  projectDir=$2
  Genome=$3
  num=$4
  
  cat $projectDir/results/$Genome/summary/summary_list.txt| awk -v num=$num 'NR > (num-1)*1000 && NR <= num*1000'| xargs 
  cat | awk -F '\t' -v projectDir=$projectDir -v genome=$Genome '{
    if ($3 + 0 == 0) {
      x = 0
      Log = projectDir "/results/" genome "/log/" $1 ".log.txt"
      while ((getline < Log) > 0) {
        if ($0 ~ "Completed: 0K bytes transferred") x++
        if ($0 ~ "Server aborted session: No such file or directory") x++
        if ($0 ~ "ncbi_error_report.xml") x++
        if ($0 ~ "Stale file handle") x++
        if ($0 ~ "NO_RUN_INFO") x++
        if ($0 ~ "NO_SRA_FOUND") x++
        if ($0 ~ "_1.fq: そのようなファイルやディレクトリはありません") x++
        if ($0 ~ "connection failed while opening file") x++
        if ($0 ~ "timeout exhausted while creating") x++
        if ($0 ~ "Warning: Could not open read file") x++
      }
      close(Log)
      if (x == 0) print genome "\t" $1
    }
  }' > tmpDirForCheckCoreDump/err.$Genome.$num.txt
  exit
fi

####################################################################################################################################
#                                                             以下、初期モード
####################################################################################################################################
projectDir=`echo $0| sed 's[/sh/checkCoreDump.sh[['`

# Rerun を検出するためだが、必要ないかも。
cat << 'DDD' > /dev/null
echo "Rerun (Rr) されたジョブを調べています..."
awk '{
  if ($0 ~ /^Job ID =/) x[FILENAME]++
} END {
  for (fn in x) {
    if (x[fn] > 1) {
      j++
      print fn
    }
  }
  if (j > 0) print "上記は再実行が必要です。\n\n"
}' $projectDir/results/*/log/*.txt
DDD


echo "残存フォルダの終了状態を調べています..."
echo ""
for Genome in `ls $projectDir/results`; do
  for SRX in `ls $projectDir/results/$Genome| grep '[SDE]RX[0-9][0-9][0-9][0-9]'`; do
    ls $projectDir/results/$Genome/$SRX| grep -c "core\."| awk -v Genome=$Genome -v SRX=$SRX -v projectDir=$projectDir -v OFT='\t' '{
      if ($1 > 0) {
        str = "Core dump"
      } else {
        logt = projectDir "/results/" Genome "/log/" SRX ".log.txt"
        while ((getline < logt) > 0) if ($0 ~ "connection failed while opening file" || $0 ~ "timeout exhausted while creating") i++
        str = (i > 0) ? "Time out" : "Unknown"
      }
      print Genome, SRX, str
    }'
  done
done


echo ""
echo "通信障害やメモリ不足による異常終了を調べています..."
echo ""

rm -rf tmpDirForCheckCoreDump
mkdir tmpDirForCheckCoreDump
for genome in `ls $projectDir/results`; do
  N=`echo $projectDir/results/$genome/summary/*RX*.txt| tr ' ' '\n'| tee tmpDirForCheckCoreDump/all.$genome.summary.txt| wc -l| awk '{printf 1 + int($1/1000)}'`
  for num in `seq $N`; do
    qsub $ql -o /dev/null -e /dev/null -N ccd $projectDir/sh/checkCoreDump.sh q $projectDir $genome $num
  done
done

while :; do
  waitN=`qstat| awk '$3 == "ccd"'| wc -l`
  if [ $waitN -eq 0 ]; then
    break
  fi
done

for genome in `ls $projectDir/results`; do
  cat tmpDirForCheckCoreDump/err.$genome.*.txt| sort
done| awk '{
  j++
  a[i] = $0
} END {
  if (j > 0) {
    print "\n下記の SRX が異常終了しました。この結果は checkCoreDump.result.txt に保存しています。"
    for (i=1; i<=j; i++) print a[i]
    for (i=1; i<=j; i++) print a[i] > "checkCoreDump.result.txt"
    print "\n再実行の必要があるため、以下を実行してください。"
    print "qsub -o /dev/null -e /dev/null chipatlas/sh/reRunSraTailor.sh chipatlas"
  } else {
    print "\n再実行の必要はないので、以下を実行してください。"
    print "sh chipatlas/sh/listForClassify.sh"
  }
}'

rm -rf tmpDirForCheckCoreDump
exit