#!/usr/bin/env bash

# This script downloads corrupts blocks and assosiated metadata
#
# Should be run as root
set -e
#set -x

VERSION=0.1

#################
## Change this ##

SUPER_BIN=dzdo
HOPS_USER=hdfshw
TMP_DIR=/tmp/corrupted_blocks # this is where the data will be downloaded. 
BUCKET_NAME=dev-hopswrks-dev
AWS_ENDPOINT_URL="https://obs-are.allstate.com"

#################

AWS_COPY="aws --endpoint-url=$AWS_ENDPOINT_URL s3 cp"
AWS_OBJECT_MEATADATA="aws --endpoint-url=$AWS_ENDPOINT_URL s3api head-object  --bucket $BUCKET_NAME --key "
MYSQL_BIN_DIR=/srv/hops/mysql/bin
MYSQL_SOCKET=/srv/hops/mysql-cluster/mysql.sock
MYSQL_RUN_CMD="$MYSQL_BIN_DIR/mysql -u root -S $MYSQL_SOCKET "

RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m' # No color

if [ "$EUID" -ne 0 ]
  then echo "Please run as root"
  exit
fi

mkdir -p $TMP_DIR
chmod 777 $TMP_DIR

$SUPER_BIN su - $HOPS_USER -c "/srv/hops/hadoop/bin/hdfs fsck / -list-corruptfileblocks 2>/dev/null | grep blk  > $TMP_DIR/corrupted_blocks_list.txt"

set +e
block_ids=()

while IFS= read -r line; do
    # blk_30011       /Projects/hello0/Models/README.md
    b=$(echo $line | awk -F " " '{print $1}' | awk -F "_" '{print $2}')
    f=$(echo $line | awk -F " " '{print $2}')
    block_ids+=("$b;$f")
done < $TMP_DIR/corrupted_blocks_list.txt


for e in "${block_ids[@]}"; do
    block_id=$(echo $e | awk -F ";" '{print $1}')
    echo "Verifying Block ID: $block_id"
    inode_id=$($MYSQL_RUN_CMD -Nse "SELECT inode_id FROM hops.hdfs_block_lookup_table WHERE block_id = $block_id;")
    gen=$($MYSQL_RUN_CMD -Nse "SELECT generation_stamp FROM hops.hdfs_block_infos WHERE block_id = $block_id AND inode_id = $inode_id;")
    block="blk_${block_id}_${gen}"
    echo "Block $block"
    prefix=$(echo "$block_id / 500" | bc)


    file_data_dir=$TMP_DIR/inode_$inode_id
    mkdir -p $file_data_dir
    chmod 777 $file_data_dir

    #download data
    $AWS_COPY s3://${BUCKET_NAME}/hopsfs-blocks-set-${prefix}/${block}.meta $file_data_dir
    $AWS_COPY s3://${BUCKET_NAME}/hopsfs-blocks-set-${prefix}/${block}.data $file_data_dir

    #download metadata
    $AWS_OBJECT_MEATADATA hopsfs-blocks-set-${prefix}/${block}.meta > $file_data_dir/${block}_meta_user_metadata.json
    $AWS_OBJECT_MEATADATA hopsfs-blocks-set-${prefix}/${block}.data > $file_data_dir/${block}_data_user_metadata.json

    # a file may have multiple corrupt blocks. 
    # get file metadata only once
    file_metadata="$file_data_dir/inode_${inode_id}_metadata.txt"
    if [ ! -f $file_metadata ]; then
      echo "Saving file meta data. Inode ID: $inode_id"
      echo "hops.hdfs_inodes" >>  $file_metadata
      $MYSQL_RUN_CMD -e "SELECT * FROM hops.hdfs_inodes where id = $inode_id;" >>  $file_metadata
      echo "hops.hdfs_block_infos" >>  $file_metadata
      $MYSQL_RUN_CMD -e "SELECT * FROM hops.hdfs_block_infos where inode_id = $inode_id;" >>  $file_metadata
      echo "hops.hdfs_replicas" >>  $file_metadata
      $MYSQL_RUN_CMD -e "SELECT * FROM hops.hdfs_replicas where inode_id = $inode_id;" >>  $file_metadata
      echo "hops.hdfs_replica_under_constructions" >>  $file_metadata
      $MYSQL_RUN_CMD -e "SELECT * FROM hops.hdfs_replica_under_constructions where inode_id = $inode_id;" >>  $file_metadata
      echo "hops.hdfs_corrupt_replicas" >>  $file_metadata
      $MYSQL_RUN_CMD -e "SELECT * FROM hops.hdfs_corrupt_replicas  where inode_id = $inode_id;" >>  $file_metadata
      echo "hops.hdfs_under_replicated_blocks" >>  $file_metadata
      $MYSQL_RUN_CMD -e "SELECT * FROM hops.hdfs_under_replicated_blocks  where inode_id = $inode_id;" >>  $file_metadata
     fi
done

echo -e  "${GREEN}Compress $TMP_DIR and attache it to the JIRA${NC}"

