diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..496ee2c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index 4e94bd2..432d873 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,10 @@ It has been tested with both: ## Fun facts -Up today, 4th of July, 2023, none of the major brands of Tape Libraries could provide a tool capable of this. I wrote the script in a day, then tested and fixed it. +Up today, 4th of July, 2023, none of the major brands of Tape Libraries could provide a tool capable of this. +I wrote the script in a day, then tested and fixed it. Total time spent: 3 days. -Total money earned: none, so I'll just put it here. \ No newline at end of file +Total money earned: none, so I'll just put it here. + +P.S. I am aware the script il poorly written, but I don't have a test environment anymore to test it properly. I'll do it in my spare time. \ No newline at end of file diff --git a/optimizer.sh b/optimizer.sh new file mode 100755 index 0000000..7d2a60e --- /dev/null +++ b/optimizer.sh @@ -0,0 +1,185 @@ +#!/bin/sh +### +### Usage: ./optimizer.sh +### +### -h | --help Shows this message +### +### optimizer.sh +### Necessary parameters: +### - sg device of the library +### - parallelism +### +### The script checks the parallelism with the available drives of the library/partition, then +### proceeds with loading *all* the tapes in the slots. Mailboxes are ignored. +### It launches several child processes. The `timeout` command tries to unload the drives, +### it succeeds when the the tape optimization is done. +### + + +help() { + sed -rn 's/^### ?//;T;p' "$0" +} +if [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]] || [[ "$1" == "" ]]; then + help + exit 1 +fi + +log() { + echo -e "$(date +'%Y-%m-%d %H:%M:%S') $1" | tee -a ${LOG_FILE} +} + +alive() { + # $1 "${LIB_SG}" + # $2 "${SLOT}" + # $3 "${DRIVE}" + # $4 "${BARCODE}" + echo "${4}" > "${TMP}/.drive${3}.lock" + local PID_ALIVE='1' + local WAIT=$((3600 + $RANDOM % 7200)) # from 1 to 2 hours + local TIMEOUT="600" #10 minuti wait before dismounting the tape + while [ $PID_ALIVE -eq 1 ]; do + log "ALIVE [$$] - Waiting ${WAIT} secondi..." + sleep "${WAIT}" + log "ALIVE [$$] - Trying to unload drive ${3} into slot ${2} (Timeout: ${TIMEOUT} seconds)..." + timeout "${TIMEOUT}" mtx -f "${1}" unload "${2}" "${3}" #kill after ${TIMEOUT} seconds + if [[ $(echo $?) == '0' ]]; then + log "ALIVE [$$] - OK. Drive ${3} unloaded." + grep -q "$4;drive$3;WIP" "${PROCESSED_BARCODES}" # Check if barcode is WIP + if [[ $(echo $?) == '0' ]]; then + log "ALIVE [$$] - Barcode $4 processed." + sed -i "s/$4;drive$3;WIP/$4;OK/g" "${PROCESSED_BARCODES}" + rm -f "${TMP}/.drive${3}.lock" + touch "${TMP}/.drive${3}.ready" + log "ALIVE [$$] - Drive ${3} newly available. Exit." + local PID_ALIVE='0' + return 0 # Fine Alive + else + log "ALIVE [$$] - ERROR. Can't find the barcode among the WIP. Exit." + return 1 + fi # Barcode processato --> OK + elif [ -f "${TMP}/.drive${3}.lock" ]; then # Drive in use + log "ALIVE [$$] - Unmount drive ${3} failed, still in use (file .lock). Checking the content." + grep -q "${4}" "${TMP}/.drive${3}.lock" + if [[ $(echo $?) == '0' ]]; then + log "ALIVE [$$] - File content OK. Waiting." + continue + else + log "ALIVE [$$] - ERROR. File content of ${TMP}/.drive${3}.lock not coherent. Exit." + return 1 + fi + elif [ -f "${TMP}/.drive${3}.ready" ]; then # Drive available + log "ALIVE [$$] - Found file ${TMP}/.drive${3}.ready. Drive available. Exit." + local PID_ALIVE='0' + return 0 + fi + done + log "Fine ALIVE [$$]" + return 0 +} + +LIB_SG=$1 +PARALLELISM=$2 + +WAIT=$((300)) +TMP='/tmp' +LIB_CONFIG="${TMP}/lib_config.txt" +PROCESSED_SLOTS="${TMP}/processed-slots.txt" +PROCESSED_BARCODES="/root/processed-barcodes.txt" + +log "***** New execution *****" + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +LOG_PATH="${SCRIPT_DIR}/log" +LOG_FILE="${LOG_PATH}/optimizer.log" +if [ ! -d "${LOG_PATH}" ]; then + echo "Creating the log directory: ${LOG_PATH}" + mkdir -vp ${LOG_PATH} +fi + +mtx -f "${LIB_SG}" status > "${LIB_CONFIG}" + +# Check if there are enough ($PARALLELISM) free drives +if [[ $(grep "Data Transfer Element" ${LIB_CONFIG} | grep 'Empty' | wc -l) -lt "${PARALLELISM}" ]]; then + log "ERROR. Not enough free drives. Exit." + exit 1 +fi + +# Clean the WIP barcodes from old executions +sed -i '/WIP/d' "${PROCESSED_BARCODES}" + +# Cleaning orphan files +rm -rf ${TMP}/.drive* +rm -rf ${PROCESSED_SLOTS} + +for (( i = 0 ; i < ${PARALLELISM} ; i++ )); do + grep -q "Data Transfer Element ${i}:Empty" ${LIB_CONFIG} + if [[ $(echo $?) == '0' ]]; then + if [[ ! -f "${TMP}/.drive${i}.ready" ]]; then + log "Creating file ${TMP}/.drive${i}.ready" + touch "${TMP}/.drive${i}.ready" + fi + fi +done + +# List full slots +SLOTS_FULL="${TMP}/slots_full.txt" +grep 'Storage Element' ${LIB_CONFIG} | grep 'Full' | grep -v IMPORT | grep -v CLN | grep -v "(" > ${SLOTS_FULL} + +COUNT=1 +OFS=$IFS +IFS=" +" +while [ $COUNT -le $(cat ${SLOTS_FULL} | wc -l) ]; do + RUNNING=$(ls -1q ${TMP}/.drive*.lock | wc -l) + if [[ "${RUNNING}" -lt "${PARALLELISM}" ]]; then + SLOT=$(sed "${COUNT}q;d" ${SLOTS_FULL} | awk '{print $3;}' | cut -d ':' -f 1) + BARCODE=$(sed "${COUNT}q;d" ${SLOTS_FULL} | awk '{print $4;}' | cut -d '=' -f 2) + grep -q "${SLOT}" "${PROCESSED_SLOTS}" #Controllo che lo slot non sia stato già processato + if [[ $(echo $?) == '0' ]]; then + echo "Slot ${SLOT} already processed." + ((COUNT++)) + continue + fi + grep -q "${BARCODE};OK" "${PROCESSED_BARCODES}" #Controllo che il barcode non sia già stato processato + if [[ $(echo $?) == '0' ]]; then + echo "Barcode ${BARCODE} already processed. Skip." + ((COUNT++)) + continue + fi + log "### SLOT ${SLOT} - BARCODE ${BARCODE}" + for (( i = 0 ; i < ${PARALLELISM} ; i++ )); do # Look for an available drive + #log "scelta drive - ciclo numero ${i}" + if [ -f "${TMP}/.drive${i}.ready" ]; then # Look for "ready" file + DRIVE=${i} + mv "${TMP}/.drive${DRIVE}.ready" "${TMP}/.drive${DRIVE}.lock" + break + fi + done + log "Load slot ${SLOT} into drive ${DRIVE}." + mtx -f "${LIB_SG}" load "${SLOT}" "${DRIVE}" + echo "${BARCODE};drive${DRIVE};WIP" >> "${PROCESSED_BARCODES}" # Barcode WIP + echo ${SLOT} >> "${PROCESSED_SLOTS}" + log "Lancio ALIVE con parametri: ${LIB_SG} - ${SLOT} - ${DRIVE} - ${BARCODE}" + alive "${LIB_SG}" "${SLOT}" "${DRIVE}" "${BARCODE}" & + RUNNING=$(ls -1q ${TMP}/.drive*.lock | wc -l) + log "MAIN - RUNNING processes: $RUNNING" + if [[ "${RUNNING}" -ge "${PARALLELISM}" ]]; then + log "MAIN (if) - Massimo Parallelismo. Aspetto ${WAIT} secondi." + sleep "${WAIT}" + else + ((COUNT++)) + log "Waiting 2 minutes to avoid too many commands." + sleep 120 + fi + else + log "MAIN - Maximum parallelism. Waiting ${WAIT} seconds." + sleep "${WAIT}" + fi +done +IFS=$OFS + +rm -rf "${LIB_CONFIG}" +rm -rf "${SLOTS_FULL}" +rm -rf "${PROCESSED_SLOTS}" + +log "***** End execution *****\n\n" \ No newline at end of file