#!/bin/bash

source "$(realpath $(dirname ${0})/.cron-init)"
log "Started"

init_workers 4

# Get dataset list
FINISHED=0
DATASET_INDEX=0
DATASETS=($(${CLI_CMD} extracts:list --status=Processed --show-datasets-only))
log "DATASET_COUNT=${#DATASETS[@]}"

# For each extract
while (( ${DATASET_INDEX} < ${#DATASETS[@]} ))
do
  # For each worker
  for WORKER_ID in ${!WORKERS[@]};
  do
    # If there are no more extracts, then leave process loop
    if (( ${DATASET_INDEX} >= ${#DATASETS[@]} ))
    then
      break 2
    fi

    # Check if worker is still busy. If not, execute the next item in the queue
    kill -0 "${WORKERS[${WORKER_ID}]}" >/dev/null 2>&1
    RETVAL=$?
    if (( ${RETVAL} != 0 ))
    then
      DATASET=${DATASETS[${DATASET_INDEX}]}
      if [ -n "${DATASET}" ]
      then
        if [ -n "${WORKERS[${WORKER_ID}]}" ]
        then
          log "DATASET=${WORKER_CONTEXT[${WORKER_ID}]}, PID=${WORKERS[${WORKER_ID}]} - Finished"
          FINISHED=$((FINISHED + 1))
        fi

        ${CLI_CMD} extracts:upload $@ ${DATASET} &
        WORKERS[${WORKER_ID}]=$!
        WORKER_CONTEXT[${WORKER_ID}]=${DATASET}

        log "DATASET=${DATASET}, PID=${WORKERS[${WORKER_ID}]} - Started"
      fi

      DATASET_INDEX=$((DATASET_INDEX + 1))
    fi
  done

  # Sleep for a few seconds until the next iteration
  sleep 0.5
done

# Wait for any remaining workers to finish
log "Reached end of queue; Waiting for remaining workers to finish"
while (( ${FINISHED} < ${#DATASETS[@]} ))
do
  for WORKER_ID in ${!WORKERS[@]};
  do
    if [ -n "${WORKERS[${WORKER_ID}]}" ]
    then
      kill -0 "${WORKERS[${WORKER_ID}]}" >/dev/null 2>&1
      RETVAL=$?
      if (( ${RETVAL} != 0 ))
      then
        log "DATASET=${WORKER_CONTEXT[${WORKER_ID}]}, PID=${WORKERS[${WORKER_ID}]} - Finished"
        WORKERS[${WORKER_ID}]=""
        FINISHED=$((FINISHED + 1))
      fi
    fi
  done

  sleep 0.5
done

log "Finished"
exit 0
