#!/bin/ksh

# version 1.0, 2002-01-07, Michael Wang <xw73@columbia.edu>.
# * first working (somewhat) version.
# version 1.1, 2002-01-11, Michael Wang <xw73@columbia.edu>.
# * added exit in trap.
# * use OPATH for external command.
# * keep the current directory unchanged (use $t).
# * eliminated the extra wait: (( the_num_stat < num_file )) && sleep 10
# * put status reporting lines together.

(: ${.sh.version}) 1>/dev/null 2>&1 || {
  i= unset i
  for i in /usr/dt/bin/dtksh /usr/local/bin/ksh; do
    $i -c ": \${.sh.version}" 2>/dev/null && exec $i "$0" "$@"
  done
  echo "ksh93 is not found."
  exit 1
}

OPATH=$PATH
PATH=$(PATH=/bin:/usr/bin getconf PATH)
typeset -x PATH

function my_getopts {
  typeset PATH= SEP=: i=
  typeset -u _I
  for i; do
    _I=$i
    eval ${_I%%?(:)=*}= unset ${_I%%?(:)=*}
    case $_I in
      *[!:]=*) eval $(IFS=$SEP; print ${_I%%=*}=\"${_I#*=}\")   ;;
      *:=*)    eval $(IFS=$SEP; print ${_I%%:=*}=\"${i#*:=}\")  ;;
    esac
    [[ $_I == PARFILE:=* ]] && . $PARFILE
  done
}

my_getopts DEBUG= HELP= SUBMIT:= COMFILE:= MULTI= "$@"

[[ $DEBUG == Y ]] && set -x

[[ $HELP == Y ]] && {
  PATH=$OPATH whence perldoc >/dev/null || {
    print "You need to have perldoc in your PATH."
    exit 1
  }
  j= unset j
  j=/tmp/${0##*/}_man.$$
  trap "rm -f $j" EXIT INT
  sed -n "/^##/s:^## \{0,1\}::p" $0 > $j
  PATH=$OPATH perldoc $j
  exit 0
}

[[ -n $SUBMIT && -x $SUBMIT ]] || {
  SUBMIT=$(PATH=$OPATH whence submit)
  [[ -n $SUBMIT && -x $SUBMIT ]] || {
    SUBMIT=/usr/local/bin/submit
    [[ -n $SUBMIT && -x $SUBMIT ]] || {
      print "ERROR: Can not find submit."
      exit 1
    }
  }
}

[[ -r $COMFILE ]] || {
  print "ERROR: Can not read command file ($COMFILE)."
  exit 1
}

: ${MULTI:=3}
[[ $MULTI == +([0-9]) ]] || {
  print "ERROR: MULTI ($MULTI) is not a number."
  exit 1
}

t= unset t
t=/tmp/${0##*/}_tmp.$$
mkdir -p $t

j= unset j
typeset -Z3 j
(( j = 0 ))
> $t/$j.pid; > $t/$j.log
print 0 > $t/$j.stat

(( num_file = 0 ))
i= unset i
while read i; do
  (( num_file++ ))
done < $COMFILE

(( num_file >= 1 )) || {
  print "ERROR: I have nothing to run."
  exit 1
}

typeset -Z3 the_num_file the_num_stat
(( the_num_stat = 0 ))

function cleanup {
  (( $1 )) && print "ERROR: Please wait for cleanup to finish."

  for i in $t/+([0-9]).pid
  do
    j=${i%.pid}
    [[ -f $j.stat ]] || kill -- -$(< $i)
    cat $j.log
  done
  rm -rf $TMPDIR
  date
  return $1
}

trap "cleanup 1; exit 1" EXIT INT

while (( the_num_stat < num_file )); do
  set -- $t/+([0-9]).stat; (( the_num_stat = $# - 1 ))
  set -- $t/+([0-9]).pid;  (( the_num_file = $# - 1 ))
  (( the_num_proc = the_num_file - the_num_stat ))

  print "submitted:finished:allowed:total=\c"
  print "$the_num_file:$the_num_stat:$MULTI:$num_file"

  (( the_num_proc >= MULTI )) && {
    print "INFO: $(date +%H:%M:%S) maximum number of session reached, wait."
    sleep 10
    continue
  }

  (( the_num_file >= num_file )) && {
    print "INFO: $(date +%H:%M:%S) wait for ${0##*/} to finish."
    (( the_num_stat < num_file )) && sleep 10
    continue
  }

  (( the_num_file++ ))
  $SUBMIT -i $t/$the_num_file.pid  \
          -l $t/$the_num_file.log  \
          -e $t/$the_num_file.stat \
          PATH=$OPATH $(sed -n ${the_num_file##+(0)}p $COMFILE)
  until [[ -r $t/$the_num_file.pid && -r $t/$the_num_file.log ]]; do
    sleep 1
  done
done

i= j= ok= err= tot= unset i j ok err tot
(( tot = -1 ))
(( ok = -1 ))
(( err = 0 ))
for i in $t/+([0-9]).stat; do
  j=$(< $i)
  (( tot++ ))
  if [[ $j == "0" ]]; then
    (( ok++ ))
  else
    (( err++ ))
  fi
done

cleanup 0
trap - EXIT

print "INFO: $tot/$num_file finished, $ok successes, $err failures."

if (( ok == num_file )); then
  print "${0##*/} finished successfully."
  exit 0
else
  print "${0##*/} failed."
  exit 1
fi

## =head1 NAME
##
## mrun - run a list of commands in multiple sessions
##
## =head1 SYNOPSIS
##
## B<mrun>
##   comfile:=I<command file> multi=I<number>|parfile=I<parameter file>
##   [debug=y|n] [submit=I<submit path>]
##
## B<mrun> help=y
##
## =head1 DESCRIPTION
##
## B<mrun> start up multiple processes, specified by the multi=I<number>
## option, to run a list of the commands in I<comfile> in parallel, in the
## order specified in the file. Whenever a command is finished, a new process
## is started so that there are always the same number processes running
## until there is no more command to run.
##
## B<submit> is used to keep track of the existence of the process,
## the logfile for the process, and finishing status. This is more efficient
## and more reliable than the ps command. The full path of submit is searched
## in the order of command line option, PATH in interactive use, and
## /usr/local/bin/submit. The technique has been used since long time ago
## in bk.oracle, and it is seperated as a generic utility.
##
## When all of the commands in the list are finished, B<mrun> will report
## the status of how many are successful and how many failed. B<mrun> is
## considered successful only if all the commands are finished successful,
## and it is considered as a failure if any one of the commands are failed.
##
## When exception happens, B<mrun> tries its best to clean up the running
## processes, and clean up the temporary files.
##
## =head1 SEE ALSO
##
## submit help=y, bk.oracle.
##
## =head1 AUTHOR
##
## Michael Wang <F<xw73@columbia.edu>>.
##
## This is free software. You may copy or redistribute it under the same
## terms as Perl. However, if you modify it, you need to send the
## modification to the author via email.