#!/bin/ksh
#
# version 5.2, 2008-05-29, Michael Wang <xw73@columbia.edu>, sponsored by
# DataBase Intelligence Group, Inc. (http://DBIGusa.com).
# * Reverted back to ksh88.
#
# version 5.1, 2001-12-20, Michael Wang <xw73@columbia.edu>.
# - Clean up and simplication, sometimes with ksh93 features (see ver4.2).
# - help=y.
#
# version 5.0, 2001-09-19, Michael Wang <xw73@columbia.edu>.
# * Added "set echo on" for all SQL statements ("set echo on" does not work in
#   interactive mode (here document, pipe).
# * Added Oracle 9i support: use "sqlplus" if "svrmgrl" is not available.
#
# version 4.3, 2001-06-07, Michael Wang <xw73@columbia.edu>.
# * when CDPATH is set, cd may generate output, directed to /dev/null.
#   oracle:ADEV:816$cd app/..
#   /u0/oracle
#
# version 4.2, 2001-06-05, Michael Wang <xw73@columbia.edu>.
# * while [[ $f == *%S* ]]; do f=${f%%%S*}@(0)$i${f#*%S}; done
# ->while [[ $f == *%S* ]]; do f=${f%%%S*}*(0)$i${f#*%S}; done
#   i.e. not 1 "0" but 0 or more "0"s. 
#
# version 4.1, 2001-01-06, Michael Wang <xw73@columbia.edu>.
# * Fixed the problem in the case that archive log format contains %S and
#   %T. This was found out by Ming Sun. 
# * Replaced sed command with ksh built-in to save time:
#   while [[ $f == *%s* ]]; do f=${f%%%s*}@($i)${f#*%s}; done
#
# version 4.0, 200-11-03, Michael Wang <xw73@columbia.edu>.
# * Fixed problem due to incomplete backup by rechecking the cksum.
# * Fixed problem due to filled up backup filesystem by defining the_avail=-1.
# * Fixed problem for infinite wait for (( THE_NOARCHIVED == 0 )) condition
#   by reducing THE_SEQ. 
# * Make PERL and FIND confuration oprions:
#   PERL points to the perl command, and FIND points to the find command
#   that supports -mmin option (GNU and AT&T find). 
# * Added [[ -d $ORACLE_SID ]] || mkdir $ORACLE_SID for each BKUP_DIRS.
# * Reducing mandatory configuration parameters by setting the default
#   values.
# * Fixed the problem when running the script without path (orahot.ksh SID)
#   by using dirname.
# * Cleaned the obsolete ksh syntax as reported by ksh93 -n option:
#   -eq within [[...]] obsolete, use ((...))
#   '=' obsolete, use '=='
# * unset LD_LIBRARY_PATH which is not needed for properly installed
#   Oracle binaries.
#
# version 3.141, 2000-10-07, Michael Wang <xw73@columbia.edu>.
# * Added options for using compress, gzip, or bzip2 as compression tool.
#
# version 3.01, 2000-08-06, Michael Wang <xw73@columbia.edu>.
# * Chang on ORACLE_HOME= line shown below
#   OLD => ORACLE_HOME=$(sed -n "s/^$ORACLE_SID:\(.*\):.*$/\1/p" $ORATAB)
#   NEW => ORACLE_HOME=$(sed -n "s/^$ORACLE_SID:\([^:]*\):.*$/\1/p" $ORATAB)
#   This accommodates the generalized oratab with more than 3 fields.
#
# version 3.0, 2000-06-11, Michael Wang <xw73@columbia.edu>.
# * Change from timestamp based "ls -1rt" to sequence number based from
#   Oracle dictionary per discussion with John Zhong and Jose Garcia. 
#   As with multiple archivers in Oracle 8i, the sequence number and and 
#   timestamp are not aligned. It is not possible to find ARCHIVED=Y log 
#   files from timestamp alone. 
#   In the new version, the current sequence number is obtained from v$log.
#   Then we wait until all sequence number less than current becomes ARCHIVED.
#   All log files with sequence number less than current are backed up, and
#   removed from log_archive_dest if no long needed by standby database. 
#
# version 2.2, 2000-02-20, Michael Wang <xw73@columbia.edu>.
# * Added compatibility with Oracle 8i which has option to use either
#   archive_log_dest or archive_log_dest_n (but not both).
#
# version 2.1, 1999-08-20, Michael Wang <xw73@columbia.edu>.
# * Replace ? to $ORACLE_HOME in log_archive_dest 
# * General code cleanup.
#
# version 2.0, 1999-03-18, Michael Wang <xw73@columbia.edu>.
# * Added handling for "%S", "%t", and "%T" in the log_archive_format. 
#   minor code clean up. 
# * Added an option "CHKFS". Under this option, redo is run only the
#   "redo filesystem" is over the REDO_THRESHOLD. 
# * "lock" checking is done first before other checking to prevent duplicate
#   process cluttering the logfile. 
# * In "lock" situation, 0 exit code with "CHKFS" option and unique exit code 
#   without "CHKFS" option to be used by orahot backup.
# * With "CHKFS" option, no file cleanup. 
# * Change KEEP_DAYS to KEEP_MINS. KEEP_MINS can be 0 meaning "no keep".
#
# version 1.0, 1999-01-30, Michael Wang <xw73@columbia.edu>.
# * rewrite based on previous versions developped in house.

OPATH=; unset OPATH
OPATH=$PATH

GETCONF=; unset GETCONF

for GETCONF in /bin/getconf /usr/bin/getconf; do
  [[ -x $GETCONF ]] && {
    PATH=$($GETCONF PATH)  # avoid ksh93 (m, l, ...) getconf
    break;                 # which has problems.
  }
done

typeset -x PATH

function my_getopts {
  typeset PATH= SEP=: i=
  typeset -u _I
  for i; do
    _I=$i
    [[ _I = *=* ]] && eval "${_I%%?(:)=*}=; unset ${_I%%?(:)=*}"
    case $_I in
      *[!:]=*) eval $(IFS=$SEP; print ${_I%%=*}=\"${_I#*=}\")   ;;
      *:=*)    eval $(IFS=$SEP; print ${_I%%:=*}=\"${i#*:=}\")  ;;
      CHKFS)   CHKFS=Y;;
      !(*=*))  SID=$i;;
    esac
  done
}

my_getopts sid:= chkfs= help= debug= "$@"

[[ $DEBUG = Y ]] && set -x

function shdoc {
  typeset version="3.1, 2008-05-28, Michael Wang <xw73@columbia.edu>."

  typeset help doc file

  typeset    tmpdir=/tmp/shdoc_$$_$RANDOM
  typeset    base tmpf i flag

  eval help=y doc=pod "$@"

  [[ -n ${OPATH:-} ]] && typeset PATH=$OPATH

  set -- $(echo $help $doc | tr '[a-z]' '[A-Z]')
  help=$1 doc=$2

  base=${file##*/}
  base=${base%.*sh}

  case ${doc} in
    (POD) mkdir -p ${tmpdir}          && tmpf=${tmpdir}/${base}.pod        ;;  
    (MAN) mkdir -p ${tmpdir}/man/man1 && tmpf=${tmpdir}/man/man1/${base}.1 ;;
  esac

  while IFS= read -r i; do
    [[ $i = "## "* ]] && {
      [[ $i = "## ${doc}_START"* ]]      && { flag=Y; continue; }
      [[ $i = "## ${doc}_STOP"*  ]]      && { flag=N; continue; }
      [[ ${doc} = POD && -z ${flag:-} ]] &&   flag=Y
    }

    if [[ $flag = "Y" ]]; then
      i=${i#"##"}
      i=${i#" "}
      printf "%s\n" "${i}"
    fi
  done < ${file} > ${tmpf}

  case $help in
    (Y)      case ${doc} in
               (POD) perldoc ${tmpf}                   ;;
               (MAN) MANPATH=${tmpdir}/man man ${base} ;;
             esac
             ;;
    (PS|PDF) case ${doc} in
               (POD) pod2man ${tmpf} ;;
               (MAN) cat     ${tmpf} ;;
             esac |
               groff -man |
                 case ${help} in
                   (PS)  cat     >${base}.ps  ;;
                   (PDF) ps2pdf - ${base}.pdf ;;
                 esac
             ;;
    (POD)    cp ${tmpf} ${base}.pod
             ;;
    (HTML)   pod2html ${tmpf} > ${base}.html
             rm pod2htmd.tmp pod2htmi.tmp
             ;;
  esac

  rm -rf ${tmpdir}
  return 0
}

[[ $HELP = @(Y|PS|PDF|POD|HTML) ]] && {
  : ${DOC:=man}
  shdoc file=$0 doc=$DOC help=$HELP
  exit 0
}

[[ -z "$SID" ]] && {
  print "ERROR: you must specify ORACLE_SID."
  exit 1
}

typeset -x ORACLE_SID=$SID

: ${CHKFS:=N}    # to be run hourly.

# $0 could be ".", convert to absolute path
PDIR=$(cd $(dirname $0)/.. >/dev/null && pwd -P) 

LOGDIR=$PDIR/operlog/$ORACLE_SID
[[ -d ${LOGDIR%/*} && ! -d ${LOGDIR} ]] && mkdir ${LOGDIR}

TMPDIR=$PDIR/temp/$ORACLE_SID
[[ -d ${TMPDIR%/*} && ! -d ${TMPDIR} ]] && mkdir ${TMPDIR}

CONFILE=$PDIR/etc/oraredo_$ORACLE_SID.cf

cd $TMPDIR

function chkdir {
  [[ -z "$1" ]] && {
    print "ERROR: No directory specified for chkdir function."
    return 1
  }
  typeset i=$1
  
  [[ -d $i ]] || {
    print "ERROR: Can not find $i directory."
    return 1
  }

  > $i/foo.$$ || {
    print "ERROR: Can not write in $i directory."
    return 1
  }
  rm $i/foo.$$
}

chkdir $LOGDIR || { print "ERROR: in checking $LOGDIR."; exit 1; }

i=; unset i
i=$LOGDIR/oraredo_$ORACLE_SID.log.$(date +%m%d)
print "Please check the logfile: $i."
exec >> $i 2>&1

date

chkdir $TMPDIR || { print "ERROR: in checking $TMPDIR."; exit 1; }

[[ -r $CONFILE ]] || {
  print "ERROR: Can not find configuration file: $CONFILE."
  exit 1
}

. $CONFILE

: ${FIND:=/usr/local/bin/find}

[[ -x $FIND ]] || {
  print "ERROR: Can not find GNU/Find ($FIND)."
  exit 1
}

: ${PERL:=/usr/local/bin/perl}

[[ -x $PERL ]] || {
  print "ERROR: Can not find Perl ($PERL)."
  exit 1
}

i=${COMPRESS_C%% *}
j=${i##*/}

if [[ $j = compress ]]
then THEZ=".Z"
     COMPRESS_D="${i%compress}uncompress -c"
elif [[ $j = gzip ]]
then THEZ=".gz"
     COMPRESS_D="$i -dc"
elif [[ $j = bzip2 ]]
then THEZ=".bz2"
     COMPRESS_D="$i -dc"
elif [[ $j = cat ]]
then THEZ=""
     COMPRESS_D="cat"
else COMPRESS_C="compress -c"
     THEZ=".Z"
     COMPRESS_D="uncompress -c"
fi

: ${KEEP_MINS:=1440}

[[ "$KEEP_MINS" = ?([+-])+([0-9]) ]] || {
  print "KEEP_MINS ($KEEP_MINS) in $CONFILE is not a number."
  exit 1
}

: ${REDO_THRESHOLD:=50}

[[ "$REDO_THRESHOLD" = +([0-9]) ]] || {
  print "REDO_THRESHOLD ($REDO_THRESHOLD) in $CONFILE is not a number."
  exit 1
}

: ${BLACKOUT:=00000-00000}

for i in $BLACKOUT
do
  j=${i%%-*}
  k=${i##*-}

  [[ "$j" = +([0-9]) && "$k" = +([0-9]) ]] || {
    print "ERROR: BLACKOUT not in correct format in $CONFILE."
    exit 1
  }

  l=$(date +%w%H%M)

  (( l >= j && l <= k )) && {
    print "INFO: blackout zone $i, quit."
    exit 0
  }
done

[[ -n "$BKUP_DIRS" ]] || {
  print "ERROR: BKUP_DIRS not defined in $CONFILE."
  exit 1
}

for i in $BKUP_DIRS
do
  [[ -d $i/$ORACLE_SID ]] || mkdir $i/$ORACLE_SID

  chkdir $i/$ORACLE_SID || {
    print "ERROR: in checking $i/$ORACLE_SID."
    exit 1
  }
done

[[ "$CHKFS" = "Y" ]] || {
  find $LOGDIR -mtime +60 -name \*log\* -type f -print -exec rm -f {} \;
  find $TMPDIR -type f -print -exec rm -f {} \;

  for i in $BKUP_DIRS; do
    (( KEEP_MINS >= 1 )) && {
      $FIND $i/$ORACLE_SID -type f -mmin +$KEEP_MINS -print -exec rm -f {} \;
    }

    (( KEEP_MINS <= 0 )) && {
      $FIND $i/$ORACLE_SID -type f -print -exec rm -f {} \;
    }
  done
}

: ${STANDBY:=N}
: ${STANDBY_DONE:=standby.done}
: ${STANDBY_TIMEOUT:=2160}
typeset -u STANDBY

[[ "$STANDBY" = @(Y|N) ]] || {
  print "ERROR: STANDBY has to be Y or N."
  exit 1
}

[[ "$STANDBY" = N ]] || [[ "$STANDBY_TIMEOUT" = +([0-9]) ]] || {
  print "STANDBY_TIMEOUT ($STANDBY_TIMEOUT) in $CONFILE is not a number."
  exit 1
}

: ${MAX_ARCH_TIME:=600}

[[ $MAX_ARCH_TIME = +([0-9]) ]] || {
  print "MAX_ARCH_TIME ($MAX_ARCH_TIME) in $CONFILE is not a number."
  exit 1
}

: ${REDO_WAIT_MINS:=60}

[[ $REDO_WAIT_MINS = +([0-9]) ]] || {
  print "REDO_WAIT_MINS ($REDO_WAIT_MINS) in $CONFILE is not a number."
  exit 1
}

ORACLE_HOME=; unset ORACLE_HOME
typeset -x ORACLE_HOME=$(awk -F: "/^$ORACLE_SID:/ {print \$2; exit}" /var/opt/oracle/oratab)
typeset -x LD_LIBRARY_PATH=
ISQL=$ORACLE_HOME/bin/svrmgrl
[[ -x $ISQL ]] || ISQL=$ORACLE_HOME/bin/sqlplus

[[ -x "$ISQL" ]] || {
  print "ERROR: Can not find $ISQL. Possibly on the other node of the cluster."
  exit 1
}

locked=; unset locked
(( locked = 0 ))
(( SECONDS = 0 ))

while (( SECONDS <= REDO_WAIT_MINS*60 )); do
  mkdir $LOGDIR/lock && {
    trap "rmdir $LOGDIR/lock" EXIT INT
    (( locked = 1 ))
    break
  }

  print "INFO: An archive log or hot backup process is running, wait."
  sleep 60
done

(( locked == 0 )) && {
  print "ERROR: Failed to obtain lock ($LOGDIR/lock)."
  exit 1
}

isql= fsql=; unset isql fsql
typeset -Z3 isql

(( isql = 1 ))
fsql=${isql}_seq

{ print "spool ${fsql}.lis"
  print "set echo on"

  [[ $ISQL = *svrmgrl ]] && print "connect internal"
  [[ $ISQL = *sqlplus ]] && print "connect / as sysdba"

  print "select to_char(sysdate,'yyyy-mm-dd hh24:mi:ss') from dual"
  print "/"

  print "SELECT 'THE_THREAD=' || thread# || ' ' || 'THE_SEQ=' || sequence# a"
  print "  FROM v\$log WHERE status='CURRENT'"
  print "/"

  print "SELECT 'THE_NOARCHIVED=' || TO_CHAR(COUNT(1))"
  print "  FROM v\$log"
  print " WHERE sequence# < (SELECT sequence# FROM v\$log WHERE status='CURRENT')"
  print "   AND archived='NO'"
  print "/"

  print "SELECT 'THE_DEST=' || value b"
  print "  FROM v\$parameter WHERE name='log_archive_dest'"
  print "/"

  print "SELECT 'THE_FORMAT=' || value c"
  print "  FROM v\$parameter WHERE name='log_archive_format'"
  print "/"

  print "spool off"
  print "exit"
} > ${fsql}.sql

[[ $ISQL = *svrmgrl ]] && $ISQL command="@${fsql}.sql"
[[ $ISQL = *sqlplus ]] && $ISQL /nolog   @${fsql}.sql

i=; unset i
i=$(grep "^THE_.*=" ${fsql}.lis)

grep -E "ORA-|MGR-" ${fsql}.lis || (( $(print "$i" | grep -c "^") != 4 )) && {
  print "ERROR: In getting current sequence number statements."
  exit 1
}

eval "$i"

[[ $THE_THREAD = +([0-9]) && $THE_SEQ = +([0-9]) ]] || {
  print "ERROR: Did not get correct thread# or sequence#."
  exit 1
}

(( THE_NOARCHIVED == 0 )) || {
  NTIME=$($PERL -e 'print time()')  # Time Now
  print "Initial Time = $NTIME"
  i_time=0
  i_arch=0

  until (( THE_NOARCHIVED == 0 )); do
    (( i_arch += 1 ))
    print "Current THE_SEQ = $THE_SEQ"
    print "Elapsed Time = $i_time"

    (( isql += 1 ))
    fsql=${isql}_arch

    { print "spool ${fsql}.lis"
      print "set echo on"

      [[ $ISQL = *svrmgrl ]] && print "connect internal"
      [[ $ISQL = *sqlplus ]] && print "connect / as sysdba"

      print "SELECT 'THE_NOARCHIVED=' || TO_CHAR(COUNT(1)) e"
      print "  FROM v\$log"
      print " WHERE sequence# < $THE_SEQ AND archived='NO'"
      print "/"

      print "spool off"
      print "exit"
    } > ${fsql}.sql

    [[ $ISQL = *svrmgrl ]] && $ISQL command="@${fsql}.sql"
    [[ $ISQL = *sqlplus ]] && $ISQL /nolog   @${fsql}.sql

    i=; unset i
    i=$(grep "^THE_NOARCHIVED=" ${fsql}.lis)

    grep -E "ORA-|MGR-" ${fsql}.lis || (( $(print "$i" | grep -c "^") != 1 )) && {
      print "ERROR: In getting THE_NOARCHIVED statements."
      exit 1
    }

    eval "$i"

    (( THE_NOARCHIVED == 0 )) || {
      if (( i_arch%2 == 1 )); then
        sleep 10
      else
        (( THE_SEQ == 1 )) || (( THE_SEQ -= 1 ))
      fi

      CTIME=$($PERL -e 'print time()')
      (( i_time = CTIME - NTIME ))

      (( i_time > MAX_ARCH_TIME )) && {
        print "ERROR: Timeout waiting non-current log files to be archived."
        exit 1
      }
    }
  done
}

[[ -z $THE_DEST ]] && {
  (( isql += 1 ))
  fsql=${isql}_dest

  { print "spool ${fsql}.lis"
    print "set echo on"

    [[ $ISQL = *svrmgrl ]] && print "connect internal"
    [[ $ISQL = *sqlplus ]] && print "connect / as sysdba"

    print "SELECT 'THE_DEST=' || destination d"
    print "  FROM v\$archive_dest"
    print " WHERE target='PRIMARY'"
    print "   AND destination LIKE '%/%'"
    print "   AND rownum <=1"
    print "/"

    print "spool off"
    print "exit"
  } > ${fsql}.sql

  [[ $ISQL = *svrmgrl ]] && $ISQL command="@${fsql}.sql"
  [[ $ISQL = *sqlplus ]] && $ISQL /nolog   @${fsql}.sql

  i=; unset i
  i=$(grep "^THE_.*=" ${fsql}.lis)

  grep -E "ORA-|MGR-" ${fsql}.lis || (( $(print "$i" | grep -c "^") != 1 )) && {
    print "ERROR: In getting THE_DEST."
    exit 1
  }
  eval "$i"
}

# THE_DEST=${THE_DEST/#\?/$ORACLE_HOME}
THE_DEST=$(echo ${THE_DEST} | sed "s:^\?:$ORACLE_HOME:")

[[ "$THE_FORMAT" = *%[stST]* ]] || {
  print "ERROR: Did not get THE_FORMAT correctly."
  exit 1
}

if [[ -d "$THE_DEST" ]]
then        #/* log_archive_dest=/path; log_archive_format=arch_SID_%s */
  THE_DESTMAT=${THE_DEST%/}/$THE_FORMAT
elif [[ -d "${THE_DEST%/*}" ]]
then        #/* log_archive_dest=/path/arch; log_archive_format=_SID_%s */
  THE_DESTMAT=${THE_DEST}${THE_FORMAT}
else 
  print "ERROR: Did not get log_archive_dest correctly."
  exit 1
fi

THE_LOGDIR=${THE_DESTMAT%/*}
set -- $(df -Pk $THE_LOGDIR)
j=${12%\%}
print "INFO: The archive log filesystem is $j% full."

# One of the four conditions: 10

[[ "$CHKFS" = "Y" ]] && (( j < REDO_THRESHOLD )) && {
  print "THERE IS NO NEED FOR ORACLE REDO BACKUP ($j% < $REDO_THRESHOLD%)."
  rmdir $LOGDIR/lock
  date
  trap - EXIT
  exit 0
}

# Other three conditons: 11, 00, 01

i_seq=; unset i_seq
(( i_seq = THE_SEQ - 1 ))

while (( i_seq >=1 )); do
  DESTMAT=$THE_DESTMAT

  # DESTMAT=${DESTMAT//%s/@($i_seq)}           # %s = sequence#
  # DESTMAT=${DESTMAT//%S/*(0)$i_seq}          # %S = sequence# (padded with 0s)
  # DESTMAT=${DESTMAT//%t/@($THE_THREAD)}      # %t = thread#
  # DESTMAT=${DESTMAT//%T/*(0)@$THE_THREAD}    # %T = thread# (padded with 0s)
  # DESTMAT=${DESTMAT//%r/+([0-9])}            # %r = logical incarnation

  DESTMAT=$(echo ${DESTMAT} | sed "s:%s:@($i_seq):g")
  DESTMAT=$(echo ${DESTMAT} | sed "s:%S:*(0)$i_seq}:g")
  DESTMAT=$(echo ${DESTMAT} | sed "s:%t:@($THE_THREAD):g")
  DESTMAT=$(echo ${DESTMAT} | sed "s:%T:*(0)$THE_THREAD:g")
  DESTMAT=$(echo ${DESTMAT} | sed "s:%r:+([0-9]):g")

  eval set -- "$DESTMAT"
  (( $# == 1 )) && eval [[ $1 = $DESTMAT ]] || break
  path_file=$1

  (( KEEP_MINS <= -1 )) && { rm $path_file; continue; }

  path=${path_file%/*}
  file=${path_file##*/}

  the_avail=-1
  for dir in $BKUP_DIRS
  do
    avail=$(df -Pk $dir | awk 'NR == 2 {print $4}')
    (( avail > the_avail )) && {
      the_avail=$avail
      THE_DIR=$dir
    }
  done

  THE_PATH=$THE_DIR/$ORACLE_SID/${path#/}
  mkdir -p $THE_PATH

  print "INFO: compressing $path_file -> $THE_PATH/$file$THEZ"
  j=; unset j
  find $BKUP_DIRS -name $file$THEZ -print | read j
  pq=0

  if [[ -n "$j" ]]; then
    THE_PATH=${j%/*}
    p=$(cksum < $path_file)
    q=$(cat $THE_PATH/$file$THEZ | $COMPRESS_D | cat | cksum)

    if [[ "$p" = "$q" ]]; then
      pq=1
    else
      cat $path_file | $COMPRESS_C | cat > $THE_PATH/$file$THEZ
    fi
  else
    cat $path_file | $COMPRESS_C | cat > $THE_PATH/$file$THEZ
  fi    # NOT UUOC (Useless Use Of Cat).

  (( pq == 0 )) && p=$(cksum < $path_file)
  print "INFO: $p = The cksum of $path_file."
  (( pq == 0 )) && q=$(cat $THE_PATH/$file$THEZ | $COMPRESS_D | cat | cksum)
  print "INFO: $q = The cksum of $COMPRESS_D $THE_PATH/$file$THEZ."

  [[ "$p" = "$q" ]] || {
    print "ERROR: compression problem: $path_file -> $THE_PATH/$file$THEZ."
    rmdir $LOGDIR/lock
    exit 1
  }

  if [[ "$STANDBY" = Y ]]; then
    i_done=$THE_LOGDIR/$STANDBY_DONE    # the standby file

    [[ -r $i_done ]] ||{
      print "ERROR: $i_done not found."
      exit 1
    }

    [[ -n $($FIND $i_done -mmin +$STANDBY_TIMEOUT -print) ]] && {
      print "ERROR: Standby file $i_done is too old."
      exit 1
    }

    s_seq=$(< $i_done)
    s_seq=${s_seq##+(0)}

    [[ $s_seq = +([0-9]) ]] || {
      print "ERROR: $i_done file does not contain a number."
      exit 1
    }

    (( s_seq >= i_seq )) && rm $path_file
  else
    rm $path_file
  fi
  (( i_seq = i_seq - 1 ))
done

i=; unset i
(( i = THE_SEQ - 1 - i_seq ))

print "INFO: $i \c"

if (( i == 1 ))
then print "file \c"
else print "files \c"
fi

if (( KEEP_MINS >= 0 ))
then print "compressed."
else print "deleted."
fi

print "ORACLE REDO BACKUP FINISHED SUCCESSFULLY."

rmdir $LOGDIR/lock
date
trap - EXIT
exit 0

## POD_START
## =head1 NAME
## 
## oraredo.ksh - make Oracle database archive log backup.
## 
## =head1 SYNOPSIS
## 
## oracold.ksh help=y
## 
## oracold.ksh MySID|sid:=MySid [chkfs|chkfs=y] [debug=y]
## 
## =head1 DESCRIPTION
## 
## B<oraredo.ksh> find the highest sequence (THE_SEQ) number below which
## all the archive log files are I<archived>. Or more precisely, the
## highest THE_SEQ that the following SQL statements return 0 rows:
## 
##   SELECT COUNT(1)
##   FROM v$log
##   WHERE sequence# < $THE_SEQ
##   AND archived='NO'
## 
## The highest possible number is the current sequence number. However on
## busy systems, the non-current redo logs may still be in the process of
## archiving (copying files to disk). In this case, oraredo.ksh waits, and
## reduce I<THE_SEQ> alternatively. (A simple listing by timestamp or
## alphabetic order are not reliable. With multiple archivers introduced
## in Oracle 8, an early started process may finish after a later started
## process.)
## 
## Once THE_SEQ is found, oraredo.ksh compresses the archived logs from
## THE_SEQ -1 and below until archived log file is no long found.
## 
## After the backup copy is verified to be good copy via I<cksum>, the
## original copy in archive log destination is removed.
## 
## Features include:
## 
## =over 4
## 
## =item *
## 
## Multiple filesystems (BKUP_DIRS in configuration file).  Dynamic
## allocate filesystem per free disk space.
## 
## =item *
## 
## Compare the cksum of the copied file and original file before
## deleting.
## 
## =item *
## 
## Dynamically find the destination and format of the archive log files.
## It is more generic, and does not depends on any convention or site
## standard.
## 
## =item *
## 
## Blackout timezone (BLACKOUT in configuration file) to skip backup at
## certain day and certain hours. This is to avoid conflict with hot
## backup and make backup filesystem more static for central backup.
## 
## =item *
## 
## chkfs|chkfs=y command line option to check if the file system for
## archive log is over certain threshod (THRESHOLD in configuration file),
## and only make backup when the threshod is exceeded. This is suitable
## for hourly backup, it prevents the archive log destination from being
## full, and to avoid frequent small backups so that the backup filesystem
## more static for central backup.
## 
## =item *
## 
## Willing to wait for other redo backup or hot backup session.
##
## =item *
##
## You can choose how long to keep the archive logs in backup directories
## (KEEP_MINS in configuration file), or just delete the logs without
## backup (set KEEP_MINS to be a negative number). The latter is suitable
## for staging or development databases which need to be in archivelog mode
## but does not want to fill the archive log destination.
## 
## =back
##
## =head1 INSTALLATION
## 
## To install, follow these easy 1-2-3 steps: 
## 
## =over 4
## 
## =item 1.
## 
## Create an install directory for redo backup <job dir>/ora_redo. And
## under ora_redo, create following sub-directories:
## 
##   script
##   etc
##   operlog/<SID>
##   temp/<SID>
## 
## Where SID is the actual ORACLE_SID name. If you have multiple SIDs,
## then just create them under operlog and temp.
## 
## =item 2. 
## 
##   cp oraredo.ksh script/oraredo.ksh
##   chmod 755 script/oraredo.ksh
## 
## =item 3.
## 
## Edit etc/oraredo_SID.cf from the sample configuration shown below. If
## you have multiple SIDs, then just create multiple files.
## 
## =back
##   # *
##   # * Where do you want to keep backup redo logs?
##   # *
##   
##   BKUP_DIRS="/bkup01/redo /bkup02/redo"
##   
##   # *
##   # * How long do you want to keep backup redo logs?
##   # * If KEEP_MINS is minus number, then archive logs are deleted
##   # * without being copies to backup directories. 
##   # * default is 1440 minutes (24 hours).
##   # *
##   
##   KEEP_MINS=1440    # in minutes
##   
##   # *
##   # * With CHKFS option, at what high watermark 
##   # * of archive_dest you want to run the backup?
##   # * Default is 50 (50%).
##   # *
##   
##   REDO_THRESHOLD=50
##   
##   # *
##   # * Is there blackout perid that you do not want run backup?
##   # * The default is no blackout. 
##   # *
##   
##   BLACKOUT="00000-00000"
##   # BLACKOUT="00000-00000 11830-20600 61830-62400 00000-00600 00000-62400"
##   #   00000-00000: no blackout
##   #   11830-20600: Monday 18:30 to Tuesday 06:00
##   #   61830-62400 00000-00600: Saturday 18:30 to Sunday 06:00
##   #   00000-62400: all blackout
##    
##   # *
##   # * Does this database support a standby database?
##   # * The default is N.
##   # *
##   
##   STANDBY="N"
##    
##   # *
##   # * IF STANDBY=Y, what file at archive_dest keeps the last shipped/applied
##   # * archived log file sequence number? (I won't delete the file until it is
##   # * no longer needed by the standby site.)
##   # * The default is standby.done.
##   # *
##   
##   STANDBY_DONE="standby.done"
##    
##   # *
##   # * At what age of STANDBY_DONE file will I stop backup and report error?
##   # * Default is 2160 (36 hours).
##   # *
##   
##   STANDBY_TIMEOUT=2160    # in minutes
##   
##   # *
##   # * What command do you use to compress a file to standard output?
##   # * Default is "compress -c".
##   # *
##   
##   COMPRESS_C="/usr/bin/compress -c"
##   # COMPRESS_C="/usr/local/bin/gzip -c"
##   # COMPRESS_C="/usr/local/bin/bzip2 -c"
##   
##   # *
##   # * Where can I find perl?
##   # * Default is /usr/local/bin/perl.
##   # *
##   
##   PERL=/usr/local/bin/perl
##   
##   # *
##   # * Where can I find "find" that supports -mmin option? 
##   # * (GNU find and AT&T find supports -mmin option.) 
##   # * Default is /usr/local/bin/find.
##   # *
##   
##   FIND=/usr/local/bin/find
##   
##   # *
##   # * How long you want to wait for non-current redo logs to be archived?
##   # * The default is 600 (10 minutes).
##   # *
##   
##   MAX_ARCH_TIME=600    # in seconds
##   
##   # *
##   # * How long you want to wait for other redo/hot backup sessions to finish?
##   # * The default is 60 (1 hour).
##   # *
##   
##   REDO_WAIT_MINS=60    # in minutes
##
## =head1 SEE ALSO
##
## orahot.ksh help=y.
##
## =head1 AUTHOR
## 
## Michael Wang <F<xw73@columbia.edu>> with help from many others
## acknowledged in version notes.
## 
## This is free software. You may copy or redistribute it under the same
## terms as Perl. However, if you modify it, you need to send the
## modification to the author via email.
## POD_STOP
##
## MAN_START
## .\" Automatically generated by Pod::Man v1.37, Pod::Parser v1.14
## .\"
## .\" Standard preamble:
## .\" ========================================================================
## .de Sh \" Subsection heading
## .br
## .if t .Sp
## .ne 5
## .PP
## \fB\\$1\fR
## .PP
## ..
## .de Sp \" Vertical space (when we can't use .PP)
## .if t .sp .5v
## .if n .sp
## ..
## .de Vb \" Begin verbatim text
## .ft CW
## .nf
## .ne \\$1
## ..
## .de Ve \" End verbatim text
## .ft R
## .fi
## ..
## .\" Set up some character translations and predefined strings.  \*(-- will
## .\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
## .\" double quote, and \*(R" will give a right double quote.  | will give a
## .\" real vertical bar.  \*(C+ will give a nicer C++.  Capital omega is used to
## .\" do unbreakable dashes and therefore won't be available.  \*(C` and \*(C'
## .\" expand to `' in nroff, nothing in troff, for use with C<>.
## .tr \(*W-|\(bv\*(Tr
## .ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
## .ie n \{\
## .    ds -- \(*W-
## .    ds PI pi
## .    if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
## .    if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\"  diablo 12 pitch
## .    ds L" ""
## .    ds R" ""
## .    ds C` ""
## .    ds C' ""
## 'br\}
## .el\{\
## .    ds -- \|\(em\|
## .    ds PI \(*p
## .    ds L" ``
## .    ds R" ''
## 'br\}
## .\"
## .\" If the F register is turned on, we'll generate index entries on stderr for
## .\" titles (.TH), headers (.SH), subsections (.Sh), items (.Ip), and index
## .\" entries marked with X<> in POD.  Of course, you'll have to process the
## .\" output yourself in some meaningful fashion.
## .if \nF \{\
## .    de IX
## .    tm Index:\\$1\t\\n%\t"\\$2"
## ..
## .    nr % 0
## .    rr F
## .\}
## .\"
## .\" For nroff, turn off justification.  Always turn off hyphenation; it makes
## .\" way too many mistakes in technical documents.
## .hy 0
## .if n .na
## .\"
## .\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
## .\" Fear.  Run.  Save yourself.  No user-serviceable parts.
## .    \" fudge factors for nroff and troff
## .if n \{\
## .    ds #H 0
## .    ds #V .8m
## .    ds #F .3m
## .    ds #[ \f1
## .    ds #] \fP
## .\}
## .if t \{\
## .    ds #H ((1u-(\\\\n(.fu%2u))*.13m)
## .    ds #V .6m
## .    ds #F 0
## .    ds #[ \&
## .    ds #] \&
## .\}
## .    \" simple accents for nroff and troff
## .if n \{\
## .    ds ' \&
## .    ds ` \&
## .    ds ^ \&
## .    ds , \&
## .    ds ~ ~
## .    ds /
## .\}
## .if t \{\
## .    ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
## .    ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
## .    ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
## .    ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
## .    ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
## .    ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
## .\}
## .    \" troff and (daisy-wheel) nroff accents
## .ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
## .ds 8 \h'\*(#H'\(*b\h'-\*(#H'
## .ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
## .ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
## .ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
## .ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
## .ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
## .ds ae a\h'-(\w'a'u*4/10)'e
## .ds Ae A\h'-(\w'A'u*4/10)'E
## .    \" corrections for vroff
## .if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
## .if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
## .    \" for low resolution devices (crt and lpr)
## .if \n(.H>23 .if \n(.V>19 \
## \{\
## .    ds : e
## .    ds 8 ss
## .    ds o a
## .    ds d- d\h'-1'\(ga
## .    ds D- D\h'-1'\(hy
## .    ds th \o'bp'
## .    ds Th \o'LP'
## .    ds ae ae
## .    ds Ae AE
## .\}
## .rm #[ #] #H #V #F C
## .\" ========================================================================
## .\"
## .IX Title "ORAREDO 1"
## .TH ORAREDO 1 "2008-05-29" "perl v5.8.5" "User Contributed Perl Documentation"
## .SH "NAME"
## oraredo.ksh \- make Oracle database archive log backup.
## .SH "SYNOPSIS"
## .IX Header "SYNOPSIS"
## oracold.ksh help=y
## .PP
## oracold.ksh MySID|sid:=MySid [chkfs|chkfs=y] [debug=y]
## .SH "DESCRIPTION"
## .IX Header "DESCRIPTION"
## \&\fBoraredo.ksh\fR find the highest sequence (\s-1THE_SEQ\s0) number below which
## all the archive log files are \fIarchived\fR. Or more precisely, the
## highest \s-1THE_SEQ\s0 that the following \s-1SQL\s0 statements return 0 rows:
## .PP
## .Vb 4
## \&  SELECT COUNT(1)
## \&  FROM v$log
## \&  WHERE sequence# < $THE_SEQ
## \&  AND archived='NO'
## .Ve
## .PP
## The highest possible number is the current sequence number. However on
## busy systems, the non-current redo logs may still be in the process of
## archiving (copying files to disk). In this case, oraredo.ksh waits, and
## reduce \fI\s-1THE_SEQ\s0\fR alternatively. (A simple listing by timestamp or
## alphabetic order are not reliable. With multiple archivers introduced
## in Oracle 8, an early started process may finish after a later started
## process.)
## .PP
## Once \s-1THE_SEQ\s0 is found, oraredo.ksh compresses the archived logs from
## \&\s-1THE_SEQ\s0 \-1 and below until archived log file is no long found.
## .PP
## After the backup copy is verified to be good copy via \fIcksum\fR, the
## original copy in archive log destination is removed.
## .PP
## Features include:
## .IP "\(bu" 4
## Multiple filesystems (\s-1BKUP_DIRS\s0 in configuration file).  Dynamic
## allocate filesystem per free disk space.
## .IP "\(bu" 4
## Compare the cksum of the copied file and original file before
## deleting.
## .IP "\(bu" 4
## Dynamically find the destination and format of the archive log files.
## It is more generic, and does not depends on any convention or site
## standard.
## .IP "\(bu" 4
## Blackout timezone (\s-1BLACKOUT\s0 in configuration file) to skip backup at
## certain day and certain hours. This is to avoid conflict with hot
## backup and make backup filesystem more static for central backup.
## .IP "\(bu" 4
## chkfs|chkfs=y command line option to check if the file system for
## archive log is over certain threshod (\s-1THRESHOLD\s0 in configuration file),
## and only make backup when the threshod is exceeded. This is suitable
## for hourly backup, it prevents the archive log destination from being
## full, and to avoid frequent small backups so that the backup filesystem
## more static for central backup.
## .IP "\(bu" 4
## Willing to wait for other redo backup or hot backup session.
## .IP "\(bu" 4
## You can choose how long to keep the archive logs in backup directories
## (\s-1KEEP_MINS\s0 in configuration file), or just delete the logs without
## backup (set \s-1KEEP_MINS\s0 to be a negative number). The latter is suitable
## for staging or development databases which need to be in archivelog mode
## but does not want to fill the archive log destination.
## .SH "INSTALLATION"
## .IX Header "INSTALLATION"
## To install, follow these easy 1\-2\-3 steps: 
## .IP "1." 4
## Create an install directory for redo backup <job dir>/ora_redo. And
## under ora_redo, create following sub\-directories:
## .Sp
## .Vb 4
## \&  script
## \&  etc
## \&  operlog/<SID>
## \&  temp/<SID>
## .Ve
## .Sp
## Where \s-1SID\s0 is the actual \s-1ORACLE_SID\s0 name. If you have multiple SIDs,
## then just create them under operlog and temp.
## .IP "2." 4
## .Vb 2
## \&  cp oraredo.ksh script/oraredo.ksh
## \&  chmod 755 script/oraredo.ksh
## .Ve
## .IP "3." 4
## Edit etc/oraredo_SID.cf from the sample configuration shown below. If
## you have multiple SIDs, then just create multiple files.
## .PP
## .Vb 1
## \&  BKUP_DIRS="/bkup01/redo /bkup02/redo"
## .Ve
## .PP
## .Vb 6
## \&  # *
## \&  # * How long do you want to keep backup redo logs?
## \&  # * If KEEP_MINS is minus number, then archive logs are deleted
## \&  # * without being copies to backup directories. 
## \&  # * default is 1440 minutes (24 hours).
## \&  # *
## .Ve
## .PP
## .Vb 1
## \&  KEEP_MINS=1440    # in minutes
## .Ve
## .PP
## .Vb 5
## \&  # *
## \&  # * With CHKFS option, at what high watermark 
## \&  # * of archive_dest you want to run the backup?
## \&  # * Default is 50 (50%).
## \&  # *
## .Ve
## .PP
## .Vb 1
## \&  REDO_THRESHOLD=50
## .Ve
## .PP
## .Vb 4
## \&  # *
## \&  # * Is there blackout perid that you do not want run backup?
## \&  # * The default is no blackout. 
## \&  # *
## .Ve
## .PP
## .Vb 6
## \&  BLACKOUT="00000-00000"
## \&  # BLACKOUT="00000-00000 11830-20600 61830-62400 00000-00600 00000-62400"
## \&  #   00000-00000: no blackout
## \&  #   11830-20600: Monday 18:30 to Tuesday 06:00
## \&  #   61830-62400 00000-00600: Saturday 18:30 to Sunday 06:00
## \&  #   00000-62400: all blackout
## .Ve
## .PP
## .Vb 4
## \&  # *
## \&  # * Does this database support a standby database?
## \&  # * The default is N.
## \&  # *
## .Ve
## .PP
## .Vb 1
## \&  STANDBY="N"
## .Ve
## .PP
## .Vb 6
## \&  # *
## \&  # * IF STANDBY=Y, what file at archive_dest keeps the last shipped/applied
## \&  # * archived log file sequence number? (I won't delete the file until it is
## \&  # * no longer needed by the standby site.)
## \&  # * The default is standby.done.
## \&  # *
## .Ve
## .PP
## .Vb 1
## \&  STANDBY_DONE="standby.done"
## .Ve
## .PP
## .Vb 4
## \&  # *
## \&  # * At what age of STANDBY_DONE file will I stop backup and report error?
## \&  # * Default is 2160 (36 hours).
## \&  # *
## .Ve
## .PP
## .Vb 1
## \&  STANDBY_TIMEOUT=2160    # in minutes
## .Ve
## .PP
## .Vb 4
## \&  # *
## \&  # * What command do you use to compress a file to standard output?
## \&  # * Default is "compress -c".
## \&  # *
## .Ve
## .PP
## .Vb 3
## \&  COMPRESS_C="/usr/bin/compress -c"
## \&  # COMPRESS_C="/usr/local/bin/gzip -c"
## \&  # COMPRESS_C="/usr/local/bin/bzip2 -c"
## .Ve
## .PP
## .Vb 4
## \&  # *
## \&  # * Where can I find perl?
## \&  # * Default is /usr/local/bin/perl.
## \&  # *
## .Ve
## .PP
## .Vb 1
## \&  PERL=/usr/local/bin/perl
## .Ve
## .PP
## .Vb 5
## \&  # *
## \&  # * Where can I find "find" that supports -mmin option? 
## \&  # * (GNU find and AT&T find supports -mmin option.) 
## \&  # * Default is /usr/local/bin/find.
## \&  # *
## .Ve
## .PP
## .Vb 1
## \&  FIND=/usr/local/bin/find
## .Ve
## .PP
## .Vb 4
## \&  # *
## \&  # * How long you want to wait for non-current redo logs to be archived?
## \&  # * The default is 600 (10 minutes).
## \&  # *
## .Ve
## .PP
## .Vb 1
## \&  MAX_ARCH_TIME=600    # in seconds
## .Ve
## .PP
## .Vb 4
## \&  # *
## \&  # * How long you want to wait for other redo/hot backup sessions to finish?
## \&  # * The default is 60 (1 hour).
## \&  # *
## .Ve
## .PP
## .Vb 1
## \&  REDO_WAIT_MINS=60    # in minutes
## .Ve
## .SH "SEE ALSO"
## .IX Header "SEE ALSO"
## orahot.ksh help=y.
## .SH "AUTHOR"
## .IX Header "AUTHOR"
## Michael Wang <\fIxw73@columbia.edu\fR> with help from many others
## acknowledged in version notes.
## .PP
## This is free software. You may copy or redistribute it under the same
## terms as Perl. However, if you modify it, you need to send the
## modification to the author via email.
## MAN_STOP