#!/bin/sh

cd `/usr/bin/dirname $0`

: ${cat:=cat}

exec 3>&2 > ${cat}maker.log 2>&1

: ${sleep:=1}
: ${url:='http://www.wis-jma.go.jp/meta/oaiprovider.jsp'}
: ${mp:='iso19139'}
: ${st:='WIS-CATALOGUE'}
: ${wopt:="-q --header='Accept-Encoding: gzip'"}
: ${mlimit:=81920}
: ${vlimit:=102400}
: ${newbn:=${cat}alogue}
: ${oldbn:=${cat}alogue}
export CATDB=z.database

ulimit -m $mlimit -v $vlimit

rm -f ${CATDB}
if stat ${oldbn}.zip
then
  unzip -p ${oldbn}.zip ${cat}obs.csv | ruby dbize.rb
  unzip -p ${oldbn}.zip ${cat}obs.csv | awk -F, '(NR==1){print $6; exit}' > z.date
  stat ${CATDB}
else
  rm -f z.date
  echo bootstrap mode
fi

if [ X"$res" = X"" ]
then

  wget $wopt -O"zoai.xml" "${url}?verb=ListRecords&metadataPrefix=${mp}&set=${st}"

  xsltproc oai2cat.xsl zoai.xml > zcat.xml
  #sh oaivalidate.sh zoai.xml > validate.log
  res=`xsltproc oai2res.xsl zoai.xml`
  xsltproc --stringparam title yes cat2csv.xsl zcat.xml > zcat.csv
  grep '^[GHY]' zcat.csv > ${cat}prod.csv || :
  grep -v '^[GHY]' zcat.csv > ${cat}obs.csv || :
  test ! -f zoai.zip || zip -q zoai.zip zoai.xml
  mv -f zoai.xml zoai.bak
  mv -f zcat.xml zcat.bak
fi

while [ X"$res" != X"" ]
do
  fssres=`echo "$res" | sed 's/[^-._A-Za-z0-9]/_/g'`
  echo "resumptionToken=$res"
  wget $wopt -O"zoai-${fssres}.xml" \
    "${url}?verb=ListRecords&resumptionToken=${res}"
  if ! xsltproc oai2cat.xsl zoai-${fssres}.xml > zcat-${fssres}.xml
  then
    echo "=== rescue shrink ${fssres} ==="
    ruby cirp_strip.rb zoai-${fssres}.xml > zoai.tmp
    if test -f zoai.zip
    then
      mv -f zoai-${fssres}.xml zoai-${fssres}.big
      zip -q zoai.zip zoai-${fssres}.big
      rm -f zoai-${fssres}.big
    fi
    mv -f zoai.tmp zoai-${fssres}.xml
    xsltproc oai2cat.xsl zoai-${fssres}.xml > zcat-${fssres}.xml
  fi
  #sh oaivalidate.sh zoai-${fssres}.xml >> validate.log
  res=`xsltproc oai2res.xsl zoai-${fssres}.xml`
  xsltproc cat2csv.xsl zcat-${fssres}.xml > zcat.csv
  grep '^[GHY]' zcat.csv >> ${cat}prod.csv || :
  grep -v '^[GHY]' zcat.csv >> ${cat}obs.csv || :
  test ! -f zoai.zip || zip -q zoai.zip zoai-${fssres}.xml
  mv -f zoai-${fssres}.xml zoai.bak
  mv -f zcat-${fssres}.xml zcat.bak
  test -f z.exit2 && exit 2
  sleep $sleep
done

if test ! -f ${CATDB}
then
  echo this time no diff - run again
  exit 0
fi

echo making diff ...

NOWDATE="`awk -F, '(NR==1){print $6; exit}' ${cat}obs.csv`" \
  PREVDATE="`cat z.date`" \
  ruby dbdif.rb ${cat}obs.csv > z.dif.html
mv -f z.dif.html ${cat}diff.html

sh distri.sh ${cat}diff.html

today=`date +%Y%m%d`
mon=`date +%Y%m`

if test -f zoai.zip
then mv -f zoai.zip zoai-${today}.zip
fi
rm -f zcat.bak zoai.bak zcat.csv ${newbn}.zip ${CATDB} z.date

zip ${newbn}.zip ${cat}obs.csv ${cat}prod.csv ${cat}diff.html
unzip -l ${newbn}.zip

dir=../../wisdata/arch/wmocat/${mon}
test -d ${dir} || mkdir -p ${dir}
cp -f ${newbn}.zip ../../wisdata/arch/wmocat/${mon}/${newbn}-${today}.zip
echo ${newbn}.zip stored wmocat/${mon}/${newbn}-${today}.zip >&3
echo ${newbn}.zip stored wmocat/${mon}/${newbn}-${today}.zip 

echo done
echo done. >&3