#!/bin/sh

surl=http://gaw.empa.ch/gawsis/xml/
prox=http://localhost/
xsdp=/var/www/html/xsd19139/gmx/gmx.xsd

export http_proxy=$prox

indx=zindex.html
mddb=ymddb.tar
mdls=ymdls.txt

if [ -f ${mddb} ]; then
  echo already ${mddb} exists
  # exit 0
fi

if [ -f ${mdls} ]; then
  cp ${mdls} z.txt
else
  wget -q -O${indx} ${surl}
  xsltproc --html indexlinks.xsl ${indx} | ruby -ruri -e \
  'u=URI(ARGV[0]); for line in $stdin; puts u + line.chomp; end' $surl \
    > z.txt
fi

: > zindex.tsv

i=0
awk '((NR % 10) == 1) && !/\+/' z.txt | while read url
do
  i=`expr 1 + $i`
  wget -q -Ozraw$i.xml ${url} \
  || wget -Ozraw$i.xml ${url} || break
  xsltproc fixgmd.xsl zraw$i.xml > zfil$i.xml || break
  tar --append --file=znew.${mddb} zraw$i.xml zfil$i.xml
  printf '%s\t%s\n' $i "`basename ${url} .xml`" >> zindex.tsv
  rm -f zraw$i.xml zfil$i.xml
done
tar --append --file=znew.${mddb} zindex.tsv z.txt
mv -f znew.${mddb} ${mddb}
rm -f z*
echo done $i