#!/bin/bash

###
### IETFMEASUREMENTS - Get various AD work load and past performance statistics
###
### Version 1.0.0
###
### Written in 2007-2011 by Jari Arkko
### Donated to the public domain.
###
### Changelog:
###
### 1.0.0	- Initial implementation
### 1.0.1	- Template improvements, html cleanup
### 2.0.0       - Added RFC publication speed measurements in millibits
###
### Usage:
###
###   ietfmeasurements [ --ad pattern ] [ --doc pattern ] [ --copy ] [ --quiet ] [ --quick ]
###

###
### Process arguments
###

THISPROG=$0
DEBUG=0
AD=".*"
DOC=".*"
COPY=0
QUIET=0
QUICK=0
FETCH=1
RUNCALC=1
TIMESTAMP=0
PREFFAMILY=IPv6

while [ $# -gt 0 ]
do
  case $1 in
    --ad)        shift;
                 AD="$1";
                 shift;;
    --doc)       shift;
                 DOC="$1";
                 shift;;
    --copy)      shift;
                 COPY=1;;
    --quiet)     shift;
                 QUIET=1;;
    --quick)     shift;
                 QUICK=1;;
    --debug)     shift;
                 DEBUG=1;;
    --nofetch)   shift;
                 FETCH=0;;
    --nocalc)    shift;
                 RUNCALC=0;;
    --timestamp) shift;
                 TIMESTAMP=1;;
    --prefer-family=IPv4)
                 shift;
                 PREFFAMILY=IPv4;;
    --prefer-family=IPv6)
                 shift;
                 PREFFAMILY=IPv6;;
    *)           echo 'ietfmeasurements: Unrecognized argument ('$1') -- exit';
                 exit 1;;
  esac
done

###
### Initialize
###

LANG=en
export LANG
#LC_ALL=en
#export LC_ALL
TIMINGDATABASEURL=http://www.arkko.com/tools/admeasurements/draft-timing.txt
NONADTIMINGDATABASEURL=http://www.arkko.com/tools/admeasurements/draft-timing-nonad.txt
ADDRAFTDATABASEURL=http://arkko.com/tools/admeasurements/adwork.txt
CURYEAR=`date +%Y`
CURMON=`date +%m`
CURDAY=`date +%d`
if [ $CURDAY -le 22 ]
then
  if [ $CURMON -eq 1 ]
  then
    CURYEARCOMPLETE=`expr $CURYEAR - 1`
    CURMONCOMPLETE=12
  else
    CURYEARCOMPLETE=$CURYEAR
    CURMONCOMPLETE=`expr $CURMON - 1`
  fi
else
  CURYEARCOMPLETE=$CURYEAR
  CURMONCOMPLETE=$CURMON
fi
tmpdir=/tmp/im_tmp
if [ $FETCH = 1 ]
then
  rm -rf $tmpdir
  mkdir $tmpdir
else
  rm -f $tmpdir/*.html
fi
cd $tmpdir
WGETOPTS="--prefer-family="$PREFFAMILY" -q"
export WGETOPTS

if [ $TIMESTAMP = 1 ]
then
  echo 'Running ietfmeasurements on '`date`'...'
fi

###
### Fetch database
###

if [ $FETCH = 1 ]
then
  echo Fetching database files...
  if wget $WGETOPTS -O draft-timing.txt $TIMINGDATABASEURL
  then
    nop=nop
  else
    echo 'ietfmeasurements: cannot access '$TIMINGDATABASEURL' - exit'
    exit 1
  fi

  if wget $WGETOPTS -O draft-timing-nonad.txt $NONADTIMINGDATABASEURL
  then
    nop=nop
  else
    echo 'ietfmeasurements: cannot access '$NONADTIMINGDATABASEURL' - exit'
    exit 1
  fi

  if wget $WGETOPTS -O adwork.txt $ADDRAFTDATABASEURL
  then
    nop=nop
  else
    echo 'ietfmeasurements: cannot access '$ADDRAFTDATABASEURL' - exit'
    exit 1
  fi
fi

###
### Quick select of the relevant docs
###

if [ $RUNCALC = 1 ]
then

  cat draft-timing.txt | grep -e "$DOC" > draft-timing-selected.txt
  mv draft-timing.txt draft-timing-orig.txt
  mv draft-timing-selected.txt draft-timing.txt

  cat draft-timing-nonad.txt | grep -e "$DOC" > draft-timing-nonad-selected.txt
  mv draft-timing-nonad.txt draft-timing-nonad-orig.txt
  mv draft-timing-nonad-selected.txt draft-timing-nonad.txt

fi

###
### FUNCTION: Common AWK functions
###

COMMONFUNCS='

function draftnameisofficial(d) {
  return(d ~ /^draft-ietf-/ ||
         d ~ /^draft-iab-/ ||
         d ~ /^draft-iesg-/ ||
         d ~ /^draft-irtf-/);
}

function dateyear(d) {
  split(d,d1a,"-");
  return(d1a[1]);
}

function quarter(d) {
  mo = datemonth(d);
  if (mo <= 3) mo = 1;
  else if (mo <= 6) mo = 4;
  else if (mo <= 9) mo = 7;
  else mo = 10;
  return(dateyear(d) "-" mo);
}

function quartername(d) {
  if (datemonth(d) <= 3) na = "Q1";
  else if (datemonth(d) <= 6) na = "Q2";
  else if (datemonth(d) <= 9) na = "Q3";
  else na = "Q4";
  return(na " " dateyear(d));
}

function quarternumeric(d) {
  if (datemonth(d) <= 3) nu = 0.0;
  else if (datemonth(d) <= 6) nu = 0.25;
  else if (datemonth(d) <= 9) nu = 0.5;
  else nu = 0.75;
  return(dateyear(d) + nu);
}

function datemonth(d) {
  split(d,d1a,"-");
  return(d1a[2]);
}

function dateyearmonth(d) {
  return(dateyear(d) "-" datemonth(d));
}

function addmonth(d) {
  y = dateyear(d);
  m = datemonth(d);
  if (m == 12) return(sprintf("%04d-%02d",y+1,1));
  else return(sprintf("%04d-%02d",y,m+1));
}

function datelt(d1,d2) {
  split(d1,d1a,"-");
  split(d2,d2a,"-");
  y1 = d1a[1] + 0;
  m1 = d1a[2] + 0;
  d1 = d1a[3] + 0;
  y2 = d2a[1] + 0;
  m2 = d2a[2] + 0;
  d2 = d2a[3] + 0;
  if (y1 < y2) return(1);
  if (y1 > y2) return(0);
  if (m1 < m2) return(1);
  if (m1 > m2) return(0);
  if (d1 < d2) return(1);
  if (d1 > d2) return(0);
  return(0);
}

function datele(d1,d2) {
  split(d1,d1a,"-");
  split(d2,d2a,"-");
  y1 = d1a[1] + 0;
  m1 = d1a[2] + 0;
  d1 = d1a[3] + 0;
  y2 = d2a[1] + 0;
  m2 = d2a[2] + 0;
  d2 = d2a[3] + 0;
  if (y1 < y2) return(1);
  if (y1 > y2) return(0);
  if (m1 < m2) return(1);
  if (m1 > m2) return(0);
  if (d1 < d2) return(1);
  if (d1 > d2) return(0);
  return(1);
}

function html_file_css_head(title,csshtmlfile,slogan) {
  csshtmlfileseensection[csshtmlfile] = 0;
  if (substr(csshtmlfile,1,5) != "/dev/") close(csshtmlfile);
  printf("<!DOCTYPE html PUBLIC %c-//W3C//DTD XHTML 1.0 Transitional//EN%c %chttp://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd%c>\n", 34, 34, 34, 34) > csshtmlfile;
  printf("<html xmlns=%chttp://www.w3.org/1999/xhtml%c xml:lang=%cen%c lang=%cen%c>\n",
         34, 34, 34, 34, 34, 34) >> csshtmlfile;
  printf("<head>\n") >> csshtmlfile;
  printf("<meta http-equiv=%cContent-Type%c content=%ctext/html;charset=utf-8%c />\n",
         34, 34, 34, 34) >> csshtmlfile;
  printf("<link href=%c../admeasurements/admeasurements.css%c rel=%cstylesheet%c type=%ctext/css%c />\n",
         34, 34, 34, 34, 34, 34) >> csshtmlfile;
  printf("<title>%s</title>\n", title) >> csshtmlfile;
  printf("</head>\n") >> csshtmlfile;
  printf("<body>\n") >> csshtmlfile;
  printf("<div class=%cpage%c>\n", 34, 34) >> csshtmlfile;
  printf("<table border=%c0%c width=%c100%%%c>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("<tbody>\n") >> csshtmlfile;
  printf("<tr><td class=%cwglist%c>\n", 34, 34) >> csshtmlfile;
  printf("   <a href=%chttp://tools.ietf.org%c><img class=%clogo%c alt=%cIETF%c src=%chttp://tools.ietf.org/images/ietflogo3h.png%c/></a><br/><br/>\n", 34, 34, 34, 34, 34, 34, 34, 34) >> csshtmlfile;
  printf("   <div class=%cmenuitem%c><a href=%chttp://www.ietf.org/%c>IETF Home</a></div>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("   <div class=%cmenuitem%c><a href=%chttp://tools.ietf.org/about%c>About Tools</a></div>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("   <div class=%cmenuitem%c><a href=%chttp://tools.ietf.org/dailydose%c>News</a></div>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("   <div class=%cmenuitem%c><a href=%chttp://tools.ietf.org/html/%c>Documents</a></div>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("   <br/>\n") >> csshtmlfile;
  printf("   <div class=%ctopitem%c><a href=%chttp://www.arkko.com/tools/stats.html%c>Statistics</a>:</div>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("   <div class=%cmenuitem%c>&nbsp; &nbsp; <a href=%chttp://www.arkko.com/tools/docstats.html%c>Docs</a></div>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("   <div class=%cmenuitem%c>&nbsp; &nbsp; <a href=%chttp://www.iana.org/about/performance/%c>IANA</a></div>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("   <div class=%cmenuitem%c>&nbsp; &nbsp; <a href=%chttp://www.arkko.com/tools/admeasurements/%c>IESG</a></div>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("   <div class=%cmenuitem%c>&nbsp; &nbsp; <a href=%chttp://www.arkko.com/tools/lifecyclestats.html%c>Lifecycle</a></div>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("   <div class=%cmenuitem%c>&nbsp; &nbsp; <a href=%chttp://www.arkko.com/tools/rfceditorstats.html%c>RFC Ed</a></div>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("   <div class=%cmenuitem%c>&nbsp; &nbsp; <a href=%chttp://rtg.ietf.org/~fenner/iesg/%c>Misc</a></div>\n", 34, 34, 34, 34) >> csshtmlfile;
  printf("</td>\n") >> csshtmlfile;
  printf("\n") >> csshtmlfile;
  printf("<td>\n") >> csshtmlfile;
  printf("\n") >> csshtmlfile;
  printf("<div class=%cadmeasurements%c>\n", 34, 34) >> csshtmlfile;
  printf("<div class=%cadmeasurements_banner%c>\n", 34, 34) >> csshtmlfile;
  printf("<h1>%s</h1>\n", title) >> csshtmlfile;
  printf("\n") >> csshtmlfile;
  printf("<div class=%cadmeasurements_slogan%c>\n", 34, 34) >> csshtmlfile;
  printf("&#8220;%s&#8221;\n", slogan) >> csshtmlfile;
  printf("</div><!-- admeasurements_slogan -->\n") >> csshtmlfile;
  printf("\n") >> csshtmlfile;
  printf("</div><!-- admeasurements_banner -->\n") >> csshtmlfile;
  printf("</div><!-- admeasurements -->\n") >> csshtmlfile;
  printf("\n") >> csshtmlfile;
  printf("<div class=%cadmeasurements%c>\n", 34, 34) >> csshtmlfile;
  printf("\n") >> csshtmlfile;
  printf("<table class=%cadmeasurements_columns%c>\n", 34, 34) >> csshtmlfile;
  printf("<tbody>\n") >> csshtmlfile;
  printf("<tr>\n") >> csshtmlfile;
  printf("<td class=%cadmeasurements_topic%c>\n", 34, 34) >> csshtmlfile;
}

function html_file_css_section(title,csshtmlfile) {
  if (csshtmlfileseensection[csshtmlfile] != 0) {
    printf("\n<div style=%cclear:both;padding:1px;%c></div>\n", 34, 34) >> csshtmlfile;
  }

  csshtmlfileseensection[csshtmlfile] = 1;

  printf("\n<h2>%s</h2>\n\n", title) >> csshtmlfile;
}

function html_file_css_section_nextcol(title,csshtmlfile) {
  csshtmlfileseensection[csshtmlfile] = 1;
  printf("\n</td>\n\n<td class=%cadmeasurements_topic%c>\n", 34, 34) >> csshtmlfile;
  printf("\n<h2>%s</h2>\n\n", title) >> csshtmlfile;
}

function html_file_css_end(htmlfile) {
  printf("</td>\n") >> htmlfile;
  printf("</tr>\n") >> htmlfile;
  printf("</tbody>\n") >> htmlfile;
  printf("</table><!-- admeasurements_columns -->\n") >> htmlfile;
  printf("</div><!-- admeasurements -->\n") >> htmlfile;
  printf("</td>\n") >> htmlfile;
  printf("</tr>\n") >> htmlfile;
  printf("</tbody>\n") >> htmlfile;
  printf("</table><!-- page -->\n") >> htmlfile;
  printf("</div><!-- page -->\n") >> htmlfile;
  printf("</body>\n") >> htmlfile;
  printf("</html>\n") >> htmlfile;
  close(htmlfile);
}

function plottojpg(file,epsfile,jpgfile) {
  system("gnuplot < " file);
  cmd = "convert " epsfile " " jpgfile;
  system(cmd);
}
'

###
### FUNCTION: Limiting to the specified AD and document
###

function limitlifecycletiming() {

  if [ x"$AD" = xall ]
  then
    adislimited=0
  else
    adislimited=1
  fi
  
  cut -f3 -d: adwork.txt | grep -e "$AD" | sort -u > focus-ads.txt
  cat draft-timing.txt draft-timing-nonad.txt | fgrep -v '#' | cut -f2 -d: | grep -e "$DOC" | sort -u > focus-docs.txt

  cat draft-timing.txt draft-timing-nonad.txt |
  awk -v quick=$QUICK -v preffamily=$PREFFAMILY -v adislimited=$adislimited '

'"$COMMONFUNCS"'

function readfocusads() {
  while ((getline ad < "focus-ads.txt") == 1) {
    focusad[ad] = ad;
  }
}

function readfocusdocs() {
  while ((getline doc < "focus-docs.txt") == 1) {
    if (doc != "") {
      focusdoc[doc] = doc;
    }
  }
}

function readaddocs() {
  while ((getline addocline < "adwork.txt") == 1) {
    split(addocline,comps,":");
    ad = comps[3];
    doc = comps[6];
    if (comps[5] == "") {
      state = comps[4];
    } else {
      state = comps[4] ":" comps[5];
    }
    docad[doc] = ad;
    docstate[doc] = state;
  }
}

BEGIN {
  FS = ":";
  readfocusads();
  readfocusdocs();
  readaddocs();
  printf("# - field 21: current state\n");
  printf("# - field 22: pages\n");
  printf("# - field 23: bytes\n");
  printf("# - field 24: publication speed in millibits/s\n");
}

/^#.*/ {
  printf("%s\n", $0);
  next;
}

/^timing:.*/ {
  draft = $2;
  iesgdays = $9;
  wgdays = $12;
  inddays = $15;
  rfceddays = $17;
  pubdate = $16;
  if (!draft in focusdoc) {
    printf("# %s not in focus\n", draft);
    next;
  } else if (!docad[draft] in focusad &&
             !adislimited) {
    printf("# %s AD %s not focus (%d)\n", draft, docad[draft], adislimited);
    next;
  } else {
    if (quick) {
      revs = 1;
    } else {
      system("docrevs --prefer-family=" preffamily " " draft " > /tmp/ietfmeasurements.revs");
      getline revs < "/tmp/ietfmeasurements.revs";
      close("/tmp/ietfmeasurements.revs");
    }
    indandwgdays = wgdays+inddays;
    totaldays = iesgdays+wgdays+inddays+rfceddays;
    ds = docstate[draft];
    if (ds == "") ds = "ID Exists";
    gsub(/:/," - ",ds);
    if (quick) {
      pages = bytes = 1;
    } else {
      system("docpages --version last --prefer-family=" preffamily " " draft " | tee /tmp/" draft ".docpages.out > /tmp/ietfmeasurements.pages");
      getline pages < "/tmp/ietfmeasurements.pages";
      gsub(/^.*:/,"",pages);
      close("/tmp/ietfmeasurements.pages");
      system("docbytes --version last --prefer-family=" preffamily " " draft " > /tmp/ietfmeasurements.bytes");
      getline bytes < "/tmp/ietfmeasurements.bytes";
      gsub(/^.*:/,"",bytes);
      close("/tmp/ietfmeasurements.bytes");
    }
    mibitspersec = (1000.0 * 8.0 * bytes) / (totaldays * 86400.0);
    printf("%s:%d:%d:%d:%.2f:%s:%d:%d:%.2f\n",
            $0,
            indandwgdays, totaldays, revs, (totaldays * 1.0) / revs,
            ds,
            pages,
            bytes,
            mibitspersec);
    next;
  }
}

END {
}
' > draft-timing-limited-all.txt

  fgrep ':RFC Published' draft-timing-limited-all.txt > draft-timing-limited.txt

  awk '
'"$COMMONFUNCS"'

BEGIN {
  FS = ":";
}

/^timing/ {
  if (draftnameisofficial($2)) printf("%s\n", $0);
}

' < draft-timing-limited.txt > draft-timing-limited-wg.txt

  awk '
'"$COMMONFUNCS"'

BEGIN {
  FS = ":";
}

/^timing/ {
  if (!draftnameisofficial($2)) printf("%s\n", $0);
}

' < draft-timing-limited.txt > draft-timing-limited-ind.txt

}

###
### FUNCTION: Main stats datafile
###

function datalifecycletiming() {

  inputfile=$1;
  outputfile=$2;
  distributionoutputfile=$3;

  awk -v distributionoutputfile=$distributionoutputfile '

'"$COMMONFUNCS"'

function distribute(days) {
  if (days < distrstep) val = 0;
  else if (days >= distrmax) val = distrmax/distrstep;
  else val = days / distrstep;
  val = sprintf("%d",val) + 0;
  #printf("distribute(%d)=%d/%f/%s\n", days, val, val, val) >> "/tmp/distrdebug.txt";
  return(val);
}

BEGIN {
  FS = ":";
  firstdate = "none";
  lastdate = "none";
  distrmax = 600;
  distrstep = 30;
  for (i = 0; i < distrmax/distrstep; i++) {
    distributionind[i] = 0;
    distributionwg[i] = 0;
    distributioniesg[i] = 0;
    distributionrfced[i] = 0;
  }
}

/^#.*/ {
  next;
}

/^timing:.*/ {
  draft = $2;
  approv = $3;
  pubreq = $4;
  addays = $5;
  prevaddays = $6;
  authordays = $7;
  otherdays = $8;
  iesgdays = $9;
  versions = $10;
  individualsubmission = $11;
  individualdays = $12;
  individualdraft = $13;
  wgsubmission = $14;
  wgdays = $15;
  rfcpub = $16;
  rfceddays = $17;

  if (individualdraft == "" && individualdays == 0) {
    if (!draftnameisofficial(draft)) {
      individualdays = wgdays;
      wgdays = 0;
    }
  }

  if (firstdate == "none" || datelt(rfcpub,firstdate)) {
    printf("debug: setting firstdate to %s due to document %s\n", rfcpub, draft) >> "debugs.txt";
    firstdate = rfcpub;
  }
  if (lastdate == "none" || datelt(lastdate,rfcpub)) {
    printf("debug: setting lastdate to %s due to document %s\n", rfcpub, draft) >> "debugs.txt";
    lastdate = rfcpub;
  }
  
  ym = dateyearmonth(rfcpub);
  printf("debug: adding entry to %s for %s\n", ym, draft) >> "debugs.txt";
  if (entrycount[ym] == "") {
    entrycount[ym] = 0;
    entryindividualcount[ym] = 0;
    entryindividualdays[ym] = 0;
    entrywgcount[ym] = 0;
    entrywgdays[ym] = 0;
    entryiesgdays[ym] = 0;
    entryrfceddays[ym] = 0;
  }
  
  entrycount[ym]++;
  if (individualdays > 0) {
    entryindividualcount[ym]++;
    entryindividualdays[ym] += individualdays;
    distributionind[distribute(individualdays)]++;
  }
  if (wgdays > 0) {
    entrywgcount[ym]++;
    entrywgdays[ym] += wgdays;
    distributionwg[distribute(wgdays)]++;
  }
  entryiesgdays[ym] += iesgdays;
  distributioniesg[distribute(iesgdays)]++;
  entryrfceddays[ym] += rfceddays;
  distributionrfced[distribute(rfceddays)]++;

  next;
}

END {
  printf("debug: firstdate = %s (%s)\n", firstdate, dateyearmonth(firstdate)) >> "debugs.txt";
  printf("debug: lastdate = %s (%s)\n", lastdate, dateyearmonth(lastdate)) >> "debugs.txt";
  for (ym = quarter(dateyearmonth(firstdate)); datele(ym,dateyearmonth(lastdate)); ym = addmonth(addmonth(addmonth(ym)))) {
    printf("debug: ym = %s\n", ym) >> "debugs.txt";
    qcount = entrycount[ym] + entrycount[ym+1] + entrycount[ym+2];
    qrfceddays = entryrfceddays[ym] + entryrfceddays[ym+1] + entryrfceddays[ym+2];
    qiesgdays = entryiesgdays[ym] + entryiesgdays[ym+1] + entryiesgdays[ym+2];
    qwgdays = entrywgdays[ym] + entrywgdays[ym+1] + entrywgdays[ym+2];
    if (entrycount[ym] > 0) {
      printf("%6.2f\t%6.2f\t%6.2f\t%6.2f # N=%d\n",
             quarternumeric(ym),
             (qrfceddays + 0.0) / qcount,
             (qiesgdays + 0.0) / qcount,
             (qwgdays + 0.0) / qcount,
             qcount);
    }
  }

  printf("#\n") > distributionoutputfile;
  for (i = 0; i < distrmax/distrstep; i++) {
    printf("%c%3d to %3s days%c\t%d\t%d\t%d\t%d\t%d\n",
           34,
           i * distrstep,
           (i + 1) * distrstep >= distrmax ? "inf" : ((i + 1) * distrstep - 1),
           34, 
	   distributionind[i],
	   distributionwg[i],
	   distributionind[i] + distributionwg[i],
	   distributioniesg[i],
	   distributionrfced[i]) >> distributionoutputfile;
  }
}
' < $inputfile > $outputfile
 
}

###
### FUNCTION: Make a report out of extreme cases
###

function reportextremes() {

  title="$1";
  unit="$2";

  awk -v title="$title" \
      -v unit="$unit" '

'"$COMMONFUNCS"'

BEGIN {
  FS = ":";
  html_file_css_head(title,"/dev/stdout","End-to-End Delay from draft-smith to an RFC");
  overrideunit = "";
}

/^ietfmeasurements:header:.*/ {
  html_file_css_section($3,"/dev/stdout");
  fieldindex = $4;
  overrideunit = $5;
  next;
}

/^ietfmeasurements:header_nextcol:.*/ {
  html_file_css_section_nextcol($3,"/dev/stdout");
  fieldindex = $4;
  overrideunit = $5;
  next;
}

/^ietfmeasurements:subheader:.*/ {
  printf("<p>%s:</p>\n", $3);
  next;
}

/^timing:.*/ {
  draft = $2;
  split($0,comps,":");
  unittouse = (overrideunit == "" ? unit : overrideunit);
  if (unittouse ~ /millibit/) {
    printf("<li><a href=%c%s-timing.html%c>%s</a> (%.2f %s)</li>\n",
           34, draft, 34, draft, comps[fieldindex],
           unittouse); 
  } else {
    printf("<li><a href=%c%s-timing.html%c>%s</a> (%d %s)</li>\n",
           34, draft, 34, draft, comps[fieldindex],
           unittouse); 
  }
}

END {
  html_file_css_end("/dev/stdout");
}
'

}

###
### FUNCTION: Make a report from a datafile
###


function reporttiming() {

  datafile=$1.dat;
  cmdfilebase=$1;
  epsfilebase=$1;
  jpgfilebase=$1;
  htmlfile=$2;
  startq=`head -1 $datafile|cut -f1`
  endq=`tail -1 $datafile|cut -f1`
  
  awk -v datafile=$datafile \
      -v cmdfilebase=$cmdfilebase \
      -v epsfilebase=$epsfilebase \
      -v jpgfilebase=$jpgfilebase \
      -v curyear=$CURYEAR -v curmonth=$CURMON -v curday=$CURDAY \
      -v startq=$startq -v endq=$endq '

'"$COMMONFUNCS"'

function fracpart(x) {
  integernum = sprintf("%d",x) + 0;
  return(x - integernum);
}

function developmentgraph(title,fieldno) {
  if (fieldno == 0) {
    cmdfile = cmdfilebase ".cmd";
    epsfile = epsfilebase ".eps";
    jpgfile = jpgfilebase ".jpg";
  } else {
    cmdfile = cmdfilebase "-" fieldno ".cmd";
    epsfile = epsfilebase "-" fieldno ".eps";
    jpgfile = jpgfilebase "-" fieldno ".jpg";
  }
  html_file_css_section(title,"/dev/stdout");
  printf("set output %c%s%c\n", 34, epsfile, 34) > cmdfile;
  printf("set terminal postscript eps enhanced %cTimes-Roman%c 24 color solid\n", 34, 34, 34, 34) >> cmdfile;
  printf("set style data histogram\n") >> cmdfile;
  printf("set style histogram rowstacked\n") >> cmdfile;
  printf("set title %cWG Document Process%c\n", 34, 34) >> cmdfile;
  printf("set xlabel %cQuarter%c\n", 34, 34) >> cmdfile;
  printf("set ylabel %cDays%c\n", 34, 34) >> cmdfile;
  printf("set style fill solid .75 border -1\n") >> cmdfile;
  printf("set boxwidth 0.8 relative\n") >> cmdfile;
  printf("set size 2.5,2.1\n") >> cmdfile;
  printf("set pointsize 2\n") >> cmdfile;
  printf("set view ,,2\n") >> cmdfile;
  printf("set xtics (") >> cmdfile;
  first = 1;
  for (q = startq; q <= endq; q += 0.25) {
    if (first) first = 0; else printf(", ") >> cmdfile;
    printf("%cQ%d\\n%4d%c %6.2f", 34, (fracpart(q) * 4) + 1, q, 34, (q - startq) / 0.25) >> cmdfile;
  }
  printf(")\n") >> cmdfile;
  if (fieldno == 0) {
    printf("plot %c%s%c using 4 t %cWG and individual time%c, %c%c using 3 t %cIESG time%c, %c%c using 2 t %cRFC Editor time%c\n", 34, datafile, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34) >> cmdfile;
  } else if (fieldno == 1) {
    printf("plot %c%s%c using 4 t %cWG and individual time%c\n", 34, datafile, 34, 34, 34) >> cmdfile;
  } else if (fieldno == 2) {
    printf("plot %c%s%c using 3 t %cIESG time%c\n", 34, datafile, 34, 34, 34) >> cmdfile;
  } else if (fieldno == 3) {
    printf("plot %c%s%c using 2 t %cRFC Editor time%c\n", 34, datafile, 34, 34, 34) >> cmdfile;
  }
  close(cmdfile);
  plottojpg(cmdfile,epsfile,jpgfile);
  printf("<img src=%c%s%c alt=%cdevelopment%c>\n", 34, jpgfile, 34, 34, 34);
}

BEGIN {
  FS = ":";

  disclaimer = "<p>";
  disclaimer = disclaimer sprintf("Statistics run taken %d.%d.%d by the ", curday, curmonth, curyear);
  disclaimer = disclaimer sprintf("<a href=%chttp://www.arkko.com/tools/ietfmeasurements.html%c>ietfmeasurements</a>", 34, 34);
  disclaimer = disclaimer " tool.";
  disclaimer = disclaimer "</p>\n";
  disclaimer = disclaimer "<p>\n";
  disclaimer = disclaimer "Please be aware that this tool measures only what has been entered to the tracker.";
  disclaimer = disclaimer " This often does not correspond to reality.";
  disclaimer = disclaimer "</p>\n";

  html_file_css_head("Lifecycle Analysis of WG RFC Production","/dev/stdout","End-to-End Delay from draft-smith to an RFC");
  developmentgraph("Development over the Years",0);
  developmentgraph("Development over the Years, WG Phase Only",1);
  developmentgraph("Development over the Years, IESG Phase Only",2);
  developmentgraph("Development over the Years, RFC Editor Phase Only",3);
}

END {
  html_file_css_section("Data Freshness and Reliability","/dev/stdout");
  printf("%s\n", disclaimer);
  html_file_css_end("/dev/stdout");
}

' < /dev/null > $htmlfile

}

###
### FUNCTION: List lifecycles for a set of given drafts
###

function listdraftlifecycles() {

  heading="$1";
  inputfile=$2;
  outputfile=$3;

  sort < $inputfile |
  awk -v heading="$heading" '

'"$COMMONFUNCS"'

BEGIN {
  FS = ":";
  html_file_css_head(heading,"/dev/stdout","End-to-End Delay from draft-smith to an RFC");
  currentheading = "";
}

/^timing:/ {
  draft = $2;
  if (draft ~ /^draft-ietf-/) {
    split(draft,comps,"-");
    thisheading = "ietf-" comps[3];
  } else {
    thisheading = substr(draft,7,1);
  }
  if (currentheading != thisheading) {
    currentheading = thisheading;
    if (currentheading == "ietf-mobike") {
      html_file_css_section_nextcol(thisheading,"/dev/stdout");
    } else {
      html_file_css_section(thisheading,"/dev/stdout");
    }
  }
  printf("<li><a href=%c%s-timing.html%c>%s</a></li>\n", 34, draft, 34, draft);
}

END {
  html_file_css_end("/dev/stdout");
}

' > $outputfile

}

###
### FUNCTION: Make a report from a distribution datafile
###


function reporttimingdistribution() {

  datafile=$1.dat;
  cmdfile=$1.cmd;
  epsfile=$1.eps;
  jpgfile=$1.jpg;
  htmlfile=$2;

  awk -v datafile=$datafile \
      -v cmdfile=$cmdfile \
      -v epsfile=$epsfile \
      -v jpgfile=$jpgfile \
      -v curyear=$CURYEAR -v curmonth=$CURMON -v curday=$CURDAY '

'"$COMMONFUNCS"'

function distributiongraph(title,fieldindex) {
  html_file_css_section(title " Processing Time Distribution","/dev/stdout");
  printf("<p>The following graph shows the distribution of processing time for categories\n");
  printf("of 0 to 30 days, 30 to 60 days, ... and 570 to infinity days.\n");
  printf("All WG documents published as RFCs are included in the count, if they were processed by one of the current IESG members.</p>\n");
  cmdfilereal = fieldindex "-" cmdfile;
  printf("set output %c%s%c\n", 34, fieldindex "-" epsfile, 34) > cmdfilereal;
  printf("set terminal postscript eps enhanced color solid\n", 34, 34) >> cmdfilereal;
  printf("set style data boxes\n") >> cmdfilereal;
  printf("set title %c%s Time%c\n", 34, title, 34) >> cmdfilereal;
  printf("set xlabel %cDays%c\n", 34, 34) >> cmdfilereal;
  printf("set ylabel %cNumber of documents%c\n", 34, 34) >> cmdfilereal;
  printf("set style fill solid .75 noborder\n") >> cmdfilereal;
  printf("set boxwidth 0.8 relative\n") >> cmdfilereal;
  printf("set size 2.3,1.7\n") >> cmdfilereal;
  printf("set pointsize 2\n") >> cmdfilereal;
  printf("set view ,,2\n") >> cmdfilereal;
  printf("set xtics (%c0-30%c 0, %c270-300%c 10, %c570-%c 20)\n",
         34, 34, 34, 34, 34, 34) >> cmdfilereal;
  printf("plot %c%s%c using %d notitle\n", 34, datafile, 34, fieldindex) >> cmdfilereal;
  close(cmdfilereal);
  plottojpg(cmdfilereal,fieldindex "-" epsfile,fieldindex "-" jpgfile);
  printf("<img src=%c%s%c alt=%cdistribution%c>\n", 34, fieldindex "-" jpgfile, 34, 34, 34);
}

BEGIN {
  FS = ":";

  disclaimer = "<p>";
  disclaimer = disclaimer sprintf("Statistics run taken %d.%d.%d by the ", curday, curmonth, curyear);
  disclaimer = disclaimer sprintf("<a href=%chttp://www.arkko.com/tools/ietfmeasurements.html%c>ietfmeasurements</a>", 34, 34);
  disclaimer = disclaimer " tool.";
  disclaimer = disclaimer "</p>\n";
  disclaimer = disclaimer "<p>\n";
  disclaimer = disclaimer "Please be aware that this tool measures only what has been entered to the tracker.";
  disclaimer = disclaimer " This often does not correspond to reality.";
  disclaimer = disclaimer "</p>\n";

  html_file_css_head("Distribution of Processing Times","/dev/stdout","End-to-End Delay from draft-smith to an RFC");
  distributiongraph("WG and individual",4);
  distributiongraph("IESG",5);
  distributiongraph("RFC Editor",6);
}

END {
  html_file_css_section("Data Freshness and Reliability","/dev/stdout");
  printf("%s\n", disclaimer);
  html_file_css_end("/dev/stdout");
}

' < /dev/null > $htmlfile

}

###
### Make the main report
###

if [ $RUNCALC = 1 ]
then

  echo Calculating lifecycle limits...
  limitlifecycletiming
  echo Calculating lifecycle times...
  datalifecycletiming draft-timing-limited-wg.txt timing-development-wg.dat timing-development-wg-distribution.dat

fi

echo Reporting timing development...
reporttiming timing-development-wg wgdocs.html
echo Reporting timing distribution...
reporttimingdistribution timing-development-wg-distribution wgdistr.html

###
### Make the report of shortest and longest process times
###

echo Sorting results...
sort -n --key=9,9 --field-separator=: < draft-timing-limited-wg.txt > draft-timing-limited-wg-sortbyiesg.txt
sort -n --key=15,15 --field-separator=: < draft-timing-limited-wg.txt > draft-timing-limited-wg-sortbywg.txt
sort -n --key=17,17 --field-separator=: < draft-timing-limited-wg.txt > draft-timing-limited-wg-sortbyrfced.txt
sort -n --key=18,18 --field-separator=: < draft-timing-limited-wg.txt > draft-timing-limited-wg-sortbyindwg.txt
sort -n --key=19,19 --field-separator=: < draft-timing-limited-wg.txt > draft-timing-limited-wg-sortbyoverall.txt
sort -n --key=23,23 --field-separator=: < draft-timing-limited.txt > draft-timing-limited-sortbypages.txt
sort -n --key=24,24 --field-separator=: < draft-timing-limited.txt > draft-timing-limited-sortbybytes.txt
sort -n --key=25,25 --field-separator=: < draft-timing-limited.txt > draft-timing-limited-sortbybitspeed.txt

awk '
BEGIN { FS = ":"; }
/^timing:.*/ { if ($13 != "") printf("%s\n", $0); }
' < draft-timing-limited-wg.txt |
sort -n --key=12,12 --field-separator=: > draft-timing-limited-wg-sortbyind.txt

awk '
'"$COMMONFUNCS"'
BEGIN {
  FS = ":";
}
/^timing:.*/ {
  if ($13 == "") printf("%s\n", $0);
}
' < draft-timing-limited-ind.txt |
sort -n --key=18,18 --field-separator=: > draft-timing-limited-ind-sortbyindad.txt

sort -n --key=19,19 --field-separator=: < draft-timing-limited-ind.txt > draft-timing-limited-ind-sortbyoverall.txt

echo Finding extreme results...
(
 echo 'ietfmeasurements:header:Overall Process Time (WG Drafts):19'
 echo 'ietfmeasurements:subheader:Shortest'
 head -25 draft-timing-limited-wg-sortbyoverall.txt;
 echo 'ietfmeasurements:subheader:Longest'
 tail -25 draft-timing-limited-wg-sortbyoverall.txt;

 echo 'ietfmeasurements:header:Overall Process Time (Individual Drafts):19'
 echo 'ietfmeasurements:subheader:Shortest'
 head -25 draft-timing-limited-ind-sortbyoverall.txt;
 echo 'ietfmeasurements:subheader:Longest'
 tail -25 draft-timing-limited-ind-sortbyoverall.txt;

 echo 'ietfmeasurements:header:Individual and WG Process Time:18'
 echo 'ietfmeasurements:subheader:Shortest'
 head -25 draft-timing-limited-wg-sortbyindwg.txt;
 echo 'ietfmeasurements:subheader:Longest'
 tail -25 draft-timing-limited-wg-sortbyindwg.txt;

 echo 'ietfmeasurements:header:Wait Before WG Adoption (WG Drafts):12'
 echo 'ietfmeasurements:subheader:Shortest'
 head -25 draft-timing-limited-wg-sortbyind.txt;
 echo 'ietfmeasurements:subheader:Longest'
 tail -25 draft-timing-limited-wg-sortbyind.txt;

 echo 'ietfmeasurements:header_nextcol:Wait Before AD Adoption (Individual Submissions):18'
 echo 'ietfmeasurements:subheader:Shortest'
 head -25 draft-timing-limited-ind-sortbyindad.txt;
 echo 'ietfmeasurements:subheader:Longest'
 tail -25 draft-timing-limited-ind-sortbyindad.txt;

 echo 'ietfmeasurements:header:WG Process Time:15'
 echo 'ietfmeasurements:subheader:Shortest'
 head -25 draft-timing-limited-wg-sortbywg.txt;
 echo 'ietfmeasurements:subheader:Longest'
 tail -25 draft-timing-limited-wg-sortbywg.txt

 echo 'ietfmeasurements:header:IESG Process Time:9'
 echo 'ietfmeasurements:subheader:Shortest'
 head -25 draft-timing-limited-wg-sortbyiesg.txt;
 echo 'ietfmeasurements:subheader:Longest'
 tail -25 draft-timing-limited-wg-sortbyiesg.txt

 echo 'ietfmeasurements:header:RFC Editor Process Time:17'
 echo 'ietfmeasurements:subheader:Shortest'
 head -25 draft-timing-limited-wg-sortbyrfced.txt;
 echo 'ietfmeasurements:subheader:Longest'
 tail -25 draft-timing-limited-wg-sortbyrfced.txt

) |
reportextremes "Shortest and  Processing Times" "days" > extremes.html

(
 echo 'ietfmeasurements:header:Number of pages (All Drafts):23:pages'
 echo 'ietfmeasurements:subheader:Shortest'
 head -50 draft-timing-limited-sortbypages.txt;
 echo 'ietfmeasurements:subheader:Longest'
 tail -50 draft-timing-limited-sortbypages.txt;

 echo 'ietfmeasurements:header:Number of bytes (All Drafts):24:bytes'
 echo 'ietfmeasurements:subheader:Shortest'
 head -50 draft-timing-limited-sortbybytes.txt;
 echo 'ietfmeasurements:subheader:Longest'
 tail -50 draft-timing-limited-sortbybytes.txt;

 echo 'ietfmeasurements:header_nextcol:Publication speed in millibits/s (All Drafts):25:millibits/s'
 echo 'ietfmeasurements:subheader:Slowest'
 head -150 draft-timing-limited-sortbybitspeed.txt;
 echo 'ietfmeasurements:subheader:Fastest'
 tail -150 draft-timing-limited-sortbybitspeed.txt;

) |
reportextremes "Document Size and Processing Times" "none" > sizematters.html

sort -n --key=10,10 --field-separator=: < draft-timing-limited-wg.txt > draft-timing-limited-wg-sortbyiesgrevs.txt
sort -n --key=20,20 --field-separator=: < draft-timing-limited-wg.txt > draft-timing-limited-wg-sortbyrevs.txt

awk '
BEGIN { FS = ":"; }
/^timing:.*/ { if ($20 > 1) printf("%s\n", $0); }
' < draft-timing-limited-wg.txt |
(LC_ALL=C; export LC_ALL; sort -g --key=21,21 --field-separator=:) > draft-timing-limited-wg-sortbyrevperiod.txt

(
 echo 'ietfmeasurements:header:Document Revisions in IESG Phase:10'
 echo 'ietfmeasurements:subheader:Most'
 tail -30 draft-timing-limited-wg-sortbyiesgrevs.txt;

 echo 'ietfmeasurements:header:Document Revisions:20'
 echo 'ietfmeasurements:subheader:Least'
 head -30 draft-timing-limited-wg-sortbyrevs.txt;
 echo 'ietfmeasurements:subheader:Most'
 tail -30 draft-timing-limited-wg-sortbyrevs.txt;

 echo 'ietfmeasurements:header_nextcol:Average Time Between Revisions:21:days'
 echo 'ietfmeasurements:subheader:Shortest'
 head -40 draft-timing-limited-wg-sortbyrevperiod.txt;
 echo 'ietfmeasurements:subheader:Longest'
 tail -40 draft-timing-limited-wg-sortbyrevperiod.txt;
) |
reportextremes "Exceptional Drafts" "revisions" > exceptional.html

###
### Make the listing of all published RFCs, drafts and their lifecycles
###

echo Making lists of drafts...
listdraftlifecycles "Lifecycle Analysis of Published RFCs" draft-timing-limited.txt rfcs.html
listdraftlifecycles "Lifecycle Analysis of All Drafts" draft-timing-limited-all.txt drafts.html

###
### Copy
###

echo Copying results...
if [ $COPY = 1 ]
then
  if [ $QUIET = 0 ]
  then
   SCPOPTS=
  else
   SCPOPTS=-q
  fi
  scp $SCPOPTS \
      /home/jar/LME/IETF/Tools/lifecyclestats.html \
      jarkko@users.piuha.net:public_html/tools/
  scp $SCPOPTS \
      draft-timing-limited-all.txt \
      jarkko@users.piuha.net:public_html/tools/draft-timing-withspeeds.txt
  scp $SCPOPTS \
      drafts.html \
      rfcs.html \
      wgdocs.html \
      wgdistr.html \
      extremes.html \
      sizematters.html \
      exceptional.html \
      *.jpg \
      jarkko@users.piuha.net:public_html/tools/lifecycle

fi

###
### Cleanup
###

exit 0