#!/usr/local/bin/perl -w
################################################################################
#        file: mk-logs.pl
#      writer: Dan Tsafrir and Dror Feitelson
# description: generates
#              http://www.cs.huji.ac.il/labs/parallel/workload/logs.html
#	       based on data in LOGS
#
#       input: the LOGS file is divided into stanzas, one for each log.
#	       each one contains the following in this order:
#	       LOG    <log name>
#	       DIR    <directory with all log-related files>
#	       NOTES  <filename>  [default: index.html]
#	       ORIG   <filename>+ [suggested: machine-year-0.gz]
#	       SWF    <filename>+ [suggested: machine-year-n.swf.gz]
#	       CLEAN  <filename>+ [suggested: machine-year-n.m-cln.swf.gz]
#	       OLD    <filename>+ [additional versions with lower n]
#	       CHANGE <date> <filename>
#		      <changelog entry>
#		      [can be multiline, terminated by the next CHANGE or by]
#	       END
#              there is also a GENCHANGE option for general changes
#	       comments are introduced by #
################################################################################


################################################################################
# use
################################################################################
use strict;
use warnings;
use lib '/cs/share/lib/perl';
use Time::ParseDate;


################################################################################
# config
################################################################################
my $g_grep_com   = 'zgrep';
my $g_no_data    = "&nbsp;";	# what to print in empty table cells (html)
my $g_icon_file  = "icons/save-icon14a.jpeg";
my $g_icon_notes = "icons/info.gif";
my $g_icon_v     = "icons/v1b.jpeg";
my $g_symb_v     = '&#8730';
my $g_font_size_table = "-1";	# size of font of content of table
my $g_font_size_fsize = "-2";	# size of font of files' size
my $g_with_old   = 0;		# include old versions in table?

# return fd to SWF file
sub swf2fd($) {
    my $swf = shift;
    my $fd;

    open($fd,"zcat <$swf |") or die "failed opening `$swf': $!\n";
    return $fd;
}


################################################################################
# read data
################################################################################


open(L, "<LOGS");
open(U, ">logs-list");

my @g_logs;
my %g_log2inf;
my %g_merge;
my @g_change;

#-------------------------------------------------------------------------------
# read a stanza from the LOGS file and put the data in a structure
#-------------------------------------------------------------------------------
sub get_next_log() {

    my $logname="";
    my %log;
    $log{dir} = $log{orig} = $log{swf} = $log{clean} = $log{old} = "";
    $log{doload} = 1;
    $log{notes} = "index.html";

    my $in_change  =0;
    my $change_log ="";
    my $change_text="";

  READLOOP:
    while (<L>) {

	# strip comments and spaces
	if (/^([^#]*)#/) {
	    $_ = $1;
	}
	if (/^\s*(.*[^\s])\s*$/) {
	    $_ = $1;
	}
	if (/^\s*$/) {
	    next;
	}

	if (/^LOG\s+(.+)$/) {
	    $logname = $1;
	    push(@g_logs, $logname);
	    $log{name} = $logname;
	}

	elsif (/^DIR\s+(.+)$/) {
	    $log{dir} = $1;
	}

	elsif (/^NOTES\s+(.+)$/) {
	    $log{notes} = $1;
	}

	elsif (/^ORIG\s+(.+)$/) {
	    $log{orig} = $1;
	}

	elsif (/^SWF\s+(.+)$/) {
	    $log{swf} = $1;
	}

	elsif (/^CLEAN\s+(.+)$/) {
	    $log{clean} = $1;
	}

	elsif (/^OLD\s+(.+)$/) {
	    $log{old} = $1;
	}

	elsif (/^NOLOAD$/) {
	    $log{doload} = 0;
	}

	elsif (/(CHANGE|END)/) {

	    if ($in_change) {
		# first need to store previous change
		my %change;
		$change{log}  = $change_log;
		$change{text} = $change_text;
		$change{date} = $in_change;
		push(@g_change, (\%change));
		$change_text = "";
	    }

	    if (/^CHANGE\s+([\d\/]+)\s+(.*)$/) {
	        $in_change  = $1;
	        $change_log = $2;
	    }

	    if (/^GENCHANGE\s+([\d\/]+)$/) {
	        $in_change  = $1;
	        $change_log = "";

		if ($logname eq "") {
		    $logname = "dummy";
		}
	    }

	    elsif (/^END$/) {
		$in_change = 0;
		if ($logname ne "dummy") {
		    $g_log2inf{$logname} = \%log;
		}
		last READLOOP;
	    }

	}

	else {
	    if ($in_change) {
		$change_text .= $_;
	    }
	    else {
	        warn ">>> what is this? $_";
	    }
	}
    }

    $g_merge{ $log{dir} }{ $log{notes} }++;
    $g_merge{ $log{dir} }{ $log{orig} }++;

    if ($log{clean} ne "") {
	print U "http://www.cs.huji.ac.il/labs/parallel/workload/$log{dir}/$log{clean}\n";
    }
    elsif ($log{swf} ne "") {
	print U "http://www.cs.huji.ac.il/labs/parallel/workload/$log{dir}/$log{swf}\n";
    }

    return( $logname ne "" );
}

while (get_next_log()) {}



# SWF column name to associated index
my %g_swf_col2index = 
    (
     'job'        =>  1,	# a counter field, starting from 1
     'submit'     =>  2,	# seconds. submittal time
     'wait'       =>  3,	# seconds. diff between submit and begin to run
     'runtime'    =>  4,	# seconds. end-time minus start-time
     'proc_alloc' =>  5,	# number of allocated processors
     'cpu_used'   =>  6,	# seconds. user+system. avg over procs
     'mem_used'   =>  7,	# KB. avg over procs.
     'proc_req'   =>  8,	# requested number of processors
     'user_est'   =>  9,	# seconds. user runtime estimation
     'mem_req'    => 10,	# KB. avg over procs.
     'status'  	  => 11,	# 0=fail; 1=completed; 5=canceled
     'uid'        => 12,	# user id
     'gid'        => 13,	# group id
     'exe_num'    => 14,	# [1,2..n] n = app# appearing in log
     'q_num'	  => 15,	# [1,2..n] n = queue# in the system
     'partition'  => 16,	# [1,2..n] n = partition# in the systems
     'prev_jobs'  => 17,	# cur job will start only after ...
     'think_time' => 18,	# seconds should elapse between termination of 
				# preceding job and submittal of this one
     );



################################################################################
# implementation
################################################################################


#-------------------------------------------------------------------------------
# Given a $log, return
#  ($month_diff, $begin, $end)
# where:
#   $month_diff = duration of log in months
#        $begin = "$month $year" of beginning of $log
#          $end = "$month $year" of end of $log
#-------------------------------------------------------------------------------
sub log2period($)
{
    my ($log) = @_;
    my $swf   = "$g_log2inf{$log}->{dir}/$g_log2inf{$log}->{swf}";
    my %res;

    foreach my $field (qw(StartTime EndTime)) {

	my $line = `$g_grep_com -w $field $swf` 
	    or die "couldn't find $field line in $swf";

	# parsing something like:
	#    ; Fri Oct  1 00:00:03 PST 1993
	my ($wday, $month, $mday, $time, $tz, $year) = 
	$line =~
          m/;\s*$field:\s*(\w+)\s+(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+([\w+-]+)\s+(\d+)/

	    or die "failed parsing '$field' line in $swf";

	$wday  = substr($wday , 0, 3);	# first 3 chars of weekday
	$month = substr($month, 0, 3);	# first 3 chars of month
	$year += 1900 if $year < 1900;	# transform 96 to 1996

	# write it in a format readable by Time::Local::parsedate()
	my $sec = parsedate("$wday $month $mday $time $year");
	$res{$field} = {
	    'sec' => $sec,
	    'str' => "$month $year",
	};
    }

    my $month_diff = 
	($res{EndTime}{sec} - $res{StartTime}{sec}) / (3600*24*(365/12));
    $month_diff = int($month_diff +0.5);
    return ($month_diff, $res{StartTime}{str}, $res{EndTime}{str});
}


#-------------------------------------------------------------------------------
# given a name of an $swf file, return ref2hash with these keys: 
# 1) load  (percentage between 0-100)
# 2) users (number)
# 3) jobs  (number)
# 4) cpus  (number)
# 5) uest  (value indicating whether at least one user's runtime estimate <> -1)
# 6) mem   (value indicating whether at least one user's memory           <> -1)
#-------------------------------------------------------------------------------
sub statistics($)
{
    my $swf = shift;
    if( ! -f $swf ) {
	return {
	    'load'  => $g_no_data,
	    'users' => $g_no_data,
	    'jobs'  => $g_no_data,
	    'cpus'  => $g_no_data,
	    'uest'  => $g_no_data,
	    'mem'   => $g_no_data,
	};
    }

    my $fd        = swf2fd($swf);
    my %users     = ();
    my %res       = ();
    my $jobs      = 0;
    my $cpus      = -1;
    my $have_ue   = 0;
    my $have_mem  = 0;
    my $time_fst  = -1;
    my $time_last = -1;
    my $work      = 0;

    my $idx_sbm  = $g_swf_col2index{submit    } - 1;
    my $idx_wt   = $g_swf_col2index{wait      } - 1;
    my $idx_uid  = $g_swf_col2index{uid       } - 1;
    my $idx_siz  = $g_swf_col2index{proc_alloc} - 1;
    my $idx_rmem = $g_swf_col2index{mem_req   } - 1;
    my $idx_umem = $g_swf_col2index{mem_used  } - 1;
    my $idx_stt  = $g_swf_col2index{status    } - 1;
    my $idx_ue   = $g_swf_col2index{user_est  } - 1;
    my $idx_rt   = ($swf !~ m/l_llnl_t3d/) 
	? $g_swf_col2index{runtime } - 1
	: $g_swf_col2index{cpu_used} - 1;
    #   in llnl_t3d the runtime is not available. jobs are gang scheduled
    #   and for some reason, dror decided to leave the runtime field -1
    #   and place the cumulative cpu time in cpu_used.

    while( <$fd> ) {

	if( m/\s*;\s*MaxProcs:\s*(\d+)\s*/ ||
	    (m/\s*;\s*MaxNodes:\s*(\d+)\s*/ && $cpus==-1) ) {
	    $cpus = $1;
	    next;
	}

	next if( m/^\s*;/o || m/^s*$/o ); # skip WS or remark
	die "bad line: $_" if ! m/^[0-9 \t-\.]+$/o;

	my @l    = split;
	my $sbm  = $l[$idx_sbm ];
	my $wt   = $l[$idx_wt  ];
	my $uid  = $l[$idx_uid ];
	my $rt   = $l[$idx_rt  ];
	my $siz  = $l[$idx_siz ] <= $cpus ? $l[$idx_siz] : $cpus;
	my $rmem = $l[$idx_rmem];
	my $umem = $l[$idx_umem];
	my $stt  = $l[$idx_stt ];
	my $ue   = $l[$idx_ue  ];
	my $end  = $sbm;
	$end += $wt if ($wt != -1);
	$end += $rt if ($rt != -1);
	my $good = 
	    ( ($siz > 0) && ($rt >= 0)&&($stt==-1||$stt==0||$stt==1||$stt==5) ) 
	    ? 1 : 0;

	$jobs++;
	$users{$uid} = 1;
	$time_fst    = $end if((($time_fst == -1) || ($end < $time_fst)) && $good );
	$time_last   = $end if(                      ($end > $time_last) && $good );
	$work       += $siz * $rt if( $good );
	$have_ue     = 1 if( $good && $ue   > 0);
	$have_mem    = 1 if(($good && $rmem > 0) || ($good && $umem > 0) );
    }

    close($fd);

    my $area    = ($time_last - $time_fst) * $cpus;
    $res{load}  = $area ? sprintf("%.1f", 100 * $work / $area) : "?";
    $res{users} = scalar( keys %users );
    $res{jobs}  = $jobs;
    $res{cpus}  = $cpus;
    $res{uest}  = $have_ue
	? "<font color=\"red\"><center>$g_symb_v</center></font>" 
	: $g_no_data;
    $res{mem}   = $have_mem
	? "<font color=\"red\"><center>$g_symb_v</center></font>" 
	: $g_no_data;

    return \%res;
}


#-------------------------------------------------------------------------------
# given a path and a file name, create an 'href' "around" it such that the file
# will be displayed as a clickable file-icon.
# transparently handles a list of file names too.
#-------------------------------------------------------------------------------
sub file2icon_link($$)
{
    my ($dir,$file) = @_;

    return $g_no_data if ($file eq "");

    my @files = split(' ', $file);

    my $rets = "";
    foreach my $f (@files) {

	my $path = "$dir/$f";

	#return $g_no_data if ! -f $file;

	my $is_html = ($file =~ m/\.html$/o);
	my $tooltip = $is_html ? "important notes" : "view/save file";
	my $icon_img= $is_html ? $g_icon_notes : $g_icon_file;
	my $icon    = "<img title=\"$tooltip\"\n" .
	  	      "\talt=\"$tooltip\"\n" .
		      "\talign=\"middle\"\n" .
		      "\tsrc=\"$icon_img\">";

	my $ret = "<a href=\"$path\">$icon</a>";

	# add size:
	if( ! $is_html ) {
	    my $ls     = `ls -lh $path` or die "failed ls-ing $path: $!\n";
	    my ($size) = $ls =~ m/^\S+\s+\S+\s+\S+\s+\S+\s+(\S+)\s+/;
	    $ret      .= " <font size=\"$g_font_size_fsize\">$size</font>";
	}
	$rets .= $ret;
    }
    return $rets;
}


#-------------------------------------------------------------------------------
# given a file name + string, create an 'href' "around" the text to point to
# the file.
#-------------------------------------------------------------------------------
sub file2text_link($$) {
    my ($file, $txt) = @_;
    return "<a href=\"$file\">$txt</a>";
}


#-------------------------------------------------------------------------------
# print given string as a 'td' html entry.
# separating commas are added to integers.
# numbers are justified to the right.
# maybe in the special formats:
#
# &SKIPPING&
#   nothing is printed
#
# &concat&${rows}&${cols}&${txt}&
#   when cell spans on more than one cell (obvious meaning)
#-------------------------------------------------------------------------------
sub td($)
{
    my ($s) = @_;
    my $attr = "";

    if( $s eq "&SKIPPING&" ) {
	# skipping this table-data (probably due to the a cell which spans
	# on this cell).
	return;
    }
    if( $s =~ m/^[\d.]+$/ ) {
	# print number right-justified
	$attr   = "align=\"right\"";
    }
    if( $s =~ m/^[\d]+$/ ) {
	# add commas
	my $rev = "";
	for(my ($i,$j)=(length($s)-1, 1); $i>=0; $i--, $j++) {
	    $rev = substr($s,$i,1) . $rev;
	    $rev = ",$rev"
		if( ($j % 3 == 0) && ($i > 0) );
	}
	$s = $rev;
    }
    if( $s =~ m/^[&]concat[&](\d+)[&](\d+)[&](.+)[&]$/s ) {
	# spanning over more than one column
	my $rows = $1;
	my $cols = $2;
	my $txt  = $3;
	$attr    = "colspan=\"$cols\" rowspan=\"$rows\"";
	$s       = $txt;
    }
    if( $s =~ m/&SKIPPING&/ ) {
	return;
    }

    print "  <td $attr><font size=\"$g_font_size_table\">$s</font></td>\n";
}


#-------------------------------------------------------------------------------
# get:
#    ($dir, $file_icon, $file)
# where $dir is a directory where one or more logs reside, potentially sharing
# their notes and orig files.  one such file is specified by $file, and an appropriate
# icon is given in $file_icon.  this function checks whether the file is indeed used
# more than once, as indicated in %g_merge; if so, the first use gets a re-created
# icon that spans the required number of rows, and the others get a skipped icon.
# if it is only used once, the given icon is used as is.
#-------------------------------------------------------------------------------
sub merge_icon($$$)
{
    my ($dir, $file_icon, $file) = @_;

    my $res;
    if( $g_merge{$dir}{$file} == 1 ) {
	$res = $file_icon;
    }
    elsif( $g_merge{$dir}{$file} == 0 ) {
	$res = "&SKIPPING&";
    }
    else {
	my $rows = $g_merge{$dir}{$file};
	my $cols = 1;
	my $txt  = $file_icon;
	$res = "&concat&${rows}&${cols}&${txt}&";
	$g_merge{$dir}{$file} = 0;
    }

    return $res;
}


#-------------------------------------------------------------------------------
# given a $log name, print the associated 'tr' entry
#-------------------------------------------------------------------------------
sub print_entry($$)
{
    my ($log,$serial)= @_;
    my $dir	     = $g_log2inf{$log}->{dir};
    if (! -d $dir) {
	warn "no directory for log $log! (should be $dir)";
	return;
    }
    my $file_swf     = $g_log2inf{$log}{swf};
    my $file_swf_cln = $g_log2inf{$log}{clean};
    my $file_orig    = $g_log2inf{$log}{orig};
    my $file_notes   = $g_log2inf{$log}{notes};
    my $file_old     = $g_log2inf{$log}{old};

    if (($file_orig    eq "") &&
	($file_swf     eq "") &&
	($file_swf_cln eq "")) {

	die "log $log has no orig, swf, or cleaned version!";
    }

    my ($month_span, $begin_time, $end_time) = log2period($log);

    my $swf_stt       = statistics("$dir/$file_swf");
    my $swf_jobs      = $swf_stt->{jobs};
    my $swf_users     = $swf_stt->{users};
    my $swf_load      = $swf_stt->{load};
    my $swf_file      = file2icon_link($dir, $file_swf);
    my $swf_cpus      = $swf_stt->{cpus};
    my $swf_have_ue   = $swf_stt->{uest};
    my $swf_have_mem  = $swf_stt->{mem};

    my $swf_cln_stt   = statistics("$dir/$file_swf_cln");
    my $swf_cln_jobs  = $swf_cln_stt->{jobs};
    my $swf_cln_users = $swf_cln_stt->{users};
    my $swf_cln_load  = $swf_cln_stt->{load};
    my $swf_cln_file  = file2icon_link($dir, $file_swf_cln);
    my $swf_cln_exists= ( -f "$dir/$file_swf_cln" ) ? 1 : 0;

    if (! $g_log2inf{$log}{doload}) {
	$swf_load = $swf_cln_load = $g_no_data;
    }

    my $orig_notes    = file2icon_link($dir, $file_notes);
    my $orig_file     = file2icon_link($dir, $file_orig);

    # handle the case in which more than one trace uses the same notes file
    # (in which case the "notes" icon is merged between successive lines)
    $orig_notes = merge_icon($dir, $orig_notes, $file_notes);
    $orig_file  = merge_icon($dir, $orig_file , $file_orig );

    #
    # print ...
    #
    print "\n";
    print "<!------------------- $log ------------------------->\n";
    print "<tr>\n";

    td( $serial );
    td( file2text_link("$dir/$file_notes",$log) );
    td( $orig_notes    );
    td( $begin_time    );
    td( $end_time      );
    td( $month_span    );
    td( $swf_cpus      );
    td( $swf_have_ue   );
    td( $swf_have_mem  );

    print "\n";
    if( $swf_cln_exists ) {
	td($swf_cln_jobs);
	td($swf_cln_users);
	td($swf_cln_load);
	td($swf_cln_file);
    } else {
	my $txt = "<i><center><span style=\"color: rgb(150, 150, 150);\">";
	$txt   .= "use original";
	$txt   .= "</i></center></span>";
	td("&concat&1&4&${txt}&");
    }

    print "\n";
    td($swf_jobs);
    td($swf_users);
    td($swf_load);
    td($swf_file);

    print "\n";
    td($orig_file);

    # older versions
    if ($g_with_old) {
	if( $g_log2inf{$log}{old} eq "" ) {
	    td($g_no_data);
	} else {
	    td( file2icon_link($g_log2inf{$log}{dir}, $g_log2inf{$log}{old}) );
	}
    }

    print "</tr>\n";
}


#-------------------------------------------------------------------------------
# print the head of the document (including the table header) and the end
#-------------------------------------------------------------------------------
sub print_header()
{
    print<<EOF
<html>

<!------------------------------------------------------------------------->
<!------------------------------------------------------------------------->
<!----------------------------  NOTE -------------------------------------->
<!----   THIS FILE IS AUTOMATICALLY GENERATED BY $0       ----->
<!----   ****************** DO NOT EDIT *******************           ----->
<!------------------------------------------------------------------------->
<!------------------------------------------------------------------------->

<head>
  <meta name="author" content="D. G. Feitelson and D. Tsafrir">
  <title>Parallel Workloads Archive: Logs</title>
  <base href="http://www.cs.huji.ac.il/labs/parallel/workload/">
  <link rel=stylesheet type="text/css" href="wl.css">
  <link rel="shortcut icon" href="ico.gif">
</head>


<body>
<h1>Logs of Real Parallel Workloads from Production Systems</h1>

<!---------------------------------- introduction ----------------------------->

This page points to detailed workload logs collected from large scale
parallel systems in production use in various places around the world.
<p>
The original logs come in different formats.
Information about the individual format for each log is given to the degree
available in the log's associated notes document.
In addition to the original format, all logs are converted to the
<a href="swf.html">Standard Workload Format (SWF)</a>.
Some of the logs also have <a href="logs.html#clean">cleaned</a> versions;
it is recommended that these be used when available.
<p>
To promote reproducibility, files are named according to the following convention.
Each file name has two parts, identifying the dataset and the file version.
The dataset identifier is constructed as &lt;site&gt;-&lt;machine&gt;-&lt;year&gt;.
The version is a serial number and possible modifier.
Number 0 is reserved for the original (unconverted) version.
The number is increased by 1 for each new conversion (in case errors were found
in a conversion and additional conversions were performed).
Modified versions are indicated by a minor number.
Thus a file name of CTC-SP2-1996-2.1-cln.swf is based on the dataset CTC-SP2-1996
(log of the SP2 machine at CTC starting from 1996), and represents a cleaned
version of conversion 2 of this dataset.
<span style="color:red">When using these logs, please specify exactly which file you are
using, in order to enable others to reproduce your work.</span>
<p>
Please send comments and additional information to
<a href="mailto:feit\@cs.huji.ac.il">feit\@cs.huji.ac.il</a>.

<p>
<!---------------------------------- logs ----------------------------->
<h2>The Log Files:</h2>

<table border="1">

<!--------------header row1---------------->
<tr>
  <th colspan="1" rowspan="2"><b><center>#		 </center></b></td>
  <th colspan="2" rowspan="2"><b><center>Name		 </center></b></td>
  <th colspan="1" rowspan="2"><b><center>From		 </center></b></td>
  <th colspan="1" rowspan="2"><b><center>To		 </center></b></td>
  <th colspan="1" rowspan="2"><b><center>Mon		 </center></b></td>
  <th colspan="1" rowspan="2"><b><center>CPUs		 </center></b></td>
  <th colspan="1" rowspan="2"><b><center>U<br>E		 </center></b></td>
  <th colspan="1" rowspan="2"><b><center>M<br>e<br>m     </center></b></td>
  <th colspan="4" rowspan="1"><b><center>Standard Format:<br><i>CLEANED</i>
							</center></b></td>
  <th colspan="4" rowspan="1"><b><center>Standard Format:<br><i>ORIGINAL</i>
							</center></b></td>
  <th colspan="1" rowspan="2"><b><center>Original<br>Log</center></b></td>
EOF
;

    if ($g_with_old) {
	print<<EOF
  <th colspan="1" rowspan="2"><b><center>Old<br>Ver</center></b></td>
EOF
    }

    print<<EOF
</tr>

<!--------------header row2---------------->
<tr>
  <!------swf cleaned------>
  <th><b><center>Jobs				</center></b></td>
  <th><b><center>Users				</center></b></td>
  <th><b><center>Util%				</center></b></td>
  <th><b><center>File				</center></b></td>
  <!------swf------>
  <th><b><center>Jobs				</center></b></td>
  <th><b><center>Users				</center></b></td>
  <th><b><center>Util%				</center></b></td>
  <th><b><center>File				</center></b></td>
</tr>

EOF
}

sub print_trailer()
{
    print<<EOF

</body>

</html>
EOF
}


#-------------------------------------------------------------------------------
# close the log table.
#-------------------------------------------------------------------------------
sub print_log_table_closure()
{
    print<<EOF
<tr><td colspan="18">
get <a href="logs-list">a list of the current (cleaned) versions of all logs</a>,
useful for downloading with wget -i
</td></tr>
</table>

<p>
<!-------------------------- EXPLANATION ------------------------->
<table border="0">
<tr>
  <td align="right">
         <img align="middle" src="$g_icon_notes"></a></td>
  <td> = Important notes associated with log</td>
</tr>
<tr>
  <td align="right">
         <img align="middle" src="$g_icon_file"></a></td>
  <td> = File to download</td>
</tr>
<tr>
  <td align="right"><b>Mon</b></td>
  <td> = Duration of log in months</td>
</tr>
<tr>
  <td align="right"><b>UE</b><br>&nbsp;</td>
  <td> = Indicates whether log contains user runtime estimates<br>
     &nbsp;&nbsp; [You can add artificial estimates to logs that lack them
     by using <a href="m_tsafrir05/">this utility</a>]</td>
</tr>
<tr>
  <td align="right"><b>Mem</b></td>
  <td> = Indicates whether log contains data about memory (requested or used)</td>
</tr>
<tr>
  <td align="right"><b>Util%</b></td>
  <td> = Utilization expressed as percentage</td>
</tr>
</table>
<p>

<a name="clean" />
The ORIGNAL SWF files reflect the data "as is", with only minor modifications
to make them self-consistent (for example, jobs that seem to start before they
arrive are modified to equate the arrival and start times).
Regrettably, the data often includes problematic and unrepresentative data,
such as significant automated administrative activity or large-scale flurries
of activity by single users.
In order to ease the use of these logs in performance evaluations, we
therefore also provide a CLEANED version of the logs, and recommend that
this version be used
(of course, you are urged to make your own decision regarding what cleaning
if at all should be applied, but if you don't want to get into this, we
suggest you use ours).
Specific information about the cleaning performed on each log is detailed
in the "usage notes" section of the log's notes document.
Further information about identifying and justifying the removal
of problematic and unrepresentative data may be found in:
<ul>
<li>
D. G. Feitelson and D. Tsafrir,
  &#147;<a href="San06ISPASS.pdf">Workload sanitation for
  performance evaluation</a>&#148;.
  In <i>IEEE Intl. Symp. Performance Analysis of Systems and Software</i>,
  pp. 221-230, Mar 2006.
<li>
D. Tsafrir and D. G. Feitelson,
  &#147;<a href="Bfly06IPDPS.pdf">Instability in parallel job
  scheduling simulation: the role of workload flurries</a>&#148;.
  In 20th <i>Intl. Parallel and Distributed Processing Symp.</i>,
  Apr 2006.
</ul>
In addition, older versions of the converted files are sometimes available
from the log's notes page.
These should only be used for reproducing or verifying previous research that
used those versions.
<p>
EOF
}


#-------------------------------------------------------------------------------
# print the control section
#-------------------------------------------------------------------------------
sub print_ctrl()
{
    my $date = `date`;

    print<<EOF
<p>
<!-------------------------- CONTROL ------------------------->
<table border=\"1\">
<tr>
  <td>Last update</td>
  <td>&nbsp;<i>$date</i></td>
</tr>
<tr>
  <td>Visit count since Aug 20, 2004</td>
  <td>
    &nbsp;
    <!-- Start of StatCounter Code -->
    <script type="text/javascript" language="javascript">
    var sc_project=366996;
    var sc_partition=1;
    </script>

    <script type="text/javascript" language="javascript" src="http://www.statcounter.com/counter/counter.js"></script><noscript><a href="http://www.statcounter.com/" target="_blank"><img  src="http://c2.statcounter.com/counter.php?sc_project=366996&amp;amp;java=0" alt="free hit counter" border="0"></a> </noscript>

     <!-- End of StatCounter Code -->
     [ see some access statistics <a
       href="http://my.statcounter.com/project/standard/stats.php?project_id=366996&guest=1"
     >here</a> ]
  </td>
</tr>
<tr>
  <td>Automatically generated by</td>
  <td>&nbsp;<a href=\"$0\">$0</a></td>
</tr>
</table>
EOF
}


#-------------------------------------------------------------------------------
# print the changelog
#-------------------------------------------------------------------------------
sub print_changelog()
{
    print<<EOF

<p>
<!------------------------------ changelog ---------------------------->
<h3>Change log:</h3>

<table border="1">

<!--------------header row1---------------->
<tr>
  <th><b><center>#		</center></b></td>
  <th><b><center>Date		</center></b></td>
  <th><b><center>Log File	</center></b></td>
  <th><b><center>Change 	</center></b></td>
</tr>
EOF
;
    my $nchange = $#g_change + 1;

    foreach my $entry (reverse sort {$a->{date} cmp $b->{date}} @g_change) {

        my $log = $entry->{log};
	# make sure log exists
        # BROKEN BECAUSE CHANGED TO LOG FILE RATHER THAN LOG NAME
	#exists $g_log2inf{$log} or die "bug: unknown logid=$g_change{$date}{log}";

	$entry->{date} =~ /(\d+)\/(\d+)\/(\d+)/;
	my $yr  = $1;
	my $mon = ("", "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[$2];
	my $day = $3;

	print
	    "<tr"   , ($log eq "")? " class=\"gen\"" : "" , ">\n",
	    "  <td>", $nchange--                          , "</td>\n",
	    "  <td>", "$day $mon $yr"                     , "</td>\n",
	    "  <td>", ($log eq "")? "&nbsp;" : $log       , "</td>\n",
	    "  <td>", $entry->{text}                      , "</td>\n",
	    "</tr>\n",
	    "\n";
    }

    print "</table>\n\n";
}


################################################################################
# do it
################################################################################

print_header();
my $serial = 0;
foreach my $log (@g_logs) {
    printf stderr "doing log $log\n";
    print_entry($log, ++$serial);
}
print_log_table_closure();
print_ctrl();
print_changelog();
print_trailer();


################################################################################
#                                    EOF
################################################################################