#!/usr/bin/perl
#
# scan job execution log files from the Cornell SP2 system, and use convert2swf.pm
# to convert the data into the standard workload format.
#

use warnings;
use strict;
use Time::Local;
use POSIX;
use convert2swf_v4;

# NOTE: there are actually 2 versions, with slight differences,
# e.g. early doesn't have the 20th field
#my $version = "early";
my $version = "prod";



############################
#
# DO THE CONVERSION:
#

my $tz_str    = "US/Eastern";
my $max_procs = 336;
my @queues = ("DSI",
	      "astro",
	      "piofs",
	      "informix",
	     );

SWF::init( $tz_str, $max_procs, \@queues );

my $format_bugs;
my $jobs_data = get_jobs();

my ($zero, $maxtime) = SWF::convert( $jobs_data, $format_bugs );

my $header = generate_header( $zero, $maxtime, $max_procs );

SWF::print( $header );



############################
# SPECIFICS for CTC workload

# the format of each entry includes the following space-separated fields:
# 0  job name
# 1  LoadLeveler class (actually time limit)
# 2  nodes
# 3  submit time (secs since epoch)
# 4  start time (secs since epoch)
# 5  end time (secs since epoch)
# 6  memory requested
# 7  node type requested (wide/thin)
# 8  mass storage requested (y/n)
# 9  adapter requested (ether/ip/us)
# 10 submit time (calendar)
# 11 start time (calendar)
# 12 end time (calendar)
# 13 resource use (sum of node CPU times)
# 14 job type (serial/parallel/PVM)
# 15 completion status (completed/removed/other)
# 16 user ID
# 17 cumulative user CPU time
# 18 cumulative system CPU time
# 19 name of script used to submit job
# 20 runtim limit for EASY (min)

my $job_cnt;

sub get_jobs {
##############
# parse input format and create a hash for each job

    my @list_of_jobs;

    #
    # scan trace and collect job info
    #
    while (<>) {

	#
	# parse line and extract info
	#

	if (/^\s*$/) {		# empty line
	    next;
	}

	$_ =~ /\s*(.*)\s*/;		# get rid of white space at ends
	$_ = $1;
	my @line = split('\s+');	# split into fields

	if (($version eq "prod") && ($#line != 20)) {
	    $format_bugs .= "format_problem $_\n";
	    warn(">>>Format problem on $_\ngot $#line fields (should be 20)");
	    next;
	}
	if (($version eq "early") && ($#line != 19)) {
	    # last two fields often not separated
	    if (($#line == 18) && ($line[18] =~ /^(\d+)(.+)$/)) {
		$line[18] = $1;
		$line[19] = $2;
	    }
	    else {
		$format_bugs .= "format_problem $_\n";
		warn(">>>Format problem on $_\ngot $#line fields (should be 19)");
		next;
	    }
	}

	my %nj;

	#
	# set the desired fields as specified in convert2swf.pl
	#

	$nj{trace}  = $_;	# used for error messages

	$nj{submit} = $line[3];
	$nj{start}  = $line[4];
	$nj{end}    = $line[5];

	$nj{procs}   = $line[2];

	# CPU time is given for all nodes, separated into user and system.
	# we need total per node.
	my $cpu;
	if ($line[2] > 0) {
	    $cpu = int( ($line[17] + $line[18]) / $line[2] );
	    $nj{cpu} = $cpu;
	}

	if ($version eq "prod") {
	    # requested runtime is given in minutes
	    $nj{req_time} = $line[20] * 60;
	}
	# no data on requested runtime in early version

	if ($version eq "early") {
	    $nj{req_mem} = $line[6] * 1024;
	}
	# all request 128M in prod version

	$nj{status} = ($line[15] eq "Co") ? 1 : 0;

	$nj{user}   = $line[16];

	$nj{app}    = $line[19];

	$nj{queue}  = $line[1];

	$list_of_jobs[$job_cnt++] = \%nj;
    }

    return \@list_of_jobs;
}


sub generate_header {
    my ($zero, $maxtime, $max_procs) = @_;

    my ($sec, $min, $hr, $mday, $mon, $year, $wday, $yday, $isdst);

    $header  = "; Version: 2.2\n";
    $header .= "; Computer: IBM SP2\n";
    $header .= "; Installation: Cornell Theory Center (CTC)\n";

    if ($version eq "early") {
	$header .= "; Acknowledge: Steve Hotovy\n";
    }
    else {
	$header .= "; Acknowledge: Dan Dwyer and Steve Hotovy\n";
    }
    $header .= "; Information: http://www.tc.cornell.edu/\n";
    $header .= ";              http://www.cs.huji.ac.il/labs/parallel/workload/\n";

    $header .= sprintf("; Conversion: Dror Feitelson (feit\@cs.huji.ac.il) %s\n",
		       strftime("%d %b %Y", localtime()));


    $header .= sprintf("; MaxJobs: %d\n", $job_cnt);
    $header .= sprintf("; MaxRecords: %d\n", $job_cnt);
    $header .= "; Preemption: No\n";

    $header .= sprintf("; UnixStartTime: %d\n", $zero);
    $header .= "; TimeZone: -18000\n";
    $header .= sprintf("; TimeZoneString: %s\n", $tz_str);

    $header .= sprintf("; StartTime: %s\n",
	   strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($zero)));

    $header .= sprintf("; EndTime:   %s\n",
	   strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($maxtime)));

    $header .= "; MaxNodes: $max_procs (batch partition; 512 total)\n";
    $header .= "; MaxProcs: $max_procs\n";

    $header .= "; Note: uses the EASY scheduler\n";

    $header .= "; Queues: these are actually LoadLeveler classes\n";
    # NOTE: do not include list of queues

    return $header;
}