#!/usr/bin/perl
#
# scan job execution log file from the Sharcnet system, and use convert2swf_v2.pm
# to convert the data into the standard workload format.
#

use warnings;
use strict;
use Time::Local;
use POSIX;
use convert2swf_v4;


############################
#
# DO THE CONVERSION:
#

my $tz_str    = "Canada/Eastern";
my $max_procs = 6828;
my $max_nodes = 2091;
my @queues    = ();

$ENV{TZ} = $tz_str;
POSIX::tzset();

SWF::init( $tz_str, $max_procs, \@queues );

my $format_bugs;
my $jobs_data = get_jobs();

my ($zero, $maxtime) = SWF::convert( $jobs_data, $format_bugs, 1 );

my $header = generate_header( $zero, $maxtime, $max_procs );

SWF::print( $header );



############################
# SPECIFICS for Sharcnet workload

# the format of each entry includes the following comma-separated fields:
#  0 job ID
#  1 system name
#  2 user ID
#  3 state (k, d, f)
#  4 job type (1, 2, 3)
#  5 submit date and time (seconds from epoch)
#  6 start date and time (seconds from epoch)
#  7 end date and time (seconds from epoch)
#  8 user time
#  9 system time
# 10 total time
# 11 cpus used
# 12 nodes used
# 13 exit status
# 14 memory used (KB)
# 15 page faults
# 16 command (sanitized to number)
# 17 flags (serial, test, mpi, normal, default, threaded, ...)
# 18 suspended time
# 19 suspension history
# 20 array index


my $job_cnt;

sub get_jobs {
##############
# parse input format and create a hash for each job

    my @list_of_jobs;

    my $proc_for_cpu = 0;
    my $run_mismatch = 0;
    my $run_suspend_mismatch = 0;
    my $was_suspended = 0;
    my $neg_suspended = 0;
    my $bad_suspended = 0;

    #
    # scan trace and collect job info
    #
    my $dummy = <>;	# skip 2 lines
    $dummy = <>;
    while (<>) {


#if ($job_cnt == 10000) {last;}

	#
	# parse line and extract info
	#

	if (/^\s*$/) {		# empty line
	    next;
	}

	$_ =~ /\s*(.*)\s*/;		# get rid of white space at ends
	$_ = $1;
	my @line = split(',');	# split into fields

	if ($#line != 20) {
	    $format_bugs .= "format_problem $_\n";
	    warn(">>>Format problem on $_\ngot $#line fields (should be 20)");
	    next;
	}

	my %nj;

	#
	# set the desired fields as specified in convert2swf.pl
	#

	$nj{trace}  = $_;	# used for error messages

	my $submit;
	if ($line[5] =~ /(-?\d+(\.\d+)?)/) {
	    $submit = $1;
	    $nj{submit}  = $submit;
	}

	my $start;
	if ($line[6] =~ /(-?\d+(\.\d+)?)/) {
	    $start = $1;
	    $nj{start}  = $start;
	}

	my $suspend;
	if ($line[18] =~ /(-?\d+(\.\d+)?)/) {
	    if ($1 > 0) {
		$suspend = $1;
		$was_suspended++;
	    }
	    elsif ($1 < 0) {
		$neg_suspended++;
	    }
	}

	my $end;
	if ($line[7] =~ /(-?\d+(\.\d+)?)/) {
	    $end = $1;

	    # if job was suspended, subtract from end time to get real runtime
	    if (defined($suspend) && ($suspend > 0)) {

		$end -= $suspend;

		# also note negative runtime
		if ((defined($start)  && ($end < $start)) ||
		    (defined($submit) && ($end < $submit))) {

		    $bad_suspended++;
		}
	    }
	    $nj{end} = $end;
	}

	my $procs;
	if ($line[11] =~ /(\d+)/) {
	    $procs = $1;
	    $nj{procs} = $procs;
	}

	my $run;
	if ($line[10] =~ /(-?\d+(\.\d+)?)/) {
	    $run = $1;
	    if ((defined($procs)) && ($procs > 0)) {
		$run /= $procs;
		if (! defined($start) || ! defined($end)) {
		    $nj{run} = $run;
		}
	    }
	}

	if (defined($start) && defined($end) && defined($run)) {
	    if ($run != $end - $start) {
		# end already adjusted by $suspend
		if ((! defined($suspend)) || ($suspend == 0)) {
		    #print "end-start=", $end-$start, " != run=$run  [dif=", abs($end-$start-$run),"]\n";
		    $run_mismatch++;
		}
		else {
		    $run_suspend_mismatch++;
		}
	    }
	}

	my $cpu = 0;
	if ($line[8] =~ /(-?\d+(\.\d+)?)/) {
	    $cpu = $1;
	    if ((defined($procs)) && ($procs > 0)) {
		$nj{cpu} = $cpu / $procs;
	    }
	    else {
		$proc_for_cpu++;
	    }
	}
	if ($line[9] =~ /(-?\d+(\.\d+)?)/) {
	    $cpu += $1;
	    if ((defined($procs)) && ($procs > 0)) {
		$nj{cpu} = $cpu / $procs;
	    }
	}

	my $status = -1;
	if ($line[3] =~ /(\w)/) {
	    if ($1 eq "d") {	# done
		$status = 1;
	    }
	    elsif ($1 eq "k") {	# killed
		$status = 5;
	    }
	    elsif ($1 eq "f") {	# failed
		$status = 0;
	    }

	    $nj{status} = $status;
	}

	my $mem;
	if ($line[14] =~ /(\d+)/) {
	    $mem = $1;
	    if ((defined($procs)) && ($procs > 0)) {
		$nj{mem} = ($mem / $procs) * 1024;
	    }
	}

	my $user;
	if ($line[2] =~ /(\w+)/) {
	    $user = $1;
	    $nj{user} = $user;
	}

	my $app;
	if ($line[16] =~ /(\w+)/) {
	    $app = $1;
	    $nj{app} = $app;
	}

	my $partition;
	if ($line[1] =~ /(\w+)/) {
	    $partition = $1;
	    $nj{partition} = $partition;
	}

	$list_of_jobs[$job_cnt++] = \%nj;
    }

    if ($was_suspended != 0) {
	$format_bugs .= "subtracted suspend from end time $was_suspended times\n";
    }
    if ($bad_suspended != 0) {
	$format_bugs .= "  this led to negative runtime $bad_suspended times\n";
    }
    if ($neg_suspended != 0) {
	$format_bugs .= "  suspended negative time ignored $neg_suspended times\n";
    }
    if ($proc_for_cpu != 0) {
	$format_bugs .= "missing processors to calculate CPU usage $proc_for_cpu times\n";
    }
    if ($run_mismatch != 0) {
	$format_bugs .= "run != end-start $run_mismatch times\n";
    }
    if ($run_suspend_mismatch != 0) {
	$format_bugs .= "run != end-start-suspend $run_suspend_mismatch times\n";
    }

    return \@list_of_jobs;
}


sub generate_header {
    my ($zero, $maxtime, $max_procs) = @_;

    my ($sec, $min, $hr, $mday, $mon, $year, $wday, $yday, $isdst);

    $header  = "; Version: 2.2\n";
    $header .= "; Computer: various\n";
    $header .= "; Installation: Sharcnet grid\n";

    $header .= "; Acknowledge: John Morton and Clayton Chrusch\n";
    $header .= "; Information: http://www.sharcnet.ca/\n";
    $header .= ";              http://www.cs.huji.ac.il/labs/parallel/workload/\n";

    $header .= sprintf("; Conversion: Dror Feitelson (feit\@cs.huji.ac.il) %s\n",
		       strftime("%d %b %Y", localtime()));


    $header .= sprintf("; MaxJobs: %d\n", $job_cnt);
    $header .= sprintf("; MaxRecords: %d\n", $job_cnt);
    $header .= "; Preemption: Yes\n";

    $header .= sprintf("; UnixStartTime: %d\n", $zero);
    $header .= "; TimeZone: -18000\n";
    $header .= sprintf("; TimeZoneString: %s\n", $tz_str);

    $header .= sprintf("; StartTime: %s\n",
	   strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($zero)));

    $header .= sprintf("; EndTime:   %s\n",
	   strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($maxtime)));

    $header .= "; MaxNodes: $max_nodes\n";
    $header .= "; MaxProcs: $max_procs\n";

    return $header;
}

