#!/usr/bin/perl
#
# scan job execution log files from the SDSC Blue Horizon system, and
# use convert2swf.pm to convert the data into the standard workload format.
#

use warnings;
use strict;
use Time::Local;
use POSIX;
use convert2swf_v4;


############################
#
# DO THE CONVERSION:
#

my $tz_str    = "US/Pacific";
my $max_procs = 1152;
my @queues    = ("interactive",
		 "express",
		 "high",
		 "normal",
		 "low",
		 "standby",
		);

log_init();
SWF::init( $tz_str, $max_procs, \@queues );

my $format_bugs;
my $jobs_data = get_jobs();

my ($zero, $maxtime) = SWF::convert( $jobs_data, $format_bugs );

my $header = generate_header( $zero, $maxtime, $max_procs );

SWF::print( $header );



##################################
# SPECIFICS for SDSC Blue workload

# the format of each entry includes the following space-separated fields:
# 0  user (sanitized as User1, User2, ...)
# 1  queue (interactive, express, high, normal, low, unknown)
# 2  total CPU time
# 3  wallclock time
# 4  service units (used in accounting)
# 5  nodes used
# 6  maximal nodes used
# 7  number of jobs run (hopefully always 1)
# 8  max memory (unused)
# 9  memory (unused)
# 10 I/O (unused)
# 11 disk space (unused)
# 12 connect time (unused)
# 13 waiting time
# 14 slowdown
# 15 priority (maps to queue)
# 16 submit date and time
# 17 start date and time
# 18 end date and time
# 19 requested wallclock time
# 20 requested memory (unused)
# 21 requested nodes
# 22 completion status

my $job_cnt;
my $ppn;


sub log_init {
##############

    # ppn is processors per node
    $ppn = 8;
}


sub get_jobs {
##############
# parse input format and create a hash for each job
#
# note: the original format provides times in human-readable format (hh:mm:ss),
# but these seem to be wrong, and specifically the result of using gmtime instead
# of localtime. so here we also need to use gmtime.

    my @list_of_jobs;

    #
    # scan trace and collect job info
    #
    while (<>) {

	#
	# parse line and extract info
	#

	if (/^\s*$/) {		# empty line
	    next;
	}

	$_ =~ /\s*(.*)\s*/;		# get rid of white space at ends
	$_ = $1;
	my @line = split('\s+');	# split into fields

	if ($#line != 22) {
	    $format_bugs .= "format_problem $_\n";
	    warn(">>>Format problem on $_\ngot $#line fields (should be 22)");
	    next;
	}

	# special checks
	if ($line[7] != 1) {
	    $format_bugs .= "many_jobs_problem [$line[7]] $_\n";
	}
	if ($line[6] < $line[5]) {
	    $format_bugs .= "maxpar_small_problem [mp=$line[6] n=$line[5]] $_\n";
	}
	if ($line[6] > $line[5]) {
	    $format_bugs .= "maxpar_big_problem [mp=$line[6] n=$line[5]] $_\n";
	}

	my %nj;

	#
	# set the desired fields as specified in convert2swf.pm
	#

	$nj{trace}  = $_;	# used for error messages


	# map status first
	my $status = 0;
	if ($line[22] == 2) {
	    $status = 5;
	}
	elsif ($line[22] == 1) {
	    $status = 1;
	}
	elsif ($line[22] != 3) {
	    # 3 means unknown...
	    $format_bugs .= "unknown_status_problem [$line[22]] $_\n";
	}
	if ($status != 0) {
	    $nj{status} = $status;
	}

	# submit time, in seconds
	my $submit;
	if ($line[16] =~ /(\d\d\d\d)-(\d\d)-(\d\d)--(\d\d):(\d\d):(\d\d)/) {
	    my $yr  = $1 - 1900;
	    my $mon = $2 - 1;
	    my $day = $3;
	    my $hr  = $4;
	    my $min = $5;
	    my $sec = $6;

	    $submit = &timegm($sec, $min, $hr, $day, $mon, $yr);
	}
	else {
	    $submit = -1;
	}
	$nj{submit} = $submit;

	# start time, in seconds
	my $start;
	if ($line[17] =~ /(\d\d\d\d)-(\d\d)-(\d\d)--(\d\d):(\d\d):(\d\d)/) {
	    my $yr  = $1 - 1900;
	    my $mon = $2 - 1;
	    my $day = $3;
	    my $hr  = $4;
	    my $min = $5;
	    my $sec = $6;

	    $start = &timegm($sec, $min, $hr, $day, $mon, $yr);
	    if ($start == 0) {
		if ($status == 1) {
		    # job is OK -- a real problem
		    $format_bugs .= "start_zero $_\n";
		}
		$start = -1;
	    }
	}
	else {
	    $start = -1;
	}
	$nj{start} = $start;

	# end time, in seconds
	my $end;
	if ($line[18] =~ /(\d\d\d\d)-(\d\d)-(\d\d)--(\d\d):(\d\d):(\d\d)/) {
	    my $yr  = $1 - 1900;
	    my $mon = $2 - 1;
	    my $day = $3;
	    my $hr  = $4;
	    my $min = $5;
	    my $sec = $6;

	    $end = &timegm($sec, $min, $hr, $day, $mon, $yr);
	}
	else {
	    $end = -1;
	}
	$nj{end} = $end;


	# more checks
	if (($start != -1) && ($end != -1)) {
	    my $wallclk = $end - $start;
	    if (($wallclk > $line[3]) && ($line[3] > 0)) {
		$format_bugs .= "wallclock_short_problem [e-s=$wallclk wc=$line[3]] $_\n";
	    }
	    my $wait = $line[13];
	    if (($submit > 0) && ($start > 0) && ($wait != -1)) {
		my $calc_wait = $start - $submit;
		if ($calc_wait > $wait) {
		    $format_bugs .= "wait_short_problem [s-s=$calc_wait w=$wait] $_\n";
		}
		if ($calc_wait < $wait) {
		    $format_bugs .= "wait_long_problem [s-s=$calc_wait w=$wait] $_\n";
		}
	    }
	}

	# number of processors:
	# allocation is in 8-processor nodes
	my $procs = $line[5] * $ppn;
	$nj{procs} = $procs;

	# divide total CPU by number of procs.
	my $cpu;
	if ($line[2] == 0) {
	    $cpu = 0;
	}
	elsif (($procs == -1) || ($procs == 0)) {
	    $cpu = -1;
	}
	else {
	    $cpu = ($line[2]/$procs);
	}
	$nj{cpu} = $cpu;

	# requested processors (also in 8-processor nodes)
	my $req_procs = $line[21] * $ppn;
	$nj{req_procs} = $req_procs;

	# requested runtime
	my $req_time = $line[19];
	$nj{req_time} = $req_time;

	# user ID
	$line[0] =~ /^User(\d+)$/;
	my $user = $1;
	$nj{user} = $user;

	# queue data
	my $queue = $line[1];
	$nj{queue} = $queue;

	$list_of_jobs[$job_cnt++] = \%nj;
    }

    return \@list_of_jobs;
}


sub generate_header {
#####################
    my ($zero, $maxtime, $max_procs) = @_;

    my ($sec, $min, $hr, $mday, $mon, $year, $wday, $yday, $isdst);


    $header  = "; Version: 2.2\n";
    $header .= "; Computer: IBM SP\n";
    $header .= "; Installation: SDSC Blue Horizon\n";

    $header .= "; Acknowledge: Travis Earheart and Nancy Wilkins-Diehr\n";
    $header .= "; Information: http://www.sdsc.edu/\n";
    $header .= "; Information: http://www.cs.huji.ac.il/labs/parallel/workload/\n";

    $header .= sprintf("; Conversion: Dror Feitelson (feit\@cs.huji.ac.il) %s\n",
		       strftime("%d %b %Y", localtime()));


    $header .= sprintf("; MaxJobs: %d\n", $job_cnt);
    $header .= sprintf("; MaxRecords: %d\n", $job_cnt);
    $header .= "; Preemption: No\n";

    $header .= sprintf("; UnixStartTime: %d\n", $zero);
    $header .= "; TimeZone: -28800\n";
    $header .= sprintf("; TimeZoneString: %s\n", $tz_str);

    $header .= sprintf("; StartTime: %s\n",
	   strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($zero)));

    $header .= sprintf("; EndTime:   %s\n",
	   strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($maxtime)));

    $header .= sprintf("; MaxNodes: %d\n", $max_procs / $ppn);
    $header .= "; MaxProcs: $max_procs\n";

    $header .= "; Information: Each node is an 8-way SMP\n";
    $header .= "; Information: 375 MHz Power3 processors\n";
    $header .= "; Information: 4 GB memory per node (shared by 8 processors)\n";
    $header .= "; Information: express queue limit: 8 nodes for 2 hours\n";
    $header .= "; Information: high / normal queue limit: 36 hours\n";

    return $header;
}