#!/usr/bin/perl
#
# scan LSF data as described in lsb.acct.man from the LANL O2K (Nirvana) log,
# and use convert2swf.pm to convert into the standard workload format.
#

use warnings;
use strict;
use Time::Local;
use POSIX;
use convert2swf_v4;


############################
#
# DO THE CONVERSION:
#

my $tz_str    = "US/Mountain";
my $max_procs = 2048;
my @queues = ("short",
	      "long",
	      "small",
	      "medium",
	      "large",
	      "xlarge",
	      "small-d",
	      "medium-d",
	      "large-d",
	      "xlarge-d",
	      "chammpq",
	      "schammpq",
	      "mchammpq",
	      "shared",
	      "ir-shared",
	      "scavenger",
	      "test"
	     );

SWF::init( $tz_str, $max_procs, \@queues );

my $format_bugs;
my $jobs_data = get_jobs();

my ($zero, $maxtime) = SWF::convert( $jobs_data, $format_bugs );

my $header = generate_header( $zero, $maxtime, $max_procs );

SWF::print( $header );


############################
# SPECIFICS for LSF workload

# the format of each entry includes the following space-separated fields.
# but note that 23 and 25 are themselves space-separated lists that could be empty,
# so the numbering in each record is erratic.
# job record:
#  0 The event type (%s), which is always "JOB_FINISH"
#  1 The version number of the log file format (%s)
#  2 The time the event was logged (in seconds since the epoch) (%d)
#  3 The ID for the job (%d)
#  4 The UNIX user ID of the submitter (%hu)
#  5 Bit flags for job processing (%d)
#  6 The number of processors initially requested for execution (%d)
#  7 The job submission time (%lu)
#  8 The job start time - the job should be started at or after this time (%lu)
#  9 The job termination deadline - the job should be terminated by this time (%lu)
# 10 The job dispatch time - time job	was dispatched for execution (%lu)
# 11 The user name of the submitter (%s)
# 12 The name of the job queue that this job was submitted to (%s)
# 13 The resource requirement specified by the user (%s)
# 14 The job dependency condition specified by the user (%s)
# 15 The pre-execution command specified by the user (%s)
# 16 The submission host name (%s)
# 17 The current working directory (%s)
# 18 The input file name (%s)
# 19 The output file name (%s)
# 20 The error output file name (%s)
# 21 The job script file name (%s)
# 22 The number of host names to which job dispatching will be limited (%d)
# 23 A list of host names to which job dispatching will be limited (%s for each); blank if the last field value is 0
# 24 The number of processors used for execution (%d)
# 25 A list of execution host names (%s for each); blank if the last field value is 0
# 26 The job status (%d).  The number 32 represents EXIT, 64 represents DONE
# 27 The CPU factor of the first execution host (%f)
# 28 The job name (%s)
# 29 The complete batch job command specified by the user (%s)
# resource usage:
# 30 The user time used (%f)
# 31 The system time used (%f)
# 32 The maximum shared text size (%d)
# 33 The integral of the shared text size over time (in kilobyte seconds) (%d)
# 34 The integral of the shared memory size over time (valid only on Ultrix) (%d)
# 35 The integral of the unshared data size over time (%d)
# 36 The integral of the unshared stack size over time (%d)
# 37 The number of page reclaims (%d)
# 38 The number of page faults (%d)
# 39 The number of times the process was swapped out (%d)
# 40 The number of block input operations (%d)
# 41 The number of block output operations (%d)
# 42 The number of characters read and written (valid only on HP-UX) (%d)
# 43 The number of System V IPC messages sent (%d)
# 44 The number of messages received (%d)
# 45 The number of signals received (%d)
# 46 The number of voluntary context switches (%d)
# 47 The number of involuntary context switches (%d)
# 48 The exact user time used (valid only on ConvexOS) (%d)
# additional fields:
# 49 The name of the user to whom job related mail was sent (%s)
# 50 The project name (%s)
# 51 The UNIX exit status of the job (%d)
# 52 The maximum number of processors specified for the job (%d)
# 53 The login shell used for the job (%s)
# EXTRA EMPTY STRING FIELD HERE
# 54 The job array index (%d)
# 55 The maximum resident memory usage in KBytes of all processes in the job (%d)
# 56 The maximum virtual memory usage in KBytes of all processes in the job (%d)


my $job_cnt;


sub get_jobs {
##############
# parse input and create a hash for each job

    my @list_of_jobs;

    #
    # scan trace and collect job info
    #
    while (<>) {

	#
	# parse line and extract info
	#

	if (/^\s*$/) {		# empty line
	    next;
	}

	$_ =~ /\s*(.*)\s*/;		# get rid of white space at ends
	$_ = $1;
	# regrettably the LSF log format includes two lists of hostnames
	# that may potentially be used (and are actually used) in the middle.
	# but these lists can also be omitted.
	# in addition, many fields are strings delimited by "", and may include
	# spaces, which are also used to delimit fields...
	# so we must first parse the line correctly, skipping the hostname lists.
	my @line = &parse();

	if ($#line != 55) {
	    # the last index is actually 57 but we skip the two node lists
	    $format_bugs .= "format_problem $_\n";
	    warn(">>>Format problem on $_\ngot $#line fields (should be 55)");
	    next;
	}

	my %nj;

	#
	# set the desired fields as specified in convert2swf.pl
	#

	$nj{trace}  = $_;		# used for error messages

	$nj{submit} = $line[7];
	$nj{start}  = $line[10];	# actual dispatch time
	$nj{end}    = $line[2];		# time of logging "job-finish" event

	$nj{procs}  = $line[23];

	# CPU time is given for all nodes, separated into user and system.
	# we need total per node.
	my $cpu;
	if ($line[23] > 0) {
	    $cpu = int( ($line[28] + $line[29]) / $line[23] );
	    $nj{cpu} = $cpu;
	}

	$nj{mem}    = $line[54];

	$nj{req_procs} = $line[6];

	# job status
	if ($line[24] == 32) {
	    $nj{status} = 0;
	}
	elsif ($line[24] == 64) {
	    $nj{status} = 1;
	}

	$nj{user}   = $line[11];
	$nj{group}  = $line[48];
	$nj{app}    = $line[26];

	$nj{queue}  = $line[12];

	$list_of_jobs[$job_cnt++] = \%nj;
    }

    return \@list_of_jobs;
}


#
# parse the horrendeous format dreamed up by LSF
#
sub parse {

    my @out;
    my $cnt = 0;
    my $next_is_cnt = 0;
    my $i = 0;
    my $in_string = 0;
    my @tokens = split(' ');
    foreach my $tok (@tokens) {

	if ($cnt) {
	    # counting fields that need to be skipped
	    # assumes they are single-token fields...
	    $cnt--;
	    # if this is the end of the first list, prepare for second one
	    if (($cnt == 0) && ($next_is_cnt == 0)) {
		$next_is_cnt = 1;
	    }
	    next;
	}

	if ($in_string) {
	    if (($tok =~ /[^\"]\"$/) || ($tok =~ /[^\"]\"\"\"$/) ||
		($tok =~ /^\"$/) || ($tok =~ /^\"\"\"$/)) {
		# string ends here
		# special cases: (1) if string includes ", this is indicated by "",
		# so if the " has another " in front of it it does NOT count. however,
		# (2) if the string ends with a " there will be 3 in a sequence, and it
		# DOES count.  also, (3) if the whole token is only a " it means the
		# string ends with a space and this is ok.
		$tok =~ /(.*)\"$/;
		$out[$i++] .= " " . $1;
		$in_string = 0;
	    }
	    else {
		# still in the middle
		$out[$i] .= " " . $tok;
	    }
	    next;
	}

	if ($tok =~ /^\"/) {
	    # start a string
	    if ($tok =~ /\"$/) {
		# whole string in a single token
		$tok =~ /^\"(.*)\"$/;
		$out[$i++] = $1;
	    }
	    else {
		$tok =~ /^\"(.*)$/;
		$out[$i] = $1;
		$in_string = 1;
	    }
	    next;
	}

	# not a string
	$out[$i++] = $tok;

	# if this has been identified as the count of the second list, use it
	if ($next_is_cnt == 1) {
	    $cnt = $tok;
	    $next_is_cnt = -1;
	    next;
	}
	# first count of how many fields need to be skipped appears
	# in field 22
	if ($i == 23) {
	    $cnt = $tok;

	    # but if 0, nothing to do, and need to prepare for second list...
	    if ($cnt == 0) {
		$next_is_cnt = 1;
	    }
	}
    }

    return @out;
}


#
# generate header
#
sub generate_header {
    my ($zero, $maxtime, $max_procs) = @_;

    my ($sec, $min, $hr, $mday, $mon, $year, $wday, $yday, $isdst);

    my $header  = "; Version: 2.2\n";
    $header .= "; Computer: Origin 2000\n";
    $header .= "; Installation: LANL\n";

    $header .= "; Acknowledge: Fabrizio Petrini\n";
    $header .= "; Information: http://www.acl.lanl.gov/\n";
    $header .= ";              http://www.cs.huji.ac.il/labs/parallel/workload/\n";

    $header .= sprintf("; Conversion: Dror Feitelson (feit\@cs.huji.ac.il) %s\n",
		       strftime("%d %b %Y", localtime()));

    $header .= sprintf("; MaxJobs: %d\n", $job_cnt);
    $header .= sprintf("; MaxRecords: %d\n", $job_cnt);
    $header .= "; Preemption: No\n";

    $header .= sprintf("; UnixStartTime: %d\n", $zero);
    $header .= "; TimeZone: -25200\n";
    $header .= sprintf("; TimeZoneString: %s\n", $tz_str);

    $header .= sprintf("; StartTime: %s\n",
	   strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($zero)));

    $header .= sprintf("; EndTime:   %s\n",
	   strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($maxtime)));

    $header .= "; MaxNodes: $max_procs\n";
    $header .= "; MaxProcs: $max_procs\n";

    $header .= "; Note: uses the LSF resource manager\n";

    # NOTE: do not include list of queues

    return $header;
}

