#!/usr/bin/perl
#
# scan job execution log files from the uBGL system at LLNL (Slurm format), and use
# convert2swf_v2.pm to convert the data into the standard workload format.
#

use warnings;
use strict;
use Time::Local;
use POSIX;
use convert2swf_v4;


############################
#
# DO THE CONVERSION:
#

my $tz_str    = "US/Pacific";
my $max_procs = 32*32*2; # two mid-planes with dimensions of 32x32 each
			 # (each mid-plane thus consists of 1024 cores)

my @queues = ();

SWF::init( $tz_str, $max_procs, \@queues );

my $format_bugs;
my $jobs_data = get_jobs();

my ($zero, $maxtime) = SWF::convert( $jobs_data, $format_bugs );

my $header = generate_header( $zero, $maxtime, $max_procs );

SWF::print( $header );


#############################
# SPECIFICS for uBGL workload

# the format of each entry includes the following space-separated fields:
#  0 JobId=<number>
#  1 UserId=xxxxxx(<number>) - the x's hide the actual user name to conserve privacy
#  2 Name=<string> - name of executable (script), could be empty
#  3 JobState=<status>
#  4 Partition=<string>
#  5 TimeLimit=<number>
#  6 StartTime=<date and time>
#  7 EndTime=<date and time>
#  8 NodeList=<string> - midplanes, could be (null)
#  9 NodeCnt=<number> - midplanes
# 10 Connection=<string>
# 11 Reboot=no - OPTIONAL FIELD, affects numbering of subsequent ones
# 12 Rotate=yes
# 13 MaxProcs=<number> - max procs to actually use
# 14 Geometry=<string> - midplanes
# 15 Start=None
# 16 Block_ID=<string> - or unassigned


my $job_cnt;
my $problem1;
my $problem2;
my $problem3;
my $problem4;
my $problem5;
my $problem6;
my $problem7;

my $completed;
my $failed;
my $cancelled;
my $timeout;
my $node_fail;
my $other_status;
my $short;
my $shortfailed;

my $nalloc;

my $user_1;
my $user_3;


sub get_jobs {
##############
# parse input format and create a hash for each job

  my @list_of_jobs;

  $job_cnt  = 0; #counter for the jobs
  $problem1 = 0; #NodeList inconsistent with NodeCnt
  $problem2 = 0; #NodeList includes more than one plane.
  $problem3 = 0; #StartTime > EndTime
  $problem4 = 0; #jobs with zero recorded runtime, but a non-FAILED status
  $problem5 = 0; #jobs that exceeded their TimeLimit
  $problem6 = 0; #jobs who weren't allocated any processors, but are non-FAILED
  $problem7 = 0; #user requested less than a full mid-plane

  #Status variables
  $completed    = 0;
  $failed       = 0;
  $cancelled    = 0;
  $timeout      = 0;
  $node_fail    = 0;
  $other_status = 0;
  $short        = 0;
  $shortfailed  = 0;

  $nalloc = 0; #Number of jobs who requested, but weren't allocated any (or insufficient number of) processors.

  #Heavy Users
  $user_1 = 0;
  $user_3 = 0;


  #
  # scan trace and collect job info
  #
  while (<>) {
    if (/^\s*$/) {		# empty line
      next;
    }

    $_ =~ /\s*(.*)\s*/;		# get rid of white space at ends
    $_ = $1;
    my @line = split('\s+');	# split into fields

    if (($#line != 15) && ($#line != 16)) {
      $format_bugs .= "format_problem $_\n";
      warn(">>>Format problem on $_\ngot $#line fields (should be 15 or 16)");
      next;
    }

    my %nj;

    #
    # set the desired fields as specified in convert2swf.pl
    #

    $nj{trace}  = $_;	# used for error messages


    my ($stime, $etime);
    #Start Time
    if ($line[6] =~ /^StartTime=(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)(.*)$/) {

      my @begtmarr;
      $begtmarr[5] = $1 - 1900;
      $begtmarr[4] = $2 - 1;
      $begtmarr[3] = $3;
      $begtmarr[2] = $4;
      $begtmarr[1] = $5;
      $begtmarr[0] = $6;
      $stime = &timelocal(@begtmarr);

      $nj{start}  = $stime;
      #$nj{submit} = $stime;
    }


    #End Time
    if ($line[7] =~ /^EndTime=(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)(.*)$/) {

      my @endtmarr;
      $endtmarr[5] = $1 - 1900;
      $endtmarr[4] = $2 - 1;
      $endtmarr[3] = $3;
      $endtmarr[2] = $4;
      $endtmarr[1] = $5;
      $endtmarr[0] = $6;
      $etime = &timelocal(@endtmarr);

      $nj{end} = $etime;
    }

    #wait = start - submit is identically 0 for this log
    #run  = end - start


    #Job Status
    my $status_id;
    if ($line[3] =~ /^JobState=(.*)/) {
      my $status = $1;

      if ($status eq "COMPLETED") {
	$status_id = 1;
	$completed++;
      }
      elsif ($status eq "FAILED") {
	$status_id = 0;
	$failed++;
      }
      elsif ($status eq "CANCELLED") {
	$status_id = 5;
	$cancelled++;
      }
      elsif ($status eq "TIMEOUT") {
	$status_id = 0;
	$timeout++;
      }
      elsif ($status eq "NODE_FAIL") {
	$status_id = 0;
	$node_fail++;
      }
      else {
	$status_id = -1;
	$other_status++;
      }
      $nj{status} = $status_id;
    }

    if (defined($etime) && defined($stime) && (($etime - $stime) < 5)) {
      $short++;
      if ($status_id == 0) {
	$shortfailed++;
      }
    }


    #Allocated and Requested Processors
    my $req_procs = -1;
    #Some but not all log lines include a "Reboot" field that should be skipped
    my $max_procs_field;
    if ($line[11] =~ /^Reboot=(.*)/) {
      $max_procs_field = 13;
    }
    else {
      $max_procs_field = 12;
    }

    if ($line[$max_procs_field] =~ /^MaxProcs=(.*)/) {
      my $max_procs_data = $1;

      if ($max_procs_data eq "None") {

	# use nodes
	if ($line[9] =~ /^NodeCnt=(.*)/) {
	  my $req_nodes = $1;
	  if ($req_nodes>1) {$problem2++;}
	  my $req_procs = $req_nodes * 1024;
	  if ($req_procs != 0) {
	    $nj{procs}     = $req_procs;
	    $nj{req_procs} = $req_procs;
	  }
	}
	# check
	if ($line[8] =~ /^NodeList=ubgl(.*)/) {
	  my @plane_list = split('x', $1);	# split into fields
	  my $allocated_procs = @plane_list * 1024;
	  if ($allocated_procs != $req_procs) {
	    $problem1++;
	  }
 	}
	elsif ($line[8] eq "NodeList=(null)") {
	  if ($status_id==1) {$problem6++;}
	}
      }
      else {
	# in case the job requests less than a full mid-plane we assume it was
	# allocated that number, not the entire mid-plane.
        if ($max_procs_data =~ /(\d+)k/) {
	  $max_procs_data = $1 * 1024;
        }
	else {
	  $problem7++;
	}
	$nj{procs}     = $max_procs_data;
	$nj{req_procs} = $max_procs_data;
      }
    }


    #Requested Time (given in minutes)
    my $req_time;
    if ($line[5] =~ /^TimeLimit=(\d+)/) {
      $req_time = $1;
      if ($req_time != 4294967294) {	# unsigned -1 value
	$nj{req_time} = $req_time * 60;
      }
    }


    #User ID
    if ($line[1] =~ /^UserId=x*\((\d+)\)$/) {
      my $user = $1;
      if ($user eq "55146") {$user_1++;}
      if ($user eq "55194") {$user_3++;}
      $nj{user} = $user;
    }


    #Executable (Application)
    if ($line[2] =~ /^Name=(.*)/) {
      my $app = $1;
      if ($app ne "") {
	$nj{app} = $app;
      }
    }


    #Partition
    if ($line[4] =~ /^Partition=(.*)/) {
      my $partition = $1;
      $nj{partition} = $partition;
    }

    #no data on CPU time
    #no data on memory
    #no data on requested memory
    #no data on queue
    #no data on group

    $list_of_jobs[$job_cnt++] = \%nj;



    #SOME CHECKS:

    #How many jobs were recorded with zero runtime, but a non-FAILED status?
    if (($etime == $stime) && ($status_id != 0)) {
      $problem4++;
    }
    #How many jobs exceeded their TimeLimit?
    if (defined($etime) && defined($stime) && defined($req_time) &&
	(($etime - $stime) > $req_time)) {
      $problem5++;
    }
  }

  $format_bugs .= "Jobs with inconsistent number of nodes (midplanes): $problem1\n";
  $format_bugs .= "Jobs using more than one midplane:  $problem2\n";
  $format_bugs .= "StartTime > EndTime:  $problem3\n";
  $format_bugs .= "RunTime is zero, but the job wasn't FAILED:  $problem4\n";
  $format_bugs .= "Number of jobs whose RunTime exceeds TimeLimit:  $problem5\n";
  $format_bugs .= "Number of jobs who weren't allocated any processors, but were COMPLETED: $problem6\n";
  $format_bugs .= "Number of jobs in which the user specifically asked for less than a full mid-plane: $problem7\n";
  $format_bugs .= "status: COMPLETED=$completed, FAILED=$failed, CANCELLED=$cancelled, " .
    "TIMEOUT=$timeout, NODE_FAIL=$node_fail, UNKNOWN STATUS=$other_status\n";
  $format_bugs .= "Short jobs: $short, of these $shortfailed failed\n";
  $format_bugs .= "user 55146: $user_1, user 55194: $user_3\n";

  return \@list_of_jobs;
}


sub generate_header {
  my ($zero, $maxtime, $max_procs) = @_;

  my ($sec, $min, $hr, $mday, $mon, $year, $wday, $yday, $isdst);

  $header  = "; Version: 2.2\n";
  $header .= "; Computer: Small BlueGene/L\n";
  $header .= "; Installation: High Performance Computing - Lawrence Livermore National Laboratory\n";

  $header .= "; Acknowledge: Moe Jette, LLNL\n";
  $header .= "; Information: https://computing.llnl.gov/\n";

  $header .= sprintf("; Conversion: Sarah Gingichashvili and Dror Feitelson (feit\@cs.huji.ac.il) %s\n",
                     strftime("%d %b %Y", localtime()));

  $header .= sprintf("; MaxJobs: %d\n", $job_cnt);
  $header .= sprintf("; MaxRecords: %d\n", $job_cnt);
  $header .= "; Preemption: No\n";

  $header .= sprintf("; UnixStartTime: %d\n", $zero);
  $header .= sprintf("; TimeZoneString: %s\n", $tz_str);

  $header .= sprintf("; StartTime: %s\n",
                     strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($zero)));

  $header .= sprintf("; EndTime:   %s\n",
                     strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($maxtime)));

  $header .= "; MaxNodes: $max_procs\n";
  $header .= "; MaxProcs: $max_procs\n";

  $header .= "; Note: Scheduler is Slurm (https://computing.llnl.gov/linux/slurm/)\n";

  return $header;
}
