#!/usr/bin/perl
#
# scan job execution log files from the Atlas system at LLNL (Slurm format), and use
# convert2swf_v2.pm to convert the data into the standard workload format.
#

use warnings;
use strict;
use Time::Local;
use POSIX;
use convert2swf_v4;


############################
#
# DO THE CONVERSION:
#

my $tz_str    = "US/Pacific";
my $proc_per_node = 8;
my $max_nodes = 1152;
my $max_procs = $max_nodes * $proc_per_node;

my @queues = ();

SWF::init( $tz_str, $max_procs, \@queues );

my $format_bugs;
my $jobs_data = get_jobs();

my ($zero, $maxtime) = SWF::convert( $jobs_data, $format_bugs );

my $header = generate_header( $zero, $maxtime, $max_procs );

SWF::print( $header );

##############################
# SPECIFICS for Atlas workload

# the format of each entry includes the following space-separated fields:
#  0 JobId=<number>
#  1 UserId=xxxxxx(<number>) - the x's hide the actual user name to conserve privacy
#  2 Name=<string> - name of executable (script), could be empty
#  3 JobState=<status>
#  4 Partition=<string>
#  5 TimeLimit=<number>
#  6 StartTime=<date and time>
#  7 EndTime=<date and time>
#  8 NodeList=<string> - could be (null)
#  9 NodeCnt=<number>
# note that each node has 8 CPUs.


my $job_cnt;
my $problem1;
my $problem2;
my $problem3;
my $problem4;

my $completed;
my $failed;
my $cancelled;
my $timeout;
my $other_status;
my $node_fail;

my $nalloc;

my $short;
my $shortfailed;

my %cores;
sub get_jobs {
##############
# parse input format and create a hash for each job

  my @list_of_jobs;
  $job_cnt   = 0; #counter for the jobs
  $problem1  = 0; #StartTime > EndTime
  $problem2  = 0; #jobs with zero recorded runtime, but a non-FAILED status
  $problem3  = 0; #jobs that exceeded their TimeLimit
  $problem4  = 0; #jobs who weren't allocated any processors, but are non-FAILED
  $short     = 0;
  $shortfailed = 0;

  #Status variables
  $completed    = 0;
  $failed       = 0;
  $cancelled    = 0;
  $timeout      = 0;
  $other_status = 0;
  $node_fail    = 0;

  $nalloc = 0; #Number of jobs who requested, but weren't allocated any (or insufficient number of) processors.

  #
  # scan trace and collect job info
  #
  while (<>) {
    if (/^\s*$/) {		# empty line
      next;
    }

    $_ =~ /\s*(.*)\s*/;		# get rid of white space at ends
    $_ = $1;
    my @line = split('\s+');	# split into fields

    #NOTE: At times the field which records the job executable name contains a string which
    #itself contains a number of spaces. The following code deals with this issue.
    while ((3 <= $#line) && ($line[3] !~ /^JobState=(.*)/)) {
      $line[2] .= " " . $line[3];
      for (my $i=3; $i<$#line; $i++) {
	$line[$i] = $line[$i+1];
      }
      delete $line[$#line];
    }

    if ($#line != 9) {
      $format_bugs .= "format_problem: $#line fields in $_\n";
      warn(">>>Format problem on $_\ngot $#line fields (should be 9)");
      next;
    }

    my %nj;

    #
    # set the desired fields as specified in convert2swf.pl
    #

    $nj{trace}  = $_;	# used for error messages


    my ($stime, $etime);
    my $j;

    #Start Time (two possible formats)
    my @begtmarr;
    if ($line[6] =~ /^StartTime=(\d+)(\W)(\d+)-(\d+):(\d+):(\d+)(.*)$/) {
      $begtmarr[5] = 2007 - 1900;
      $begtmarr[4] = $1 - 1;
    }
    elsif ($line[6] =~ /^StartTime=(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)(.*)$/) {
      $begtmarr[5] = $1 - 1900;
      $begtmarr[4] = $2 - 1;
    }
    $begtmarr[3] = $3;
    $begtmarr[2] = $4;
    $begtmarr[1] = $5;
    $begtmarr[0] = $6;
    $stime = &timelocal(@begtmarr);
    $nj{start}  = $stime;
    #no data on submit time


    #End Time (two possible formats)
    my @endtmarr;
    if ($line[7] =~ /^EndTime=(\d+)(\W)(\d+)-(\d+):(\d+):(\d+)(.*)$/) {
      $endtmarr[5] = 2007 - 1900;
      $endtmarr[4] = $1 - 1;
    }
    elsif ($line[7] =~ /^EndTime=(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)(.*)$/) {
      $endtmarr[5] = $1 - 1900;
      $endtmarr[4] = $2 - 1;
    }
    $endtmarr[3] = $3;
    $endtmarr[2] = $4;
    $endtmarr[1] = $5;
    $endtmarr[0] = $6;
    $etime = &timelocal(@endtmarr);
    $nj{end}    = $etime;

    #wait = start - submit is identically 0 for this log;
    #run = end - start;

    #Job Status
    my $status_id;
    if ($line[3] =~ /^JobState=(.*)/) {
      my $status = $1;
      if ($status eq "COMPLETED") {
        $status_id = 1;
        $completed++;
      }
      elsif ($status eq "FAILED") {
        $status_id = 0;
        $failed++;
      }
      elsif ($status eq "CANCELLED") {
        $status_id = 5;
        $cancelled++;
      }
      elsif ($status eq "TIMEOUT") {
        $status_id = 0;
        $timeout++;
      }
      elsif ($status eq "NODE_FAIL") {
        $status_id = 0;
        $node_fail++;
      }
      else {
        $status_id = -1;
        $other_status++;
      }
      $nj{status} = $status_id;
    }

    if (defined($etime) && defined($stime) && (($etime - $stime) < 5)) {
      $short++;
      if ($status_id == 0) {
	$shortfailed++;
      }
    }

    #Allocated and Requested Processors
    my $allocated_procs;
    my $next;
    my @next_req;
    # Allocated
    if ($line[8] =~ m/NodeList=([a-z]*)(\[?)([0-9,-]+)(\]?)/) {
      $allocated_procs = 0;
      my @alloc_list = split(',', $3);	# split into fields
      for ($j=0; $j<@alloc_list; $j++) { #list of nodes, some ranges of nodes, some single nodes
        $next = $alloc_list[$j];
        @next_req = split('-',$next); #in case it's a range of nodes
        if (@next_req==2) {$allocated_procs += ($next_req[1]-$next_req[0]+1);} #range
        elsif (@next_req==1) {$allocated_procs++;} #single node
      }
      $allocated_procs *= $proc_per_node;
      if ($allocated_procs!=0) {
        $nj{procs} = $allocated_procs;
      }
    }
    elsif ($line[8] eq "NodeList=(null)") {
      if ($status_id != 0) {
        $problem4++;
      }
    }

    # Requested
    if ($line[9] =~ /^NodeCnt=(.*)/) {
	my $req_procs = $1 * $proc_per_node;
      if ($req_procs != 0) {
        $nj{req_procs} = $req_procs;
      }
    }

    #Requested Time (given in minutes)
    my $req_time;
    if ($line[5] =~ /^TimeLimit=(\d+)/) {
      $req_time = $1;
      if ($req_time != 4294967294) {	# unsigned -1 value
	$nj{req_time} = $req_time * 60;
      }
    }

    #User ID
    if ($line[1] =~ /^UserId=x*\((\d+)\)$/) {
      my $user = $1;
      $nj{user}  = $user;
    }

    #Executable (Application) Number
    if ($line[2] =~ /^Name=(.*)/) {
      my $app = $1;
      if ($app ne "") {
	  $nj{app} = $app;
      }
    }

    #Partition
    if ($line[4] =~ /^Partition=(.*)/) {
      my $partition = $1;
      $nj{partition} = $partition;
    }

    #no data on CPU time
    #no data on memory
    #no data on requested memory
    #no data on queue
    #no data on group

    $list_of_jobs[$job_cnt++] = \%nj;



    #SOME CHECKS:

    #How many jobs were recorded with zero runtime, but a non-FAILED status?
    if (defined($nj{start}) && (defined($nj{end})) && ($etime == $stime) && ($status_id != 0)) {
      $problem2++;
    }
    #How many jobs exceeded their TimeLimit?
    if (defined($etime) && defined($stime) && defined($req_time) &&
	($req_time!=0) && (($etime - $stime) > $req_time*60)) {
      $problem3++;
    }
    #Is the number of allocated processors sufficient?
    if (defined($nj{procs}) && defined($nj{req_procs}) && ($nj{procs} < $nj{req_procs})) {
      $nalloc++;
    }
  }

  $format_bugs .= "StartTime > EndTime:  $problem1\n";
  $format_bugs .= "RunTime is zero, but the job wasn't FAILED:  $problem2\n";
  $format_bugs .= "Number of jobs whose RunTime exceeds TimeLimit:  $problem3\n";
  $format_bugs .= "Number of jobs who weren't allocated any processors, but were non-FAILED: $problem4\n";
  $format_bugs .= "Number of jobs who requested, but weren't allocated any (or insufficient number of) processors: $nalloc\n";

  return \@list_of_jobs;
}


sub generate_header {
  my ($zero, $maxtime, $max_procs) = @_;

  my ($sec, $min, $hr, $mday, $mon, $year, $wday, $yday, $isdst);

  $header  = "; Version: 2.2\n";
  $header .= "; Computer: Linux cluster (Atlas)\n";
  $header .= "; Installation: High Performance Computing - Lawrence Livermore National Laboratory\n";

  $header .= "; Acknowledge: Moe Jette, LLNL\n";
  $header .= "; Information: https://computing.llnl.gov/?set=resources&page=OCF_resources#atlas\n";

  $header .= sprintf("; Conversion: Sarah Gingichashvili and Dror Feitelson (feit\@cs.huji.ac.il) %s\n",
                     strftime("%d %b %Y", localtime()));

  $header .= sprintf("; MaxJobs: %d\n", $job_cnt);
  $header .= sprintf("; MaxRecords: %d\n", $job_cnt);
  $header .= "; Preemption: No\n";

  $header .= sprintf("; UnixStartTime: %d\n", $zero);
  $header .= "; TimeZone: 3600\n";
  $header .= sprintf("; TimeZoneString: %s\n", $tz_str);

  $header .= sprintf("; StartTime: %s\n",
                     strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($zero)));

  $header .= sprintf("; EndTime:   %s\n",
                     strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($maxtime)));

  $header .= "; MaxNodes: $max_nodes\n";
  $header .= "; MaxProcs: $max_procs\n";

  $header .= "; Note: Scheduler is Slurm (https://computing.llnl.gov/linux/slurm/)\n";

  return $header;
}
