#!/opt/pkgs/bin/perl -w
#
# Take the data-logs from nss and plots them using gnuplot, Grace or
# the perl module Chart::
#
# $Id: plot-nss.pl,v 2.1 2001/05/25 14:40:09 stoffel Exp stoffel $
#
# $Log: plot-nss.pl,v $
# Revision 2.1  2001/05/25 14:40:09  stoffel
# Better version checking of gnuplot
# rework the structure of program
# initial work on adding new field to log file format for backup level info.
# cleanups on the key format.  Still in progress
#
# Require gnuplot:  http://www.gnuplot.org

require 5.000;
use Time::Local;
use Date::DateCalc qw(:all);
use Getopt::Long;
use Chart::Lines;
use Chart::Points;
use Chart::Bars;
use Chart::LinesPoints;
use Chart::Mountain;
use Chart::StackedBars;

&Getopt::Long::config("no_ignore_case");

my @log;
$debug = 1;
$| = 1;
my $gnuplot="/home/stoffel/bin/gnuplot";
my $grace="/home/stoffel/bin/xmgrace";
my $plottool = "Chart";

my $do_print = 0;
$plot = "nss-plot";
$prtcmd ="/bin/lp -c";

my $with_imp = 0;

&main;
exit;

######################################################################
sub main {

  $ret = GetOptions('s' => \$by_size,
		    't' => \$by_time,
		    'D:i' => \$debug,
		    'o=s' => \$ofile,
		    'P:s' => \$do_print,
		    'U=s' => \$plottool,
		    'X' => \$use_x,
		   );
  
  &usage if (!$ret);
  &usage if ($#ARGV < 0);
  
  print "main()\n" if ($debug > 0);

  if ($plottool eq "gnuplot") {
    &gnuplot_version_ok || die "Error: you must have a newer version of gnuplot to use this program.  3.5 patchlevel beta 349 is the minimum\n\n";
  }

  if ($plottool eq "grace") {
    warn "Warning!  Grace is barely supported...\n\n";
  }
  if ($plottool eq "Chart") {
    warn "Warning!  Chart:: is barely supported...\n\n";
  }
  if ($plottool eq "xmgr") {
    die "Error!  xmgr not supported yet.\n";
  }

  # Suck in the data from each file, stuff into a hash, then prompt
  # the user for data filtering needs.  The actual plot routines are
  # later on, and will need to be redone due to this change.

  foreach $file (@ARGV) {
    @log = &read_log($file);
    $file =~ s/^.*\/(\w+)/$1/;
    ($server,$savegrp) = split(/\./,$file);
    print "server: $server    Savegroup: $savegrp\n" if ( $debug > -1);
    print STDERR "  read $#log lines\n" if $debug;

    @{$data{$server}{$savegrp}} = @log;
  }

  # Do data reductions here, before we pass them off to plotting
  # functions. 

  %data = &normalize_data(%data);

  # Now plot the data.  How do we tell the tools how to plot?  

  if ($plottool eq "gnuplot") {
    &plot_gnuplot_log(%data);
  }
  elsif ($plottool eq "grace") {
    &plot_grace_log(%data);
  }
  elsif ($plottool eq "Chart") {
    &plot_Chart_log(%data);
  }
}


#---------------------------------------------------------------------
# GNUPLOT routines
#---------------------------------------------------------------------
sub plot_gnuplot_log {
  print "  plot_gnuplot_log()\n" if $debug > 0;

  my %data = @_;
  my @s;
  
  my $cfile = "/tmp/cfile.$$";
  my $dfile = "/tmp/dfile.$$";
  $ofile = "/tmp/plot.$$.ps" if (!$ofile); 
  
  $num = &create_gnuplot_dfiles("$dfile",%data);
  &create_gnuplot_cfile($cfile,$dfile,$num,$by_size,$by_time,$use_x);
  
  print `$gnuplot $cfile`;
  if (-s "/tmp/$plot.$$.ps") { 
    `$prtcmd /tmp/$plot.$$.ps`; 
  }
}

#---------------------------------------------------------------------
sub create_gnuplot_dfiles {
  print "  create_gnuplot_dfiles()\n" if $debug > 0;

  my $dfile = shift @_;
  my @log = @_;
  
  my $l;
  my @d;
  my @dd;
  my $day;
  my $time;
  my $size;
  
  # write the data file
  
  foreach $day (sort keys %data) {
	foreach $p (sort keys %dp) {
	  my @log = @{$data{$day}{$p}};
	  $x++;
	  foreach $l (@log) {
		@d = split(/\|\|/,$l);
		@dd = split(" ", $d[0]);    
		$dd[1] = &decode_month($dd[1]);
		$day="$dd[3]:$dd[2]:$dd[1]:$dd[4]";
		$time = &decimal_hours($d[2]);
		$size = $d[5];

		print DFILE "$day $time $size\n";
		print "day: $d[0] ($day)\n" if ($debug > 1);
	  }
	}
  }
  open(DFILE, "> $dfile") ||
	die "Error: Can't open $dfile for writing: $!\n";
  close DFILE;

  return $x;
}

#---------------------------------------------------------------------
# Create the configuration control file

sub create_gnuplot_cfile {
  print " create_gnuplot_cfile()\n" if $debug;

  my $cfile = shift @_;
  my $dfilebase = shift @_;
  my $numdfiles = shift @_;
  my $by_size = shift @_;
  my $by_time = shift @_;
  my $use_x = shift @_;
  
  open(CFILE, "> $cfile") ||
    die "Error: Can't open $cfile for writing: $!\n";

  # Create the control file, using some of the data from dfile for
  # labels, etc.

  print CFILE <<"EOCF";

set title "Server: $server    Savegroup: $savegrp"
set xlabel "Start date of Savegroup"
set xtics rotate
set xdata time
set timefmt "%H:%M:%S:%d:%m:%Y"
set format x "%m/%d/%Y"
set key top left
set key box
set grid ytics lw 1
EOCF


  if ($by_size && $by_time) {
    print CFILE <<"EOCF";
set ylabel "Backup Size (Mb)"
set y2label "Backup Time (Hrs)"
set ytics
set y2tics
set logscale y2 10
EOCF
  }
  else {
    if ($by_size) {

      print CFILE <<"EOCF";
set ylabel "Backup Size (Mb)"
set ytics
EOCF
    }
    if ($by_time) {
      
      print CFILE <<"EOCF";
set ylabel "Backup Time (Hrs)"
set ytics
EOCF
    }
  }

  if ($with_imp) {
    $impulse = "with impulses"
  } 
  else {
    $impulse = "";
  }
  
  if ($use_x) {
    print CFILE <<"EOCF";

set terminal x11

plot "$dfile" using 1:2 axes x1y2 title "Hrs" with points, "$dfile" using 1:3 axes x1y1 title "Mbs" $impulse

pause -1  "Hit return to continue"
quit

EOCF
    
  }
  else {
    print "     Output to $cfile.\n";
    print CFILE <<"EOCF";

set output "/tmp/$plot.$$.ps"
set terminal postscript landscape mono 10

plot "$dfile" using 1:2 axes x1y2 title "Hrs" with points pt 6, "$dfile" using 1:3 axes x1y1 title "Mbs" with points pt 7, "$dfile" using 1:2 axes x1y2 notitle $impulse lt 0, "$dfile" using 1:3 axes x1y1 notitle $impulse lt 0

quit

EOCF
    
  }
  close(CFILE);
}

#---------------------------------------------------------------------
# We need to have version 3.5, patchlevel beta 349 or *higher*

sub gnuplot_version_ok {
  
  print " gnuplot_version_ok()\n" if $debug > 0;

  my $gver;

  print "gnuplot: $gnuplot\n" if ($debug > -1);
  $gver = `echo show version | gnuplot 2>&1`;
  
  $gver =~ /version (\d+\.\d+)/;

  if ($1 > 3.5) {
    return 1;
  }
  elsif ( $1 < 3.5) {
    return 0;
  }
  else {
    $gver =~ m/patchlevel beta (\d+)/;

    if ($1 >= 349) {
      return 1;
    }
    else {
      return 0;
    }
  }
}


#---------------------------------------------------------------------
# GRACE routines
#---------------------------------------------------------------------

sub plot_grace_log {
  print "  plot_grace_log()\n" if $debug > 0;

  my @log = @_;
  my @s;
  
  my $dfile = "/tmp/dfile.$$";
  $ofile = "/tmp/nss-plot-grace.$$.ps" if (!$ofile); 
  
  &create_grace_dfile($dfile,@log);  
  
  print `$grace $dfile`;

  if ((-s "$ofile") && ($do_print)) { 
    `$prtcmd $ofile`; 
    print " Printed $ofile.\n";
  }
}

#---------------------------------------------------------------------
# create the combined control and data file for use with 'grace'.

sub create_grace_dfile {

  my $dfile = shift @_;
  my @d = @_;

  my @legends = qw( Hours Megabytes Hours Megabytes );


  open(DFILE, "> $dfile") ||
    die "Error! Can't open $dfile for writing: $!\n";

  print DFILE <<"EOCF";

\@g0 on
#\@g0 type Chart
\@g0 stacked true
\@with g0
\@ world xmin 0
\@ world xmax 11
\@ world ymin 0
\@ world ymax 300
\@ stack world 0, 0, 0, 0
\@ title "Backups - time vs. size and speed."
\@ title size 1.5
\@ subtitle "stoffel\@lucent.com"
\@ subtitle size 1.0
\@ xaxes scale Normal
\@ yaxes scale Normal
\@ xaxis on
\@ xaxis ticklabel on
\@ xaxis ticklabel format yymmdd
\@ xaxis ticklabel prec 0
\@ yaxis on
\@ frame type 1
\@ frame linestyle 1
\@ frame linewidth 1
\@ frame color 1
EOCF

  foreach ($x = 0; $x <= 3; $x++) {
    print DFILE <<"EOCF";
\@    s$x type xy
\@    s$x symbol $x+1
\@    s$x symbol size 0.850000
\@    s$x symbol color $x+2
\@    s$x symbol pattern 1
\@    s$x symbol fill color $x+2
\@    s$x symbol fill pattern 1
\@    s$x symbol linewidth 1
\@    s$x symbol linestyle 1
\@    s$x symbol char 65
\@    s$x symbol char font 2
\@    s$x symbol skip 0
\@    s$x line type 1
\@    s$x line linestyle 1
\@    s$x line linewidth 1
\@    s$x line color 4
\@    s$x line pattern 1
\@    s$x baseline type 0
\@    s$x baseline off
\@    s$x dropline on
\@    s$x fill type 2
\@    s$x fill rule 0
\@    s$x fill color 7
\@    s$x fill pattern 1
\@    s$x avalue off
\@    s$x comment "copy of set $x"
\@    s$x legend  "$legends[$x]"
EOCF

}
  
  foreach $l (@log) {
    @d = split(/\|\|/,$l);
    
    # Now grab the start time and break it into chunks then format it
    # for grace to parse nicely in ISO8601 format.
    # IN:  Tue Jul 10 23:59:00 2001
    # OUT: 1999-12-31T23:59:59.999
    #
    
    @dd = split(" ",$d[0]);
    $dd[1] = &decode_month($dd[1]);

    # ISO8601 format
    $day="$dd[4]-$dd[1]-$dd[2]T$dd[3].000";

    $time = &decimal_hours($d[2]);
    $size = $d[5];
    
    if ($dd[4] > 2001) {
      if ($d[6] =~ m/full/) {
	push @timefull, "\t$day\t$time\n";
	push @sizefull, "\t$day\t$size\n";
      }
      elsif ($d[6] =~ m/incr/) {
	push @timeincr, "\t$day\t$time\n";
	push @sizeincr, "\t$day\t$size\n";
      }
    }
  }
  
  print DFILE "\@target G0.S0\n";
  print DFILE "\@type bar\n";

  foreach $l (@timefull) {
    print DFILE "$l";
  }
  print DFILE "&\n\n";

  print DFILE "\@target G0.S1\n";
  print DFILE "\@type bar\n";

  foreach $l (@sizefull) {
    print DFILE "$l";
  }
  print DFILE "&\n\n";

  print DFILE "\@target G0.S2\n";
  print DFILE "\@type bar\n";

  foreach $l (@timeincr) {
    print DFILE "$l";
  }
  print DFILE "&\n\n";

  print DFILE "\@target G0.S3\n";
  print DFILE "\@type bar\n";

  foreach $l (@sizeincr) {
    print DFILE "$l";
  }
  print DFILE "&\n\n";

  close(DFILE);

  print STDERR "Output to $dfile\n";
}  

#---------------------------------------------------------------------
# Chart:: routines
#---------------------------------------------------------------------

sub plot_Chart_log {

  my %data = @_;

  print STDERR "\nplot_Chart_log()\n" if $debug;

  my @log;
  my @legends;
  
  my %layout = ( 'title' => 'Incremental Backups vs Size',
		 'x_label' => 'YYYY-MM-DD',
		 'x_ticks' => 'vertical',
		 'f_x_tick' => \&Chart_format_xlabel,
		 'skip_x_ticks' => '5',
		 'y_ticks' => '5',
		 'f_y_tick' => \&Chart_format_ylabel,
		 'y_label' => 'Gigabytes',
		 'legend' => 'bottom',
		 'legend_labels' => \@legends,
		 'grid_lines' => 'true',
	       );
  
  $xsize = 800;
  $ysize = 600;
  #$c = Chart::Lines->new($xsize,$ysize);
  $c = Chart::StackedBars->new($xsize,$ysize);
  
  print STDERR "\n  building legends...\n" if $debug;
  undef @legends;
  foreach $p (sort keys %dp) {
    $p =~ s/-/ /;
    push @legends, $p;
    print "    $p\n";
  }
  
  $c->set(%layout);
  
  # Go through each date, put all data sets into graph.  This is
  # tricky, since missing data points need to be listed as a zero.
  
  print STDERR "  data loop:\n" if $debug;
  
  foreach $day (sort keys %data) {
    print STDERR "  day: $day\n" if $debug;
    if ($day =~ m/^2002-/) {
      undef @line;
      push @line, $day;
      # loop through list of all keys for each date, making sure we zero
      # out non-existent entries.
      foreach $p (sort keys %dp) {
	if (!defined $data{$day}{$p}) {
	  push @line, 0;
	}
	else {
	  push @line, $data{$day}{$p};
	}
      }
      $c->add_pt(@line);
    }
  }
  
  $c->jpeg("/tmp/chart.jpg");
  print STDERR "Wrote output to /tmp/chart.jpg\n";
}


#---------------------------------------------------------------------
# Nothing for now...

sub Chart_format_xlabel {

  my $l = shift @_;
  $l =~ s/T\d\d:.*$//;

  return $l;
}

#---------------------------------------------------------------------
# Integers only

sub Chart_format_ylabel {

  return int(shift);
}



#---------------------------------------------------------------------
# Helper Routines
#---------------------------------------------------------------------
# Normalize the data and sort it by date.

sub normalize_data {

  my %data = @_;

  my %sizebydate;

  print STDERR " loop through \%data\n" if $debug;
  foreach $s (sort keys %data) {
    print STDERR "  server: $s\n" if $debug;

    foreach $sg (sort keys %{$data{$s}}) {
      print STDERR "   savegrp: $sg\n" if $debug;
      $dp{"$s-$sg"}++;
      @lines = @{$data{$s}{$sg}};

      foreach $l (@lines) {
	my @d = split(/\|\|/,$l);

	my @dd = split(" ",$d[0]);
	$dd[1] = &decode_month($dd[1]);
	
	my $day=sprintf("%4d-%02d-%02d",$dd[4],$dd[1],$dd[2]);

	$sizebydate{$day}{"$s-$sg"} = $d[5];
      }
    }
  }
  return %sizebydate;
}

#---------------------------------------------------------------------
sub usage {
  print "Usage:\n";
  print "  $0 [-D [num]] [-s] [-t] [-X] [-U gnuplot|xmgr|grace] [-o output] file ...\n";
  exit;
}

#---------------------------------------------------------------------
sub decimal_hours {
  my $t = shift @_;
  my ($h, $m);
  
  ($h,$m) = split(":",$t);
  return (sprintf "%.2f",($h+($m/60)));
}

#---------------------------------------------------------------------
#
# Record format:
# - one line per record
# - each field is seperated by ||
#
# - fields: start_time stop_time total_time num_clients num_failed
#           savegroup_size_mb
#
sub read_log {

  print "read_log()\n" if $debug > 0;
  my @l;

  my $file = shift @_;
  my %seen;
  
  open(LOG,"<$file") || die "Error: Can't open $file for reading: $!\n";
  while(<LOG>) {
    # make sure we don't pull in double lines from the log.
    next if (m/\|\|9$/ or m/full$/);
    push @l, $_ unless $seen{$_}++;
  }
  close LOG;
  
  return @l;
}