#!/usr/local/bin/perl -w
#
# Program: nss - (N)SR (S)avegroup (S)ummary
#
# Author: John Stoffel (john@stoffel.org)
#
# HTTP:  http://jfs.ecotarium.org/sources/nss/
#
# License: GPLv2 license, as found at
#   http://www.gnu.org/licenses/licenses.html#GPL
#
# Description: parses the output of the savegroup report from Legato
# Networker Save & Restore backup software, and gives back the number
# of hosts backed up at each level, amount of data saved at each level
# and number of hosts that didn't backup at all, etc.
#
# Usage: ./nss -h
#
# $Id: nss.pl,v 3.2 2003/04/01 19:31:18 stoffel Exp $

require 5.000;
use Time::Local;
use Getopt::Long;
use File::Path;
use File::Basename;
&Getopt::Long::config("no_ignore_case");

# Default directories and filename formats, edit for your site or see
# how to override in the usage message.

$logdir = "/nsr/logs/nsslogs";
$saveinputdir = "/nsr/logs/Backup";
$tmpdir = "/tmp";
$mailtofile = "$tmpdir/nss-mail.$$";

# See the docs for how to customize these.
$saveinputfile = "%S/%G-%D";
$subject = "Backups %E: %S - %G";

# Command paths
$sendmail = "/usr/lib/sendmail -t";
$mminfo = "/usr/sbin/nsr/mminfo";

#
# Do not edit below this line!
#

$mailto = "";
$DEBUG = -1;
$sizetop = -1;
$timetop = -1;
$logdata = -1;
$saveinput = -1;
$unsuccessful = 1;
$tapereport = -1;
$warnings = -1;

$server = "";
$savegrp = "";
$clients = 0;
$ignore_skip_level = 0;

&main;

#---------------------------------------------------------------------
# Subroutines
#---------------------------------------------------------------------

sub main {

  $ret = GetOptions('m=s', \$mailto,
		    'S=s', \$subject,
		    's:i', \$sizetop,
		    'o',   \$saveinput,
		    'O=s', \$saveinputdir,
		    'F=s', \$saveinputfile,
		    't:i', \$timetop,
		    'T',   \$tapereport,
		    'u:i', \$unsuccessful,
		    'l:i', \$logdata,
		    'L=s', \$logdir,
		    'd:i', \$DEBUG,
		    'v',   \$verbose,
		    'h',   \$usage,
		    'w',   \$warnings,
		    'server=s',  \$server,
		   );
  
  &usage if ($usage || !$ret);
  
  &init;
  
  if ($mailto) {
    open(OUT,">$mailtofile") || die "Couldn't write file $mailtofile: $!\n";
    print OUT "To: $mailto\n";
    print OUT "Subject: $subject\n";
    print OUT "\n";
  } 
  else {
    open(OUT,">-") || die "Couldn't open stdout: $!\n";
  }
  
  # Main parsing routine.
  &parseinput;
  
  # Print the verbose client info if asked with -v switch
  &print_verbose_clients if ($verbose);
  
  &sum_client_totals;
  &scale_client_totals;
  
  &print_report_summary;
  &print_level_report;
  &print_tape_report($server) if ($tapereport > 0);
  &print_unsuccessful if ($unsuccessful > 0);
  &print_top_n_size($sizetop) if ($sizetop > 0);
  &print_top_n_time($timetop) if ($timetop > 0);
  &print_warnings if ($warnings > 0);
  
  close(OUT);
  
  &save_log_data($logdir) if ($logdata > -1);

  if ($saveinput > -1) {
    &save_input_data($saveinputdir,$saveinputfile,$server,$savegrp,$start,@input) 
  }

  if ($mailto) {
    open(OUT,"|$sendmail") || die "Couldn't email message: $!\n";
    open(IN,"<$mailtofile") || die "Couldn't read file $mailtofile: $!\n";
    
    while(<IN>) {
      if (m/^Subject:/) {
	s/%S/$server/;
	s/%G/$savegrp/;
	s/%E/$sucess/;
      }
      print OUT;
    }
  }
}
 
#---------------------------------------------------------------------
# Get some basic info from the headers and list of failed clients, if
# any.  So we assume we're always starting from the head of the file.

sub parseinput {

  my $lc=0;

  my $state_initial = 1;
  my $state_successful = 3;
  my $state_unsuccessful = 2;
  
  my $state = $state_initial;

 LOOP:  
  while (<>) {
    push @input, $_ if $saveinput;

    next LOOP if m/^$/;
    $lc++;
    chomp;
    
    # if we have leading log file stuff, strip it out.  See below.
    if ($strip) {
      print STDERR "strip before: $_\n" if ($DEBUG >= 4);
      s/$strip//o;
      print STDERR "strip after: $_\n\n" if ($DEBUG >= 4);
    }
    
    if ($state == $state_initial) {
      
      # This is *the* main line we need to start off with.  If there's
      # leading stuff before the NetWorker word, make sure to strip it
      # out from all following lines.
      
      if ((/(^.*)NetWorker(.*?)Savegroup:/) ||
          (/(^.*)Solstice Backup(.*?)Savegroup:/)) {
        if ($1) {
          $strip = $1;
          $strip =~ s/\d/\\d/g;
          print STDERR "\nstrip = $strip\n" if ($DEBUG >=3);
        }

        m/\) (.+) (completed|aborted)/;
        $savegrp = $1;
        
        m/,\s+(\d+) client/;
        $clients = $1;
        
        m/client.* \((All Succeeded|.* Failed)\)/;
        if ($1 eq "All Succeeded") {
          @failed = ();
        }
        else {
          $f = $1;
          $f =~ s/(.*) Failed$/$1/;
          @failed = split(/,/,$f);
        }
      }
      
      if (/^Start time:/) {
        $restart = 0;
        $start = $_;
        $start =~ s/Start time:\s+(\w\w\w.*)$/$1/;
      }
      
      if (/^Restart time:/) {
        $restart = 1;
        $start = $_;
        $start =~ s/Restart time:\s+(\w\w\w.*)$/$1/;
      }
      
      if (/^End time:/) {
        $end=$_;
        $end =~ s/End time:\s+(\w\w\w.*)$/$1/;
      }
      
      if (/(\w+)\s*:\s*bootstrap\s*/) {
        $server = $1;
      }
      
      if (/^--- Successful Save Sets ---/) {
        $state = $state_successful;
	next LOOP;
      }
      if (/^--- Unsuccessful Save Sets ---/) {
        $state = $state_unsuccessful;
        next LOOP;
      }
    }
    
    # We're parsing the failed section of the header here.
    if ($state == $state_unsuccessful) {
      if (/^--- Successful Save Sets ---/) {
        $state = $state_successful;
        next LOOP;
      }
      print STDERR "(state_unsuccessful)\n" if ($DEBUG > 1);
      
      # If all clients fail, or the server has problems, we can try to
      # figure out the server name from this info.
      
      if (/.*:index save: Cannot open save session with (\w+)$/) {
        $server = $1;
        print STDERR "(state_unsuccessful) server: server=$server\n" if ($DEBUG > 1);       
      }
      push @undone, $_;
    }

    if ($state == $state_successful) {
      
      # skip general warning messages if not saving them
          
          if (m/^\*/) {
                next LOOP if (!$warnings);
                m/^\*\s*(\S+.*)/;
                push @warnings, $1;
                next LOOP;
          }
      
      # skip a specific warning message
      next LOOP if m/no cycles found in media db/;
      
      if (m/\s+([\w\.-]+): (.+)\s+level=(.+)\,\s+(\d+|\d+\.\d+) (\w\w) (\d\d:\d\d:\d\d)\s+(\d+)/) {
        $cl = $1;
        $dir = $2;
        $level = $3;
        $total = $4;
        $scale = $5;
        $time = $6;
        $files = $7;
        
	next LOOP if ($level =~ m/skip/ && $ignore_skip_level);

        if (!exists($sum{"$level"})) { 
          $sum{"$level"} = 0; 
        }
        
	# Scale to Kilobytes for all read-in data.
        if ($scale eq "MB") {
          $total = $total * 1024;
        }
        elsif ($scale eq "GB") {
          $total = $total * 1024 * 1024;
        }
        elsif ($scale eq "TB") {
          $total = $total * 1024 * 1024 * 1024;
        }
        
        $sum{"$level"} = $sum{"$level"} + $total;
        
        $clients{$cl}->{$dir}->{'level'} = $level;
        $clients{$cl}->{$dir}->{'total'} = $total;
        $clients{$cl}->{$dir}->{'time'} = $time;
        $clients{$cl}->{$dir}->{'files'} = $files;
        
        # lets look to see if we can figure out which host is actually
        # the server.  We can't just look for /nsr/index in $dir because
        # it might not be where they really store their indexes.
        
        if ($level eq "9" || $level eq "full") {
          print STDERR "server-cl: $cl level=$level dir=$dir\n" if ($DEBUG > 1);
          if (($dir =~ m=/(Networker|nsr)/index/=) ||
              ($dir =~ m/index:\w+/)) {
            $server = $cl;
            print STDERR "server-cl: server=$server\n" if ($DEBUG > 1);
          }
        }
      }
    }    
  }
  
  # If we got no lines of input before EOF, die.
  die "No input read.\n\n" if ($lc < 1);
}

#---------------------------------------------------------------------
# Save a data file which we can then graph later.  We're looking for a
# couple of things to show:
# - amount of data backed up in a savegroup over time.
# - time it takes to backup data in a saveset (mb/hour)
#
# Record format:
# - one line per record
# - each field is seperated by ||
#
# - fields: start_time, stop_time, total_time, num_clients, num_failed,
#           savegroup_size_mb, backup_level
#

sub save_log_data {
  my $ld = shift @_;

  my $sg = $savegrp;
  my $logfile;

  $sg =~ s/^\s+//g;
  $sg =~ s/\s+$//g;
  $sg =~ s/\s/_/g;

  # Turn spaces in underscores in savegroup name
  $sg =~ s/\s/_/g;

  if ($server eq "") {
    $logfile = "$ld/SERVER-UNKOWN.$sg";
  }
  else {
    $logfile = "$ld/$server.$sg";
  }
  my $total = $all/1024;

  my $ok = 1;
  if (!-d $ld) {
    eval {
      mkpath($ld,0,0755); 
    };
    if ($@) {
      warn "Warning: couldn't mkdir $ld: $!\n";
      $ok = 0;
    }
  }
  if ($ok) {
    if (-e $logfile) {
      open(LOG,">> $logfile") || 
        warn "Warning: Couldn't open $logfile for appending: $!\n";
    }
    else {
      open(LOG,"> $logfile") || 
        warn "Warning: Couldn't open $logfile for creating: $!\n";
    }
    
    # Make sure we save data in a consistent scale, in this case Kilobytes.
    foreach $s (sort keys %sum) {
      printf LOG ("%s||%s||%02d:%02d||%d||%d||%.2f||%s\n", $start,$end,$ehour,$emin,$clients,$failures,$sum{$s},$s);
    }
    close(LOG);
  }
}

#---------------------------------------------------------------------
#
sub save_input_data {

  print "save_input_data()\n" if ($DEBUG > -1);

  my $d = shift @_;
  my $f = shift @_;
  my $srv = shift @_;
  my $sg = shift @_;
  my $date = shift @_;
  my @i = @_;

  # Create the directory if needed
  my $ok = 1;

  # Ugly code to just make the date look good.
  my @d=split(" ",$date);
  splice(@d,3,1);
  splice(@d,0,1);
  $date=join("-",$d[2],$d[0],$d[1]);
  
  # Build full path to where we save data, then do substiutions.
  my $t = "$d/$f";
  $t =~ s/%G/$sg/;
  $t =~ s/%S/$srv/;
  $t =~ s/%D/$date/;
  
  # The passed in $filename specification can include directory
  # seperators, so we need to get the true directory to test for and
  # create after all pre-processing is done.

  my $dir =  dirname($t);

  if (! -d $dir) {
    eval {
      mkpath($dir,0,0755);
    };
    if ($@) {
      warn "Warning: couldn't mkdir $dir: $@\n";
      warn "         Skipping creation of saved input.\n";
      $ok = 0;
    }
  }

  if ($ok) {
    # If the savefile exists, then don't overwrite but increment a
    # counter. 
    my $c = 0;
    my $tt = $t;
    while (-e $tt) {
      $c++;
      $tt = "$t..$c";
    }
    $t = $tt;
    
    if (open(SAVE,">$t")) { 
      print SAVE @i;
      close(SAVE);
    }
    else {
      warn "Warning! Can't write input to $t: $!\n";
    }
  }
}

#---------------------------------------------------------------------
sub print_verbose_clients {

  foreach $c (sort keys %clients) {
    print OUT "client-> $c\n";
    foreach $d (keys %{$clients{$c}}) {
      print OUT "\tdir-> $d = $clients{$c}->{$d}->{'total'} KB ($clients{$c}->{$d}->{'level'})\n";
    }
    print OUT "\n" if ($verbose);
  }
}

#---------------------------------------------------------------------
sub print_warnings {

  if (@warnings) {
        print OUT "\n            Warning Messages\n";
        print OUT "-----------------------------------------------------------\n";
        foreach (@warnings) {
          print OUT "$_\n";
        }
        print OUT "\n";
  }
}

#---------------------------------------------------------------------
sub print_tape_report {

  my $s = shift @_;
  if ($s ne "") {

    # Check if we can resolve the server name, don't do anything if we
    # can'
    if (gethostbyname($s)) {

      # Master query
      my @t = `$mminfo -s $s -r '%used,pool' -q near`;
      
      # Get rid of the header line.
      shift @t;
      
      foreach (@t) { 
        m/^(.{6})(.*)$/;
        my $pool = $2;
        my $used = $1;
        $used =~ s/\s//g;
        
        # Initialize the variables if need be.
        $f{$pool} = 0 if (!exists $f{$pool});
        $e{$pool} = 0 if (!exists $e{$pool});
        $p{$pool} = 0 if (!exists $p{$pool});
        
        # Networker 5.x uses "" to mean 0%, 6.x now uses 0% and even
        # 0.1%, which makes this a pain...
        if (!$used || $used =~ m/^0%$/) { $e{$pool}++; $te++; next; }
        if ($used =~ m/%/) { $p{$pool}++; $tp++; next; }
        if ($used =~ m/full/) { $f{$pool}++; $tf++; next; }
      }
      
      my $w = 9;
      foreach $i (keys %f) { 
        if (length($i) > $w) { 
          $w = length($i);
        } 
      }
      
      printf OUT "\n   %s Tape Report \n\n",(' ' x ($w/2));
      printf OUT " Pool%s Full  Partial Empty\n",(' ' x ($w-4));
      printf OUT " %s ----- ------- -----\n", ('-' x $w);
      foreach $k (sort keys %f) {
        printf OUT " %${w}.${w}s   %3i     %3i   %3i\n",$k,$f{$k},$p{$k},$e{$k};
      }
      printf OUT " %s ----- ------- -----\n", ('-' x $w);
      printf OUT " %${w}.${w}s   %3i     %3i   %3i\n\n","  Total",$tf,$tp,$te;
    }
  }
}

#---------------------------------------------------------------------
sub print_level_report {

  my $x;
  print OUT "Level   Size in $scalename\n";
  print OUT "----- ------------\n";
  for ($x = 0; $x <= $#levelname; $x++) {
    printf OUT "%5s %-12.2f\n",$levelname[$x],$levelsize[$x];
  }
  print OUT "----- ------------\n";
  printf OUT "Total %-12.2f %s\n\n", ($all/$scale),$scalename;
}

#---------------------------------------------------------------------
sub print_top_n_size {
  my $x = shift;

  # total up each client's amount of saved data
  foreach $c (keys %clients) {
    $total{$c} = 0;
    foreach $d (keys %{$clients{$c}}) {
      $total{$c} += $clients{$c}->{$d}->{'total'}
    }
  }
  
  print OUT "\n             Top $x hosts by size.\n\n";
  print OUT "       Hostname                 size in Mb\n";
  print OUT " ------------------------- ---------------\n";

  foreach $c (sort { $total{$b} <=> $total{$a} } keys %total) {
        if ($x-- > 0 ) {
          printf OUT " %25s: %14.1f\n",$c, ($total{$c} / 1024);
        }
  }
}


#---------------------------------------------------------------------
sub print_top_n_time {
  my $x = shift;

  # total up each client's amount of saved data
  foreach $c (keys %clients) {
    foreach $d (keys %{$clients{$c}}) {
      $timetotal{"$c:$d"} = $clients{$c}->{$d}->{'time'}
    }
  }
  
  print OUT "\n               Top $x hosts by time.\n\n";
  print OUT " Time     Hostname                      Path\n";
  print OUT " -------- ----------------------------- --------------------------------------\n";

  foreach $c (sort timesort keys %timetotal) {
    if ($x-- > 0 ) {
      ($h,$p) = split(":",$c,2);
      printf OUT " %8s %-29s %-38s\n", $timetotal{$c}, $h, $p;
    }
  }
}

#---------------------------------------------------------------------
sub print_unsuccessful {

  if (@undone) {
        print OUT "\n       Unsuccessful Save Sets\n";
        print OUT " ----------------------------------------\n";
        foreach $line (@undone) {
          print OUT " $line\n";
        }
        print OUT "\n";
  }
}

#---------------------------------------------------------------------
sub print_report_summary {

  select((select(OUT),
	  $~ = "OUT",
	  #$^ = "My_Top_Format"
	 )[0]);
  
  write OUT;
}

#---------------------------------------------------------------------
sub mytimelocal {
  my $in = shift @_;
  my $ret;

  my ($day, $monname, $numday, $date, $time, $year, $hour, $min, $sec);
  my %moncvt = ( "Jan" => 0,
                 "Feb" => 1,
                 "Mar"=>  2,
                 "Apr" => 3,
                 "May" => 4,
                 "Jun" => 5,
                 "Jul" => 6,
                 "Aug" => 7,
                 "Sep" => 8,
                 "Oct" => 9,
                 "Nov" => 10,
                 "Dec" => 11,
               );
  
  ($day, $monname, $numday, $time, $year) = split(/\s+/,$in);
  $mon = $moncvt{$monname};
  
  ($hour, $min, $sec) = split(":",$time);
  
  if ($DEBUG > -1) {
    print STDERR "debug: sec: $sec, ";
    print STDERR "min: $min, ";
    print STDERR "hour: $hour, ";
    print STDERR "numday: $numday, ";
    print STDERR "mon: $mon, ";
    print STDERR "year: $year\n";
  }
  $ret=timelocal($sec,$min,$hour,$numday,$mon,$year);
  print STDERR "timelocal return: $ret\n" if ($DEBUG > -1);

  if ($ret == -1) {
    $ret=timelocal($sec,$min,$hour,$numday,$mon,$year-1900);
    print STDERR "timelocal return: $ret\n" if ($DEBUG > -1);
  }

return $ret;
}

#---------------------------------------------------------------------
# takes two times in the "hh:mm:ss" format and compares them.

sub timesort {
  my (@a, @b);
  my ($a1, $b1);

  @aa = split(":",$timetotal{$a});
  @bb = split(":",$timetotal{$b});

  $a1 = 0;
  $b1 = 0;
  $a1 = ($aa[0] * 3600) + ($aa[1] * 60) + $aa[2] if ($#aa == 2);
  $b1 = ($bb[0] * 3600) + ($bb[1] * 60) + $bb[2] if ($#bb == 2);

  return($b1 <=> $a1);
}

#---------------------------------------------------------------------
sub init {
  if ($sizetop == 0) { $sizetop = 5; }
  if ($timetop == 0) { $timetop = 5; }
}

#---------------------------------------------------------------------
sub usage {
  print STDERR <<USAGE;

Usage: $0 [-h] [-d [num]] [-v] [-s [num]] [-t [num]] [-T] [-o] [-u]
          [-w] [-O dir] [-F file] [-l] [-L dir] [-m user] [-S subject] 
          [file] 
          
   -d [n]     - prints various levels of debug info.
   -F file    - file to save data to.  Default is:
                  $saveinputfile
   -h         - show this usage message.
   -l         - log data for use with plot-nss.pl.
   -L dir     - directory to log summary data to.  Default is:
                  $logdir
   -m user    - mail output to user, defaults to STDOUT.
   -o         - save input data to a file.  Used when you have setup NSS 
                to be the savegrp notification program.
   -O dir     - directory to save input data to.  Default is:
                  $saveinputdir
   -s [n]     - lists top N hosts in terms of amount of data saved.  
                Default is 5.
   -S subject - mail message subject.  Default is:
                  "$subject"
   -t [n]     - lists top N filesystems in terms of time to save.  Default 
                is 5.
   -T         - use mminfo to generate a *current* tape report on the backup 
                server.  Not usefull for reports re-run later.
   -u         - suppress printing of unsuccessful save sets info. 
   -v         - prints verbose info, default no.
   -w         - prints warnings, default no.
   [file]     - takes input from either STDIN (default )or a file.

 Notes:  
   When using nss to process savegroup notifications directly, make sure to
   either mail the output using -m, or save it to a log directory using -l,
   or use both together.

   The -S, -O and -F flags can use the following tokens to create files 
   or directories according to your savegroups.

     %D - date in YYYY-MM-DD format.
     %E - Exit status of savegroup, "SUCCEEDED" or "FAILURES".
     %G - savegroup name. 
     %S - server name.

USAGE

  exit;
}

#---------------------------------------------------------------------

format OUT =

 Server:    @<<<<<<<<<<<<<<<<<<<<<<<<<<<<    Clients:  @>>
            $server,                         $clients
 Savegroup: @<<<<<<<<<<<<<<<<<<<<<<<<<<<<    Failures: @>>
            $savegrp,                        $failures

@<<<<<<<<< @<<<<<<<<<<<<<<<                  Failed Clients      
$start_lab,$start
End:       @<<<<<<<<<<<<<<<                  --------------------------------
           $end
Elapsed: @>> hours, @> minutes               ^<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
         $ehour,    $emin,                   $failedc
Thruput: @#####.## Kb/sec                    ^<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
         ($all / $elapsed),                  $failedc
      or @#####.## Mb/hr                     ^<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
         ((3600 * $all)/(1024 * $elapsed)),  $failedc

.
#---------------------------------------------------------------------
# Sum up client totals for master total.
sub sum_client_totals {

  $all = 0;
  foreach $s (sort keys %sum) {
    $all = $all + $sum{$s};
  }
  
  if ($restart) {
    $start_lab = "Restart:";
  } 
  else {
    $start_lab = "Start:";
  }
  
  $elapsed = &mytimelocal($end) - &mytimelocal($start);
  print STDERR "debug: Elapsed: $elapsed (secs)\n" if ($DEBUG > -1);
  $emin = int($elapsed / 60);
  $ehour = int($emin / 60);
  $emin = $emin - ($ehour * 60);
  
  $failedc = join(", ",@failed);
  $failures = $#failed + 1;
  
  if ($failures > 0) { 
    $sucess = "FAILURES";
  } 
  else { 
    $sucess = "SUCCEEDED"; 
  };
}

#---------------------------------------------------------------------
sub scale_client_totals {

  if ($all < (1024 * 1024)) {
    $scalename = "Mb";
    $scale = 1024;
  }
  else {
    $scalename = "Gb";
    $scale = (1024 * 1024);
  }
  
  $x=0;
  foreach $s (sort keys %sum) {
    $levelname[$x] = $s;
    $levelsize[$x++] = ($sum{$s}/$scale)
  }
}