#!/usr/bin/perl -w
######################################################################
#
# $Id: webjob-jqd-list-jobs,v 1.16 2012/01/07 08:01:19 mavrik Exp $
#
######################################################################
#
# Copyright 2007-2012 The WebJob Project, All Rights Reserved.
#
######################################################################
#
# Purpose: List jobs in the specified job queues and states.
#
######################################################################

use FindBin qw($Bin $RealBin); use lib ("$Bin/../lib/perl5/site_perl", "$RealBin/../lib/perl5/site_perl", "/usr/local/webjob/lib/perl5/site_perl", "/opt/local/webjob/lib/perl5/site_perl");

use strict;
use File::Basename;
use Getopt::Std;
use WebJob::JqdRoutines;
use WebJob::KvpRoutines 1.029;
use WebJob::Properties 1.035;

BEGIN
{
  ####################################################################
  #
  # The Properties hash is essentially private. Those parts of the
  # program that wish to access or modify the data in this hash need
  # to call GetProperties() to obtain a reference.
  #
  ####################################################################

  my (%hProperties);

  ####################################################################
  #
  # Initialize regex variables.
  #
  ####################################################################

  $hProperties{'CommonRegexes'} = PropertiesGetGlobalRegexes();

  ####################################################################
  #
  # Define helper routines.
  #
  ####################################################################

  sub GetProperties
  {
    return \%hProperties;
  }
}

######################################################################
#
# Main Routine
#
######################################################################

  ####################################################################
  #
  # Punch in and go to work.
  #
  ####################################################################

  my ($phProperties);

  $phProperties = GetProperties();

  $$phProperties{'Program'} = basename(__FILE__);

  ####################################################################
  #
  # Get Options.
  #
  ####################################################################

  my (%hOptions);

  if (!getopts('ad:e:G:i:Ss:', \%hOptions))
  {
    Usage($$phProperties{'Program'});
  }

  ####################################################################
  #
  # The show all flag, '-a', is optional.
  #
  ####################################################################

  $$phProperties{'ShowAll'} = (exists($hOptions{'a'})) ? 1 : 0;

  ####################################################################
  #
  # A job queue directory, '-d', is optional.
  #
  ####################################################################

  $$phProperties{'JobQueueDirectory'} = (exists($hOptions{'d'})) ? $hOptions{'d'} : "/var/webjob/spool/jqd";

  ####################################################################
  #
  # A comma delimited list of excludes, '-e', is optional.
  #
  ####################################################################

  $$phProperties{'ExcludeList'} = (exists($hOptions{'e'})) ? $hOptions{'e'} : "";

  ####################################################################
  #
  # A group file, '-G', is optional.
  #
  ####################################################################

  $$phProperties{'GroupsFile'} = (exists($hOptions{'G'})) ? $hOptions{'G'} : "/var/webjob/config/jqd/groups";

  ####################################################################
  #
  # A comma delimited list of includes, '-i', is required.
  #
  ####################################################################

  my (@aIncludes);

  $$phProperties{'IncludeList'} = (exists($hOptions{'i'})) ? $hOptions{'i'} : "";

  if ($$phProperties{'IncludeList'} eq "")
  {
    Usage($$phProperties{'Program'});
  }

  ####################################################################
  #
  # The show stats flag, '-S', is optional.
  #
  ####################################################################

  $$phProperties{'ShowStats'} = (exists($hOptions{'S'})) ? 1 : 0;

  ####################################################################
  #
  # A comma delimited list of job states, '-s', is optional.
  #
  ####################################################################

  $$phProperties{'QueueStateList'} = (exists($hOptions{'s'})) ? $hOptions{'s'} : "todo";

  if ($$phProperties{'QueueStateList'} =~ /^all$/i)
  {
    push(@{$$phProperties{'QueueStates'}}, "hold", "todo", "sent", "open", "done", "pass", "fail", "foul");
  }
  else
  {
    foreach my $sState (split(/,/, $$phProperties{'QueueStateList'}))
    {
      if ($sState =~ /^(hold|todo|sent|open|done|pass|fail|foul)$/i)
      {
        push(@{$$phProperties{'QueueStates'}}, lc($1));
      }
      else
      {
        print STDERR "$$phProperties{'Program'}: Error='Invalid job state ($sState).'\n";
        exit(2);
      }
    }
  }

  ####################################################################
  #
  # If there isn't at least one argument left, it's an error.
  #
  ####################################################################

  if (scalar(@ARGV) < 1)
  {
    Usage($$phProperties{'Program'});
  }

  ####################################################################
  #
  # Make sure the job queue directory exists.
  #
  ####################################################################

  if (!-d $$phProperties{'JobQueueDirectory'})
  {
    print STDERR "$$phProperties{'Program'}: Error='Base directory ($$phProperties{'JobQueueDirectory'}) does not exist.'\n";
    exit(2);
  }

  ####################################################################
  #
  # Create forward/reverse queue maps.
  #
  ####################################################################

  my (%hForwardQueueMap, %hQueueMapArgs, %hReverseQueueMap);

  %hQueueMapArgs =
  (
    'ForwardQueueMap'   => \%hForwardQueueMap,
    'JobQueueDirectory' => $$phProperties{'JobQueueDirectory'},
    'KeyRegex'          => "(?:" . $$phProperties{'CommonRegexes'}{'ClientId'} . "|" . $$phProperties{'CommonRegexes'}{'ClientSuppliedFilename'} . ")",
    'ReverseQueueMap'   => \%hReverseQueueMap,
    'SharedLock'        => 1,
    'ValueRegex'        => "\\d+",
  );
  if (!JqdSetupQueueMaps(\%hQueueMapArgs))
  {
    print STDERR "$$phProperties{'Program'}: Error='$hQueueMapArgs{'Error'}'\n";
    exit(2);
  }

  ####################################################################
  #
  # Create group queue map, and conditionally fill it with groups.
  #
  ####################################################################

  my (%hGroupMap);

  %hGroupMap = ();

  if (-f $$phProperties{'GroupsFile'})
  {
    my $sKeyRegex = $$phProperties{'CommonRegexes'}{'Group'};
    my $sUnitRegex = "(?:" . $$phProperties{'CommonRegexes'}{'GroupToken'} . "|" . $$phProperties{'CommonRegexes'}{'ClientId'} . ")";
    my $sValueRegex = "(?:|$sUnitRegex(?:,$sUnitRegex)*)"; # Empty groups are allowed.
    my %hLArgs =
    (
      'File'           => $$phProperties{'GroupsFile'},
      'Properties'     => \%hGroupMap,
      'Template'       => { $sKeyRegex => $sValueRegex, },
      'VerifyValues'   => 1,
    );
    if (!KvpGetKvps(\%hLArgs))
    {
      print STDERR "$$phProperties{'Program'}: Error='$hLArgs{'Error'}'\n";
      exit(2);
    }
  }

  ####################################################################
  #
  # Resolve the include and exclude lists.
  #
  ####################################################################

  my (%hExcludes, %hIncludes, %hResolverArgs);

  %hResolverArgs =
  (
    'ForwardMap'     => \%hForwardQueueMap,
    'GroupMap'       => \%hGroupMap,
    'QueueMap'       => \%hIncludes,
    'QueueList'      => $$phProperties{'IncludeList'},
    'RecursionStack' => [],
  );
  if (!JqdResolveQueueList(\%hResolverArgs))
  {
    print STDERR "$$phProperties{'Program'}: IncludeList='$$phProperties{'IncludeList'}' Error='$hResolverArgs{'Error'}'\n";
    exit(2);
  }

  %hResolverArgs =
  (
    'ForwardMap'     => \%hForwardQueueMap,
    'GroupMap'       => \%hGroupMap,
    'QueueMap'       => \%hExcludes,
    'QueueList'      => $$phProperties{'ExcludeList'},
    'RecursionStack' => [],
  );
  if (!JqdResolveQueueList(\%hResolverArgs))
  {
    print STDERR "$$phProperties{'Program'}: ExcludeList='$$phProperties{'ExcludeList'}' Error='$hResolverArgs{'Error'}'\n";
    exit(2);
  }

  ####################################################################
  #
  # Bash the excludes against the includes, and remove the overlap.
  #
  ####################################################################

  foreach my $sExclude (sort(keys(%hExcludes)))
  {
    if (exists($hIncludes{$sExclude}))
    {
      delete($hIncludes{$sExclude});
    }
  }

  ####################################################################
  #
  # Do some work.
  #
  ####################################################################

  foreach my $sJobNameRegex (@ARGV)
  {
    if ($$phProperties{'ShowStats'})
    {
      print join("|", "client", @{$$phProperties{'QueueStates'}}), "\n";
    }
    foreach my $sQueue (sort(keys(%hIncludes)))
    {
      ################################################################
      #
      # Set the queue directory.
      #
      ################################################################

      my ($sJqdQueueDirectory);

      $sJqdQueueDirectory = $$phProperties{'JobQueueDirectory'} . "/" . $sQueue;
      if (!-d $sJqdQueueDirectory)
      {
        next;
      }

      ################################################################
      #
      # Acquire the change lock. This is an exclusive lock.
      #
      ################################################################

      my (%hQueueLockArgs);

      %hQueueLockArgs =
      (
        'LockFile' => $sJqdQueueDirectory . "/" . "change.lock",
        'LockMode' => "+<", # The file must exist or this will fail.
      );
      if (!JqdLockFile(\%hQueueLockArgs))
      {
        print STDERR "$$phProperties{'Program'}: Queue='$sQueue' Error='$hQueueLockArgs{'Error'}'\n";
        next;
      }

      ################################################################
      #
      # Conditionally print jobs that match the specified expression,
      # but only do this for the specified job states.
      #
      ################################################################

      my (%hStats);

      foreach my $sState (@{$$phProperties{'QueueStates'}})
      {
        my $sStateDirectory = $sJqdQueueDirectory . "/" . $sState;
        if (!opendir(DIR, $sStateDirectory))
        {
          print STDERR "$$phProperties{'Program'}: Queue='$sQueue' Error='Directory ($sStateDirectory) could not be opened ($!).'\n";
          next;
        }
        my @aJobFiles = map("$sStateDirectory/$_", sort(grep(/$sJobNameRegex/, readdir(DIR))));
        closedir(DIR);
        $hStats{$sState} = 0;
        foreach my $sJobFile (@aJobFiles)
        {
          next unless(-f $sJobFile);
          $hStats{$sState}++;
          print "$sJobFile\n" unless ($$phProperties{'ShowStats'});
        }
      }

      ################################################################
      #
      # Conditionally print job statistics that match the specified
      # expression, but only do this for the specified job states.
      #
      ################################################################

      if ($$phProperties{'ShowStats'})
      {
        my $sCount = 0;
        foreach my $sState (@{$$phProperties{'QueueStates'}})
        {
          $sCount += $hStats{$sState};
        }
        if ($$phProperties{'ShowAll'} || $sCount)
        {
          print "$sQueue";
          print "|$hStats{'hold'}" if exists($hStats{'hold'});
          print "|$hStats{'todo'}" if exists($hStats{'todo'});
          print "|$hStats{'sent'}" if exists($hStats{'sent'});
          print "|$hStats{'open'}" if exists($hStats{'open'});
          print "|$hStats{'done'}" if exists($hStats{'done'});
          print "|$hStats{'pass'}" if exists($hStats{'pass'});
          print "|$hStats{'fail'}" if exists($hStats{'fail'});
          print "|$hStats{'foul'}" if exists($hStats{'foul'});
          print "\n";
        }
      }

      ################################################################
      #
      # Release the change lock.
      #
      ################################################################

      JqdUnlockFile(\%hQueueLockArgs);
    }
  }

  ####################################################################
  #
  # Shutdown and go home.
  #
  ####################################################################

  1;


######################################################################
#
# Usage
#
######################################################################

sub Usage
{
  my ($sProgram) = @_;
  print STDERR "\n";
  print STDERR "Usage: $sProgram [-aS] [-d jqd-dir] [-e queue[,queue[...]]] [-G group-file] [-s {all|state[,state[...]]}] -i queue[,queue[...]] job-regex [job-regex ...]\n";
  print STDERR "\n";
  exit(1);
}


=pod

=head1 NAME

webjob-jqd-list-jobs - List jobs in the specified job queues and states.

=head1 SYNOPSIS

B<webjob-jqd-list-jobs> B<[-aS]> B<[-d jqd-dir]> B<[-e queue[,queue[...]]]> B<[-G group-file]> B<[-s {all|state[,state[...]]}]> B<-i queue[,queue[...]]> B<job-regex [job-regex ...]>

=head1 DESCRIPTION

This utility lists jobs in the specified job queues and states.  Each
B<job-regex> argument is intepreted as a Perl regular expression.

=head1 OPTIONS

=over 4

=item B<-a>

Print statistics for all queues -- even if the tally for each job
state is zero.  This option has no effect unless B<-S> is specified as
well.

=item B<-d jqd-dir>

Specifies the base directory where queues live.  This directory must
exist.  The default value is /var/webjob/spool/jqd.

=item B<-e queue[,queue[...]]>

Specifies one or more queues or queue groups that are to be excluded
from the operation.  Note that group names must be prefixed with '%'.
For example, a group called 'test' must be specified as '%test'.

=item B<-G group-file>

Specifies an alternate group file.  The default group file is
/var/webjob/config/jqd/groups.

=item B<-i queue[,queue[...]]>

Specifies one or more queues or queue groups that are to be included
in the operation.  Note that group names must be prefixed with '%'.
For example, a group called 'test' must be specified as '%test'.

=item B<-S>

List job statistics about (rather than jobs in) the specified queues.
Currently, this option prints the number of jobs in each job state.

=item B<-s {all|state[,state[...]]}>

Restrict output to the specified list of job states.  Currently, the
following states are supported: 'hold', 'todo', 'sent', 'open',
'done', 'pass', 'fail', and 'foul'.  You may also use the keyword
'all' as an alias for all states.

=back

=head1 AUTHOR

Klayton Monroe

=head1 SEE ALSO

webjob-jqd-create-job(1), webjob-jqd-delete-job(1)

=head1 LICENSE

All documentation and code are distributed under same terms and
conditions as B<WebJob>.

=cut
