#!/usr/bin/perl -w
######################################################################
#
# $Id: webjob-jqd-create-job,v 1.41 2012/01/07 08:01:18 mavrik Exp $
#
######################################################################
#
# Copyright 2007-2012 The WebJob Project, All Rights Reserved.
#
######################################################################
#
# Purpose: Add a job to the specified job queues.
#
######################################################################

use FindBin qw($Bin $RealBin); use lib ("$Bin/../lib/perl5/site_perl", "$RealBin/../lib/perl5/site_perl", "/usr/local/webjob/lib/perl5/site_perl", "/opt/local/webjob/lib/perl5/site_perl");

use strict;
use Digest::MD5;
use Digest::SHA1;
use File::Basename;
use File::Copy;
use File::Path;
use Getopt::Std;
use Time::HiRes qw(gettimeofday);
use WebJob::FdaRoutines;
use WebJob::JqdRoutines 1.024;
use WebJob::KvpRoutines 1.029;
use WebJob::Properties 1.035;
use WebJob::TimeRoutines;
use WebJob::VersionRoutines;

BEGIN
{
  ####################################################################
  #
  # The Properties hash is essentially private. Those parts of the
  # program that wish to access or modify the data in this hash need
  # to call GetProperties() to obtain a reference.
  #
  ####################################################################

  my (%hProperties);

  ####################################################################
  #
  # Initialize regex variables.
  #
  ####################################################################

  $hProperties{'CommonRegexes'} = PropertiesGetGlobalRegexes();

  ####################################################################
  #
  # Initialize platform-specific variables.
  #
  ####################################################################

  if ($^O =~ /MSWin(32|64)/i)
  {
    $hProperties{'OsClass'} = "WINX";
  }
  else
  {
    $hProperties{'OsClass'} = "UNIX";
  }

  ####################################################################
  #
  # Define helper routines.
  #
  ####################################################################

  sub GetProperties
  {
    return \%hProperties;
  }
}

######################################################################
#
# Main Routine
#
######################################################################

  ####################################################################
  #
  # Punch in and go to work.
  #
  ####################################################################

  my ($phProperties);

  $phProperties = GetProperties();

  $$phProperties{'Program'} = basename(__FILE__);

  ####################################################################
  #
  # Get Options.
  #
  ####################################################################

  my (%hOptions);

  if (!getopts('c:e:f:hi:j:m:o:p:S:T:t:', \%hOptions))
  {
    Usage($$phProperties{'Program'});
  }

  ####################################################################
  #
  # The command file, '-c', is optional.
  #
  ####################################################################

  $$phProperties{'CommandFile'} = (exists($hOptions{'c'})) ? $hOptions{'c'} : undef;

  ####################################################################
  #
  # A comma delimited list of excludes, '-e', is optional.
  #
  ####################################################################

  $$phProperties{'ExcludeList'} = (exists($hOptions{'e'})) ? $hOptions{'e'} : "";

  ####################################################################
  #
  # The job file, '-f', is required.
  #
  ####################################################################

  $$phProperties{'JobFile'} = (exists($hOptions{'f'})) ? $hOptions{'f'} : undef;

  if (!defined($$phProperties{'JobFile'}))
  {
    Usage($$phProperties{'Program'});
  }

  ####################################################################
  #
  # The put job on hold flag, '-h', is optional.
  #
  ####################################################################

  $$phProperties{'PutJobOnHold'} = (exists($hOptions{'h'})) ? 1 : 0; # This is a legacy option, and it is being phased out.

  ####################################################################
  #
  # A comma delimited list of includes, '-i', is required.
  #
  ####################################################################

  $$phProperties{'IncludeList'} = (exists($hOptions{'i'})) ? $hOptions{'i'} : "";

  if ($$phProperties{'IncludeList'} eq "")
  {
    Usage($$phProperties{'Program'});
  }

  ####################################################################
  #
  # A job group, '-j', is optional.
  #
  ####################################################################

  $$phProperties{'JobGroup'} = (exists($hOptions{'j'})) ? $hOptions{'j'} : "void";

  if ($$phProperties{'JobGroup'} !~ /^$$phProperties{'CommonRegexes'}{'Group'}$/)
  {
    print STDERR "$$phProperties{'Program'}: Error='The job group ($$phProperties{'JobGroup'}) do not pass muster.'\n";
    exit(2);
  }

  ####################################################################
  #
  # The default job file permissions, '-m', are optional.
  #
  ####################################################################

  $$phProperties{'JobFilePermissions'} = (exists($hOptions{'m'})) ? $hOptions{'m'} : "0644";

  if ($$phProperties{'JobFilePermissions'} !~ /^0[0-7]{1,3}$/)
  {
    print STDERR "$$phProperties{'Program'}: Error='The job file permissions ($$phProperties{'JobFilePermissions'}) do not pass muster.'\n";
    exit(2);
  }

  ####################################################################
  #
  # The option list, '-o', is optional.
  #
  ####################################################################

  my @sSupportedOptions =
  (
    'OverwritePoundFiles',
    'PutJobOnHold',
    'UseExistingPoundFiles',
    'UseZeroNonce',
  );
  foreach my $sOption (@sSupportedOptions)
  {
    $$phProperties{$sOption} = 0 unless ($$phProperties{$sOption});
  }

  $$phProperties{'Options'} = (exists($hOptions{'o'})) ? $hOptions{'o'} : undef;

  if (defined($$phProperties{'Options'}))
  {
    foreach my $sOption (split(/,/, $$phProperties{'Options'}))
    {
      foreach my $sSupportedOption (@sSupportedOptions)
      {
        $sOption = $sSupportedOption if ($sOption =~ /^$sSupportedOption$/i);
      }
      if (!exists($$phProperties{$sOption}))
      {
        print STDERR "$$phProperties{'Program'}: Error='Unknown or unsupported option ($sOption).'\n";
        exit(2);
      }
      $$phProperties{$sOption} = 1;
    }
  }

  ####################################################################
  #
  # The job priority, '-p', is optional.
  #
  ####################################################################

  $$phProperties{'JobPriority'} = (exists($hOptions{'p'})) ? $hOptions{'p'} : 50;

  if ($$phProperties{'JobPriority'} !~ /^\d{1,2}$/)
  {
    print STDERR "$$phProperties{'Program'}: Error='The job priority ($$phProperties{'JobPriority'}) does not pass muster.'\n";
    exit(2);
  }

  ####################################################################
  #
  # A webjob server home directory, '-S', is optional.
  #
  ####################################################################

  $$phProperties{'WebJobServerHome'} = (exists($hOptions{'S'})) ? $hOptions{'S'} : "/var/webjob";

  ####################################################################
  #
  # A job tag, '-T', is optional.
  #
  ####################################################################

  $$phProperties{'JobTag'} = (exists($hOptions{'T'})) ? $hOptions{'T'} : undef;

  ####################################################################
  #
  # The job type, '-t', is required.
  #
  ####################################################################

  $$phProperties{'JobType'} = (exists($hOptions{'t'})) ? $hOptions{'t'} : undef;

  if (!defined($$phProperties{'JobType'}))
  {
    Usage($$phProperties{'Program'});
  }

  if ($$phProperties{'JobType'} =~ /^(?:p|parallel)$/i)
  {
    $$phProperties{'JobTypeCode'} = "p";
  }
  elsif ($$phProperties{'JobType'} =~ /^(?:s|serial)$/i)
  {
    $$phProperties{'JobTypeCode'} = "s";
  }
  else
  {
    Usage($$phProperties{'Program'});
  }

  ####################################################################
  #
  # If any arguments remain, it's an error.
  #
  ####################################################################

  if (scalar(@ARGV) > 0)
  {
    Usage($$phProperties{'Program'});
  }

  ####################################################################
  #
  # Make final adjustments to the run time environment.
  #
  ####################################################################

  my ($sError);

  if (!AdjustRunTimeEnvironment($phProperties, \$sError))
  {
    print STDERR "$$phProperties{'Program'}: Error='$sError'\n";
    exit(2);
  }

  ####################################################################
  #
  # Create forward/reverse queue maps.
  #
  ####################################################################

  my (%hForwardQueueMap, %hQueueMapArgs, %hReverseQueueMap);

  %hQueueMapArgs =
  (
    'ForwardQueueMap'   => \%hForwardQueueMap,
    'JobQueueDirectory' => $$phProperties{'JqdDirectory'},
    'KeyRegex'          => "(?:" . $$phProperties{'CommonRegexes'}{'ClientId'} . "|" . $$phProperties{'CommonRegexes'}{'ClientSuppliedFilename'} . ")",
    'ReverseQueueMap'   => \%hReverseQueueMap,
    'SharedLock'        => 1,
    'ValueRegex'        => "\\d+",
  );
  if (!JqdSetupQueueMaps(\%hQueueMapArgs))
  {
    print STDERR "$$phProperties{'Program'}: Error='$hQueueMapArgs{'Error'}'\n";
    exit(2);
  }

  ####################################################################
  #
  # Create group queue map, and conditionally fill it with groups.
  #
  ####################################################################

  my (%hGroupMap);

  %hGroupMap = ();

  if (-f $$phProperties{'JqdGroupsFile'})
  {
    my $sKeyRegex = $$phProperties{'CommonRegexes'}{'Group'};
    my $sUnitRegex = "(?:" . $$phProperties{'CommonRegexes'}{'GroupToken'} . "|" . $$phProperties{'CommonRegexes'}{'ClientId'} . ")";
    my $sValueRegex = "(?:|$sUnitRegex(?:,$sUnitRegex)*)"; # Empty groups are allowed.
    my %hLArgs =
    (
      'File'           => $$phProperties{'JqdGroupsFile'},
      'Properties'     => \%hGroupMap,
      'Template'       => { $sKeyRegex => $sValueRegex, },
      'VerifyValues'   => 1,
    );
    if (!KvpGetKvps(\%hLArgs))
    {
      print STDERR "$$phProperties{'Program'}: Error='$hLArgs{'Error'}'\n";
      exit(2);
    }
  }

  ####################################################################
  #
  # Resolve the include and exclude lists.
  #
  ####################################################################

  my (%hExcludes, %hIncludes, %hResolverArgs);

  %hResolverArgs =
  (
    'ForwardMap'     => \%hForwardQueueMap,
    'GroupMap'       => \%hGroupMap,
    'QueueMap'       => \%hIncludes,
    'QueueList'      => $$phProperties{'IncludeList'},
    'RecursionStack' => [],
  );
  if (!JqdResolveQueueList(\%hResolverArgs))
  {
    print STDERR "$$phProperties{'Program'}: IncludeList='$$phProperties{'IncludeList'}' Error='$hResolverArgs{'Error'}'\n";
    exit(2);
  }

  %hResolverArgs =
  (
    'ForwardMap'     => \%hForwardQueueMap,
    'GroupMap'       => \%hGroupMap,
    'QueueMap'       => \%hExcludes,
    'QueueList'      => $$phProperties{'ExcludeList'},
    'RecursionStack' => [],
  );
  if (!JqdResolveQueueList(\%hResolverArgs))
  {
    print STDERR "$$phProperties{'Program'}: ExcludeList='$$phProperties{'ExcludeList'}' Error='$hResolverArgs{'Error'}'\n";
    exit(2);
  }

  ####################################################################
  #
  # Bash the excludes against the includes, and remove the overlap.
  #
  ####################################################################

  foreach my $sExclude (sort(keys(%hExcludes)))
  {
    if (exists($hIncludes{$sExclude}))
    {
      delete($hIncludes{$sExclude});
    }
  }

  ####################################################################
  #
  # Make sure all queues exist.
  #
  ####################################################################

  my (@aMissingQueues);

  foreach my $sQueue (sort(keys(%hIncludes)))
  {
    my $sQueueDirectory = $$phProperties{'JqdDirectory'} . "/" . $sQueue;
    if (!-d $sQueueDirectory)
    {
      push(@aMissingQueues, $sQueue);
    }
  }

  if (scalar(@aMissingQueues) > 0)
  {
    print STDERR "$$phProperties{'Program'}: Error='One or more queues (" . join(",", @aMissingQueues) . ") do not exist. No queues can be manipulated until this is fixed.'\n";
    exit(2);
  }

  ####################################################################
  #
  # Parse the job file.
  #
  ####################################################################

  my (%hJobProperties, %hLArgs);

  %hLArgs =
  (
    'File'           => $$phProperties{'JobFile'},
    'Properties'     => \%hJobProperties,
    'RequiredKeys'   => ['Command', 'CommandAlias', 'CommandLine'],
    'RequireAllKeys' => 0,
    'Template'       => PropertiesGetGlobalTemplates()->{'jqd.job'},
    'VerifyValues'   => 1,
  );
  push(@{$hLArgs{'RequiredKeys'}}, 'CommandMd5', 'CommandSha1') if ($$phProperties{'UseExistingPoundFiles'});
  if (!KvpGetKvps(\%hLArgs))
  {
    print STDERR "$$phProperties{'Program'}: Error='$hLArgs{'Error'}'\n";
    exit(2);
  }

  ####################################################################
  #
  # Set the job epoch.
  #
  ####################################################################

  my ($sSeconds, $sMicroseconds) = gettimeofday();

  $hJobProperties{'Created'} = SecondsToDateTime($sSeconds, 0);

  ####################################################################
  #
  # Set the job creator.
  #
  ####################################################################

  my ($sCreatorUsername, $sCreatorUid);

  if ($$phProperties{'OsClass'} =~ /^UNIX$/)
  {
    my $sCreatorTty = `tty`;
    if (defined($sCreatorTty) && $sCreatorTty =~ /^\/dev\//)
    {
      $sCreatorTty =~ s/[\r\n]*$//;
      $sCreatorUid = (stat($sCreatorTty))[4];
      if (!defined($sCreatorUid))
      {
        print STDERR "$$phProperties{'Program'}: Error='Unable to determine user ID (UID).'\n";
        exit(2);
      }
    }
    else
    {
      $sCreatorUid = $<;
    }
  }
  else
  {
    $sCreatorUid = $<;
  }

  $sCreatorUsername = (getpwuid($sCreatorUid))[0];
  if (!defined($sCreatorUid))
  {
    print STDERR "$$phProperties{'Program'}: Error='Unable to determine username.'\n";
    exit(2);
  }

  $hJobProperties{'Creator'} = $sCreatorUsername;

  ####################################################################
  #
  # Set the job group.
  #
  ####################################################################

  $hJobProperties{'JobGroup'} = $$phProperties{'JobGroup'};

  ####################################################################
  #
  # Generate a nonce. This is used to help prevent name collisions.
  #
  ####################################################################

  my ($sNonce);

  if ($$phProperties{'UseZeroNonce'})
  {
    $sNonce = 0;
  }
  else
  {
    $sNonce = int(rand(2**32));
  }

  ####################################################################
  #
  # Finalize owner, group, and mode.
  #
  ####################################################################

  my (%hOgmArgs, $sJobFileUid, $sJobFileGid, $sJobFileMode);

  %hOgmArgs = ( 'Owner' => $$phProperties{'JobFileOwner'} );
  $sJobFileUid = FdaOwnerToUid(\%hOgmArgs);
  if (!defined($sJobFileUid))
  {
    print STDERR "$$phProperties{'Program'}: Error='$hOgmArgs{'Error'}'\n";
    exit(2);
  }

  %hOgmArgs = ( 'Group' => $$phProperties{'JobFileGroup'} );
  $sJobFileGid = FdaGroupToGid(\%hOgmArgs);
  if (!defined($sJobFileGid))
  {
    print STDERR "$$phProperties{'Program'}: Error='$hOgmArgs{'Error'}'\n";
    exit(2);
  }

  %hOgmArgs = ( 'Permissions' => $$phProperties{'JobFilePermissions'}, 'Umask' => undef );
  $sJobFileMode = FdaPermissionsToMode(\%hOgmArgs);
  if (!defined($sJobFileMode))
  {
    print STDERR "$$phProperties{'Program'}: Error='$hOgmArgs{'Error'}'\n";
    exit(2);
  }

  ####################################################################
  #
  # Conditionally put the command file in the POUND.
  #
  ####################################################################

if ($$phProperties{'UseExistingPoundFiles'})
{
  my $sRawUniqDstDir = sprintf("%s/%s/%s/%s/%s",
    $$phProperties{'PoundDirectory'},
    substr($hJobProperties{'CommandMd5'}, 0, 2),
    substr($hJobProperties{'CommandMd5'}, 2, 2),
    substr($hJobProperties{'CommandMd5'}, 4, 2),
    substr($hJobProperties{'CommandMd5'}, 6, 2),
    );
  my $sRawUniqDstPath = sprintf("%s/%s.%s", $sRawUniqDstDir, $hJobProperties{'CommandMd5'}, substr($hJobProperties{'CommandSha1'}, 0, 8));
  if (! -f $sRawUniqDstPath)
  {
    print STDERR "$$phProperties{'Program'}: Error='The command file does not exist in the POUND.'\n";
    exit(2);
  }
  $hJobProperties{'CommandPath'} = $sRawUniqDstPath;
  $hJobProperties{'CommandSize'} = -s _;
  if ($$phProperties{'DsvRequireSignatures'} =~ /^[Yy]$/)
  {
    my $sPoundSignatureFile = $sRawUniqDstPath . ".sig";
    if (!-f $sPoundSignatureFile)
    {
      print STDERR "$$phProperties{'Program'}: Error='The command signature file does not exist in the POUND.'\n";
      exit(2);
    }
  }
}
else
{
  if (!defined($$phProperties{'CommandFile'}))
  {
    if ($$phProperties{'JobFile'} =~ /^(.+)[.]job$/)
    {
      $$phProperties{'CommandFile'} = $1;
    }
    else
    {
      print STDERR "$$phProperties{'Program'}: Error='Unable to derive command file from job file. Try specifying one with the \"-c\" option.'\n";
      exit(2);
    }
  }
  if (!-f $$phProperties{'CommandFile'})
  {
    print STDERR "$$phProperties{'Program'}: Error='The command file does not exist or is not a regular file.'\n";
    exit(2);
  }
  $hJobProperties{'CommandSize'} = -s _;
  if (!defined($hJobProperties{'CommandSize'}))
  {
    print STDERR "$$phProperties{'Program'}: Error='Unable to determine the size of the command file ($!).'\n";
    exit(2);
  }
  if ($hJobProperties{'CommandSize'} < 1)
  {
    print STDERR "$$phProperties{'Program'}: Error='The command file appears to be empty.'\n";
    exit(2);
  }
  if (!open(FH, "< $$phProperties{'CommandFile'}"))
  {
    print STDERR "$$phProperties{'Program'}: Error='Unable to open the command file ($!).'\n";
    exit(2);
  }
  $hJobProperties{'CommandMd5'} = Digest::MD5->new->addfile(*FH)->hexdigest;
  if (!defined($hJobProperties{'CommandMd5'}))
  {
    print STDERR "$$phProperties{'Program'}: Error='Unable to compute command file MD5.'\n";
    exit(2);
  }
  seek(FH, 0, 0);
  $hJobProperties{'CommandSha1'} = Digest::SHA1->new->addfile(*FH)->hexdigest;
  if (!defined($hJobProperties{'CommandSha1'}))
  {
    print STDERR "$$phProperties{'Program'}: Error='Unable to compute command file SHA1.'\n";
    exit(2);
  }
  close(FH);
  if ($$phProperties{'DsvRequireSignatures'} =~ /^[Yy]$/)
  {
    $$phProperties{'CommandSignatureFile'} = $$phProperties{'CommandFile'} . ".sig";
    my $sCommandLine = qq("$$phProperties{'DsvToolExe'}" -c -t "$$phProperties{'DsvDirectory'}");
    $sCommandLine .= qq( "$$phProperties{'CommandFile'}") if ($$phProperties{'VersionNumber'} <= 0x10300000); # Use legacy syntax for any version before 1.3.0.
    $sCommandLine .= qq( "$$phProperties{'CommandSignatureFile'}" 2>&1);
    my @aOutput = qx($sCommandLine);
    my $sStatus = ($? >> 8) & 0xff;
    if ($sStatus != 0)
    {
      print STDERR "$$phProperties{'Program'}: Error='Unable to verify command file signature.'\n";
      exit(2);
    }
  }
  my $sRawUniqDstDir = sprintf("%s/%s/%s/%s/%s",
    $$phProperties{'PoundDirectory'},
    substr($hJobProperties{'CommandMd5'}, 0, 2),
    substr($hJobProperties{'CommandMd5'}, 2, 2),
    substr($hJobProperties{'CommandMd5'}, 4, 2),
    substr($hJobProperties{'CommandMd5'}, 6, 2),
    );
  my $sRawUniqDstPath = sprintf("%s/%s.%s", $sRawUniqDstDir, $hJobProperties{'CommandMd5'}, substr($hJobProperties{'CommandSha1'}, 0, 8));
  if (!-d $sRawUniqDstDir)
  {
    eval { mkpath($sRawUniqDstDir, 0, 0755) };
    if ($@)
    {
      my $sMessage = $@; $sMessage =~ s/[\r\n]+/ /g; $sMessage =~ s/\s+/ /g; $sMessage =~ s/\s+$//;
      print STDERR "$$phProperties{'Program'}: Error='Unable to create POUND sub-directories ($sMessage).'\n";
      exit(2);
    }
  }
  $$phProperties{'CommandFileInstalled'} = 0;
  if (!-f $sRawUniqDstPath || $$phProperties{'OverwritePoundFiles'})
  {
    if (!copy($$phProperties{'CommandFile'}, $sRawUniqDstPath))
    {
      print STDERR "$$phProperties{'Program'}: Error='Unable to copy command file to the POUND ($!).'\n";
      exit(2);
    }
    $$phProperties{'CommandFileInstalled'} = 1;
  }
  $hJobProperties{'CommandPath'} = $sRawUniqDstPath;
  if ($$phProperties{'DsvRequireSignatures'} =~ /^[Yy]$/)
  {
    my $sPoundSignatureFile = $sRawUniqDstPath . ".sig";
    if (!-f $sPoundSignatureFile || $$phProperties{'OverwritePoundFiles'} || $$phProperties{'CommandFileInstalled'})
    {
      if (!copy($$phProperties{'CommandSignatureFile'}, $sPoundSignatureFile))
      {
        print STDERR "$$phProperties{'Program'}: Error='Unable to copy command signature file to the POUND ($!).'\n";
        exit(2);
      }
    }
  }
}

  ####################################################################
  #
  # Do some work.
  #
  ####################################################################

  my (@aResults, %hLogArgs, %hQueueLockArgs);

  %hLogArgs =
  (
    'Command'       => $hJobProperties{'Command'},
    'CommandSize'   => $hJobProperties{'CommandSize'},
    'Creator'       => $hJobProperties{'Creator'},
    'JobGroup'      => $hJobProperties{'JobGroup'},
    'LogFile'       => $$phProperties{'JqdLogFile'},
    'NewQueueState' => ($$phProperties{'PutJobOnHold'}) ? "hold" : "todo",
    'OldQueueState' => undef,
    'Pid'           => $$,
    'PoundName'     => sprintf("%s.%s", $hJobProperties{'CommandMd5'}, substr($hJobProperties{'CommandSha1'}, 0, 8)),
    'Program'       => $$phProperties{'Program'},
  );

  foreach my $sQueue (sort(keys(%hIncludes)))
  {
    ##################################################################
    #
    # Generate the todo job file name.
    #
    ##################################################################

    my ($sJqdQueueDirectory, $sTodoJobFile, $sTodoJobName);

    $sJqdQueueDirectory = $$phProperties{'JqdDirectory'} . "/" . $sQueue;

    if (!defined($$phProperties{'JobTag'}))
    {
      $sTodoJobName = sprintf("%s%03d_%02d_%10d_%06d_%08x", $$phProperties{'JobTypeCode'}, $hIncludes{$sQueue}, $$phProperties{'JobPriority'}, $sSeconds, $sMicroseconds, $sNonce);
    }
    else
    {
      $sTodoJobName = sprintf("%s%03d_%s", $$phProperties{'JobTypeCode'}, $hIncludes{$sQueue}, $$phProperties{'JobTag'});
    }

    $sTodoJobFile = $sJqdQueueDirectory . "/" . (($$phProperties{'PutJobOnHold'}) ? "hold" : "todo") . "/" . $sTodoJobName;

    ##################################################################
    #
    # Update log arguments.
    #
    ##################################################################

    $hLogArgs{'Message'} = undef;
    $hLogArgs{'Queue'} = $sQueue;
    $hLogArgs{'QueueTag'} = $sTodoJobName;
    $hLogArgs{'Result'} = "fail";

    ##################################################################
    #
    # Acquire the change lock. This is an exclusive lock.
    #
    ##################################################################

    %hQueueLockArgs =
    (
      'LockFile' => $sJqdQueueDirectory . "/" . "change.lock",
      'LockMode' => "+<", # The file must exist or this will fail.
    );
    if (!JqdLockFile(\%hQueueLockArgs))
    {
      $hLogArgs{'Message'} = $hQueueLockArgs{'Error'};
      push(@aResults, "fail|$sTodoJobFile");
      next;
    }

    ##################################################################
    #
    # If the queue is in the locked state, we're done since no jobs
    # may be created.
    #
    ##################################################################

    if (JqdIsQueueLocked({ 'Directory' => $sJqdQueueDirectory }))
    {
      print STDERR "$$phProperties{'Program'}: Queue='$sQueue' Warning='Queue is locked. No jobs may be created.'\n";
      next;
    }

    ##################################################################
    #
    # If a job already exists with the same name, warn the caller.
    #
    ##################################################################

    if (-f $sTodoJobFile)
    {
      $hLogArgs{'Message'} = "Another job with the same name ($sTodoJobFile) is already assigned to the specified queue.";
      push(@aResults, "fail|$sTodoJobFile");
      next;
    }

    ##################################################################
    #
    # Schedule the job.
    #
    ##################################################################

    %hLArgs =
    (
      'File'           => $sTodoJobFile,
      'Properties'     => \%hJobProperties,
      'Template'       => PropertiesGetGlobalTemplates()->{'jqd.job'},
      'VerifyValues'   => 1,
    );

    if (!KvpSetKvps(\%hLArgs))
    {
      $hLogArgs{'Message'} = $hLArgs{'Error'};
      unlink($sTodoJobFile);
      push(@aResults, "fail|$sTodoJobFile");
      next;
    }

    if (!chmod($sJobFileMode, $sTodoJobFile))
    {
      $hLogArgs{'Message'} = "Unable to set permissions (" . sprintf("%04o", $sJobFileMode) . ") for $sTodoJobFile ($!).";
      unlink($sTodoJobFile);
      push(@aResults, "fail|$sTodoJobFile");
      next;
    }

    if (!chown($sJobFileUid, $sJobFileGid, $sTodoJobFile))
    {
      $hLogArgs{'Message'} = "Unable to set owner/group ($sJobFileUid/$sJobFileGid) for $sTodoJobFile ($!).";
      unlink($sTodoJobFile);
      push(@aResults, "fail|$sTodoJobFile");
      next;
    }

    $hLogArgs{'Result'} = "pass";
    $hLogArgs{'Message'} = "Job created successfully.";
    push(@aResults, "pass|$sTodoJobFile");
  }
  continue
  {
    ##################################################################
    #
    # Conditionally generate a log message.
    #
    ##################################################################

    if (defined($hLogArgs{'Message'}))
    {
      JqdLogMessage(\%hLogArgs);
      if ($hLogArgs{'Result'} ne "pass")
      {
        print STDERR "$$phProperties{'Program'}: Queue='$sQueue' Error='$hLogArgs{'Message'}'\n";
      }
    }

    ##################################################################
    #
    # Release the change lock.
    #
    ##################################################################

    JqdUnlockFile(\%hQueueLockArgs);
  }

  ####################################################################
  #
  # Report the results.
  #
  ####################################################################

  foreach my $sResult (@aResults)
  {
    print $sResult, "\n";
  }

  ####################################################################
  #
  # Shutdown and go home.
  #
  ####################################################################

  1;


######################################################################
#
# AdjustRunTimeEnvironment
#
######################################################################

sub AdjustRunTimeEnvironment
{
  my ($phProperties, $psError) = @_;

  ####################################################################
  #
  # Pull in specific server-side properties.
  #
  ####################################################################

  my (%hConfig, %hLArgs);

  %hLArgs =
  (
    'File'           => $$phProperties{'WebJobServerHome'} . "/config/server.cfg",
    'Properties'     => \%hConfig,
    'RequireAllKeys' => 0,
    'Template'       => PropertiesGetGlobalTemplates()->{'webjob.server'},
    'VerifyValues'   => 1,
  );
  if (!KvpGetKvps(\%hLArgs))
  {
    $$psError = $hLArgs{'Error'};
    return undef;
  }
  $$phProperties{'DsvRequireSignatures'} = $hConfig{'DsvRequireSignatures'} || "N";
  $$phProperties{'JobFileGroup'} = $hConfig{'WebJobGroup'} || "wheel";
  $$phProperties{'JobFileOwner'} = $hConfig{'ApacheOwner'} || "www";
  $$phProperties{'WebJobHome'} = $hConfig{'WebJobDirectory'} || "/usr/local/webjob";

  ####################################################################
  #
  # Initialize paths that will be used during execution.
  #
  ####################################################################

  $$phProperties{'DsvDirectory'} = "$$phProperties{'WebJobServerHome'}/config/dsv";
  $$phProperties{'DsvToolExe'} = "$$phProperties{'WebJobHome'}/bin/webjob-dsvtool";
  $$phProperties{'JqdDirectory'} = "$$phProperties{'WebJobServerHome'}/spool/jqd";
  $$phProperties{'JqdGroupsFile'} = "$$phProperties{'WebJobServerHome'}/config/jqd/groups";
  $$phProperties{'JqdLogFile'} = "$$phProperties{'WebJobServerHome'}/logfiles/jqd.log";
  $$phProperties{'PoundDirectory'} = "$$phProperties{'WebJobServerHome'}/db/pound/commands";

  ####################################################################
  #
  # Make sure we have the necessary prerequisites.
  #
  ####################################################################

  foreach my $sDirectory ($$phProperties{'DsvDirectory'}, $$phProperties{'JqdDirectory'}, $$phProperties{'PoundDirectory'})
  {
    if (!-d $sDirectory)
    {
      $$psError = "Directory ($sDirectory) does not exist.";
      return undef;
    }
  }

  foreach my $sFile ($$phProperties{'JqdGroupsFile'}, $$phProperties{'JqdLogFile'})
  {
    if (!-f $sFile)
    {
      $$psError = "File ($sFile) does not exist or is not regular.";
      return undef;
    }
  }

  foreach my $sExecutable ($$phProperties{'DsvToolExe'})
  {
    if (!-x $sExecutable)
    {
      $$psError = "File ($sExecutable) does not exist or is not executable.";
      return undef;
    }
  }

  ####################################################################
  #
  # Determine which version of webjob-dsvtool is in play.
  #
  ####################################################################

  my ($sCommandLine, $sOutput, $sStatus);

  $sCommandLine = qq("$$phProperties{'DsvToolExe'}" --version);
  $sOutput = qx($sCommandLine);
  $sStatus = ($? >> 8) & 0xff;
  if ($sStatus != 0)
  {
    my $phGlobalExitCodes = PropertiesGetGlobalExitCodes();
    my $sFailure = (exists($$phGlobalExitCodes{'webjob-dsvtool'}{$sStatus})) ? $$phGlobalExitCodes{'webjob-dsvtool'}{$sStatus} : $sStatus;
    $$psError = "Command ($sCommandLine) failed ($sFailure).";
    return undef;
  }
  $sOutput =~ s/-dsvtool//; # Hack 1 to make version string look like it came from webjob.
  $sOutput =~ s/,.*$/ dsv/; # Hack 2 to make version string look like it came from webjob.
  %hLArgs =
  (
    'VersionOutput' => $sOutput,
  );
  if (!VersionParseOutput(\%hLArgs))
  {
    $$psError = $hLArgs{'Error'};
    return undef;
  }
  $$phProperties{'VersionNumber'} = $hLArgs{'VersionNumber'};

  1;
}


######################################################################
#
# Usage
#
######################################################################

sub Usage
{
  my ($sProgram) = @_;
  print STDERR "\n";
  print STDERR "Usage: $sProgram [-h] [-c command-file] [-e queue[,queue[...]]] [-j job-group] [-m mode] [-o option[,option[,...]]] [-p priority] [-S webjob-server-home] [-T tag] -f job-file -i queue[,queue[...]] -t {p|parallel|s|serial}\n";
  print STDERR "\n";
  exit(1);
}


=pod

=head1 NAME

webjob-jqd-create-job - Add a job to the specified job queues.

=head1 SYNOPSIS

B<webjob-jqd-create-job> B<[-h]> B<[-c command-file]> B<[-e queue[,queue[...]]]> B<[-j job-group]> B<[-m mode]> B<[-o option[,option[,...]]]> B<[-p priority]> B<[-S webjob-server-home]> B<[-T tag]> B<-f file> B<-i queue[,queue[...]]> B<-t {p|parallel|s|serial}>

=head1 DESCRIPTION

This utility reads/validates a job file, conditionally copies the
corresponding command/signature files to the POUND, and then adds the
job to the specified queues.  A job file is simply a text file that
contains key/value pairs for the following properties:

=over 4

=item B<Command> (required)

Specifies the actual (i.e., on-disk) name of the target program.

=item B<CommandAlias> (required)

Specifies the actual name of the target program except when
server-side GET hooks are enabled.  If GET hooks are not in use, this
value should be the same as the value for B<Command>.

=item B<CommandLine> (required)

Specifies the actual command line to execute including any arguments.
This value has the following syntax:

    <CommandAlias> [arguments]

As stated above, the values for B<Command> and B<CommandAlias> differ
only in the case where server-side GET hooks are enabled, which is
beyond the scope of this man page.

=item B<CommandMd5> (conditionally required)

Specifies the MD5 hash of the target program.  This key/value pair is
required if the B<UseExistingPoundFiles> option is set. Otherwise, it
should be omitted.

=item B<CommandSha1> (conditionally required)

Specifies the SHA1 hash of the target program.  This key/value pair is
required if the B<UseExistingPoundFiles> option is set. Otherwise, it
should be omitted.

=item B<Comment> (optional)

Specifies a comment that describes the job or provides some other
relevant information.  Note that the value for this property must fit
on a single line.

=back

The following is an example job file for a the testenv command, which
takes no arguments:

    --- testenv.job ---
    Command=testenv
    CommandAlias=testenv
    CommandLine=testenv
    Comment=This is a job to check various environment variables.
    --- testenv.job ---

Note that the queued job file will be assigned file permissions
according to the B<-m> option and ownership according to the
'ApacheOwner' and 'WebJobGroup' properties located in the server's
main config file (typically /var/webjob/config/server.cfg).  Also, if
the queued job file is not writable by nph-webjob.cgi, then that
instance of the script will freeze the associated queue and abort, so
it's important that queued job file permissions and ownerships are set
correctly.

=head1 OPTIONS

=over 4

=item B<-c command-file>

Specifies the command file.  This is a relative or full path to the
actual binary/script being queued.  By default, the command file is
derived from the job file (assuming it has an extension of '.job').
See the B<-f> option for details about the job file.

=item B<-e queue[,queue[...]]>

Specifies one or more queues or queue groups that are to be excluded
from the operation.  Note that group names must be prefixed with '%'.
For example, a group called 'test' must be specified as '%test'.

=item B<-f job-file>

Specifies the job file.  The command contained in this file will be
assigned to each of the specified queues.  If the job file has an
extension of '.job', the extension will be removed and the resulting
value is assumed to be the path/name of the command file.  See the
B<-c> option for details about the command file.

=item B<-h>

This is a legacy option, and it is being phased out.  Please use the
B<PutJobOnHold> option instead (see B<-o>).

=item B<-i queue[,queue[...]]>

Specifies one or more queues or queue groups that are to be included
in the operation.  Note that group names must be prefixed with '%'.
For example, a group called 'test' must be specified as '%test'.

=item B<-j job-group>

Specifies the name of the group to which this job belongs.  Note that
jobs are assigned to the 'void' group by default.

=item B<-m mode>

Specifies the job file's default mode (i.e., permissions).  The default
mode is 0644.

=item B<-o option,[option[,...]]>

Specifies the list of options to apply.  Currently, the following
options are supported:

=over 4

=item OverwritePoundFiles

Indicates that existing files in the POUND are to be overwritten.
Typically, this is not necessary since the POUND's directory structure
is based on hash values.  However, there are cases where this option
is useful (e.g., updating signature files or repairing files that have
become corrupt since being added to the POUND).

=item PutJobOnHold

Indicates that the job is to be created and placed in the 'hold'
state.  Jobs in this state will appear in queue listings, but they are
not visible to queue workers.  By default, all jobs are initially
placed in the 'todo' state, which means they are visible to queue
workers.

=item UseExistingPoundFiles

Indicates that the necessary files already exist in the POUND and only
they may be used when creating the job.  If the necessary files do not
exist, the script will abort.  To use this option, the job file must
include the B<CommandMd5> and B<CommandSha1> key/value pairs.

=item UseZeroNonce

Typically, a pseudo random nonce is used when creating tags to help
prevent name collisions.  This option suppresses that behavior, and
uses a value of zero instead.

=back

=item B<-p priority>

Specifies the job priority.  Priorities can be in the range [0-99]
with 0 being the highest priority.  The default priority is 50.

=item B<-S webjob-server-home>

Specifies the WebJob Server home directory.  The default value is
/var/webjob.

=item B<-T tag>

Specifies a particular tag that is to be assigned to the job.  This
can be used to repeat a partially failed job assignment.  For example,
you could have a situation where 9/10 jobs queue up fine, but one
fails.  In that case, you may want to queue the final job up using the
tag that was generated on the previous execution.  The format of the
tag is as follows:

  DD_DDDDDDDDDD_DDDDDD_HHHHHHHH

where 'DD', 'DDDDDDDDDD', and 'DDDDDD' are decimal numbers, and
'HHHHHHHH' is a hex number.  The first number, 'DD', represents the
job priority.  The second and third numbers, 'DDDDDDDDDD' and
'DDDDDD', represent the time in time in seconds and microseconds.  The
fourth number, 'HHHHHHHH', represents a pseudo random nonce.

=item B<-t {p|parallel|s|serial}>

Specifies the job type.  Parallel jobs are executed in the background,
and serial jobs are executed in the foreground.

=back

=head1 AUTHOR

Klayton Monroe

=head1 SEE ALSO

nph-webjob.cgi(1), webjob-jqd-delete-job(1), webjob-jqd-list-jobs(1), webjob-setup-server(1)

=head1 LICENSE

All documentation and code are distributed under same terms and
conditions as B<WebJob>.

=cut
