#!/usr/bin/python
# Copyright (C) 2015 Ryan Fisher, Gary Hemming
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

###################################################################
import os
import socket
import atexit
import pwd
import re
import urlparse
import time
import sys
import commands
import StringIO
import logging
import logging.handlers
import exceptions
import binascii
from optparse import OptionParser
from dqsegdb import apicalls


try:
  import pyRXP
except ImportError, e:
  print >> sys.stderr, """
Error: unable to import the pyRXP module.

You must have pyRXP installed and in your PYTHONPATH to run %s.

%s
""" %(sys.argv[0], e)

try:
  from glue import gpstime
  from glue import ldbd
  from glue import segments
  from glue import git_version
  from glue.segmentdb import segmentdb_utils
  from glue.ligolw import table
  from glue.ligolw import lsctables
  from glue.ligolw import ligolw 
  from glue.ligolw.utils import process
  from glue.ligolw import types as ligolwtypes 
except ImportError, e:
  print >> sys.stderr, """
Error: unable to import modules from glue.

Check that glue is correctly installed and in your PYTHONPATH.

%s
""" % e
  sys.exit(1)
##################################################################
__author__ = "Ryan Fisher <rpfisher@syr.edu>"
__date__ = git_version.date
__version__ = git_version.id
__src__ = git_version.status

PROGRAM_NAME = sys.argv[0].replace('./','')
PROGRAM_PID  = os.getpid()

try:
  USER_NAME = os.getlogin()
except:
  USER_NAME = pwd.getpwuid(os.getuid())[0]


##################################################################
#            Command Line
##################################################################
def parse_command_line():
  parser = OptionParser()
  
  parser.add_option("-p", "--ping", action = "store_true", default = None, help = "Ping the target server")
  parser.add_option("-t", "--segment-url", action = "store", metavar = "URL", default = None, help = "Users have to specify protocol 'https://' for a secure connection in the segment database url. For example, '--segment-url=https://segdb.ligo.caltech.edu'. No need to specify port number'.")
  parser.add_option("-o", "--output", action = "store", metavar = "FILE", default = None,help = "Write segments to FILE rather than the segment database")
  parser.add_option("-j", "--identity", action = "store", metavar = "IDENTITY", default = None,help = "Set the subject line of the server's service certificate to IDENTITY")
  parser.add_option("-I", "--insert", action = "store_true", default = 0, help = "Insert segments to the segment database")
  parser.add_option("-A", "--append", action = "store_true", default = 0, help = "Append segments to an existing segment type")
  parser.add_option("-H", "--ignore-future", action = "store_true", default = 0, help = "Allow insert or append to add segments to future times.")
  parser.add_option("-F", "--ignore-append-check", action = "store_true", default = 0, help = "Allow --append to insert segments prior to the latest existing segment.")
  parser.add_option("-D", "--debug", action = "store_true", default = 0, help = "Set debug flag True")
  parser.add_option("-i", "--ifos", action = "store", metavar = "IFOS", default = None,  help = "Set the segment interferometer to IFOS (e.g. H1)")
  parser.add_option("-n", "--name", action = "store", metavar = "NAME", default = None,  help = "Set the name of the segment to NAME (e.g. DMT-BADMONTH)")
  parser.add_option("-v", "--version", action = "store", metavar = "VERSION", default = None,  help = "Set the numeric version of the segment to VERSION (e.g. 1)")
  parser.add_option("-e", "--explain", action = "store", metavar = "EXPLAIN", default = None,  help = "Set the segment_definer:comment to COMMENT. This should explaining WHAT this flag mean (e.g. \"Light dip 10%\"). Required when --Insert/-I is specified.")
  parser.add_option("-c", "--comment", action = "store", metavar = "COMMENT", default = None,  help = "Set the segment_summary:comment to COMMENT. This should explaining WHY these segments were inserted (e.g. \"Created from hveto results\")")
  parser.add_option("-S", "--summary-file", action = "store", metavar = "FILE", default = None, help = "Read the segment_summary rows from FILE. This should be a file containing the gps start and end times that the flag was defined (i.e. the union of on and off)")
  parser.add_option("-G", "--segment-file", action = "store", metavar = "FILE", default = None,  help = "Read the segment rows from FILE. This should containin the gps start and end times when the flag was active")
  
  
  return parser.parse_args()
  
    
(options, args) = parse_command_line()
    
# Make sure all necessary command line arguments are properly given
errmsg = ""
if not options.segment_url:
  errmsg += "Error: --segment-url must be specified\n"
if not options.ifos:
  errmsg += "Error: --ifos must be specified\n"
if not options.name:
  errmsg += "Error: --name must be specified\n"
if not options.version:
  errmsg += "Error: --version must be specified\n"
if (not options.explain) and (not options.append):
  errmsg += "Error: --explain must be specified\n"
if not options.comment:
  errmsg += "Error: --comment must be specified\n"
if not options.summary_file:
  errmsg += "Error: --summary-file must be specified\n"
if not options.segment_file:
  errmsg += "Error: --segment-file must be specified\n"
if options.insert and options.append:
  errmsg += "Error: Exactly one of [--insert | --append] can be specified\n"
if (not options.insert) and (not options.append):
  errmsg += "One of --help, --insert, or --append must be specified\n"
#o if (not options.ping) and (not options.insert) and (not options.append):
#o   errmsg += "One of --ping, --help, --insert, --append must be specified\n"
# --ignore-append-check is too dangerous to allow for direct DB access;
# user must dump to XML, check it, and be authorized to insert via ldbdc
if (not options.output and options.ignore_append_check):
  errmsg += "--ignore-append-check may only be used with --output-file\n"
if options.debug:
    debug=True
else:
    debug=False

  
if len(errmsg) and not options.ping:
  print >> sys.stderr, errmsg
  print >> sys.stderr, "Run\n    %s --help\nfor more information." % sys.argv[0]
  sys.exit(1)


#o ########################################################################
#o # open connection to LDBD(W)Server
#o myClient = segmentdb_utils.setup_database(options.segment_url)
#o 
#o if options.ping:
#o    try:
#o      print myClient.ping()
#o    except Exception, e:
#o      print "Error ping --segment-url", str(e)
#o      sys.exit(1)
#o else:
#o ########################################################################
def dtd_uri_callback(uri):
  if uri in ['http://www.ldas-sw.ligo.caltech.edu/doc/ligolwAPI/html/ligolw_dtd.txt',
    'http://ldas-sw.ligo.caltech.edu/doc/ligolwAPI/html/ligolw_dtd.txt']:
    # if the XML file contains a http pointer to the ligolw DTD at CIT then
    # return a local copy to avoid any network problems
    return 'file://localhost' + os.path.join( os.environ["GLUE_PREFIX"],
      'etc/ligolw_dtd.txt' )
  else:
    # otherwise just use the uri in the file
    return uri

def publishAndExit(filepath,debug=False):
  result=callInsertMultipleDQXMLThreaded(filepath)
  if not result:
    raise Exception("Call to InsertMultipleDQXMLFileThreaded returned failure status:  Publishing unsuccessful!")
  else:
    # We're done here
    print "Insert worked!"
    sys.exit()

def del_file(name):
  os.remove(name)

def callInsertMultipleDQXMLThreaded(filepath):
  logger = logging.getLogger('ligolw_publish_dqxml_dqsegdb')
  log_file=filepath.split('.xml')[0]+'.log'
  handler = logging.handlers.RotatingFileHandler(log_file, 'a', 1024**3, 3)
  formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
  handler.setFormatter(formatter)
  logger.addHandler(handler)
  logger.setLevel(eval("logging." + "DEBUG"))
  infiles=[filepath]
  result=apicalls.InsertMultipleDQXMLFileThreaded(infiles,logger,options.segment_url,hackDec11=False,debug=False,threads=1)
  return result


#Create an xml parser, a ligo_lw document parser, and the document
xmlparser = pyRXP.Parser()
lwtparser = ldbd.LIGOLwParser()
segment_md = ldbd.LIGOMetadata(xmlparser,lwtparser)

########################################################################
# Construct local table process, process_params and segment_definer 
########################################################################
# Table process and process_params will be popullated by calling the 
# process/process_params utility

#myClient = segmentdb_utils.setup_database(options.segment_url)
doc = ligolw.Document()
doc.appendChild(ligolw.LIGO_LW())

ligolwtypes.FromPyType[type(True)] = ligolwtypes.FromPyType[type(0)] 

process_id = process.register_to_xmldoc(doc, PROGRAM_NAME, options.__dict__,
             version = __version__,
             cvs_entry_time = __date__, comment = 'ligolw_segment_insert_dqsegdb client insert or append').process_id

##########################################################################
#             Process segment-file and summary-file 
##########################################################################
# create the total time interval and storage for the active segments
intervals = segments.segmentlist()
active_segments = segments.segmentlist()

#-----------------Check version and type existence-------------------------

# Query server for current highest version and flag name
o=urlparse.urlparse(options.segment_url)        
protocol=o.scheme        
server=o.netloc

version=int(options.version)

if debug:
  print "Determining max version"
maxVersion=apicalls.dqsegdbMaxVersion(protocol,server,options.ifos,options.name)
if debug:
  print "maxVersion=%d" % maxVersion
  print "Script call provided version=%d" % version


if maxVersion == 0:
  # Flag doesn't exist in db
  # If I use my publisher, this is fine, since it will take care of it for me
  maxEndTime=0 ## Because this flag doesn't exist yet!
elif maxVersion > version:
  # Fail!
  print "Version in DB (%d) is greater than that specified (%d), exiting with error." % (maxVersion,version)
  raise ValueError
elif maxVersion == version and options.insert:
  # Fail!
  print "Version in DB (%d) is equal to that specified (%d) for insertion, exiting with error." % (maxVersion,version)
  raise ValueError
else:
  # Proceed to grab maxEndTime from server
  if debug:
    print "Version approved by checks, determining max end time:"
  if options.append:
    data_dict,query_url1=apicalls.dqsegdbQueryTimeless(protocol,server,options.ifos,options.name,options.version,include_list_string='known,metadata')
    maxEndTime=max([i[1] for i in data_dict['known']])
  elif options.insert and version == maxVersion + 1:
    maxEndTime=0 # The version is greater than the existing versionsi bye one, so we're ok to go forward regardless of what the summary segments are in the payload!
  else:
    print "Specified version is not next version available to insert into database;  Existing max version is %d and provided version was %d." % (maxVersion,version)
    raise ValueError



#-----------------Determine max_end_time of existing data in database-------------------------

#data_dict,query_url1=apicalls.dqsegdbQueryTimeless(protocol,server,options.ifos,options.name,options.version,include_list_string='known,metadata')
#maxEndTime=max([i[1] for i in data_dict['known']])

### Fix!!! (future upgrade):  If I'm going to allow an xml file input, I'm going to have to do the following check on the xml file instead, implement that when I allow that.  

#-----------------Check summary-file---------------------------------
# get and check the summary intervals to be inserted
fh = open(options.summary_file, 'r')
line_no = 1
for line in fh.readlines():
  sum_line = line.strip().split(" ")
  if len(sum_line) != 2:
     print "\nERROR: invalid format in your --summary-file row number: %d " % line_no
     print "Invalid interval is %s" % str(sum_line)
     sys.exit(1)
  current_time = str(gpstime.GpsSecondsFromPyUTC(time.time()))

  # make sure gps time is 9 digits
  #if (len(sum_line[0])!=9 or len(sum_line[1])!=9):
  #   print "\nERROR: gps time must be 9 digits"
  #   print "Invalid interval in your --summary-file row number: %d " % line_no
  #   print "Invalid interval is %s" % str(sum_line)
  #   sys.exit(1)
  ## make sure interval start time is less than and not equal to end time
  if (int(sum_line[0]) >= int(sum_line[1])):
     print "\nERROR: start_time MUST be less than the end_time"
     print "Invalid interval in your --summary-file row number: %d " % line_no
     print "Invalid interval is %s" % str(sum_line)
     sys.exit(1)
  # make sure interval start time is greater than the max end time in the database
  # unless --ignore-append-check is specified or we are inserting a new version
  elif (int(sum_line[0]) < maxEndTime) and options.append and (not options.ignore_append_check):
     print "\nERROR: summary start_time MUST be greater than the max summary end_time %d in the database" % maxEndTime
     print "Invalid interval in your --summary-file row number: %d" % line_no
     print "Invalid interval is %s" % str(sum_line)
     sys.exit(1)
  # make sure interval end time is less than current time
  elif  (int(sum_line[1]) > int(current_time)) and not options.ignore_future:
     print "\nERROR: summary end_time cannot be greater than the current time"
     print "Invalid interval in your --summary-file row number: %d " % line_no
     print "Invalid interval is %s" % str(sum_line)
     sys.exit(1)
  # if interval is valid, append it to intervals list
  else:
    this_sum = segments.segment(int(sum_line[0]),int(sum_line[1]))
    intervals.append(this_sum)
    line_no += 1
intervals.coalesce()

#-----------------Check segment-file---------------------------------
# get and check the active segments to be inserted
fh = open(options.segment_file, 'r')
line_no = 1
for line in fh.readlines():
  seg_line = line.strip().split(" ")
  if len(seg_line) != 2:
     print "\nERROR: invalid format in your --segment-file row number: %d " % line_no
     print "Invalid segment is %s" % str(seg_line)
     sys.exit(1)
  # make sure gps time is 9 digits
  #if (len(seg_line[0]) != 9 or len(seg_line[1])!=9): 
  #   print "\nERROR: gps time must be 9 digits"
  #   print "Invalid segment in your --segment-file row number: %d" % line_no
  #   print "Invalid segment is %s" % str(seg_line)
  #   sys.exit(1)
  # make sure segment start_time is less than end_time:
  if int(seg_line[0])>=int(seg_line[1]):
    print "\nERROR: segment start_time MUST be less than the end_time"
    print "Invalid segment in your --segment-file row number: %d" % line_no
    print "Invalid segment is %s" % str(seg_line)
    sys.exit(1)
  # make sure segment falls in the summary  intervals specified in the --summary-file
  this_seg = segments.segment(int(seg_line[0]),int(seg_line[1])) 
  if this_seg not in intervals:
    print "\nERROR: segment cannot fall outside of the summary intervals specified in your --summary-file"
    print "Invalid segment in your --segment-file row number: %d" % line_no
    print "Invalid segment is %s" % str(seg_line)
    sys.exit(1)
  # Otherwise, append this segment to the list of active segments
  active_segments.append(this_seg)
  line_no += 1
active_segments.coalesce()

if debug:
    print "Beginning INSERT part of code"
#######################################################################################
#                            Process INSERT or APPEND
#######################################################################################
if options.insert or options.append:
  # For the new DQSEGDB, the creator_db is gone and the seg_def_id is determined server side from the ifo,name,version tuple.  Thus, inserts and appends appear identical to the server, and only the checks done above need to differentiate them.
  #--------------------------------------------------------------------------#
  # create segment_definer table
  seg_def_table = lsctables.New(lsctables.SegmentDefTable,columns = [
                'segment_def_id', 'process_id','ifos','name', 'version','comment'])
  doc.childNodes[0].appendChild(seg_def_table)
  segment_definer = lsctables.SegmentDef()

  seg_def_id = seg_def_table.get_next_id()
  segment_definer.segment_def_id = seg_def_id
  segment_definer.process_id = process_id
  segment_definer.ifos = options.ifos
  segment_definer.name = options.name
  segment_definer.version = int(options.version)
  segment_definer.comment = options.explain
  seg_def_table.append(segment_definer)

  #---------------- create local segment table ------------#
  segment_table = lsctables.New(lsctables.SegmentTable, columns = [
                  'segment_id','process_id','segment_def_id','start_time','end_time'])
  doc.childNodes[0].appendChild(segment_table)

  for this_seg in active_segments:
    segment = lsctables.Segment()
    seg_id = segment_table.get_next_id()

    segment.segment_id = seg_id
    segment.process_id = process_id
    segment.segment_def_id = seg_def_id
    segment.start_time = int(this_seg[0])
    segment.end_time = int(this_seg[1])

    segment_table.append(segment)

  
  #-------------- create local segment_summary table ------------#
  seg_sum_table = lsctables.New(lsctables.SegmentSumTable, columns = [
                  'segment_sum_id', 'process_id', 'segment_def_id', 'start_time', 'end_time', 'comment'])
  doc.childNodes[0].appendChild(seg_sum_table)

  for this_sum in intervals:
     segment_summary = lsctables.SegmentSum()
     seg_sum_id = seg_sum_table.get_next_id()
     
     segment_summary.segment_sum_id = seg_sum_id
     segment_summary.process_id = process_id
     segment_summary.segment_def_id = seg_def_id
     segment_summary.start_time = this_sum[0]
     segment_summary.end_time = this_sum[1]
     segment_summary.comment = options.comment
 
     seg_sum_table.append(segment_summary)

  fake_file = StringIO.StringIO()
  doc.write(fake_file)

  if options.output:
    if debug:
      print "Writing out to file and then exiting."
    fp = open(options.output,"w")
    fp.write(fake_file.getvalue())
    fp.close()
  else:
    if debug:
      print "Writing to temporary file, then calling callInsertMultipleDQXMLThreaded"
    filepath='/tmp/ligolw_segment_insert_'+str(time.time())+'.xml'
    if not debug:
      atexit.register(del_file,filepath) # 
    fp = open(filepath,'w')
    fp.write(fake_file.getvalue())
    fp.close()
    publishAndExit(filepath,debug)
    #result=callInsertMultipleDQXMLThreaded(filepath)
    #if not result:
    #    raise Exception("Call to InsertMultipleDQXMLFileThreaded returned failure status:  Publishing unsuccessful!")
    #else:
    #    # We're done here
    #    if debug:
    #      print "Insert worked!"
    #    sys.exit()


