#!/usr/bin/python
#
# Copyright (C) 2008  Drew Keppel
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.


#
# =============================================================================
#
#				   Preamble
#
# =============================================================================
#


from optparse import OptionParser
try:
	import sqlite3
except ImportError:
	# pre 2.5.x
	from pysqlite2 import dbapi2 as sqlite3
import sys


from glue import iterutils
from glue.ligolw import ligolw
from glue.ligolw import lsctables
from glue.ligolw import dbtables
from glue.ligolw import utils as ligolw_utils
from pylal import db_thinca_rings
from pylal import git_version
from pylal import ligolw_sqlutils as sqlutils
from pylal import rate
from pylal.xlal.datatypes.ligotimegps import LIGOTimeGPS
import scipy
import numpy
lsctables.LIGOTimeGPS = LIGOTimeGPS


class StatsContentHandler(ligolw.LIGOLWContentHandler):
	pass
lsctables.use_in(StatsContentHandler)


__author__ = "Drew G. Keppel <drew.keppel@ligo.org>"
__version__ = "git id %s" % git_version.id
__date__ = git_version.date


#
# =============================================================================
#
#				 Command Line
#
# =============================================================================
#


def parse_command_line():
	parser = OptionParser(
		version = "Name: %%prog\n%s" % git_version.verbose_msg,
		usage = "%prog [options] [file ...]",
		description = "%prog approximates a likelihood ratio computation for inspiral searches taking into account 2D correlations."
	)
	parser.add_option("-t", "--tmp-space", metavar="path", help="Path to a directory suitable for use as a work area while manipulating the database files.  The database files will be worked on in this directory, and then moved to the final location when complete.  This option is intended to improve performance when running in a networked environment, where there might be a local disk with higher bandwidth than is available to the filesystem on which the final output will reside.")
	parser.add_option("--output-table", metavar="TABLE", default="coinc_event", help="Table to store likelihood in. Must be one of {coinc_event|coinc_inspiral}. Default is coinc_event.")
	parser.add_option("--output-column", metavar="COLUMN", default="likelihood", help="Column to store likelihood in. Default is likelihood.")
	parser.add_option("--output-dir", metavar="path", help="Path to directory to store output files.")
	parser.add_option("--output-bins", metavar="file", help="Output the binned-arrays for use in the likelihood ratio computation to a file.")
	parser.add_option("--input-bins", metavar="file", action="append", help="Input the binned-arrays to be used for likelihood ratio computation from a file.")
	parser.add_option( "--max-distance", metavar = "Mpc", type="float", help = "The maximum injected distance to use for training likeliness.")
	parser.add_option("--compute-likelihoods", action="store_true", help="Compute the likelihood ratios for coincident triggers and update the database files.")
	parser.add_option("--smooth-bins", action="store_true", help="Smooth the binned-array objects before output or before use.")
	parser.add_option("-v", "--verbose", action="store_true", help="Be verbose.")
	options, filenames = parser.parse_args()

	if options.output_table not in ("coinc_event", "coinc_inspiral"):
		raise ValueError("--output-table %s invalid.  must be one of coinc_event, coinc_inspiral" % options.output_table)

	return options, (filenames or [None])


#
# =============================================================================
#
#				 Book-Keeping
#
# =============================================================================
#


def binned_ratios_extractor(xmldoc):
	for name in [elem.Name.replace(u":pylal_rate_binnedratios", u"") for elem in xmldoc.getElementsByTagName(ligolw.LIGO_LW.tagName) if elem.hasAttribute("Name") and "pylal_rate_binnedratios" in elem.Name]:
		yield name, rate.BinnedRatios.from_xml(xmldoc, name)

class Summaries(object):
	def __init__(self, Nbins):

		#
		# setting up required attributes
		#

		self.Nbins = Nbins
		self.rings = None
		self.offset_vectors = None
		self.ifos = set()
		self.dbls = set()
		self.likelihoods = {}

		#
		# setup bin types and min and max values for the parameters
		#
                
		self.minmax = {}
		bin_types = {}

		bin_types['snr'] = 'atanlog'
		self.minmax['snr'] = (1, 1e4)

		bin_types['chisq'] = 'atanlog'
		self.minmax['chisq'] = (1, 1e8)

		bin_types['bank_chisq'] = 'atanlog'
		self.minmax['bank_chisq'] = (1, 1e8)

		bin_types['cont_chisq'] = 'atanlog'
		self.minmax['cont_chisq'] = (1, 1e8)

		bin_types['mtotal'] = 'atan'
		self.minmax['mtotal'] = (0, 100)

		#
		# here we define the single detector 2D projections we will
		# perform
		#

		self.sngl_slices = [('snr', 'chisq'), ('snr', 'mtotal'), ('chisq', 'mtotal'), ('snr', 'bank_chisq'), ('bank_chisq', 'mtotal'), ('snr', 'cont_chisq'), ('cont_chisq', 'mtotal'), ('chisq', 'bank_chisq'), ('chisq', 'cont_chisq'), ('bank_chisq', 'cont_chisq')]

		self.atanlog_atan_bins = [sngl_slice for sngl_slice in self.sngl_slices if bin_types[sngl_slice[0]] == 'atanlog' and bin_types[sngl_slice[1]] == 'atan']
		self.atanlog_atanlog_bins = [sngl_slice for sngl_slice in self.sngl_slices if bin_types[sngl_slice[0]] == 'atanlog' and bin_types[sngl_slice[1]] == 'atanlog']
		self.atan_atan_bins = [sngl_slice for sngl_slice in self.sngl_slices if bin_types[sngl_slice[0]] == 'atan' and bin_types[sngl_slice[1]] == 'atan']

		self.sngl_bins = {}
		self.sngl_1D_bins = {}
		self.sngl_1D_count = {}
		sngls_set = set()
		for col1,col2 in self.sngl_slices:
			sngls_set.add(col1)
			sngls_set.add(col2)

		#
		# store the location of all the triggers
		#

		self.idx_sngls = {}
		for n,key in enumerate(list(sngls_set) + ['ifo', 'weight', 'coinc_event_id', 'filename']):
			self.idx_sngls[key] = n

	def calc_delta_t_inj(self, trigger1_end_time, trigger1_end_time_ns, trigger2_end_time, trigger2_end_time_ns):
		#
		# not used yet, only using single detector statistics
		#
		try:
			return abs((trigger1_end_time - trigger2_end_time) + (trigger1_end_time_ns - trigger2_end_time)*1e-9)
		except: print "calc_delta_t_inj() failed"

	def calc_delta_t(self, trigger1_ifo, trigger1_end_time, trigger1_end_time_ns, trigger2_ifo, trigger2_end_time, trigger2_end_time_ns, time_slide_id):
		#
		# not used yet, only using single detector statistics
		#
		trigger1_true_end_time = lsctables.LIGOTimeGPS(trigger1_end_time, trigger1_end_time_ns)
		trigger2_true_end_time = lsctables.LIGOTimeGPS(trigger2_end_time, trigger2_end_time_ns)
		# find the instruments that were on at trigger 1's end time and
		# find the ring that contains this trigger
		try:
			[ring] = [segs[segs.find(trigger1_end_time)] for segs in self.rings.values() if trigger1_end_time in segs]
		except ValueError:
			# FIXME THERE SEEMS TO BE A BUG IN  THINCA!  Occasionally thinca records a trigger on the upper boundary
			# of its ring.  This would make it outside the ring which is very problematic.  It needs to be fixed in thinca
			# for now we'll allow the additional check that the other trigger is in the ring and use it.
			print >>sys.stderr, "trigger1 found not on a ring, trying trigger2"
			[ring] = [segs[segs.find(trigger2_end_time)] for segs in self.rings.values() if trigger2_end_time in segs]
			# now we can unslide the triggers on the ring
		try:
			trigger1_true_end_time = SnglInspiralUtils.slideTimeOnRing(trigger1_true_end_time, self.offset_vectors[time_slide_id][trigger1_ifo], ring)
			trigger2_true_end_time = SnglInspiralUtils.slideTimeOnRing(trigger2_true_end_time, self.offset_vectors[time_slide_id][trigger2_ifo], ring)
			out = abs(trigger1_true_end_time - trigger2_true_end_time)
			return float(out)
		except:
			print "calc delta t failed",trigger1_true_end_time, trigger2_true_end_time, ring
			return float(abs(trigger1_true_end_time - trigger2_true_end_time)) % 1

	def setup_binned_ratios(self):
		#
		# setting up the binned ratio objects
		#
		for ifo in self.ifos:
			self.sngl_bins[ifo] = {}
			for key in self.sngl_slices:
				if key in self.atanlog_atanlog_bins:
					self.sngl_bins[ifo][key] = rate.BinnedRatios(rate.NDBins((rate.ATanLogarithmicBins(self.minmax[key[0]][0], self.minmax[key[0]][1], self.Nbins), rate.ATanLogarithmicBins(self.minmax[key[1]][0], self.minmax[key[1]][1], self.Nbins))))
				elif key in self.atanlog_atan_bins:
					self.sngl_bins[ifo][key] = rate.BinnedRatios(rate.NDBins((rate.ATanLogarithmicBins(self.minmax[key[0]][0], self.minmax[key[0]][1], self.Nbins), rate.ATanBins(self.minmax[key[1]][0], self.minmax[key[1]][1], self.Nbins))))
				elif key in self.atan_atan_bins:
					self.sngl_bins[ifo][key] = rate.BinnedRatios(rate.NDBins((rate.ATanBins(self.minmax[key[0]][0], self.minmax[key[0]][1], self.Nbins), rate.ATanBins(self.minmax[key[1]][0], self.minmax[key[1]][1], self.Nbins))))
				else:
					print >>sys.stderr,"ERROR: binning not implemented for ",key
					sys.exit(1)

	def add_bkg_sngl(self, values):
		#
		# add a background single to its appropriate bins
		#
		for key in self.sngl_slices:
			self.sngl_bins[values[self.idx_sngls['ifo']]][key].incdenominator((values[self.idx_sngls[key[0]]], values[self.idx_sngls[key[1]]]), weight=values[self.idx_sngls['weight']])

	def add_inj_sngl(self, values):
		#
		# add an injection single to its appropriate bins
		#
		for key in self.sngl_slices:
			self.sngl_bins[values[self.idx_sngls['ifo']]][key].incnumerator((values[self.idx_sngls[key[0]]], values[self.idx_sngls[key[1]]]), weight=values[self.idx_sngls['weight']])

	def add_bkg_dbl(self, filename, ifos, values):
		#
		# add a background double to its appropriate bins
		#
		self.dbls.add(frozenset(ifos))
		self.bkg_dbls.setdefault(frozenset(ifos), []).append(tuple([filename] + list(values)))

	def add_inj_dbl(self, filename, ifos, values):
		#
		# add an injection double to its appropriate bins
		#
		self.dbls.add(frozenset(ifos))
		self.inj_dbls.setdefault(frozenset(ifos), []).append(tuple([filename] + list(values)))

	def filter_binned_ratios(self, window_length=30):
		#
		# smooth the binned array objects
		#
		for ifo in self.ifos:
			for key in self.sngl_slices:
				tablename = "".join(ifo)+"-"
				tablename += "-vs-".join(key[::-1])
				if options.verbose:
					print >>sys.stderr, "\tsmoothing %s ..." % tablename
				rate.filter_binned_ratios(self.sngl_bins[ifo][key], rate.gaussian_window(self.sngl_bins[ifo][key].numerator.bins.shape[0]/window_length, self.sngl_bins[ifo][key].numerator.bins.shape[1]/window_length, sigma=2*window_length))

	def pdfify_binned_ratios(self):
		#
		# normalize binned arrays to form pdfs
		#
		for ifo in self.ifos:
			for key in self.sngl_slices:
				self.sngl_bins[ifo][key].to_pdf()

	def output_bins(self, filename):
		#
		# save the binned array objects to xml
		#
		xmldoc = lsctables.ligolw.Document()
		xmldoc.appendChild(lsctables.ligolw.LIGO_LW())
		for key1 in self.sngl_bins.keys():
			for key2 in self.sngl_bins[key1].keys():
				tablename = "".join(key1)+"-"
				tablename += "-vs-".join(key2[::-1])
				xmldoc.childNodes[-1].appendChild(self.sngl_bins[key1][key2].to_xml(tablename))
		for key1 in self.sngl_1D_bins.keys():
			for key2 in self.sngl_1D_bins[key1].keys():
				tablename = "".join(key1)+"-"+"".join(key2)
				xmldoc.childNodes[-1].appendChild(self.sngl_1D_bins[key1][key2].to_xml(tablename))
		
		ligolw_utils.write_filename(xmldoc, filename, gz=filename.endswith(".gz"), verbose=options.verbose)

	def input_bins(self, filenames):
		#
		# read the binned array objects from xml
		#
		missing = 0
		for filename in filenames:
			xmldoc = ligolw_utils.load_filename(filename, verbose=options.verbose, contenthandler=StatsContentHandler)
			for name, ratios in binned_ratios_extractor(xmldoc):
				if '-vs-' in name:
					ifo,key1,junk,key2 = name.split('-')
					self.sngl_bins.setdefault(ifo, {})
					if (key2,key1) in self.sngl_bins[ifo].keys():
						self.sngl_bins[ifo][(key2,key1)] += ratios
					else:
						self.sngl_bins[ifo][(key2,key1)] = ratios
				else:
					print >>sys.stderr, "Binned array file %s unknown table. Skipping table %s"%(filename,name)


	def check_input_bins(self):
		#
		# check we have everything we need from the binned array objects from xml
		#
		missing = 0
		for filename in filenames:
			check_bins = {}
			for ifo in self.ifos:
				check_bins[ifo] = {}
				for key in self.sngl_slices:
					check_bins[ifo][key] = (ifo in self.sngl_bins.keys()) and (key in self.sngl_bins[ifo].keys())

			#
			# check for missing binned array objects
			#

			for ifo in self.ifos:
				for sngl_slice in self.sngl_slices:
					if not check_bins[ifo][sngl_slice]:
						print >>sys.stderr, "Binned array files do not contain %s-%s-vs-%s 2D slice."%(ifo,sngl_slice[1],sngl_slice[0])
						missing += 1
		if missing:
			sys.exit(1)


	def create_1D_slices(self):
		#
		# create the appropriate 1D slices from the 2D projections
		#
		for ifo in self.ifos:
			self.sngl_1D_bins[ifo] = {}
			self.sngl_1D_count[ifo] = {}
			for key in self.sngl_slices:
				for dim,subkey in enumerate(key):
					if subkey not in self.sngl_1D_bins[ifo].keys():
						self.sngl_1D_bins[ifo][subkey] = rate.marginalize_ratios(self.sngl_bins[ifo][key], (dim+1)%2)
						self.sngl_1D_count[ifo][subkey] = 0
					else:
						self.sngl_1D_count[ifo][subkey] += 1

	def compute_sngl_likelihood(self, values):
		#
		# compute the likelihood for the single detector triggers
		#
		log_likelihood_2D = 0.
		log_likelihood_1D = 0.
		for key in self.sngl_slices:
			log_likelihood_2D += scipy.log(self.sngl_bins[values[self.idx_sngls['ifo']]][key][(values[self.idx_sngls[key[0]]],values[self.idx_sngls[key[1]]])])
		for key in self.sngl_1D_bins[values[self.idx_sngls['ifo']]].keys():
			log_likelihood_1D += self.sngl_1D_count[values[self.idx_sngls['ifo']]][key] * scipy.log(self.sngl_1D_bins[values[self.idx_sngls['ifo']]][key][(values[self.idx_sngls[key]],)])
		self.likelihoods[(values[self.idx_sngls['filename']],values[self.idx_sngls['coinc_event_id']])] *= scipy.exp(log_likelihood_2D - log_likelihood_1D)

	def likelihood_map(self, filename, coinc_event_id):
		#
		# map the likehoods to the appropriate triggers
		#
		return self.likelihoods[(filename, coinc_event_id)]


#
# =============================================================================
#
#				SQLITE Queries
#
# =============================================================================
#

def single_inj(values):
	#
	# the SQL query for single detector injection triggers
	#
	query = """
SELECT"""
	keys = values.items()
	keys.sort(cmp=lambda x,y: cmp(x[1],y[1]))
	for value in keys:
		if value[0] == 'coinc_event_id' or value[0] == 'filename':
			continue
		if not value == keys[0]:
			query += ""","""
		if value[0] == 'weight':
			query += """
	-- Work out the correction factor for injection population distances
	-- FIXME this breaks if more than one inspinj job
	CASE (SELECT value FROM process_params WHERE program =="inspinj" AND param =="--d-distr")
		WHEN "log10" THEN  sim_inspiral.distance * sim_inspiral.distance * sim_inspiral.distance
		WHEN "uniform" THEN  sim_inspiral.distance * sim_inspiral.distance
		ELSE 1.0 END"""
			continue

		query += """
	snglA.""" + value[0]

	query += """,
	coinc_inspiral.coinc_event_id
FROM
	coinc_inspiral
	JOIN coinc_event_map AS mapA ON (mapA.coinc_event_id == coinc_inspiral.coinc_event_id)
	JOIN sngl_inspiral AS snglA ON (snglA.event_id == mapA.event_id)
	JOIN coinc_event_map AS mapB ON (mapB.event_id == coinc_inspiral.coinc_event_id)
	JOIN coinc_event_map AS mapC ON (mapC.coinc_event_id == mapB.coinc_event_id)
	JOIN sim_inspiral ON (sim_inspiral.simulation_id == mapC.event_id)
	JOIN coinc_event AS sim_coinc_event ON (sim_coinc_event.coinc_event_id == mapC.coinc_event_id)
	JOIN coinc_definer ON (coinc_definer.coinc_def_id == sim_coinc_event.coinc_def_id)
WHERE
	coinc_definer.search == 'inspiral'
	AND coinc_definer.search_coinc_type == 2
	AND mapA.table_name == 'sngl_inspiral'
	AND mapB.table_name == 'coinc_event'
	AND mapC.table_name == 'sim_inspiral'"""
	if options.max_distance:
		query += """
	AND sim_inspiral.distance < %f"""%(options.max_distance)
	query += """
	-- require coinc to not be background (= at least one of its time slide offsets is non-zero)
	-- FIXME this has to call a function to get coinc_def id
	AND NOT EXISTS (
		SELECT
			*
		FROM
			time_slide
		WHERE
			time_slide.time_slide_id == sim_coinc_event.time_slide_id
			AND time_slide.offset != 0
	)
"""

	return query

def double_inj():
	#
	# not in use, only using single detector statistics
	#
	return """
SELECT
	coinc_inspiral.coinc_event_id,
	calc_delta_t_inj(snglA.end_time, snglA.end_time_ns, snglB.end_time, snglB.end_time_ns),
	abs(2*(snglA.mchirp - snglB.mchirp)/(snglA.mchirp+snglB.mchirp)),
	abs(2*(snglA.eta - snglB.eta)/(snglA.eta+snglB.eta)),
	snglA.snr,
	snglB.snr,
	snglA.chisq,
	snglB.chisq,
	snglA.rsqveto_duration,
	snglB.rsqveto_duration,
	snglA.bank_chisq,
	snglB.bank_chisq,
	snglA.cont_chisq,
	snglB.cont_chisq,
	-- Work out the correction factor for injection population distances
	-- FIXME this breaks if more than one inspinj job
	CASE (SELECT value FROM process_params WHERE program =="inspinj" AND param =="--d-distr")
		WHEN "log10" THEN  sim_inspiral.distance * sim_inspiral.distance * sim_inspiral.distance
		WHEN "uniform" THEN  sim_inspiral.distance * sim_inspiral.distance
		ELSE 1.0 END
FROM
	coinc_inspiral
	JOIN coinc_event_map AS mapA ON (mapA.coinc_event_id == coinc_inspiral.coinc_event_id)
	JOIN coinc_event_map AS mapB ON (mapB.coinc_event_id == coinc_inspiral.coinc_event_id)
	JOIN sngl_inspiral AS snglA ON (snglA.event_id == mapA.event_id)
	JOIN sngl_inspiral AS snglB ON (snglB.event_id == mapB.event_id)
	JOIN coinc_event_map AS mapC ON (mapC.event_id == coinc_inspiral.coinc_event_id)
	JOIN coinc_event_map AS mapD ON (mapD.coinc_event_id == mapC.coinc_event_id)
	JOIN sim_inspiral ON (sim_inspiral.simulation_id == mapD.event_id)
	JOIN coinc_event AS sim_coinc_event ON (sim_coinc_event.coinc_event_id == mapD.coinc_event_id)
	JOIN coinc_event AS insp_coinc_event ON (insp_coinc_event.coinc_event_id == mapA.coinc_event_id)
	JOIN coinc_definer ON (coinc_definer.coinc_def_id == sim_coinc_event.coinc_def_id)
WHERE
	coinc_definer.search == 'inspiral'
	AND coinc_definer.search_coinc_type == 2
	AND mapA.table_name == 'sngl_inspiral'
	AND mapB.table_name == 'sngl_inspiral'
	AND mapC.table_name == 'coinc_event'
	AND mapD.table_name == 'sim_inspiral'
	AND snglA.ifo == ?
	AND snglB.ifo == ?
	-- require coinc to not be background (= at least one of its time slide offsets is non-zero)
	-- FIXME this has to call a function to get coinc_def id
	AND NOT EXISTS (
		SELECT
			*
		FROM
			time_slide
		WHERE
			time_slide.time_slide_id == coinc_event.time_slide_id
			AND time_slide.offset != 0
	)
"""

def single_bkg(values):
	#
	# the SQL query for single detector background triggers
	#
	query = """
SELECT"""
	keys = values.items()
	keys.sort(cmp=lambda x,y: cmp(x[1],y[1]))
	for value in keys:
		if value[0] in ['coinc_event_id', 'filename']:
			continue
		if not value == keys[0]:
			query += ""","""
		if value[0] == 'weight':
			query += """
	1.0"""
			continue

		query += """
	snglA.""" + value[0]

	query += """,
	coinc_inspiral.coinc_event_id,
	EXISTS (
		SELECT
			*
		FROM
			time_slide
		WHERE
			time_slide.time_slide_id == coinc_event.time_slide_id
			AND time_slide.offset != 0
	)
FROM
	coinc_inspiral
	JOIN coinc_event_map AS mapA ON (mapA.coinc_event_id == coinc_inspiral.coinc_event_id)
	JOIN sngl_inspiral AS snglA ON (snglA.event_id == mapA.event_id)
	JOIN coinc_event ON (mapA.coinc_event_id == coinc_event.coinc_event_id)
	JOIN coinc_definer ON (coinc_definer.coinc_def_id == coinc_event.coinc_def_id)
WHERE
	coinc_definer.search == 'inspiral'
	AND coinc_definer.search_coinc_type == 0
	AND mapA.table_name == 'sngl_inspiral'
"""

	return query

def double_bkg():
	#
	# not in use, only using single detector statistics
	#
	return """
SELECT
	coinc_inspiral.coinc_event_id,
	calc_delta_t(snglA.ifo, snglA.end_time, snglA.end_time_ns, snglB.ifo, snglB.end_time, snglB.end_time_ns, coinc_event.time_slide_id),
	abs(2*(snglA.mchirp - snglB.mchirp)/(snglA.mchirp+snglB.mchirp)),
	abs(2*(snglA.eta - snglB.eta)/(snglA.eta+snglB.eta)),
	snglA.snr,
	snglB.snr,
	snglA.chisq,
	snglB.chisq,
	snglA.rsqveto_duration,
	snglB.rsqveto_duration,
	snglA.bank_chisq,
	snglB.bank_chisq,
	snglA.cont_chisq,
	snglB.cont_chisq,
	EXISTS (
		SELECT
			*
		FROM
			time_slide
		WHERE
			time_slide.time_slide_id == coinc_event.time_slide_id
			AND time_slide.offset != 0
	)
FROM
	coinc_inspiral
	JOIN coinc_event_map AS mapA ON (mapA.coinc_event_id == coinc_inspiral.coinc_event_id)
	JOIN coinc_event_map AS mapB ON (mapB.coinc_event_id == coinc_inspiral.coinc_event_id)
	JOIN sngl_inspiral AS snglA ON (snglA.event_id == mapA.event_id)
	JOIN sngl_inspiral AS snglB ON (snglB.event_id == mapB.event_id)
	JOIN coinc_event ON (mapA.coinc_event_id == coinc_event.coinc_event_id)
	JOIN coinc_definer ON (coinc_definer.coinc_def_id == coinc_event.coinc_def_id)
WHERE
	coinc_definer.search == 'inspiral'
	AND coinc_definer.search_coinc_type == 0
	AND mapA.table_name == 'sngl_inspiral'
	AND mapB.table_name == 'sngl_inspiral'
	AND snglA.ifo == ?
	AND snglB.ifo == ?
"""


def single_all(values):
	#
	# the SQL query for all single detector triggers
	#
	query = """
SELECT"""
	keys = values.items()
	keys.sort(cmp=lambda x,y: cmp(x[1],y[1]))
	for value in keys:
		if value[0] in ['coinc_event_id', 'filename']:
			continue
		if not value == keys[0]:
			query += ""","""
		if value[0] == 'weight':
			query += """
	1.0"""
			continue

		query += """
	snglA.""" + value[0]

	query += """,
	coinc_inspiral.coinc_event_id
FROM
	coinc_inspiral
	JOIN coinc_event_map AS mapA ON (mapA.coinc_event_id == coinc_inspiral.coinc_event_id)
	JOIN sngl_inspiral AS snglA ON (snglA.event_id == mapA.event_id)
	JOIN coinc_event ON (mapA.coinc_event_id == coinc_event.coinc_event_id)
	JOIN coinc_definer ON (coinc_definer.coinc_def_id == coinc_event.coinc_def_id)
WHERE
	coinc_definer.search == 'inspiral'
	AND coinc_definer.search_coinc_type == 0
	AND mapA.table_name == 'sngl_inspiral'
"""

	return query

#
# =============================================================================
#
#				     Main
#
# =============================================================================
#


#
# command line
#


options, filenames = parse_command_line()


if options.input_bins and not options.compute_likelihoods:
	print >>sys.stderr, "If --input-bins is specified, --compute-likelihoods bust also be given."
	sys.exit(1)


#
# initialize book-keeping
#


likelihood_bins = Summaries(200)


#
# read binned array objects from file
#


if options.input_bins:
	if options.verbose:
		print >>sys.stderr, "importing binned ratio objects ..."
	likelihood_bins.input_bins(options.input_bins)


#
# validating output_table and output_column
#


output_table = sqlutils.validate_option(options.output_table)
output_column = sqlutils.validate_option(options.output_column)


#
# iterate over database files accumulating statistics
#


if options.verbose:
	print >>sys.stderr, "collecting initial likelihood statistics ..."

for n, filename in enumerate(filenames):
	#
	# open the database
	#

	if options.verbose:
		print >>sys.stderr, "%d/%d: %s" % (n + 1, len(filenames), filename)
	working_filename = dbtables.get_connection_filename(filename, tmp_path = options.tmp_space, verbose = options.verbose)
	connection = sqlite3.connect(working_filename)

	if options.verbose:
		print >>sys.stderr, "\textracting all ifos ..."
	# FIXME get ifos from search summary table
	likelihood_bins.ifos |= set(ifo for ifo, in connection.cursor().execute("""SELECT DISTINCT ifo FROM sngl_inspiral"""))

	if options.verbose:
		print >>sys.stderr, "\textracting coinc_event_id from everything in %s ..."%(output_table)
	for coinc_event_id in connection.cursor().execute("""
SELECT
	%s.coinc_event_id
FROM
	%s
	"""%(output_table, output_table)):
		likelihood_bins.likelihoods.setdefault((filename, coinc_event_id[0]), 0.)

	#
	# done extracting info so close database
	#

	connection.close()
	dbtables.discard_connection_filename(filename, working_filename, verbose = options.verbose)

#
# end loop over input database files
#

if options.verbose:
	print >>sys.stderr, "initializing every %s to 1.0  ..."%(output_column)
for key in likelihood_bins.likelihoods.keys():
	likelihood_bins.likelihoods[key] = 1.0

if options.input_bins:
	if options.verbose:
		print >>sys.stderr, "checking validity of binned ratio objects ..."
	likelihood_bins.check_input_bins()
else:
	if options.verbose:
		print >>sys.stderr, "setting up binned ratio objects ..."
	likelihood_bins.setup_binned_ratios()

	if options.verbose:
		print >>sys.stderr, "extracting background and injection statistics ..."
	for n, filename in enumerate(filenames):
		#
		# open the database
		#

		if options.verbose:
			print >>sys.stderr, "%d/%d: %s" % (n + 1, len(filenames), filename)
		working_filename = dbtables.get_connection_filename(filename, tmp_path = options.tmp_space, verbose = options.verbose)
		connection = sqlite3.connect(working_filename)

		#
		# extract triggers using appropriate sqlite queries determined
		# by the existence of a sim_inspiral table in the database
		#

		if "sim_inspiral" in dbtables.get_table_names(connection):
			#
			# loop over all single ifos and add sngls to
			# appropriate list
			#

			if options.verbose:
				print >>sys.stderr, "\textracting all sngls to populate binned ratio numerator ..."
			for values in connection.cursor().execute(single_inj(likelihood_bins.idx_sngls)):
				likelihood_bins.add_inj_sngl(values)

			#
			# loop over all double combinations of ifos and add
			# dbls to appropriate list
			#

#			if options.verbose:
#				print >>sys.stderr, "\textracting all dbls ..."
#			connection.create_function("calc_delta_t_inj", 4, likelihood_bins.calc_delta_t_inj)
#			for ifos in list(iterutils.choices(likelihood_bins.ifos, 2)):
#				if options.verbose:
#					print >>sys.stderr, "\t\t%s ..." % ("".join(ifos))
#				for values in connection.cursor().execute(double_inj, ifos):
#					likelihood_bins.add_inj_dbl(filename, ifos, values)
		#
		# loop over all single ifos and add sngls to
		# appropriate list
		#

		if options.verbose:
			print >>sys.stderr, "\textracting all sngls to populate binned ratio denominator ..."
		for values in connection.cursor().execute(single_bkg(likelihood_bins.idx_sngls)):
			if values[-1]:
				likelihood_bins.add_bkg_sngl(values)

		#
		# loop over all double combinations of ifos and add
		# dbls to appropriate list
		#

#		if options.verbose:
#			print >>sys.stderr, "\textracting all dbls ..."
#		likelihood_bins.rings = db_thinca_rings.get_thinca_rings_by_available_instruments(connection)
#		likelihood_bins.offset_vectors = lsctables.TimeSlideTable.get_table(dbtables.get_xml(connection)).as_dict()
#		connection.create_function("calc_delta_t", 7, likelihood_bins.calc_delta_t)
#		for ifos in list(iterutils.choices(likelihood_bins.ifos, 2)):
#			if options.verbose:
#				print >>sys.stderr, "\t\t%s ..." % ("".join(ifos))
#			for values in connection.cursor().execute(double_bkg, ifos):
#				if values[-1]:
#					background.add_bkg_dbl(filename, ifos, values[0:-1])
#				else:
#					background.add_zero_lag_dbl(filename, ifos, values[0:-1])

		#
		# done extracting info so close database
		#

		connection.close()
		dbtables.discard_connection_filename(filename, working_filename, verbose = options.verbose)

	#
	# end loop over input database files
	#

	if options.smooth_bins:
		if options.verbose:
			print >>sys.stderr, "smoothing binned ratios ..."
		likelihood_bins.filter_binned_ratios()

	if options.output_bins:
		if options.verbose:
			print >>sys.stderr, "writing binned ratios objects to file ..."
		likelihood_bins.output_bins(options.output_bins)

if not options.compute_likelihoods:
	sys.exit(0)

if options.smooth_bins:
	if options.verbose:
		print >>sys.stderr, "smoothing binned ratios ..."
	likelihood_bins.filter_binned_ratios()
if options.verbose:
	print >>sys.stderr, "normalizing binned ratios ..."
likelihood_bins.pdfify_binned_ratios()
if options.verbose:
	print >>sys.stderr, "creating 1D binned ratios ..."
likelihood_bins.create_1D_slices()

if options.verbose:
	print >>sys.stderr, "computing likelihoods ..."
for n, filename in enumerate(filenames):
	#
	# open the database
	#

	if options.verbose:
		print >>sys.stderr, "%d/%d: %s" % (n + 1, len(filenames), filename)
	working_filename = dbtables.get_connection_filename(filename, tmp_path = options.tmp_space, verbose = options.verbose)
	connection = sqlite3.connect(working_filename)

	#
	# extract triggers using appropriate sqlite queries determined
	# by the existence of a sim_inspiral table in the database
	#

	# 
	# loop over all sngls and compute likelihoods
	#

	if options.verbose:
		print >>sys.stderr, "\textracting all sngls for likelihood computation ..."
	for values in connection.cursor().execute(single_all(likelihood_bins.idx_sngls)):
		likelihood_bins.compute_sngl_likelihood(tuple(list(values) + [filename]))

	# 
	# loop over all double combinations of ifos and add dbls to
	# appropriate list 
	#

#	if options.verbose:
#		print >>sys.stderr, "\textracting all dbls ..."
#	likelihood_bins.rings = db_thinca_rings.get_thinca_rings_by_available_instruments(connection)
#	likelihood_bins.offset_vectors = lsctables.TimeSlideTable.get_table(dbtables.get_xml(connection)).as_dict()
#	connection.create_function("calc_delta_t", 7, likelihood_bins.calc_delta_t)
#	for ifos in list(iterutils.choices(likelihood_bins.ifos, 2)):
#		if options.verbose:
#			print >>sys.stderr, "\t\t%s ..." % ("".join(ifos))
#		for values in connection.cursor().execute(double_bkg, ifos):
#			if values[-1]:
#				background.add_bkg_dbl(filename, ifos, values[0:-1])
#			else:
#				background.add_zero_lag_dbl(filename, ifos, values[0:-1])

	#
	# done extracting info so close database
	#

	connection.close()
	dbtables.discard_connection_filename(filename, working_filename, verbose = options.verbose)

#
# end loop over input database files
#

#
# iterate over database files assigning likelihoods to coincs
#

if options.verbose:
	print >>sys.stderr, "updating likelihoods in databases ..."
for n, filename in enumerate(filenames):
	#
	# open the database
	#

	if options.verbose:
		print >>sys.stderr, "%d/%d: %s" % (n + 1, len(filenames), filename)
	working_filename = dbtables.get_connection_filename(filename, tmp_path = options.tmp_space, verbose = options.verbose)
	connection = sqlite3.connect(working_filename)

	#    
	# Create the output column if it doesn't exist
	#

	sqlutils.create_column(connection, output_table, output_column)

	#
	# prepare the database
	#

	connection.create_function("likelihood_map", 2, likelihood_bins.likelihood_map)
	if options.verbose:
		print >>sys.stderr, "\trecording likelihood ..."
	connection.cursor().execute("""
UPDATE
	%s
SET
	%s = likelihood_map(
			?,
			%s.coinc_event_id
	)
	"""%(output_table, output_column, output_table), (filename,))
	connection.commit()

	#
	# close the database
	#

	connection.close()
	dbtables.put_connection_filename(filename, working_filename, verbose = options.verbose)
