#!/usr/bin/python -u

# Copyright (C) 2012 Duncan M. Macleod
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

"""Interferometer summary information generator.

This script generates a nested web page and content for
reviewing interferometer performance and status.

Comments should be e-mailed to detchar+code@ligo.org.
Bugs should be noted at https://bugs.ligo.org/redmine/projects/detchar/.

"""

from __future__ import division

import sys
import os
import re
import optparse
import time
import ConfigParser
import datetime
import urlparse
import shutil
import urllib
import calendar
import getpass
import numpy
import inspect
import glob
import math
import operator
import warnings
warnings.filterwarnings("ignore", "column name", UserWarning)

from scipy import (stats, signal)
from dateutil.relativedelta import relativedelta

if not os.getenv("DISPLAY", None):
    import matplotlib
    matplotlib.use("agg", warn=False)

from collections import defaultdict
from matplotlib import pyplot

# import SwigLAL modules
import lal
import lalframe
import lalsimulation
import lalburst

# import laldetchar
import laldetchar
from laldetchar import triggers
from laldetchar.triggers import (omega, kw, daily_ihope, excesspower, trigfind)

# import PyLAL modules
from pylal import (git_version, seriesutils, plotutils, htmlutils,
                   omegautils, hacrutils, rangeutils, hdf5utils, snglcluster,
                   SnglInspiralUtils, ligolw_sicluster, ligolw_bucluster)
from pylal.dq import (dqSegmentUtils, dqTriggerUtils, stateutils, summary)

# import GLUE modules
from glue import (segmentsUtils, markup, datafind, gpstime)
from glue.lal import Cache, CacheEntry
from glue.LDBDWClient import LDBDClientException
from glue.ligolw import (ilwd, ligolw, lsctables, table, utils as ligolw_utils)
from glue.segmentdb import segmentdb_utils
from glue.segments import (segment as Segment, segmentlist as SegmentList,
                           segmentlistdict as SegmentListDict)

# set metadata
__author__  = "Duncan Macleod <duncan.macleod@ligo.org>"
__version__ = git_version.id
__date__    = git_version.date

# global variables and constants
VERBOSE = False
PROFILE = False
START = None
HDF_DATA_GROUP = "data"
HDF_SPECTRUM_GROUP = "spectrum"
HDF_SPECTROGRAM_GROUP = "spectrogram"
LIGOTimeGPS = lal.LIGOTimeGPS
re_cchar = re.compile("[\W_]+")
re_state = re.compile("run-\w+-time")
re_xml = re.compile("(xml|xml.gz)\Z")
re_etg_kw = re.compile("\A(kw|kleinewelle)\Z", re.I)
re_etg_omega = re.compile("\Aomega\Z", re.I)
re_etg_di = re.compile("\Adaily*ihope\Z", re.I)
re_etg_ep = re.compile("\A(ep|excesspower|gstlal_excesspower)\Z", re.I)

# set plot formatting
plotutils.set_rcParams()


#
# =============================================================================
#
# utility functions
#
# =============================================================================
#

def start_job_timer():
    """Starts the job timer.
    """
    global START
    START = time.time()

def elapsed_time():
    """@returns the time elapsed since the job started (in seconds).
    """
    return time.time()-START


def print_verbose(message, verbose=True, stream=sys.stdout, profile=True):
    """Prints the given message to the stream.

    @param message
        string to print
    @param verbose
        flag to print or not, default: False (don't print)
    @param stream
        file object stream in which to print, default: stdout
    @param profile
        flag to print timestamp, default: False
    """
    if stream != sys.stderr:
        profile &= PROFILE
        verbose &= VERBOSE
    if profile and message.endswith("\n"):
        message = "%s (%.2f)\n" % (message.rstrip("\n"), elapsed_time())
    if verbose:
        stream.write(message)
        stream.flush()


def natsort(l, key=None):
    """@returns a copy of the list l sorted the way that humans expect.

    @param l
        iterable to sort
    @param key
        sorting key
    """
    k = key and map(key, l) or l
    convert = lambda text: int(text) if text.isdigit() else text
    alphanum_key = lambda key: [convert(c) for c in
                                re.split('([0-9]+)', k[l.index(key)])]
    return sorted(l, key=alphanum_key)


# file-reading functions
def read_segment_file(filename):
    """@returns a glue.segments.segmentlist read from the given filename.

    @param filename
        path of XML or segwizard-format file to read
    """
    with open(filename, "r") as f:
        if re_xml.search(filename):
            return dqSegmentUtils.fromsegmentxml(f)
        else:
            # FIXME: set coltype=LIGOTimeGPS when segmentlists play
            # nicely with lal.LIGOTimeGPS
            return segmentsUtils.fromsegwizard(f, coltype=float, strict=False)


def read_segment_cache(cache):
    """@returns a SegmentList read from the given Cache object

    @param cache
        glue.lal.Cache of segment files from which to read
    """
    out = SegmentList()
    for entry in cache.pfnlist():
        out.extend(read_segment_file(entry))
    return out


def scp(remotefile, localfile):
    """Copy a (remote) file a local directory

    @param remotefile
        URL/path of file to copy
    @param localfile
        local path (file or directory) to write

    @returns path to local file
    """
    if os.path.isdir(localfile):
       localfile = os.path.join(localfile, os.path.basename(remotefile))
    url = urlparse.urlparse(remotefile)
    if url.path.startswith("/~"):
        return remotefile
    if (not url.scheme or url.scheme == "file"):
        if (os.path.isfile(url.path) and not os.path.isfile(localfile) or
                not os.path.samefile(url.path, localfile)):
            shutil.copy(url.path, localfile)
        elif not os.path.isfile(url.path):
            raise IOError("Cannot find %s" % url.path)
    else:
        urllib.urlretrieve(remotefile, localfile)
    return localfile


# plot-related functions
def subplot_durations(start, end, mode=summary.SUMMARY_MODE_GPS):
    """@returns the durations for tab subplots for this mode

    @param start
        GPS start time of job
    @param end
        GPS end time of job
    @param mode
        summary mode for this job
    """
    startd = datetime.datetime(*lal.GPSToUTC(int(start))[:6])
    endd   = datetime.datetime(*lal.GPSToUTC(int(end))[:6])
    if mode == summary.SUMMARY_MODE_GPS:
         d = int(end-start)
         if d <= 601:
             dt = datetime.timedelta(minutes=1)
         elif d <= 7201:
             dt = datetime.timedelta(minutes=10)
         elif d <= 86401:
             dt = datetime.timedelta(hours=1)
         elif d <= 259201:
             dt = datetime.timedelta(hours=6)
         else:
             dt = datetime.timedelta(days=1)
    elif mode == summary.SUMMARY_MODE_DAY:
        dt = datetime.timedelta(hours=1)
    elif mode in [summary.SUMMARY_MODE_WEEK, summary.SUMMARY_MODE_MONTH]:
        dt = datetime.timedelta(days=1)
    elif mode == summary.SUMMARY_MODE_YEAR:
        dt = datetime.timedelta(months=1)
    else:
        raise ValueError("Invalid summary mode: %d." % mode)

    dlist = []
    while startd < endd:
        e = min(endd, startd+dt)
        dlist.append(float(lal.UTCToGPS(e.timetuple()) -
                           lal.UTCToGPS(startd.timetuple())))
        startd += dt
    return dlist


def parse_plot_section(cp, section, params):
    """Parses the given ConfigParser cp for entries relating to plotting
    in the given section.

    @param cp
        INI configuration file object
    @param section
        name of INI section to parse
    @param params
        dict of default plotting parameters to update

    @returns an updated dict of plotting keyword arguments
    """
    m = re.search("trigger", section, re.I) and "column" or "axis"
    column = params.get(m, "_").lower()
    xcolumn = params.get("x%s" % m, "_").lower()
    ycolumn = params.get("y%s" % m, "_").lower()
    colorcolumn = params.get("color%s" % m, "_").lower()
    colnames = [column, xcolumn, ycolumn, colorcolumn]
    columns = ["x", "x", "y", "color"]

    # nonplot parameters
    nonplot = ["segment-length", "overlap", "time-step", "response",\
               "threshold"]
    noncolumn = ["linewidth", "color", "linecolor"]

    # add channel specific params
    args = [(key.split('-',1),val) for key,val in cp.items(section)\
                        if key.split('-')[0].lower() in colnames]
    cp.add_section('tmp')
    for key,val in args:
        if key[1] in nonplot:
            continue
        if key[0].lower() in colnames:
            idx = colnames.index(key[0].lower())
            col = columns[idx]
            if key[1] == 'log':
                key[1] = '%s%s' % (key[1], col)
            elif key[1] not in noncolumn:
                key[1] = '%s%s' % (col, key[1])
        cp.set('tmp', key[1], val)
        if key == 'labels':
            cp.set('tmp', key[1], '%s_%s' % (section, val))
    params.update(plotutils.parse_plot_config(cp, 'tmp'))
    cp.remove_section('tmp')

    if not re.search("trigger", section, re.I):
        params.pop("xaxis", None)
        params.pop("yaxis", None)
        params.pop("coloraxis", None)

    return params


# job configuration
_state_order = ["all", "locked", "science"]
def load_run_states(start_time, end_time, cp, sec="states"):
    """@returns a list of SummaryStates parsed from the configuration.

    @param start
        GPS start time of job
    @param end
        GPS end time of job
    @param cp
        INI configuration file object
    @param sec
        name of section hosting state definitions, default: "states"
    """
    # read the states from the configuration
    states = natsort(cp.items(sec), key=lambda x: x[0])
    stateidx = list(range(len(states)))
    statelist = []
    # format each state into the correct object, in the correct order
    for i,(name,definition) in enumerate(states):
        s = summary.SummaryState(name)
        s.span = start_time, end_time
        s.definition = definition
        statelist.append(s)
        for j,_state in enumerate(_state_order):
           if s.match(_state):
              stateidx[i] = -len(_state_order)+j
    statelist = zip(*sorted(zip(statelist, stateidx), key=lambda x: x[1]))[0]
    return statelist


def parse_tab_states(statelist, cp, sec):
    """@returns a list of SummaryStates configured for this tab

    @param statelist
        global list of SummaryStates
    @param cp
        INI configuration file object
    @param sec
        section name for this tab
    """
    options = cp.options(sec)
    runstates = [state[4:-5] for state in options if re_state.match(state)]
    if len(runstates) == 0:
        runstates = ["all"]
    tabstatelist = list()
    for i,runstate in enumerate(runstates):
        found = False
        for state in statelist:
            if state.match(re_cchar.sub("_", runstate).lower()):
                tabstatelist.append(state)
                found = True
                break
        if not found:
            raise ValueError("Run state '%s' not found in configuration"
                             % runstate)

    return tabstatelist


def find_tab_parent(cp, sec, valid_parents, default=None):
    """@returns the name of the parent tab parsed from the config

    @param cp
        INI configuration file object
    @param sec
        section name for this tab
    @param valid_parents
        list of valid parent names
    @param default
        default parent if none configured
    """
    if cp.has_option(sec, "parent"):
        parent = cp.get(sec, "parent")
    elif re.match("data-\w\w\w-", sec):
        match = re.match("data-\w\w\w-", sec)
        parent = match.group()[5:8]
    else:
        parent = default
    if parent not in valid_parents:
        raise ValueError("\"%s\" is an unrecognised parent name." % parent)
    return parent


def build_html_base(cp, ifo, directory, gpspair=None):
    """@returns the url for the HTML <base> href attribute.

    @param cp
         configuration file object
    @param ifo
        prefix of interferometer
    @param directory
        path to output directory for job
    @param gpspair
        "${GPSSTART}-${GSPEND}" segment for job
    """
    root = htmlutils.get_web_server()
    if cp.has_option('html', '%s-base' % ifo.lower()):
        base = cp.get('html', '%s-base' % ifo.lower()).rstrip('/')
    else:
        directory = os.path.abspath(directory)
        user = getpass.getuser()
        if not root:
            base = 'file://%s' % directory
        else:
            if root and re.search('atlas', root):
                public_folder = "WWW"
            else:
                public_folder = "public_html"
            if re.search(public_folder, directory):
                base = os.path.join(root,'~%s' % user,\
                                    directory.split(public_folder,1)[-1][1:],
                                    '')
            else:
                base = "file://%s" % directory
    # strip the domain url from the base (makes pages more dynamic)
    if base.startswith(root):
        base = base[len(root):]
    base = os.path.normpath(base)
    # appent the GSP pair if relevant
    if (not gpspair or base.endswith(gpspair)):
        return os.path.join(base, "")
    else:
        return os.path.join(base, gpspair, "")


def parse_flag_name(flag):
    """@returns the (ifo, name, version) tuple parsed for this flag
    """
    name = flag
    if re.match("\w\d:", name):
        ifo,name = name.split(":", 1)
    else:
        ifo = None
    if re.search(":\d+\Z", name):
        name, version = name.rsplit(":", 1)
        version = int(version)
    else:
        version = None
    name = re_cchar.sub("_", name.upper()).replace("_", "-", 1)
    return ifo, name, version


def PyLALAddVectorSequence(*args):
    """@returns a new VectorSequence from the inputs

    All arguments should be lal.XXXVectorSequence objects.
    """
    datatype = seriesutils.typecode(type(args[0]))
    TYPESTR  = seriesutils._typestr[datatype]
    data = numpy.concatenate([arg.data for arg in args], axis=0)
    CreateVectorSequence = getattr(lal, "Create%sVectorSequence" % TYPESTR)
    out = CreateVectorSequence(*data.shape)
    out.data = data
    return out


def find_contiguous_frames(cache, on_missing="warn"):
    """@returns a list of Caches representing contiguous chunks of
    available frame data

    @param cache
        glue.lal.Cache of frames to check
    @param on_missing
        what to do when we find missing segments
    """
    cachelist = []
    cache.sort(key=lambda e: e.segment[0])
    cache,m = cache.checkfilesexist(on_missing=on_missing)

    tmp = type(cache)()
    p = None
    for e in cache:
        if type(p) == type(None):
            tmp.append(e)
        elif e.segment[0] == p.segment[1]:
            tmp.append(e)
        elif len(tmp):
            tmp.sort(key=lambda e: e.segment[0])
            cachelist.append(tmp)
            tmp = type(cache)()
        p = e
    if len(tmp):
        cachelist.append(tmp)
    return cachelist


def zpkresponse(zeros, poles, gain, f0=0, deltaF=1, length=100):
    """@returns the absolute frequency response to the given ZPK filter
    parameters over frequency.

    @param zeros
        list of zeros for filter
    @param poles
        list of poles for filter
    @param gain
        single gain for filter
    @param f0
        minimum frequency
    @param deltaF
        frequency step
    @param length
        number of frequency points to return
    """
    f = (numpy.arange(length)*deltaF + f0)
    if not zeros or poles:
        return numpy.ones_like(f) * gain
    else:
        lti   = signal.lti(numpy.asarray(zeros), numpy.asarray(poles), gain)
        try:
            fresp = map(lambda w: numpy.polyval(lti.num, w*1j)/\
                                  numpy.polyval(lti.den, w*1j), f)
        except TypeError:
            fresp = map(lambda w: numpy.polyval(lti.num, w*1j)/\
                                  numpy.polyval([lti.den], w*1j), f)
        fresp = numpy.asarray(fresp)
    return abs(fresp)


def PyLALConcatenateTimeSeries(*serieslist, **kwargs):
    """@returns a new TimeSeries by contatenating the inputs.

    @param serieslist
        all TimeSeries objects to concatenate
    """
    if len(serieslist) == 0:
        name = kwargs.pop("name", None)
        epoch = kwargs.pop("epoch", lal.GPSTimeNow())
        return lal.CreateREAL4TimeSeries(name, epoch, 0, 1,\
                                         lal.lalStrainUnit, 0)
    if len(serieslist) == 1:
        return serieslist[0]
    fillwith = kwargs.pop("fillwith", 0.0)

    # sort
    serieslist = sorted(serieslist, key=lambda series: int(series.epoch))

    # get attributes
    epoch = serieslist[0].epoch
    deltaT = serieslist[0].deltaT
    f0 = serieslist[0].f0
    name = serieslist[0].name
    sampleUnits = serieslist[0].sampleUnits
    # get end time
    duration = float(serieslist[-1].epoch + serieslist[-1].data.length * deltaT\
                     - epoch)

    # make series
    datatype = seriesutils.typecode(type(serieslist[0]))
    TYPESTR = seriesutils._typestr[datatype]
    CreateTimeSeries = getattr(lal, "Create%sTimeSeries" % TYPESTR)
    series = CreateTimeSeries(name, epoch, f0, deltaT, sampleUnits,\
                              int(duration/deltaT))
    series.data.data = (numpy.ones(series.data.length)*fillwith)\
                           .astype(series.data.data.dtype.type)

    # loop over data appending in place
    tarray = numpy.arange(series.data.length) * deltaT + float(epoch)

    idx = 0
    idx2 = 0
    for timeseries in serieslist:
       tepoch = float(epoch) + (float(timeseries.epoch-epoch)//deltaT*deltaT)
       idx  = numpy.nonzero(tarray==tepoch)[0][0]
       idx2 = idx + timeseries.data.length
       try:
           series.data.data[idx:idx2] = timeseries.data.data
       except ValueError as e:
           warnings.warn("Caught ValueError: %s" % e)
           break
       idx = idx2
    return series

def parse_data_config(cp, cpsec, channels, prefix=""):
    """
    Parse the given configparser.configparser section cpsec for data
    processing options relating to the given list of channels.

    @param cp
         configuration file object
    @param cpsec
        name of INI section configuring this Tab
    @param channels: list of channel names read for the DataSummaryTab.
    @param prefix: prefix string for data processing options

    @return: list of parameter dicts for each channel
    @rtype: C{list} of C{dict}s
    """

    if prefix:
        prefix += "-"

    numchannels = len(channels)
    out = [dict() for i in range(numchannels)]

    # load frametypes
    if cp.has_option(cpsec, "%sframe-type" % prefix):
        frametype = cp.get(cpsec, "%sframe-type" % prefix).split(',')
    else:
        frametype = [None]
    if len(frametype) == 1:
        frametype = [frametype[0]] * numchannels
    if len(frametype) != numchannels:
        raise ValueError("Wrong number of frame-types in [%s]." % cpsec)
    for i in range(numchannels):
        out[i]["frametype"] = frametype[i]

    # load time calibration
    if cp.has_option(cpsec, "%scalibration" % prefix):
        calibration = map(eval,\
                          cp.get(cpsec, "%scalibration" % prefix).split(','))
    else:
        calibration = [None]
    if len(calibration) == 1:
        calibration = [calibration[0]] * numchannels
    if len(calibration) != numchannels:
        raise ValueError("Wrong number of calibration functions in [%s]."\
                         % cpsec)
    for i in range(numchannels):
        out[i]["calibration"] = calibration[i]

    # load time freqresponse
    if cp.has_option(cpsec, "%sfrequency-response" % prefix):
        freqresponse = map(eval, cp.get(cpsec,\
                                    "%sfrequency-response" % prefix).split(';'))
    else:
        freqresponse = [None]
    if len(freqresponse) == 1:
        freqresponse = [freqresponse[0]] * numchannels
    if len(freqresponse) != numchannels:
        raise ValueError("Wrong number of frequency response functions in"+\
                         " [%s]." % cpsec)
    for i in range(numchannels):
        out[i]["frequency-response"] = freqresponse[i]

    # load sampling
    if cp.has_option(cpsec, "%sresample" % prefix):
        resample = map(float, cp.get(cpsec, "%sresample" % prefix).split(','))
    else:
        resample = [0]
    if len(resample) == 1:
        resample = [resample[0]] * numchannels
    if len(resample) != numchannels:
        raise ValueError("Wrong number of frame-types in [%s]." % cpsec)
    for i in range(numchannels):
        out[i]["resample"] = resample[i]

    # load labels
    if cp.has_option(cpsec, "labels"):
        labels = cp.get(cpsec, "labels").split(',')
    elif cp.has_option(cpsec, "label"):
        labels = cp.get(cpsec, "label").split(",")
    else:
        labels = channels
    for i in range(numchannels):
        out[i]["label"] = labels[i]

    # load spectr(um/ogram) params
    for p in ["time-step", "segment-length", "segment-overlap"]:
        if numchannels == 1 and cp.has_section(channels[0]) and \
                cp.has_option(channels[0], p):
            x = cp.getfloat(channels[0], p)
        elif cp.has_option(cpsec, "spectrum-%s" % p):
            x = cp.getfloat(cpsec, "spectrum-%s" % p)
        elif cp.has_option("spectrum", p):
            x = cp.getfloat("spectrum", p)
        else:
            continue
        for i in range(numchannels):
            out[i][p] = x

    return out
#
# =============================================================================
#
# data-acquisition functions
#
# =============================================================================
#

def get_segments(cp, flag, segment_summary, cache=None, buffer_=0, query=True):
    """@returns the SegmentList for the given flag.

    @param cp
         configuration file object
    @param flag
         name of data-quality flag to load
    @param segment_summary
         SegmentList of summary segments in which to restrict output
    @param cache
         glue.lal.Cache of files from which to read
    @param buffer_
         amount by which to contract segments
    @param query
         load new segments from SegDB or cache (True) or load only
         from existing global memory (False), default: True
    """
    global G_SEGMENTS, G_SEGMENT_SUMMARY

    if isinstance(flag, basestring):
        flags = flag.split(',')
    else:
        flags = flag

    query_summary = (segment_summary -
                     reduce(operator.and_,
                            [G_SEGMENT_SUMMARY.get(f, SegmentList())
                             for f in flags]))
    # FIXME: remove when SegmentLists play with lal.LIGOTimeGPS
    query_summary = SegmentList([Segment(map(float, seg)) for
                                   seg in query_summary])
    query &= (abs(query_summary) != 0)
    # restrict query to NOW
    if query:
        query_summary &= SegmentList([Segment(query_summary[0][0],
                                              float(lal.GPSTimeNow()))])
    new_segments = []
    new_summary = []
    if query:
        # get cache
        if cache is not None:
            segcache = cache
        elif cp.has_option("segfind", "segment-cache"):
            segcache = cp.get("segfind", "segment-cache")
        else:
           segcache = None

        # get database
        if not segcache:
            segment_url = cp.get("segfind", "segment-url")

        # load segments from cache
        if segcache is not None:
            for f in flags:
                desc = re_cchar.sub("_", parse_f(name(f)[1]))
                segcache = segcache.sieve(segmentlist=query_summary,
                                          description=desc)
                new_segments.append(read_segment_cache(segcache) &
                                    query_summary)
                print_verbose("Loaded %d new segments from cache for %s.\n"
                              % (len(new_segments[-1]), f))
        # load segments from the segment database
        else:
            qstart, qend = query_summary.extent()
            try: # FIXME: consider removing
                new_segments, new_summary = dqSegmentUtils.grab_segments(
                                               qstart, qend, flags, segment_url,
                                               segment_summary=True)
            except LDBDClientException as e:
                print_verbose("WARNING: Caught LDBDClientException: %s\n" % e,
                              stream=sys.stderr)
                print_verbose("No segments have been loaded, but processing "
                              "will continue.\n", stream=sys.stderr)
                new_segments = new_summary = [SegmentList()]*len(flags)
            for i,f in enumerate(flags):
                print_verbose("Loaded %d new segments from %s for %s.\n"
                              % (len(new_segments[i]), segment_url, f),
                              verbose=query or abs(new_summary[i]))
    else:
        new_segments = [SegmentList()]*len(flags)
        new_summary = [G_SEGMENT_SUMMARY.get(f, SegmentList()) for f in flags]


    # apply buffer
    if buffer_:
        new_segments = [sl.contract(buffer_) for sl in new_segments]

    # record these segments
    for f,new_segs,new_summ in zip(flags, new_segments, new_summary):
        if G_SEGMENTS.has_key(f):
            G_SEGMENT_SUMMARY[f] += new_summ
            G_SEGMENTS[f] += new_segs
        else:
            G_SEGMENT_SUMMARY[f] = new_summ
            G_SEGMENTS[f]  = new_segs
        G_SEGMENT_SUMMARY[f].coalesce()
        G_SEGMENTS[f].coalesce()

    out = []
    for f in flags:
        # return the required segments
        out.append(segment_summary & G_SEGMENTS[f])
        if len(out) and f != 'G1:HVETO:1':
            minlength = (cp.has_option("segfind", "minimum-segment-length") and
                         cp.getfloat("segfind", "minimum-segment-length") or 0)
            out[-1] = type(out[-1])([seg for seg in out[-1] if
                                 float(abs(seg)) >= minlength])
    if isinstance(flag, basestring): 
        return out[0]
    else:
        return out


def get_state_segments(cp, state, veto_def_table, mode=0):
    """Load the Segments that define the given SummaryState

    @param cp
        INI configuration file object
    @param state
        IFO state object to modify
    @param veto_def_table
        LIGOLw table holding veto definitions

    No return.
    """
    print_verbose("\n----------\nLoading segments for the '%s' run state...\n"\
                  % state.name, verbose=not state.set)
    seg_sum_list = SegmentList([state.span])

    # if state has a definition, get it's segments
    if state.definition:
        # find flags
        div = re.compile("[&!]")
        divs = div.findall(state.definition)
        keys = div.split(state.definition)
        # load flags and vetoes
        flags = []
        vetoes = []
        for i,key in enumerate(keys):
            # get veto bit
            append = (key.startswith("-") or (i!=0 and divs[i-1] == "!") ) \
                and vetoes.append or flags.append
            if key.lower() == "vetoes":
                if veto_def_table:
                    for v in veto_def_table:
                        flag = "%s:%s:%d" % (v.ifo, v.name, v.version)
                        append(flag)
            elif re.match("cat/d", key, re.I): 
                if veto_def_table:
                    cat = int(key[4])
                    for v in veto_def_table:
                        if v.category == cat:
                            flag = "%s:%s:%d" % (v.ifo, v.name, v.version)
                            append(flag)
            elif re.match("ccat\d", key, re.I) :
                if veto_def_table:
                    cat = int(key[4])
                    for v in veto_def_table:
                        if v.category <= cat:
                            flag = "%s:%s:%d" % (v.ifo, v.name, v.version)
                            append(flag)
            else:
                append(key)
        # read new segments
        newsegs = seg_sum_list
        for flag in vetoes:
            # FIXME, hey! padding from the veto definer file is missing?!
            if not state.set:
                load_archive_segments(cp, flag, seg_sum_list, mode=mode)
            newsegs -= get_segments(cp, flag, seg_sum_list)
        for flag in flags:
            if not state.set:
                load_archive_segments(cp, flag, seg_sum_list, mode=mode)
            newsegs &= get_segments(cp, flag, seg_sum_list)
        state.segments = newsegs
    else:
        # otherwise use all time
        state.segments = seg_sum_list
    # set state-specific variables restrictions
    cpsec = "state-%s" % (state.name)
    if cp.has_section(cpsec):
        for param in ["segment-buffer", "event-buffer",
                      "minimum-segment-length"]:
            if cp.has_option(cpsec, param):
                setattr(state, param.replace("-", "_"),
                        cp.getfloat(cpsec, param))
    # apply minimum segment length
    state.segments.contract(state.segment_buffer)
    if state.minimum_segment_length:
        state.segments = SegmentList([
                             s for s in state.segments if
                             abs(s) >= state.minimum_segment_length])
    state.set = True
    return


def get_triggers(cp, channel, etg, segment_list, cluster=False, minsnr=0,
                 query=True, **trigfind_args):
    """@returns a LIGOLw Table of triggers for the given channel

    @param cp
         configuration file object
    @param channel
        name of channel to read
    @param etg
        trigger generator to read
    @param segment_list
        list of segments to read
    @param cluster
        apply clustering to triggers after they are read
    @param minsnr
        minimum SNR threshold
    @param query
        search for new triggers (True), or find in memory (False)
    """
    global G_TRIGGERS, G_CLUSTERED_TRIGGERS
    etg = etg.lower()
    # add keys to global memory holder
    if not G_TRIGGERS.has_key(etg):
        G_TRIGGERS[etg] = dict()
    if not G_TRIGGERS[etg].has_key(channel):
        G_TRIGGERS[etg][channel] = dict()
    if not G_CLUSTERED_TRIGGERS.has_key(etg):
        G_CLUSTERED_TRIGGERS[etg] = dict()
    if not G_CLUSTERED_TRIGGERS[etg].has_key(channel):
        G_CLUSTERED_TRIGGERS[etg][channel] = dict()

    # get job latency
    etgsec = re_cchar.sub("-", etg.lower())
    if cp.has_option(etgsec, "latency"):
        latency = cp.getfloat(etgsec, "latency")
    else:
        latency = 0

    # get columns
    columns = (cp.has_option(etgsec, "table-columns") and
               cp.get(etgsec, "table-columns").split(",") or None)
    if columns:
        required = ["process_id", "search", "channel"]
        for col in required:
            if col not in columns:
                columns.append(col)

    # construct query segments
    span = SegmentList([Segment(0, NOW-latency)])
    if cluster:
        cachedSegments = SegmentList(G_CLUSTERED_TRIGGERS[etg][channel].keys()).coalesce()
    else:
        cachedSegments = SegmentList(G_TRIGGERS[etg][channel].keys()).coalesce()
    query_segments = ((segment_list & span) - cachedSegments)
    query &= abs(query_segments) != 0

    if query:
        span = query_segments.extent()
        # get cache
        if (not cp.has_option(etg.lower(), "skip-cache-file") and
                cp.has_option("trigfind", "trigger-cache")):
            trigcache = cp.get("trigfind", "trigger-cache", raw=True)
        else:
            trigcache = None

        # get parameters
        ifo = channel.split(":",1)[0]
        etgsec = re_cchar.sub("-", etg.lower())

        trigtable = triggers.utils.new_ligolw_table(etg, columns=columns)

        # find trigger files
        if trigcache is not None:
            desc  = ("%s*%s"
                     % (re_cchar.sub("*", channel.split(':',1)[1].upper()),
                        etg))
            trigcache = trigcache.sieve(description=desc)
        elif etg.lower() != "hacr":
            if etg.lower() in ['dmtomega', 'excesspower', 'omicron',
                               'kw', 'kleinewelle']:
                trigcache = trigfind.find_trigger_urls(channel, etg,
                                                       span[0], span[1],
                                                       **trigfind_args)
            elif etg.lower() in ['omega', 'omegaspectrum']:
                # get omega online cache parameters defaults
                cachekwargs = dict()
                if cp.has_option(etg.lower(), 'duration'):
                    cachekwargs['duration'] = float(cp.get(etg.lower(),
                                                           'duration'))
                if cp.has_option(etg.lower(),'overlap'):
                    cachekwargs['overlap'] = float(cp.get(etg.lower(),
                                                          'overlap'))

                # overrid defaulte settings with channel specific if provided
                if cp.has_section('%s' %  channel):
                    if cp.has_option(channel,'duration'):
                        cachekwargs['duration'] = float(cp.get(channel,
                                                               'duration'))
                    if cp.has_option(channel,'overlap'):
                        cachekwargs['overlap'] = float(cp.get(channel,
                                                              'overlap'))
              
                # get cache
                if etg.lower() == "omega":
                    trigcache = omegautils.get_cache(span[0], span[1], ifo,
                                                     channel, **cachekwargs)
                elif etg.lower() == "omegaspectrum":
                    trigcache = omegautils.get_cache(span[0], span[1], ifo,
                                                     channel, mask='SPECTRUM',
                                                     **cachekwargs)
            elif etg.lower() in ["daily_ihope", "dailyihope", "ihopedaily",
                                 "ihope_daily"]:
                dicluster = (cp.has_option("daily_ihope", "clustering") and
                             cp.get("daily_ihope", "clustering") or None)
                trigcache = daily_ihope.find_daily_cache(
                                span[0], span[1], ifo, clustering=dicluster,
                                **trigfind_args)
            else:
                raise NotImplementedError("A default file finder for %s has"
                                          "not been written yet." % etg)

        n = 0
        nc = 0
        # load HACR triggers from database
        if not trigcache and etg.lower() == "hacr":
            host = cp.get("hacr", "host")
            # query once for the span
            trigtable = hacrutils.get_triggers(span[0], span[1], channel, host,
                                               columns=columns)
            if cluster:
                trigtable = cluster_table(trigtable, cluster)
                add_clustered_triggers(trigtable, channel, etg, span)
                nc += len(trigtable)
            else:
                add_triggers(trigtable, channel, etg, span)
                n += len(trigtable)
        # load files segment by segment in an attempt to speed up loading
        elif trigcache:
            trigcache,_ = trigcache.checkfilesexist(on_missing="warn")
            for seg in query_segments:
                segcache = trigcache.sieve(segment=seg)
                if len(segcache) == 0:
                    continue
                if etg.lower() == "omega" or etg.lower() == "omegaspectrum":
                    trigtable = omegautils.fromlalcache(segcache, start=seg[0],
                                                        end=seg[1],channel=channel,
                                                        columns=columns,
                                                        verbose=VERBOSE)
                elif etg.lower() in ['excesspower', 'daily_ihope', 'omicron']:
                    trigtable = triggers.from_files(segcache, etg,
                                                    columns=columns,
                                                    start=seg[0], end=seg[1],
                                                    verbose=VERBOSE)
                else:
                    trigtable = triggers.from_files(
                                    segcache, etg, columns=columns,
                                    channel=channel, start=seg[0], end=seg[1],
                                    verbose=VERBOSE)
                if etg.lower() == "excesspower":
                    for i,t in enumerate(trigtable):
                        if t.snr > 0:
                            trigtable[i].snr = t.snr**(1/2)
                        t.peak_frequency = t.central_freq
                        t.set_ms_period(t.get_period())
                        t.set_ms_band(t.get_band())
                        t.ms_hrss = t.amplitude
                        t.ms_snr = t.snr
                        t.ms_confidence = t.confidence
                if cluster:
                    trigtable = cluster_table(trigtable, cluster)
                    add_clustered_triggers(trigtable, channel, etg, seg)
                    nc += len(trigtable)
                else:
                    add_triggers(trigtable, channel, etg, seg)
                    n += len(trigtable)

        if cluster:
            print_verbose("Loaded and clustered %d new triggers for %s.\n"
                          % (nc, channel))
        else:
            print_verbose("Loaded %d new triggers for %s.\n" % (n, channel))
    else:
        trigtable = triggers.utils.new_ligolw_table(etg, columns=columns)


    # FIXME: convert segments to floats
    segment_list = SegmentList([Segment(map(float, seg))\
                                    for seg in segment_list])

    # return the required trigger files
    trigtable = table.new_from_template(trigtable)
    if re.search("burst", trigtable.tableName):
        get_time = lambda t: t.get_peak()
    else:
        get_time = lambda t: t.get_end()
    if cluster:
        for seg in sorted(G_CLUSTERED_TRIGGERS[etg][channel].keys()):
            if not segment_list.intersects_segment(seg):
                continue
            trigtable.extend(t for t in G_CLUSTERED_TRIGGERS[etg][channel][seg]
                             if float(t.snr)>=minsnr and
                             float(get_time(t) in segment_list))
    else:
        for seg in sorted(G_TRIGGERS[etg][channel].keys()):
            if not segment_list.intersects_segment(seg):
                continue
            trigtable.extend(t for t in G_TRIGGERS[etg][channel][seg] if
                             float(t.snr)>=minsnr and
                             float(get_time(t) in segment_list))
    return trigtable


# get multiple channels from a single cache
def get_multi_triggers(cp, channels, etg, segment_list, cluster=False,
                       minsnr=0, query=True, **trigfind_args):
    """@returns an ordered list of LIGOLw Tables of triggers for each
    entry in the channels list.

    @param cp
         configuration file object
    @param channel
        name of channel to read
    @param etg
        trigger generator to read
    @param segment_list
        list of segments to read
    @param cluster
        apply clustering to triggers after they are read
    @param minsnr
        minimum SNR threshold
    @param query
        search for new triggers (True), or find in memory (False)

    """
    multi_enabled = ["kw", "kleinewelle"]
    etg = etg.lower()
    global_segments = SegmentList([])
    for channel in channels:
        try:
            if cluster:
                global_segments.extend(SegmentList(
                                     G_CLUSTERED_TRIGGERS[etg][channel].keys()))
            else:
                global_segments.extend(SegmentList(
                                           G_TRIGGERS[etg][channel].keys()))
        except KeyError:
            pass
    global_segments.sort()
    global_segments.coalesce()

    # get job latency
    etgsec = re_cchar.sub("-", etg.lower())
    if cp.has_option(etgsec, "latency"):
        latency = cp.getfloat(etgsec, "latency")
    else:
        latency = 0
    span = SegmentList([Segment(0, NOW-latency)])
    query_segments = ((segment_list & span) - global_segments).coalesce()

    if query and abs(query_segments) > 0 and etg.lower() in multi_enabled:
        ifo = channels[0][:2]
        span = query_segments.extent()
        # get columns
        columns = (cp.has_option(etgsec, "table-columns") and
                   cp.get(etgsec, "table-columns").split(",") or None)
        if columns:
            required = ["process_id", "search", "channel"]
            for col in required:
                if col not in columns:
                    columns.append(col)
        # get cache
        if (not cp.has_option(etg.lower(), "skip-cache-file") and
               cp.has_option("trigfind", "trigger-cache")):
            trigcache = cp.get("trigfind", "trigger-cache", raw=True)
        else:
            trigcache = None

        n = dict((c,0) for c in channels)
        nc = dict((c,0) for c in channels)
        # read mulit_channel KW files
        if not trigcache:
            if etg.lower() in ["kw", "kleinewelle"]:
                trigcache = kw.find_dmt_cache(span[0], span[1], ifo,
                                              check_files=True, **trigfind_args)
        for seg in query_segments:
            if etg.lower() in ["kw", "kleinewelle"]:
                segcache = trigcache.sieve(segment=seg)
                multi_trigs = triggers.from_files(segcache, etg, start=seg[0],
                                                  end=seg[1], channel=channels,
                                                  columns=columns,
                                                  verbose=VERBOSE)
            # catalogue new triggers for each channel
            for channel in channels:
                trigtable = multi_trigs[channel]
                if cluster:
                    trigtable = cluster_table(trigtable, cluster)
                    add_clustered_triggers(trigtable, channel, etg, seg)
                    nc[channel] += len(trigtable)
                else:
                    add_triggers(trigtable, channel, etg, seg)
                    n[channel] += len(trigtable)
        for channel in channels:
            if cluster:
                print_verbose("Loaded and clustered %d new triggers for "
                              "%s.\n" % (nc[channel], channel))
            else:
                print_verbose("Loaded %d new triggers for %s.\n"
                              % (n[channel], channel))
        query = False

    out = []
    for channel in channels:
        out.append(get_triggers(cp, channel, etg, segment_list, cluster=cluster,
                                minsnr=minsnr, query=query, **trigfind_args))

    return out


def get_data(cp, channel, segment_list, frametype=None, calibration=None,
             resample=False, usends=False, superchannel=None,
             query=True, store=True):
    """@returns a list of TimeSeries containing data for a channel

    Data can be acquired from either frames OR NDS

    @param cp
         configuration file object
    @param channel
        name of channel to read
    @param segment_list
        list of segments to read
    @param frametype
        type of frame set (as used by GWDataFind)
    @param calibration
        function to apply to data for calibration
    @param resample
        sampling frequency at which to downsample the raw data
    @param usends
        read data from NDS, rather than frames, default: False
    @param superchannel
        name of conglomerate data channel from which to parse GEO
        control channel data
    @param query
        search for new triggers (True), or find in memory (False)
    @param store
        store data in memory (True) or return without storing (False)
    """
    global G_DATA
    if not G_DATA.has_key(channel):
        G_DATA[channel] = {}
    if len(segment_list) == 0:
        return []
    cached_segments = SegmentList(sorted(G_DATA[channel].keys())).coalesce()
    query_segments = segment_list - cached_segments
    query_segments.coalesce()
    query &= float(abs(query_segments)) != 0

    # get cache
    if cp.has_option("datafind", "data-cache"):
        datacache = cp.get("datafind", "data-cache", raw=True)
    else:
        datacache = None

    # get NDS info
    if usends:
        ndsserver = cp.get("datafind", "nds-server")
        ndsport = cp.getint("datafind", "nds-port")

    # get data type
    if inspect.stack()[1][3] == "process_state_vector":
        datatype = -1
    else:
        datatype = lal.LAL_D_TYPE_CODE

    if query:
        # sieve for the relevant frame type
        if not usends and datacache is not None:
            datacache = datacache.sieve(ifos=channel[0], description=frametype,
                                        exact_match=True)
        # or query the GWDataFindServer
        elif not usends:
            if cp.has_option("datafind", "datafind-server"):
                server, port = cp.get("datafind", "datafind-server").split(":")
                if not port:
                    port = None
                connection = datafind.GWDataFindHTTPConnection(
                                 host=server, port=port)
            else:
                connection = datafind.GWDataFindHTTPConnection()
            # KLUDGE for GEO trend files, 1 frame type,
            # but three different frames with different data
            if (channel[0] == "G" and
                (frametype[0] == "T" or frametype[0] == "M")):
                tmpcache = connection.find_frame_urls(channel[0], frametype[0],
                                                       span[0], span[1],
                                                       urltype="file",
                                                       on_gaps="warn")
                datacache = []
                for cacheentry in tmpcache:
                    if (len(frametype) == 1 and
                        re.search("framedata",cacheentry.url)):
                        datacache.append(cacheentry.url)
                    if (re.search("RDS3",frametype) and
                        re.search("RDS3",cacheentry.url)):
                        datacache.append(cacheentry.url)
                    if (re.search("RDS9",frametype) and
                        re.search("RDS9",cacheentry.url)):
                        datacache.append(cacheentry.url)
                # convert to cache list format
                datacache = Cache.from_urls(datacache)
            # No kludge if not GEO trend files
            else:
                datacache = connection.find_frame_urls(channel[0], frametype,
                                                   span[0], span[1],
                                                   urltype="file",
                                                   on_gaps="warn")
            connection.close()

        if datacache:
            if len(datacache) == 0:
                print_verbose("warning: no frames found for observatory %s and "
                              "frametype %s.\n" % (channel[0], frametype),
                              stream=sys.stderr, verbose=True)
            data_segments = SegmentList()
            for i,e in enumerate(datacache):
                datacache[i].segment = Segment(map(float, e.segment))
                data_segments.append(datacache[i].segment)
            data_segments.coalesce()
            query_segments &= data_segments

        # loop over segments
        for i,seg in enumerate(query_segments):
            if not usends and frametype == ('T', 'H1_T') and abs(seg) < 1:
                continue
            elif not usends and frametype in ('M', 'H1_M') and abs(seg) < 60:
                continue
            # break data up into manageable chunks if using resampling.
            s = seg[0]
            if resample:
                dt = 7200
            else:
                dt = numpy.inf
            while s < seg[1]:
                e = s + dt
                seg2 = Segment(float(s), float(min(seg[1], e))) # FIXME

                # get data from NDS
                if usends:
                    if superchannel:
                        sseries = seriesutils.fromNDS(superchannel, seg2[0],
                                                      abs(seg),
                                                      server=ndserver,
                                                      port=ndsport)
                        series = geoutils.fromsuperchannel(sdata, channel,
                                                           superchannel)
                    else:
                        series = seriesutils.fromNDS(channel, seg2[0], abs(seg),
                                                     server=ndserver,
                                                     port=ndsport)
                    if resample:
                        sampling = 1/series.deltaT
                        seriesutils.resample(series, resample)
                    if calibration:
                        series.data.data = calibration(series.data.data)
                    add_timeseries(series)

                # parse cache for this segment
                if not usends:
                    segcache = datacache.sieve(segment=seg2)
                    cachelist = find_contiguous_frames(segcache)
                    if len(cachelist) == 0:
                        s += dt
                        continue
                    elif len(cachelist) > 1:
                        print_verbose("WARNING: there are gaps in the frames, "
                                      "%d contiguous data blocks found.\n"
                                      % len(cachelist))
                    for cache in cachelist:
                        frstart = min(e.segment[0] for e in cache)
                        if frstart < seg2[0]:
                            frstart = seg2[0]
                        frend = max(e.segment[1] for e in cache)
                        if frend > seg2[1]:
                            frend = seg2[1]
                        frdur = frend - frstart
                        if superchannel:
                            sseries = seriesutils.fromlalcache(
                                          cache, superchannel, frstart,
                                          frdur, datatype=datatype)
                            series = geoutils.fromsuperchannel(sdata, channel,
                                                               superchannel)
                        else:
                            series = seriesutils.fromlalcache(cache, channel,
                                                              frstart, frdur,
                                                              datatype=datatype)
                        # apply resampling
                        if resample:
                            sampling = 1/series.deltaT
                            seriesutils.resample(series, resample)
                        # apply calibration
                        if calibration:
                            series.data.data = calibration(series.data.data)
                        add_timeseries(series)
                    s += dt
            print_verbose("Data recovered from frames for [%d ... %d) (%d/%d)\n"
                          % (seg[0], seg[1], i+1, len(query_segments)))

    # grab the data we want
    globsegs = SegmentList(sorted(G_DATA[channel].keys())).coalesce()
    globsegs.sort()
    serieslist = []
    if len(globsegs):
        datatype = seriesutils.typecode(type(G_DATA[channel][globsegs[0]]))
        TYPESTR  = seriesutils._typestr[datatype]
        cut = getattr(lal, "Cut%sTimeSeries" % TYPESTR)
    else:
        return []

    for seg in (segment_list & globsegs):
        for gseg in globsegs:
            # if global segment is exactly what we want
            if gseg == seg:
                serieslist.append(G_DATA[channel][gseg])
                if not store:
                    del G_DATA[channel][gseg]
                break
            # or if our segment is a subsegment of the global
            elif gseg.intersects(seg):
                gseries = G_DATA[channel][gseg]
                t = (numpy.arange(gseries.data.length) * gseries.deltaT +
                     float(gseries.epoch))
                try:
                    idx0 = int((t >= seg[0]).nonzero()[0][0])
                except IndexError:
                    continue
                n = int((min(seg[1], gseg[1]) - t[idx0]) / gseries.deltaT)
                if n == gseries.data.length - 1:
                    serieslist.append(gseries)
                else:
                    serieslist.append(cut(gseries, idx0, n))
                if not store:
                    del G_DATA[channel][gseg]
                break

    return serieslist


def get_spectrum(cp, channel, *args, **kwargs):
    """@returns the median, 5th percentile, and 95th percentile
    frequency series spectra for the given channel.

    @param cp
         configuration file object
    @param channel
        name of channel to read
    @param step
        length (seconds) of single average spectrum in spectrogram
    @param seglen
        length (seconds) of segment in average spectrum calculation
    @param stride
        length (seconds) of overlap in average spectrum calculation
    @param seglist
        list of segments to read
    @param frametype
        type of frame set (as used by GWDataFind)
    @param calibration
        function to apply to data for calibration
    @param fresponse: function to apply to spectrogram as sensor
        frequency response
    @param resample
        sampling frequency at which to downsample the raw data
    @param usends
        read data from NDS, rather than frames, default: False
    @param query
        search for new spectrogram (True), or find in memory (False)
    """
    global G_SPECTRUM
    # KLUDGE: skipping spectrum caching, as sometimes different segment lists can be requested
    # if G_SPECTRUM.has_key(channel):
    #     return G_SPECTRUM[channel]

    # get spectrogram
    sequencelist = get_spectrogram(cp, channel, *args, **kwargs)

    # take epoch from seglist
    epoch  = lal.LIGOTimeGPS(len(sequencelist) and args[3][0][0] or NOW)

    # flatten spectrum
    if len(sequencelist):
        f0 = sequencelist[0]['f0']
        deltaF = sequencelist[0]['deltaF']
        f_array = sequencelist[0]['f_array']
        spectrogram = numpy.concatenate(([s['data'].data for s in sequencelist
                                          if s['data'].data.shape[0]]), axis=0)
    else:
        seglen = args[2]
        f0 = 0
        deltaF = 1
        spectrogram = numpy.zeros((1, seglen//2+1))
        f_array = numpy.zeros((1, seglen//2+1))[0]

    # take median
    out = []
    for p in [50, 5, 95]:
        array = numpy.array([stats.scoreatpercentile(spectrogram[:,i], p)
                             for i in range(spectrogram.shape[1])])
        # out.append(seriesutils.fromarray(array, name=channel, epoch=epoch,
        #                                  deltaT=deltaF, f0=f0,
        #                                  frequencyseries=True))
        A = seriesutils.fromarray(array, name=channel, epoch=epoch,
                                         deltaT=deltaF, f0=f0,
                                         frequencyseries=True)
        A.f_array = f_array
        out.append(A)

    G_SPECTRUM[channel] = out
    return out


def build_omega_spectrogram(trigs, start, end, time_step, spectype='snr'):

  """
    Construct the two-dimensional spectrogram from a table of OmegaSpectrum
    triggers
  """

  # get number of segments
  numseg = float(end-start)//time_step

  # set up data array
  spectrogram = None

  # loop over time steps and calculate spectrum
  if verbose:
    sys.stdout.write("Constructing %d segment spectrogram...    " % numseg)
    sys.stdout.flush()

  f = numpy.array([])
  S = numpy.array([]) 

  # Initialize iterators over spec triggers
  trigStart = 0
  trigStop = 0

  for i in range(numseg):

    # get data chunk
    # f,S = MedianOmegaSpectrum(trigs, start+i*time_step, start+(i+1)*time_step)

    # downselect triggers in time
    span = Segment(start+i*time_step, start+(i+1)*time_step)
    segtrigs = triggers.utils.new_ligolw_table('omegaspectrum')
    if re.search("burst", segtrigs.tableName):
        get_time = lambda t: t.get_peak()
    else:
        get_time = lambda t: t.get_end()
    while trigStop<len(trigs)\
          and get_time(trigs[trigStop]) < span[1]:
      trigStop = trigStop + 1
    while trigStart<len(trigs)\
          and get_time(trigs[trigStart]) < span[0]:
      trigStart = trigStart + 1
    segtrigs.extend(t for t in trigs[trigStart:trigStop])

    # get the median spectra out of downselected triggers
    if len(f) == 0:
      f = numpy.unique1d(segtrigs.get_column('peak_frequency'))
    S = numpy.ones(len(f))
    P = segtrigs.get_column(spectype)
    if len(P) != 0:
      for j,freq in enumerate(f):
        S[j] = numpy.median(P[f == freq])
      f,S = list(map(numpy.asarray, zip(*sorted(zip(f,S)))))

    # set up spectrogram array for first time
    if len(f) > 0 and (spectrogram is None or spectrogram.length == 0):
        func = getattr(lal, "CreateREAL8VectorSequence")                                                                                     
        spectrogram = func(int(numseg), int(len(f))) 
        # fill with zeros on spectrogram creation  
        for j in range(int(numseg)):
            spectrogram.data[j,:] = numpy.zeros(len(f)).astype(float)

    # append spectrum to spectrogram
    if len(S) > 0:
      spectrogram.data[i,:] = S.astype(float)
    if verbose:
      sys.stdout.write("\b\b\b%.2d%%" % int((i+1)/numseg*100))
      sys.stdout.flush()

  if verbose:
   sys.stdout.write("\n")
   sys.stdout.flush()

  if spectrogram is not None:
      spectrogram.epoch = lal.LIGOTimeGPS(start)
      spectrogram.deltaT = time_step
      spectrogram.f_array = f
      spectrogram.name = str(Segment(start, end))
      spectrogram.sampleUnits = lal.lalSecondUnit

  return spectrogram

def get_spectrogram(cp, channel, step, seglen, stride, segment_list,
                    frametype=None, calibration=None, fresponse=None,
                    resample=None, usends=False, superchannel=None,
                    query=True, spectype='snr'):
    """@returns a list of spectrograms for the given channel.

    @param cp
         configuration file object
    @param channel
        name of channel to read
    @param step
        length (seconds) of single average spectrum in spectrogram
    @param seglen
        length (seconds) of segment in average spectrum calculation
    @param stride
        length (seconds) of overlap in average spectrum calculation
    @param segment_list
        list of segments to read
    @param frametype
        type of frame set (as used by GWDataFind)
    @param calibration
        function to apply to data for calibration
    @param fresponse: function to apply to spectrogram as sensor
        frequency response
    @param resample
        sampling frequency at which to downsample the raw data
    @param usends
        read data from NDS, rather than frames, default: False
    @param query
        search for new spectrogram (True), or find in memory (False)
    @param spectype
        Type of spectra to use, e.g. spectrum variance, kurtosis... only used for omegaspectra for the moment
    """
    global G_SPECTROGRAM,G_DATA
    # shorten segment list to the correct size for the given step, 
    segment_list.coalesce()
    tmp = SegmentList()
    for seg in segment_list:
        testseg = Segment(math.ceil(float(seg[0])/step)*step, math.floor(float(seg[1])/step)*step)
        if abs(testseg) >= step:
            tmp.append(testseg)
    segment_list = tmp.coalesce()

    # check what segments we already have for this channel
    if not G_SPECTROGRAM.has_key(channel):
        G_SPECTROGRAM[channel] = {}
        segment_summary = SegmentList()
    else:
        segment_summary = SegmentList(sorted(G_SPECTROGRAM[channel].keys()))
        segment_summary.coalesce()
    # KLUDGE! killing completely spectrogram caching
    G_SPECTROGRAM[channel] = {}
    segment_summary = SegmentList()

    # get new segments
    query_segments = segment_list - segment_summary

    # check if there is anything new to query for
    query &= (abs(query_segments) != 0)

    # get cache
    if cp.has_option("datafind", "data-cache"):
        datacache = cp.get("datafind", "data-cache", raw=True)
    else:
        datacache = None

    # get NDS info
    if usends:
        ndsserver = cp.get("datafind", "nds-server")
        ndsport = cp.getint("datafind", "nds-port")

    # If frametype looks like an etg name, get spectra from the etg
    # files, otherwise read in data for frames and compute
    # spectra. Using kludge to detect if frametype is a name of a
    # known etg, just look for section named after it.
    if cp.has_section(frametype):
        trigs = get_triggers(cp, channel, frametype, query_segments)
        for seg in query_segments:
            spectrogram = build_omega_spectrogram(trigs, seg[0], seg[1], step,
                                                  spectype=spectype)
            if spectrogram is not None:
                add_spectrogram(spectrogram, seg, channel, spectrogram.epoch,
                                step, f_array = spectrogram.f_array)
                print_verbose("%d-segment spectrogram generated for [%d ... %d)\n"
                              % (spectrogram.length, seg[0], seg[1]))
            else:
                print_verbose("No data found for spectrogram in [%d ... %d)\n"
                              % (seg[0], seg[1]))

    else:
        # get data
        #store = G_DATA.has_key(channel)
        store = True
        serieslist = get_data(cp, channel, query_segments, frametype=frametype,
                              calibration=calibration, usends=usends,
                              resample=resample, superchannel=superchannel,
                              query=query, store=store)

        # make spectrogram
        for series in serieslist:
             dt = series.deltaT
             if not series.data or series.data.length < seglen//dt:
                 print_verbose("Data segment at %d too short for spectrogram, "
                               "skipping.\n" % (int(series.epoch)),
                               stream=sys.stderr, verbose=True)
                 continue

             try:
                 out = seriesutils.compute_average_spectrogram(series, step//dt,
                                                               seglen//dt,
                                                               stride//dt)
             except ValueError:
                 continue
             spectrogram, deltaF, f0 = out
             spectrogram.data = spectrogram.data**(1/2)
             if fresponse is not None:
                 if isinstance(fresponse, type(lambda:None)):
                     spectrogram.data = fresponse(spectrogram.data)
                 elif hasattr(fresponse, "__contains__") and len(fresponse) == 3:
                     zeros,poles,gain = fresponse
                     fresponse = zpkresponse(zeros, poles, gain, f0=f0,
                                             deltaF=deltaF,
                                             length=spectrogram.vectorLength)
                     spectrogram.data *= fresponse
             seg = Segment(float(series.epoch),
                           float(series.epoch + series.data.length * dt))
             add_spectrogram(spectrogram, seg, series.name, series.epoch,
                             step, f0=f0, deltaF=deltaF)
             print_verbose("%d-segment spectrogram generated for [%d ... %d)\n"
                            % (spectrogram.length, seg[0], seg[1]))

    # return the correct data
    globsegs = SegmentList(sorted(G_SPECTROGRAM[channel].keys())).coalesce()
    globsegs.sort()
    spectrograms = []
    for s in (segment_list & globsegs):
        try:
            if abs(s) >= step:
                spectrograms.append(G_SPECTROGRAM[channel][s])
        except KeyError:
            print_verbose("WARNING: Requested spectrogram segment %s not "
                          "found in global list\n" % (str(s)),
                          stream=sys.stderr)
    return sorted(spectrograms, key=lambda s: float(s['epoch']))


#
# =============================================================================
#
# Global data storage functions
#
# =============================================================================
#

def add_triggers(trigtable, channel, etg, seg):
    """Append the given glue.ligolw.Table trigtable to the global trigger
    holder for the given channel and segment.

    @param trigtable
        LIGOLw trigger table to store in memory
    @param channel
        name of channel to store
    @param etg
        trigger generator
    @param seg
        valid GPS [start,stop) interval for this trigger table
    """
    global G_TRIGGERS
    # add channel
    etg = etg.lower()
    if not G_TRIGGERS.has_key(etg):
        G_TRIGGERS[etg] = dict()
    if not G_TRIGGERS[etg].has_key(channel):
        G_TRIGGERS[etg][channel] = dict()

    # cast segment
    seg = Segment(seg[0], seg[1])

    # get segments for channel
    seglist = SegmentList(sorted(G_TRIGGERS[etg][channel].keys()))
    seglist.coalesce()

    # if completely new, add it
    disjoint = all([seg.disjoint(gseg) for gseg in seglist])
    if len(seglist) == 0 or disjoint:
        G_TRIGGERS[etg][channel][seg] = trigtable

    # otherwise it must exist inside the extent, which means it probably touches
    # one of the existing segments
    else:
        for gseg in seglist:
            if not gseg.disjoint(seg):
                seg += gseg
                G_TRIGGERS[etg][channel][seg] =\
                    G_TRIGGERS[etg][channel][gseg] + trigtable
                if seg != gseg:
                    del G_TRIGGERS[etg][channel][gseg]
                trigtable = G_TRIGGERS[etg][channel][seg]
    return


def add_clustered_triggers(trigtable, channel, etg, seg):
    """Append the given clustered glue.ligolw.Table trigtable to the
    global trigger holder for the given channel, etg and segment.

    @param trigtable
        LIGOLw trigger table to store in memory
    @param channel
        name of channel to store
    @param etg
        trigger generator
    @param seg
        valid GPS [start,stop) interval for this trigger table
    """
    global G_CLUSTERED_TRIGGERS
    etg = etg.lower()
    # add channel
    if not G_CLUSTERED_TRIGGERS.has_key(etg):
        G_CLUSTERED_TRIGGERS[etg] = dict()
    if not G_CLUSTERED_TRIGGERS[etg].has_key(channel):
        G_CLUSTERED_TRIGGERS[etg][channel] = dict()

    # get segments for channel
    seglist = SegmentList(sorted(G_CLUSTERED_TRIGGERS[etg][channel].keys()))
    seglist.coalesce()

    # if completely new, add it
    disjoint = all([seg.disjoint(gseg) for gseg in seglist])
    if len(seglist)==0 or disjoint:
        G_CLUSTERED_TRIGGERS[etg][channel][seg] = trigtable

    # otherwise it must touch at least one of the segments
    else:
        for gseg in seglist:
            if not gseg.disjoint(seg):
                seg += gseg
                G_CLUSTERED_TRIGGERS[etg][channel][seg] = (
                    G_CLUSTERED_TRIGGERS[etg][channel][gseg])
                G_CLUSTERED_TRIGGERS[etg][channel][seg].extend(trigtable)
                trigtable = G_CLUSTERED_TRIGGERS[etg][channel][seg]
                if seg != gseg:
                    del G_CLUSTERED_TRIGGERS[etg][channel][gseg]
    return


def add_timeseries(series):
    """Add a REAL4TimeSeries object data to the global data holder.

    @param series: TimeSeries to store
    """
    global G_DATA
    # add channel
    channel = series.name
    if not G_DATA.has_key(channel):
        G_DATA[channel] = dict()

    # get data type
    datatype = seriesutils.typecode(type(series))
    TYPESTR  = seriesutils._typestr[datatype]

    # get segments for channel
    seglist = SegmentList(sorted(G_DATA[channel].keys()))
    seglist.coalesce()

    # get segment for this timeseries
    seg = Segment(float(series.epoch),
                  float(series.epoch+series.data.length*series.deltaT))

    new = True
    resize = getattr(lal, "Resize%sTimeSeries" % TYPESTR)
    for i,gseg in enumerate(seglist):
        # if it postpends an existing segment
        if gseg[1] == seg[0]:
            new = False
            seg += gseg
            G_DATA[channel][seg] = G_DATA[channel][gseg]
            l1 = G_DATA[channel][seg].data.length
            l2 = series.data.length
            G_DATA[channel][seg] = resize(G_DATA[channel][seg], 0, l1+l2)
            G_DATA[channel][seg].data.data[l1:] = series.data.data
            series = G_DATA[channel][seg]
        # if it prepends an existing segment
        if gseg[0] == seg[1]:
            new = False
            seg += gseg
            G_DATA[channel][seg] = series
            l1 = G_DATA[channel][gseg].data.length
            l2 = series.data.length
            resize(G_DATA[channel][seg], 0, l1+l2)
            G_DATA[channel][seg].data.data[l2:] =\
                G_DATA[channel][gseg].data.data
            series = G_DATA[channel][seg]
    if new:
        G_DATA[channel][seg] = series


def add_spectrogram(sequence, seg, channel, epoch, deltaT, f0=0, deltaF=0, f_array=None):
    """Add a REAL4VectorSequence to the global data holder.
    """
    # add channel
    if not G_SPECTROGRAM.has_key(channel):
        G_SPECTROGRAM[channel] = dict()

    # get segments for channel
    seglist = SegmentList(sorted(G_SPECTROGRAM[channel].keys()))
    seglist.coalesce()

    # add segment
    new = True
    for gseg in seglist:
        # if it postpends an existing segment
        if gseg[1] == seg[0]:
            new = False
            seg += gseg
            G_SPECTROGRAM[channel][seg] = G_SPECTROGRAM[channel][gseg]
            G_SPECTROGRAM[channel][seg]['data'] = PyLALAddVectorSequence(
                                               G_SPECTROGRAM[channel][gseg]['data'],
                                               sequence)
            del G_SPECTROGRAM[channel][gseg]
            sequence = G_SPECTROGRAM[channel][seg]['data']
        # if it prepends an existing segment
        if gseg[0] == seg[1]:
            new = False
            seg += gseg
            G_SPECTROGRAM[channel][seg] = G_SPECTROGRAM[channel][gseg]
            G_SPECTROGRAM[channel][seg]['epoch'] = epoch
            G_SPECTROGRAM[channel][seg]['data'] = PyLALAddVectorSequence(
                                                sequence,
                                                G_SPECTROGRAM[channel][gseg]['data'])
            del G_SPECTROGRAM[channel][gseg]
            sequence = G_SPECTROGRAM[channel][seg]['data']

    if new:
        G_SPECTROGRAM[channel][seg] = {}
        G_SPECTROGRAM[channel][seg]['data'] = sequence
        G_SPECTROGRAM[channel][seg]['epoch'] = epoch
        G_SPECTROGRAM[channel][seg]['deltaT'] = deltaT
        G_SPECTROGRAM[channel][seg]['f0'] = f0
        G_SPECTROGRAM[channel][seg]['deltaF'] = deltaF
        G_SPECTROGRAM[channel][seg]['f_array'] = f_array


#
# =============================================================================
#
# Archival functions
#
# =============================================================================
#

def archive_data(filename, **hdfargs):
    """Write all stored data to an HDF5 format file

    @param filename
        filepath in which to write
    @keyword hdfargs
        keyword arguments for h5py module datasets
    """
    with hdf5utils.h5py.File(filename, "w") as h5file:
        hdfargs.setdefault("compression", "gzip")
        hdfargs.setdefault("compression_opts", 1)

        # write timeseries data
        for channel in G_DATA.keys():
            series = PyLALConcatenateTimeSeries(*G_DATA[channel].values())
            if series.data.length > 0:
                hdf5utils.writeTimeSeries(h5file, series,
                                          group=HDF_DATA_GROUP, **hdfargs)

        # write spectrum data
        for speclist in G_SPECTRUM.values():
            for spectrum in speclist:
                if spectrum.data.length > 0:
                    hdf5utils.writeFrequencySeries(h5file, spectrum,
                                                   group=HDF_SPECTRUM_GROUP,
                                                   **hdfargs)

        # write spectrogram data
        for channel in G_SPECTROGRAM.keys():
            group = "%s/%s" % (HDF_SPECTROGRAM_GROUP, channel)
            h5group = hdf5utils._create_groups(h5file, group)
            for seg in G_SPECTROGRAM[channel].keys():
                # format spectrogram data
                sequence = G_SPECTROGRAM[channel][seg]['data']
                epoch = G_SPECTROGRAM[channel][seg]['epoch']
                deltaT = G_SPECTROGRAM[channel][seg]['deltaT']
                f0 = G_SPECTROGRAM[channel][seg]['f0']
                deltaF = G_SPECTROGRAM[channel][seg]['deltaF']
                metadata = {"epoch":float(epoch), "dx":deltaT,
                            "f0":f0, "dy":deltaF}
                if sequence.length > 0 and sequence.vectorLength > 0:
                    hdf5utils.arrayToDataset(h5group, "-".join(map(str, seg)),
                                             sequence.data, metadata,
                                             **hdfargs)


def archive_segments(outfile, param_dict):
    """Write the stored segments to the given file object as xml

    @param outfile
        output file path
    @param param_dict
        dict of (option, value) pairs from command line
    """
    xmldoc = ligolw.Document()
    xmldoc.appendChild(ligolw.LIGO_LW())
    ifo = param_dict["ifo"]
    process = ligolw_utils.process.register_to_xmldoc(xmldoc, __file__,
                                                      param_dict,
                                                      version=__version__,
                                                      ifos=ifo)
    process.start_time = gpstime.GpsSecondsFromPyUTC(START)
    for flag in G_SEGMENTS.keys():
        ifo, name, version = parse_flag_name(flag)
        seg_def_id = segmentdb_utils.add_to_segment_definer(xmldoc,
                                                            process.process_id,
                                                            ifo, name, version)
        segmentdb_utils.add_to_segment(xmldoc, process.process_id, seg_def_id,
                                       G_SEGMENTS[flag].coalesce())
        segmentdb_utils.add_to_segment_summary(
            xmldoc, process.process_id, seg_def_id,
            G_SEGMENT_SUMMARY[flag].coalesce())
    process.end_time = lal.GPSTimeNow().gpsSeconds
    ligolw_utils.write_filename(xmldoc, outfile, gz=outfile.endswith("gz"),
                                verbose=False)


def archive_triggers(outdir, start, end, param_dict, clustered=False):
    """Write the stored triggers to the xml
    """
    ifo = param_dict["ifo"]
    if clustered:
        channels = [(etg, channel) for etg in G_CLUSTERED_TRIGGERS.keys() for
                    channel in G_CLUSTERED_TRIGGERS[etg].keys()]
    else:
        channels = [(etg, channel) for etg in G_TRIGGERS.keys() for
                    channel in G_TRIGGERS[etg].keys()]

    for etg,channel in channels:
        if ((clustered and not len(G_CLUSTERED_TRIGGERS[etg][channel].keys()))
            or (not clustered and not len(G_TRIGGERS[etg][channel].keys()))):
            continue
        retg = re_cchar.sub("_", etg).upper()
        if clustered:
            retg = "%s_CLUSTERED" % retg
        rchannel = channel
        if re.match("[A-Z]\d:", channel):
            rchannel = rchannel[3:]
        rchannel = re_cchar.sub("_", rchannel).upper()
        outfile = os.path.join(outdir, "%s-%s_%s-%d-%d.xml.gz"
                                       % (ifo, rchannel, retg, int(start),
                                          int(end-start)))
        xmldoc = ligolw.Document()
        xmldoc.appendChild(ligolw.LIGO_LW())
        process = ligolw_utils.process.register_to_xmldoc(xmldoc, __file__,
                                                          param_dict,
                                                          version=__version__,
                                                          ifos=ifo)
        seg_table = lsctables.New(lsctables.SegmentTable)
        xmldoc.childNodes[-1].appendChild(seg_table)
        global_triggers = (clustered and
                           G_CLUSTERED_TRIGGERS[etg][channel].iteritems() or
                           G_TRIGGERS[etg][channel].iteritems())
        for i,(seg,trig_table) in enumerate(global_triggers):
            try:
                event_table = table.get_table(xmldoc, trig_table.tableName)
            except ValueError:
                event_table = table.new_from_template(trig_table)
                xmldoc.childNodes[-1].appendChild(event_table)
            except AttributeError:
                warnings.warn("AttributeError caught when archiving triggers. "
                              "%s table for %s has type %s instead of "
                              "ligolw.Table" % (etg, seg, type(trig_table)))
                continue
            sid = ilwd.get_ilwdchar_class(table.StripTableName(
                                              seg_table.tableName),
                                          "process_id")
            pid = ilwd.get_ilwdchar_class(table.StripTableName(
                                              event_table.tableName),
                                          "process_id")
            seg = map(LIGOTimeGPS, map(float, seg))
            segment = lsctables.Segment()
            segment.start_time = seg[0].gpsSeconds
            segment.start_time_ns = seg[0].gpsNanoSeconds
            segment.end_time = seg[1].gpsSeconds
            segment.end_time_ns = seg[1].gpsNanoSeconds
            segment.segment_id = sid(i)
            segment.segment_def_id = None
            segment.segment_def_cdb = None
            segment.creator_db = None
            segment.process_id = process.process_id
            seg_table.append(segment)
            # FIXME: store segment information in process_id
            for t in trig_table:
                t.process_id = pid(int(segment.segment_id))
            event_table.extend(trig_table)
        process.start_time = gpstime.GpsSecondsFromPyUTC(START)
        process.end_time = lal.GPSTimeNow().gpsSeconds
        ligolw_utils.write_filename(xmldoc, outfile, gz=outfile.endswith("gz"),
                                    verbose=False)


def load_archive_data(cp, channel, seglist, mode=0, verbose=False):
    """Load data archived by a previous run of this code, e.g. daily
    range to use in a monthly run.

    @param cp
         configuration file object
    @param channel
        name of channel to read
    @param seglist
        list of segments to read
    @param mode: summary mode for this job
    @param verbose
        print verbose output, default: False
    """
    # return blank is no segments
    if not len(seglist):
        return []

    # restrict archived data to NOW
    start,end = seglist.extent()
    span = Segment(start, min(end, NOW))
    seglist &= SegmentList([span])

    # otherwise get start date
    start_date = datetime.datetime(*lal.GPSToUTC(int(start))[:6])

    # get file for mode
    datadir = "data"
    if mode == summary.SUMMARY_MODE_GPS:
        globstr = os.path.join(datadir, "*.hdf")
    elif mode == summary.SUMMARY_MODE_DAY:
        globstr = os.path.join("archive_daily",
                               "%d%.2d%.2d"
                                   % (start_date.year, start_date.month,
                                      start_date.day),
                               datadir, "*.hdf")
    elif mode == summary.SUMMARY_MODE_WEEK\
    or mode == summary.SUMMARY_MODE_MONTH:
        globstr = os.path.join("archive_monthly",
                               "%d%.2d*" % (start_date.year, start_date.month),
                               datadir, "*.hdf")
    elif mode == summary.SUMMARY_MODE_YEAR:
        globstr = os.path.join("archive_monthly", "%d*" % (start_date.year),
                               datadir, "*.hdf")
    else:
        raise RuntimeError("Mode not recognised.")

    # make cache
    cache = Cache.from_urls(glob.glob(globstr)).sieve(segmentlist=seglist)

    #
    # read data from file
    #

    # TODO : this block reads all the DATA from the HDF5 file
    #        it should probably read only the data requested

    for e in cache:
        # open file
        f = e.path

        # get segment
        seg = e.segment & span
        start = seg[0]
        duration = float(abs(seg))

        # get data
        try:
            h5file = hdf5utils.h5py.File(f, 'r')
        except hdf5utils.h5py.h5e.LowLevelIOError as e:
            warnings.warn('Caught exception: %s' % str(e))
            pass
        else:
            if HDF_DATA_GROUP in list(h5file)\
            and channel in list(h5file[HDF_DATA_GROUP]):
                series = hdf5utils.readTimeSeries(h5file, channel,
                                                  group=HDF_DATA_GROUP,
                                                  start=start,
                                                  duration=duration)
                add_timeseries(series)
                seg = Segment(float(series.epoch),
                              float(series.epoch + series.data.length
                                                   * series.deltaT))
                print_verbose("Data recovered from archive for %s: %s\n"
                              % (channel, str(seg)), verbose=verbose)

    return


def load_archive_spectrogram(cp, channel, seglist, mode=0, verbose=False):
    """
    Load a spectrogram archived by a previous run of this code,
    e.g. daily h(t) to use in a monthly run.

    @param cp
         configuration file object
    @param channel
        name of channel to read
    @param seglist
        list of segments to read
    @param mode: summary mode for this job
    @param verbose
        print verbose output, default: False
    """
    # return blank if no segments
    if not len(seglist):
        return []

    # restrict archived data to NOW
    start,end = seglist.extent()
    span = Segment(start, min(end, NOW))
    seglist &= SegmentList([span])

    # otherwise get start date
    start_date = datetime.datetime(*lal.GPSToUTC(int(start))[:6])

    # get file for mode
    datadir = "data"
    if mode == summary.SUMMARY_MODE_GPS:
        globstr = os.path.join(datadir, "*.hdf")
    elif mode == summary.SUMMARY_MODE_DAY:
        globstr = os.path.join("archive_daily",
                               "%d%.2d%.2d"
                                   % (start_date.year, start_date.month,
                                      start_date.day),
                               datadir, "*.hdf")
    elif mode == summary.SUMMARY_MODE_WEEK\
    or mode == summary.SUMMARY_MODE_MONTH:
        globstr = os.path.join("archive_monthly",
                               "%d%.2d*" % (start_date.year, start_date.month),
                               datadir, "*.hdf")
    elif mode == summary.SUMMARY_MODE_YEAR:
        globstr = os.path.join("archive_monthly", "%d*" % (start_date.year),
                               datadir, "*.hdf")
    else:
        raise RuntimeError("Mode not recognised.")

    # make cache
    cache = Cache.from_urls(glob.glob(globstr)).sieve(segmentlist=seglist)

    #
    # read data from file
    #

    # TODO : this block reads all the DATA from the HDF5 file
    #        it should probably read only the data requested

    for e in cache:
        # open file
        f = e.path
        h5file =  hdf5utils.h5py.File(f, "r")

        # load list of all spectrograms for this channel
        group = "%s/%s" % (HDF_SPECTROGRAM_GROUP, channel)
        try:
            datasets = list(h5file[group])
        except KeyError:
            datasets = []
        datasegs = [map(float, s.split("-")) for s in datasets]
        datasegs = SegmentList(map(Segment, datasegs))

        for dataseg,dataset in zip(datasegs, datasets):
            # get required segment
            seg = dataseg & span
            start = seg[0]
            duration = float(abs(seg))
            out = hdf5utils.readVectorSequence(h5file, dataset, group=group,
                                               start=start, duration=duration)
            spectrogram, epoch, deltaT, f0, deltaF = out
            seg = Segment(float(epoch),
                                   float(epoch + spectrogram.length*deltaT))
            add_spectrogram(spectrogram, seg, channel, epoch, deltaT, f0=f0,
                            deltaF=deltaF)
        h5file.close()
        if len(datasegs):
            seg = datasegs.extent()
            print_verbose("Spectrogram recovered from archive for %s: "
                          "[%d ... %d)\n"
                          % (channel, seg[0], seg[1]), verbose=verbose)

    return


def load_archive_spectrum(cp, channel, seglist, mode=0):
    """
    Load spectrum archived by a previous run of this code, e.g. daily range
    to use in a monthly run.

    @param cp
         configuration file object
    """
    # return blank is no segments
    if not len(seglist):
        return []

    # restrict archived data to NOW
    start,end = seglist.extent()
    now  = min(span[1], NOW)
    span = Segment(start, min(end, NOW))
    seglist &= SegmentList([span])

    # otherwise get start date
    start = datetime.datetime(*lal.GPSToUTC(int(seglist[0][0]))[:6])

    # get file for mode
    datadir = "data"
    if mode == summary.SUMMARY_MODE_GPS:
        globstr = os.path.join(datadir, "*.hdf")
    elif mode == summary.SUMMARY_MODE_DAY:
        globstr = os.path.join("archive_daily",
                               "%d%.2d%.2d"
                                   % (start.year, start.month, start.day),
                               datadir, "*.hdf")
    elif mode == summary.SUMMARY_MODE_WEEK\
    or mode == summary.SUMMARY_MODE_MONTH:
        globstr = os.path.join("archive_monthly",
                               "%d%.2d*" % (start.year, start.month),
                               datadir, "*.hdf")
    elif mode == summary.SUMMARY_MODE_YEAR:
        globstr = os.path.join("archive_monthly", "%d*" % (start.year),
                               datadir, "*.hdf")
    else:
        raise RuntimeError("Mode not recognised.")

    # make cache
    cache = Cache.from_urls(glob.glob(globstr)).sieve(segmentlist=seglist)

    #
    # read data from file
    #

    # TODO : this block reads all the DATA from the HDF5 file
    #        it should probably read only the data requested

    for e in cache:
        f = e.path
        series = hdf5utils.readFrequencySeries(f, channel,
                                               group=HDF_DATA_GROUP)
        add_timeseries(series)


def load_archive_segments(cp, flag, seglist, mode=0, verbose=False):
    """Load segments archived by a previous run of this code, e.g. daily
    state vector to use in a monthly run.

    @param cp
         configuration file object
    @param flag: name of flag to read
    @param seglist
        list of valid segments to restrict return
    @param mode: identifier of run mode
    @param verbose
        print verbose output, default: False
    """
    # return blank is no segments
    if not len(seglist):
        return []

    # restrict archived data to NOW
    start,end = seglist.extent()
    span = Segment(start, min(end, NOW))
    seglist &= SegmentList([span])

    # otherwise get start date
    start = datetime.datetime(*lal.GPSToUTC(int(start))[:6])

    # get file for mode
    datadir = "segments"
    tag = "%s-SUMMARY_PAGE_SEGMENTS-" % flag[:2]
    if mode == summary.SUMMARY_MODE_GPS:
        globstr = os.path.join(datadir, "%s*.xml.gz" % tag)
    elif mode == summary.SUMMARY_MODE_DAY:
        globstr = os.path.join("archive_daily",
                               "%d%.2d%.2d"
                                   % (start.year, start.month, start.day),
                               datadir, "%s*.xml.gz" % tag)
    elif mode == summary.SUMMARY_MODE_WEEK\
    or mode == summary.SUMMARY_MODE_MONTH:
        globstr = os.path.join("archive_daily",
                               "%d%.2d*" % (start.year, start.month),
                               datadir, "%s*.xml.gz" % tag)
    elif mode == summary.SUMMARY_MODE_YEAR:
        globstr = os.path.join("archive_monthly", "%d*" % (start.year),
                               datadir, "%s*.xml.gz" % tag)
    else:
        raise RuntimeError("Mode not recognised.")

    cache = Cache.from_urls(glob.glob(globstr)).sieve(segmentlist=seglist)
    if cache:
        ifo, name, version = parse_flag_name(flag)
        for fp in cache.pfnlist():
            with open(fp, "r") as f:
                xmldoc, digest = ligolw_utils.load_fileobj(
                                     f, gz=fp.endswith(".gz"))
                seg_def_table = table.get_table(
                                    xmldoc, lsctables.SegmentDefTable.tableName)
                seg_def_ids = [row.segment_def_id for row in seg_def_table if
                               row.name == name and row.version == version]
                seg_sum_table = table.get_table(
                                    xmldoc, lsctables.SegmentSumTable.tableName)
                seg_sum_list = SegmentList()
                for row in seg_sum_table:
                    if row.segment_def_id in seg_def_ids:
                        if hasattr(row, "start_time_ns"):
                            seg = Segment(map(float, row.get()))
                        else:
                            seg = Segment(row.start_time, row.end_time)
                        seg_sum_list.append(seg)
                seg_table = table.get_table(xmldoc,
                                            lsctables.SegmentTable.tableName)
                seg_list = SegmentList()
                for row in seg_table:
                    if row.segment_def_id in seg_def_ids:
                        if hasattr(row, "start_time_ns"):
                            seg = Segment(map(float, row.get()))
                        else:
                            seg = Segment(row.start_time, row.end_time)
                        seg_list.append(seg)
                if G_SEGMENTS.has_key(flag):
                    G_SEGMENT_SUMMARY[flag].extend(seg_sum_list)
                    G_SEGMENTS[flag].extend(seg_list)
                else:
                    G_SEGMENT_SUMMARY[flag] = seg_sum_list
                    G_SEGMENTS[flag] = seg_list
                G_SEGMENT_SUMMARY[flag].coalesce()
                G_SEGMENTS[flag].coalesce()


def load_archive_triggers(cp, ifo, channel, etg, segment_list, mode=0,
                          clustered=False, columns=[]):
    """Read the archive for the given channel, and load its triggers
    into global memory
    """
    global G_TRIGGERS, G_CLUSTERED_TRIGGERS
    # add keys to global memory holder
    if not G_TRIGGERS.has_key(etg):
        G_TRIGGERS[etg] = dict()
    if not G_TRIGGERS[etg].has_key(channel):
        G_TRIGGERS[etg][channel] = dict()
    if not G_CLUSTERED_TRIGGERS.has_key(etg):
        G_CLUSTERED_TRIGGERS[etg] = dict()
    if not G_CLUSTERED_TRIGGERS[etg].has_key(channel):
        G_CLUSTERED_TRIGGERS[etg][channel] = dict()

    # return blank is no segments
    if not len(segment_list):
        return

    # restrict archived data to NOW
    start,end = segment_list.extent()
    span = Segment(start, min(end, NOW))
    query_segments = segment_list & SegmentList([span])

    # restrict archived data to new segments
    query_segments -= SegmentList(sorted(G_TRIGGERS[etg][channel].keys()))

    # otherwise get start date
    start = datetime.datetime(*lal.GPSToUTC(int(start))[:6])

    # format file search
    retg = re_cchar.sub("_", etg).upper()
    if clustered:
        retg = "%s_CLUSTERED" % retg
    rchannel = channel
    if re.match("[A-Z]\d:", rchannel):
        rchannel = rchannel[3:]
    rchannel = re_cchar.sub("_", rchannel).upper()
    tag = "%s-%s_%s" % (ifo, rchannel, retg)
    datadir = "triggers"
    if mode == summary.SUMMARY_MODE_GPS:
        globstr = os.path.join(datadir, "%s*.xml.gz" % tag)
    elif mode == summary.SUMMARY_MODE_DAY:
        globstr = os.path.join("archive_daily",
                               "%d%.2d%.2d"
                                   % (start.year, start.month, start.day),
                               datadir, "%s*.xml.gz" % tag)
    elif mode == summary.SUMMARY_MODE_WEEK\
    or mode == summary.SUMMARY_MODE_MONTH:
        globstr = os.path.join("archive_daily",
                               "%d%.2d*" % (start.year, start.month),
                               datadir, "%s*.xml.gz" % tag)
    elif mode == summary.SUMMARY_MODE_YEAR:
        globstr = os.path.join("archive_monthly", "%d*" % (start.year),
                               datadir, "%s*.xml.gz" % tag)
    else:
        raise RuntimeError("Mode not recognised.")

    # get columns
    etgsec = re_cchar.sub("-", etg.lower())
    columns = (cp.has_option(etgsec, "table-columns") and
               cp.get(etgsec, "table-columns").split(",") or None)
    if columns:
        required = ["process_id", "search", "channel"]
        for col in required:
            if col not in columns:
                columns.append(col)


    cache = Cache.from_urls(glob.glob(globstr)).sieve(
                segmentlist=query_segments)
    if cache:
        for fp in cache.pfnlist():
            with open(fp, "r") as f:
                xmldoc, digest = ligolw_utils.load_fileobj(
                                     f, gz=fp.endswith(".gz"))
                # load segment table
                try:
                    seg_table = table.get_table(
                                    xmldoc, lsctables.SegmentTable.tableName)
                except:
                    seg_table = []
                # load trigger table
                if etg.lower() in ["omega", "kw", "kleinewelle", "excesspower",
                                   "gstlal_excesspower"]:
                    events = lsctables.New(lsctables.SnglBurstTable,
                                            columns=columns)
                    events.extend(table.get_table(
                                      xmldoc,
                                      lsctables.SnglBurstTable.tableName))
                elif re.search("(inspiral|ihope)", etg, re.I):
                    events = lsctables.New(lsctables.SnglInspiralTable,
                                           columns=columns)
                    events.extend(table.get_table(
                                     xmldoc,
                                     lsctables.SnglInspiralTable.tableName))
                # map triggers to segments
                for row in seg_table:
                    segment_events = lsctables.New(type(events),
                                                   columns=columns)
                    segment_events.extend(
                        t for t in events if
                        int(t.process_id) == int(row.segment_id))
                    if clustered:
                        add_clustered_triggers(segment_events, channel,
                                               etg, row.get())
                    else:
                        add_triggers(segment_events, channel, etg, row.get())

#
# =============================================================================
#
# Clustering functions
#
# =============================================================================
#

def cluster_table(trigtable, dt):
    """@returns a clustered version of the given table
    """
    tablecopy = table.new_from_template(trigtable)
    tablecopy.extend(trigtable)
    if isinstance(trigtable, lsctables.SnglBurstTable):
        testfunc = ligolw_bucluster.ExcessPowerTestFunc
        clusterfunc = ligolw_bucluster.OmegaClusterFunc
        sortfunc = ligolw_bucluster.ExcessPowerSortFunc
        bailoutfunc = ligolw_bucluster.ExcessPowerBailoutFunc
    elif isinstance(trigtable, lsctables.SnglInspiralTable):
        testfunc = lambda a, b: SnglInspiralUtils.CompareSnglInspiral(a, b, dt)
        clusterfunc = ligolw_sicluster.SnglInspiralCluster
        sortfunc = SnglInspiralUtils.CompareSnglInspiralByEndTime
        bailoutfunc = testfunc
    snglcluster.cluster_events(tablecopy, testfunc, clusterfunc, sortfunc,
                               bailoutfunc, verbose=False)
    return tablecopy

# =============================================================================
# Process segments
# =============================================================================

def process_segments(tab, cp, cpsec, verbose=False,
                     htmlonly=False, doplots=True, dosubplots=True):
    """
    Process and summarise segments. Modifies SegmentSummaryTab in place.

    @param tab
        SegmentSummaryTab object to process
    @param cp
         configuration file object
    @param cpsec
        name of INI section configuring this Tab
    @param verbose
        print verbose output, default: False
    @param doplots
        generate plots, default: True
    @param dosubplots
        generate subplots, default: True
    @param htmlonly
        generate enough data for HTML only, default: False
    """
    print_verbose("\n----------\nProcessing %s segments for the %s state...\n"
                  % (tab.name, tab.state.name),
                  verbose=verbose, profile=False)

    #
    # setup
    #

    if not os.path.isdir(tab.directory):
        os.makedirs(tab.directory)

    #
    # get flags
    #

    flags  = cp.get(cpsec, "data-quality-flags").split(',')
    if cp.has_option(cpsec, "labels"):
        labels = cp.get(cpsec, "labels").split(',')
    elif cp.has_option(cpsec, "label"):
        labels = cp.get(cpsec, "label").split(",")
    else:
        labels = flags

    #
    # get archived segments
    #

    if cp.has_option(cpsec, "use-archive")\
    and (cp.get(cpsec, "use-archive").lower() in ["", "true"]):
        for flag,label in zip(flags, labels):
            load_archive_segments(cp, flag, tab.segments,
                                  mode=tab.mode, verbose=verbose)
        print_verbose("Archive segments loaded.\n", verbose=verbose)

    #
    # get new segments
    #

    new_segments = SegmentListDict()
    if htmlonly:
        for flag,label in zip(flags, labels):
        # parse flag for composition
            name = re_cchar.sub("_", label.upper())
            if not name.startswith(tab.ifo):
                name = "%s-%s" % (tab.ifo, name)
            f = os.path.join(tab.datadirectory, "%s_%s-%d-%d.txt"
                                             % (name, tab.state.tag,
                                                tab.start_time, abs(tab.span)))
            if os.path.isfile(f):
                tab.add_segments(label, read_segment_file(f))
    else:
        all_flags = []
        for flag in flags:
            all_flags.extend(re.split("[&!]", flag))
        get_segments(cp, all_flags, tab.segments)
        for flag,label in zip(flags, labels):
            divs = re.findall("[&!]", flag)
            comp_flags = re.split("[&!]", flag)
            comp_segs = SegmentList(tab.segments)
            for i,comp_flag in enumerate(comp_flags):
                seglist = get_segments(cp, comp_flag, tab.segments, query=False)
                if len(divs) == 0:
                    comp_segs = seglist
                elif i!=0 and divs[i-1] == "!":
                    comp_segs -= seglist
                else:
                    comp_segs &= seglist
            label = re.sub("&", " and ", label)
            label = re.sub("!", " minus ", label)
            tab.add_segments(label, comp_segs)

    #
    # make plots
    #

    # get plot sections
    plots = [p for p in cp.items(cpsec) if p[0].startswith("plot-")]

    # if we got no plots, pick up the defaults
    if len(plots) == 0:
        plots = [("plot-segments", 0), ("plot-duty-cycle", 1)]

    # sort plots by their given order
    plots.sort(key=lambda o: str(o[1]).isdigit() and (int(o[1])+1) or 1000)

    for plot,_ in plots:
        plottag = re_cchar.sub("_", plot[5:].upper())
        outfile = os.path.join(tab.directory, "%s-%s_%s_%s-%d-%d.png"
                                     % (tab.ifo,
                                        re_cchar.sub("_", tab.name.upper()),
                                        tab.state.tag, plottag,
                                        tab.start_time, abs(tab.span)))
        plotparams = parse_plot_section(cp, cpsec,
                                        plotutils.parse_plot_config(cp, plot))
        if htmlonly or not doplots:
            tab.plots.append((outfile, plotparams.pop("description", None)))
        else:
            if re.search("duration", plot, re.I):
                tab.plotduration(outfile, **plotparams)
            elif re.search("duty-cycle", plot, re.I):
                tab.plotdutycycle(outfile, **plotparams)
            elif re.search("histogram", plot, re.I):
                tab.plothistogram(outfile, **plotparams)
            else:
                tab.plotsegments(outfile, **plotparams)
            print_verbose("%s saved.\n" % outfile, verbose=verbose)

    #
    # subplots
    #

    if cp.has_option(cpsec, "sub-plot"):
        subplot = cp.get(cpsec, "sub-plot")
        substart = tab.start_time
        deltas = subplot_durations(tab.start_time, tab.end_time, tab.mode)

        for dt in deltas:
            plottag = re_cchar.sub("_", subplot[5:].upper())
            basefile = "%s-%s_%s_%s-%d-%d.png"\
                       % (tab.ifo, re_cchar.sub("_", tab.name.upper()),
                          tab.state.tag, plottag, substart, dt)
            plotparams = parse_plot_section(cp, cpsec,
                                       plotutils.parse_plot_config(cp,subplot))
            d = datetime.datetime(*lal.GPSToUTC(int(substart))[:6])
            # test if plot exists somewhere else
            if tab.mode == summary.SUMMARY_MODE_WEEK\
            or tab.mode == summary.SUMMARY_MODE_MONTH:
                outdir = os.path.join("archive_daily", d.strftime("%Y%m%d"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            elif tab.mode == summary.SUMMARY_MODE_YEAR:
                outdir = os.path.join("archive_monthly", d.strftime("%Y%m"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            else:
                outfile = os.path.join(tab.directory, basefile)

            # otherwise make it as normal
            if htmlonly or not dosubplots:
                tab.subplots.append((outfile,\
                                     plotparams.pop("description", None)))
            else:
                if re.search("duration", subplot, re.I):
                    tab.plotduration(outfile, subplot=True, **plotparams)
                elif re.search("duty-cycle", subplot, re.I):
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plotdutycycle(outfile, subplot=True, **plotparams)
                elif re.search("histogram", subplot, re.I):
                    tab.plothistogram(outfile, subplot=True, **plotparams)
                else:
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plotsegments(outfile, subplot=True, **plotparams)
            substart += dt
        print_verbose("%d subplots saved.\n" % len(tab.subplots),\
                      verbose=verbose)

    if not htmlonly:
        tab.finalize()
        tab.frametohtml()



# =============================================================================
# Process state vector
# =============================================================================

def process_state_vector(tab, cp, cpsec, verbose=False, htmlonly=False,\
                        doplots=True, dosubplots=True):
    """
    Process and summarise state vector data.

    @param tab
        StateVectorSummaryTab object to process
    @param cp
         configuration file object
    @param cpsec
        name of INI section configuring this Tab
    @param verbose
        print verbose output, default: False
    @param doplots
        generate plots, default: True
    @param dosubplots
        generate subplots, default: True
    @param htmlonly
        generate enough data for HTML only, default: False
    """
    global G_SEGMENTS,G_SEGMENT_SUMMARY
    print_verbose("\n----------\nProcessing %s state vector for the %s "\
                  "state...\n" % (tab.name, tab.state.name),\
                  verbose=verbose, profile=False)

    #
    # setup
    #

    if not os.path.isdir(tab.directory):
        os.makedirs(tab.directory)

    # get bits
    bitmask = [(int(bit), mask) for (bit,mask) in cp.items(cpsec) if
               bit.isdigit()]
    tab.add_bitmask(bitmask)
    compobits = [(bit,mask) for (bit,mask) in cp.items(cpsec) if
                 not bit.isdigit() and re.sub("[!,]", "", str(bit)).isdigit()]
    tab.add_derived_bits(compobits)
    allbits = dict()
    allbits.update(tab.bitmask)
    allbits.update(tab.derived_bitmask)

    # get params
    channel = cp.get(cpsec, "channel")
    frametype = cp.get(cpsec, "frame-type")

    # load archived data
    if cp.has_option(cpsec, "use-archive") \
    and (cp.get(cpsec, "use-archive").lower() in ["", "true"]):
        load_archive_data(cp, channel, tab.segments, mode=tab.mode,
                          verbose=verbose)
        print_verbose("Archive data loaded.\n", verbose=verbose)

    # get new data
    if not htmlonly:
        # initialise empty segmentlists
        for flag in tab.bitmask.values():
            tab.add_segments(flag, [])

        # get and parse data
        data = get_data(cp, channel, tab.segments, frametype=frametype,
                        store=False)
        for series in data:
            state_segs = stateutils.tosegmentlistdict(series,tab.bitmask)
            for bit in tab.bits:
                flag = tab.bitmask[bit]
                tab.add_segments(flag, state_segs[flag])
                if (cp.has_option(cpsec, 'store-segments') and not
                    G_SEGMENTS.has_key(flag)):
                    G_SEGMENT_SUMMARY[flag] = SegmentList([tab.span])
                    G_SEGMENTS[flag] = state_segs[flag]

        # restrict tab segments to available data
        if len(data):
            defstart = float(data[0].epoch)
            defend = float(data[-1].epoch +
                           data[-1].data.length * data[-1].deltaT)
            tab.segments &= SegmentList([Segment(defstart, defend)])

        # construct derived bits
        if len(tab.derived_bitmask.keys()):
            for bitset,flag in tab.derived_bitmask.iteritems():
                bitset = re.split(",", bitset)
                seglist = SegmentList(tab.segments)
                for bit in bitset:
                    if bit.startswith("!"):
                        seglist -= tab.segdict[tab.bitmask[int(bit[1:])]]
                    else:
                        seglist &= tab.segdict[tab.bitmask[bit]]
                tab.add_segments(flag, seglist)

    #
    # make plots
    #

    # get plot sections
    plots = [p for p in cp.items(cpsec) if p[0].startswith("plot-")]

    # if we got no plots, pick up the defaults
    if len(plots) == 0:
        plots = [("plot-segments", "0")]

    # sort plots by their given order
    plots.sort(key=lambda o: o[1].isdigit() and (int(o[1])+1) or 1000)

    for plot,_ in plots:
        plottag = re_cchar.sub("_", plot[5:].upper())
        outfile = os.path.join(tab.directory, "%s-%s_%s_%s-%d-%d.png"\
                                     % (tab.ifo,\
                                        re_cchar.sub("_", tab.name.upper()),\
                                        tab.state.tag, plottag,\
                                        tab.start_time, abs(tab.span)))
        plotparams = parse_plot_section(cp, cpsec,\
                                        plotutils.parse_plot_config(cp, plot))
        if htmlonly:
            tab.plots.append((outfile, plotparams.pop("description", None)))
        else:
            if plot == 'plot-segment-duration':
                tab.plotduration(outfile, **plotparams)
            elif plot == 'plot-duty-cycle':
                tab.plotdutycycle(outfile, **plotparams)
            else:
                tab.plotsegments(outfile, **plotparams)
            print_verbose("%s saved.\n" % outfile, verbose=verbose)

    #
    # subplots
    #

    if cp.has_option(cpsec, "sub-plot"):
        subplot = cp.get(cpsec, "sub-plot")
        deltas = subplot_durations(tab.start_time, tab.end_time, tab.mode)
        substart = tab.start_time

        for dt in deltas:
            plottag = re_cchar.sub("_", subplot[5:].upper())
            basefile = "%s-%s_%s_%s-%d-%d.png"\
                       % (tab.ifo, re_cchar.sub("_", tab.name.upper()),\
                          tab.state.tag, plottag, substart, dt)
            plotparams = plotutils.parse_plot_config(cp, subplot)
            plotparams = parse_plot_section(cp, cpsec, plotparams)
            d = datetime.datetime(*lal.GPSToUTC(int(substart))[:6])
            # test if plot exists somewhere else
            if tab.mode == summary.SUMMARY_MODE_WEEK\
            or tab.mode == summary.SUMMARY_MODE_MONTH:
                outdir = os.path.join("archive_daily", d.strftime("%Y%m%d"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            elif tab.mode == summary.SUMMARY_MODE_YEAR:
                outdir = os.path.join("archive_monthly", d.strftime("%Y%m"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            else:
                outfile = os.path.join(tab.directory, basefile)

            # otherwise make it as normal
            if htmlonly or not dosubplots:
                tab.subplots.append((outfile, plotparams.pop("description",\
                                                             None)))
            else:
                if re.search("duration", subplot, re.I):
                    tab.plotduration(outfile, subplot=True, **plotparams)
                elif re.search("duty-cycle", subplot, re.I):
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plotdutycycle(outfile, subplot=True, **plotparams)
                elif re.search("histogram", subplot, re.I):
                    tab.plothistogram(outfile, subplot=True, **plotparams)
                else:
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plotsegments(outfile, subplot=True, **plotparams)
            substart += dt
        print_verbose("%d subplots saved.\n" % len(tab.subplots),\
                      verbose=verbose)

    if not htmlonly:
        tab.finalize()
        tab.frametohtml()

# =============================================================================
# Process triggers
# =============================================================================

def process_triggers(tab, cp, cpsec, verbose=False, doplots=True,\
                     dosubplots=True, htmlonly=False):
    """
    Process some triggers.

    @param tab
        StateVectorSummaryTab object to process
    @param cp
         configuration file object
    @param cpsec
        name of INI section configuring this Tab
    @param verbose
        print verbose output, default: False
    @param doplots
        generate plots, default: True
    @param dosubplots
        generate subplots, default: True
    @param htmlonly
        generate enough data for HTML only, default: False
    """
    if cp.has_option(cpsec, "channel"):
        channels = cp.get(cpsec, "channel").split(',')
    elif cp.has_option(cpsec, "channels"):
        channels = cp.get(cpsec, "channels").split(',')
    else:
        raise ValueError("No channels configured in [%s]." % cpsec)
    tab.etg = cp.has_option(cpsec, "trigger-generator")\
              and cp.get(cpsec, "trigger-generator") or cpsec[9:]
    print_verbose("\n----------\nProcessing %s %s triggers for the %s state"\
                  "...\n" % (" and ".join(channels), tab.etg, tab.state.name),\
                  verbose=verbose, profile=False)

    #
    # setup
    #

    if not os.path.isdir(tab.directory):
        os.makedirs(tab.directory)

    tab.cluster = cp.has_option(cpsec, "cluster-triggers")\
                  and cp.get(cpsec, "cluster-triggers") or False

    #
    # get archived triggers
    #

    if (cp.has_option(cpsec, "use-archive") and
            (cp.get(cpsec, "use-archive").lower() in ["", "true"])):
        for channel in channels:
            load_archive_triggers(cp, ifo, channel, tab.etg, tab.segments,
                                  mode=tab.mode, clustered=tab.cluster)
        print_verbose("Archive triggers loaded.\n", verbose=verbose)

    #
    # get triggers
    #

    # get SNR threshold
    snr = cp.has_option(cpsec, "snr-threshold")\
          and cp.getfloat(cpsec, "snr-threshold") or 0

    # get triggers
    if not htmlonly:
        # get trigfind args
        trigfind_args = dict()
        etgsec = tab.etg.lower()
        for opt,val in cp.items(etgsec)+cp.items(cpsec):
            if opt.startswith("trigfind-"):
                trigfind_args[opt[9:]] = val
        for chan in channels:
            trigtable = get_triggers(cp, chan, tab.etg, tab.segments,\
                                     cluster=tab.cluster, minsnr=snr,
                                     **trigfind_args)
            tab.add_triggers(chan, trigtable)

    print_verbose("All triggers loaded.\n", verbose=verbose)

    #
    # plot triggers
    #

    # get plot sections
    plots = [p for p in cp.items(cpsec) if p[0].startswith("plot-")]

    # if we got no plots, pick up the defaults
    if len(plots) == 0:
        plots = [("plot-time-frequency-snr", "0")]

    # sort plots by their given order
    plots.sort(key=lambda o: o[1].isdigit() and (int(o[1])+1) or 1000)

    for plot,_ in plots:
        plottag = re_cchar.sub("_", plot[5:].upper())
        outfile = os.path.join(tab.directory, "%s-%s_%s_%s-%d-%d.png"\
                                     % (tab.ifo,\
                                        re_cchar.sub("_", tab.name.upper()),\
                                        tab.state.tag, plottag,\
                                        tab.start_time, abs(tab.span)))
        plotparams = parse_plot_section(cp, cpsec,\
                                        plotutils.parse_plot_config(cp, plot))
        if htmlonly or not doplots:
            tab.plots.append((outfile, plotparams.pop("description", None)))
        else:
            # TODO: implement multi-threading plotting
            if re.search("hist", plot, re.I):
                tab.plothistogram(outfile, **plotparams)
            elif re.search("rate", plot, re.I):
                tab.plotrate(outfile, **plotparams)
            elif re.search("auto", plot, re.I):
                tab.plotautocorrelation(outfile, **plotparams)
            else:
                tab.plottable(outfile, **plotparams)
            print_verbose("%s saved.\n" % outfile, verbose=verbose)

    #
    # subplots
    #

    if cp.has_option(cpsec, "sub-plot"):
        subplot = cp.get(cpsec, "sub-plot")
        substart = tab.start_time
        deltas = subplot_durations(tab.start_time, tab.end_time, tab.mode)
        plottag = re_cchar.sub("_", subplot[5:].upper())

        for dt in deltas:
            basefile = "%s-%s_%s_%s-%d-%d.png"\
                       % (tab.ifo, re_cchar.sub("_", tab.name.upper()),
                          tab.state.tag, plottag, substart, dt)
            plotparams = plotutils.parse_plot_config(cp, subplot)
            plotparams = parse_plot_section(cp, cpsec, plotparams)
            d = datetime.datetime(*lal.GPSToUTC(int(substart))[:6])
            # test if plot exists somewhere else
            if tab.mode == summary.SUMMARY_MODE_WEEK\
            or tab.mode == summary.SUMMARY_MODE_MONTH:
                outdir = os.path.join("archive_daily", d.strftime("%Y%m%d"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            elif tab.mode == summary.SUMMARY_MODE_YEAR:
                outdir = os.path.join("archive_monthly", d.strftime("%Y%m"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            else:
                outfile = os.path.join(tab.directory, basefile)

            # otherwise make it as normal
            if htmlonly or not dosubplots:
                tab.subplots.append((outfile, plotparams.pop("description",\
                                                             None)))
            else:
                if re.search("hist", subplot, re.I):
                    tab.plothistogram(outfile, subplot=True, **plotparams)
                elif re.search("rate", subplot, re.I):
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plotrate(outfile, subplot=True, **plotparams)
                elif re.search("auto", subplot, re.I):
                    tab.plotautocorrelation(outfile, subplot=True, **plotparams)
                else:
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plottable(outfile, subplot=True, **plotparams)
            substart += dt
        print_verbose("%d subplots saved.\n" % len(tab.subplots),\
                      verbose=verbose)

    if not htmlonly:
        tab.finalize()
        tab.frametohtml()

def process_auxiliary_triggers(tab, cp, cpsec, verbose=False, doplots=True,\
                               htmlonly=False):
    """
    Process some auxiliary channel triggers relative to an analysis channel.

    @param tab
        AuxTriggerSummaryTab object to process
    @param cp
         configuration file object
    @param cpsec
        name of INI section configuring this Tab
    @param verbose
        print verbose output, default: False
    @param doplots
        generate plots, default: True
    @param htmlonly
        generate enough data for HTML only, default: False
    """
    tab.etg = cp.has_option(cpsec, "trigger-generator")\
              and cp.get(cpsec, "trigger-generator")\
              or cpsec[12:].split()[0]
    print_verbose("\n----------\nProcessing %s auxiliary channel triggers for "\
                  "the %s state...\n" % (tab.etg, tab.state.name),\
                  verbose=verbose, profile=False)

    #
    # setup
    #

    if not os.path.isdir(tab.directory):
        os.makedirs(tab.directory)

    etgsec = re_cchar.sub("-", tab.etg.lower())
    tab.cluster = ((cp.has_option(cpsec, "cluster-window") and
                    cp.getfloat(cpsec, "cluster-window")) or
                   (cp.has_option(etgsec, "cluster-window") and
                    cp.getfloat(etgsec, "cluster-window")) or
                   False)

    # get SNR threshold
    snr = cp.has_option(cpsec, "snr-threshold")\
          and cp.getfloat(cpsec, "snr-threshold") or 0

    # get channels
    tab.mainchannel = (cp.has_option(cpsec, "main-channel") and
                       cp.get(cpsec, "main-channel")) or None
    auxsec = cp.has_option(cpsec, "auxiliary-channels")\
             and cp.get(cpsec, "auxiliary-channels")\
             or "%s-channels" % tab.etg.lower()
    channels = [c for i,c in cp.items(auxsec) if c != tab.mainchannel]
    channels.sort()
    if tab.mainchannel:
        allchannels = [tab.mainchannel] + channels
    else:
        allchannels = channels

    # get coinc params
    if tab.mainchannel:
        dt = cp.getfloat(cpsec, "coincidence-window")
        shifts = map(float, cp.get(cpsec, "coincidence-time-slides").split(","))

    #
    # get archived triggers
    #

    if (cp.has_option(cpsec, "use-archive") and
            (cp.get(cpsec, "use-archive").lower() in ["", "true"])):
        for channel in allchannels:
            load_archive_triggers(cp, ifo, channel, tab.etg, tab.segments,
                                  mode=tab.mode, clustered=tab.cluster)
        print_verbose("Archive triggers loaded.\n", verbose=verbose)

    #
    # get triggers
    #

    # get trigfind args
    trigfind_args = dict()
    etgsec = re_cchar.sub("-", tab.etg.lower())
    for opt,val in cp.items(etgsec)+cp.items(cpsec):
        if opt.startswith("trigfind-"):
            trigfind_args[opt[9:]] = val

    # get triggers
    if not htmlonly:
        trigtables = get_multi_triggers(cp, allchannels, tab.etg, tab.segments,
                                        cluster=tab.cluster,
                                        minsnr=snr, **trigfind_args)
        for chan,trigtable in zip(allchannels, trigtables):
            tab.add_triggers(chan, trigtable)

    print_verbose("All triggers loaded.\n", verbose=verbose)

    #
    # get plots
    #

    # get plot sections
    plots = [p for p in cp.items(cpsec) if p[0].startswith("plot-")]

    # if we got no plots, pick up the defaults
    if len(plots) == 0:
        plots = [("plot-time-frequency-snr", "0")]

    # sort plots by their given order
    plots.sort(key=lambda o: o[1].isdigit() and (int(o[1])+1) or 1000)

    #
    # process and plot auxiliary
    #

    sigmaplot = None
    channelparams = parse_data_config(cp, cpsec, channels)
    for chan, params in zip(channels, channelparams):
        tab.auxplots[chan] = list()
        for plot,_ in plots:
            # format plot
            plottag = re_cchar.sub("_", plot[5:].upper())
            outfile = os.path.join(tab.directory, "%s-%s_%s_%s-%d-%d.png"\
                                     % (tab.ifo,\
                                         re_cchar.sub("_", chan[3:].upper()),\
                                        tab.state.tag, plottag,\
                                        tab.start_time, abs(tab.span)))
            plotparams = parse_plot_section(cp, cpsec,\
                                          plotutils.parse_plot_config(cp, plot))

            # skip if htmlonly
            if htmlonly or not doplots:
                if re.search("slide", plot, re.I):
                    sigmaplot = plot
                tab.auxplots[chan].append((outfile,\
                                           plotparams.pop("description", None)))
                continue

            # process data
            if re.search("slide", plot, re.I):
                # find significance of coincidences
                tab.compute_coinc_significance(chan, dt=dt, shifts=shifts)
                sigmaplot = plot
            if re.search("coinc", plot, re.I):
                # find coincs
                tab.get_coincs(chan, tab.mainchannel, dt=dt)
                tab.get_coincs(tab.mainchannel, chan, dt=dt)
                print_verbose("(%d,%d) coincs in (%s,%s).\n"\
                              % (tab.numcoincs[(chan, tab.mainchannel)],\
                                 tab.numcoincs[(tab.mainchannel, chan)],\
                                 chan, tab.mainchannel), verbose=verbose)
                tab.plotcoincs(outfile, chan, **plotparams)
            elif re.search("slide", plot, re.I):
                # find significance of coincidences
                tab.compute_coinc_significance(chan, dt=dt, shifts=shifts)
                tab.plotsigma(outfile, chan, **plotparams)
            elif re.search("hist", plot, re.I):
                if len(tab.segments):
                    plotparams.setdefault("normalize", tab.segments.extent())
                tab.plothistogram(outfile, chan, **plotparams)
            elif re.search("rate", plot, re.I):
                tab.plotrate(outfile, channels=chan, **plotparams)
            elif re.search("auto", plot, re.I):
                tab.plotautocorrleation(outfile, channels=chan, **plotparams)
            else:
                tab.plottable(outfile, chan, **plotparams)
        print_verbose("%d plots saved for %s.\n" % (len(plots), chan),\
                      verbose=verbose)

    #
    # make summary plot
    #

    if sigmaplot:
        plottag = re_cchar.sub("_", plot[5:].upper())
        outfile = os.path.join(tab.directory, "%s-%s_%s_%s-%d-%d.png"\
                                         % (tab.ifo,\
                                            re_cchar.sub("_", name.upper()),\
                                            tab.state.tag, plottag,\
                                            tab.start_time, abs(tab.span)))
        if htmlonly or not doplots:
            tab.plots.append((outfile, plotparams.pop("description", None)))
        else:
            plotparams = parse_plot_section(cp, cpsec,\
                                          plotutils.parse_plot_config(cp, plot))
            plotparams.pop("channel", None)
            tab.plotsigma(outfile, channel=None, **plotparams)
            print_verbose("%s saved.\n" % outfile, verbose=verbose)
    else:
        tab.plots.append((tab.auxplots[channels[0]][0][0], None))

    if not htmlonly:
        tab.finalize()
        tab.frametohtml()

# =============================================================================
# HVeto round class
# =============================================================================

class HVetoRound(object):

  def __init__(self):
    self.channel = None
    self.etg = None
    self.freqband = [0, numpy.inf]
    self.veto_segments = SegmentList([])
    self.significance = numpy.nan
    self.fakesignificance = None
    self.dt = None
    self.snr = None
    self.deadtime = [None]
    self.efficiency = [None]
    self.use_percentage = None
    self.safety = [None]
    self.veto_file = ""
    self.vetoed_h_trigs = lsctables.New(lsctables.SnglBurstTable)
    self.vetoing_aux_trigs = lsctables.New(lsctables.SnglBurstTable)


def write_hveto_results(filename, rounds):

  fobj = open(filename, 'w')

  # write headers
  fobj.write('#N channel significance t_win snr use deadtime safety file\n')

  N = max(rounds.keys())

  for i in range(1, N+1):
    fobj.write(' '.join(map(str, [i, '%s;%s;%g-%gHz' % (rounds[i].channel,rounds[i].etg,\
                                                        rounds[i].freqband[0],rounds[i].freqband[1]),\
                                    rounds[i].significance,\
                                    rounds[i].dt, rounds[i].snr,\
                                    ','.join(map(str, rounds[i].efficiency)),\
                                    ','.join(map(str, rounds[i].deadtime)),\
                                    ','.join(map(str, rounds[i].safety)),\
                                    os.path.abspath(rounds[i].veto_file)])))
    fobj.write('\n')
  fobj.close()

def load_hveto_results(filename):

  rounds = {}
  with open(filename, 'r') as fobj:
    for line in fobj:
      if re_cchar.match(line): continue
      i, c, sig, dt, snr, eff, deadtime, safety, f = line.split(' ')
      i = int(i)
      rounds[i] = HVetoRound()
      csplit = c.split(';')
      if len(csplit) > 2:
        rounds[i].freqband = map(float, csplit[2][:-2].split('-'))
      else:
        rounds[i].freqband = [-numpy.inf, numpy.inf]
      if len(csplit) > 1:
        rounds[i].etg = csplit[1]
      else:
        rounds[i].etg = 'HACR'
      rounds[i].channel       = csplit[0]
      rounds[i].significance  = float(sig)
      rounds[i].dt            = float(dt)
      rounds[i].snr           = float(snr)
      rounds[i].efficiency    = map(float, eff.split(','))
      rounds[i].deadtime      = map(float, deadtime.split(','))
      if safety=='None':
        rounds[i].safety      = [None]
      else:
        rounds[i].safety      = map(float, safety.split(','))
      rounds[i].veto_file     = f.rstrip('\n')
      hveto_day_xml = ligolw_utils.load_filename(rounds[i].veto_file)
      rounds[i].veto_segments = SegmentList([])
      for seg in hveto_day_xml.childNodes[0].childNodes[-1]:
          rounds[i].veto_segments.append(seg.get())
      rounds[i].veto_segments.coalesce()

  return rounds


def write_round_xml( vetosegs, vetotrigs, winner, ifo):
	"""
	Write out the products from this round of hveto: veto segments and the vetoed triggers.
	"""
	# Create a new document
	xmldoc = ligolw.Document()
	xmldoc.appendChild( ligolw.LIGO_LW() )

	# Append the process information
	procrow = ligolw_utils.process.append_process( xmldoc )

	# Add the vetoed triggers
	xmldoc.childNodes[0].childNodes.append( vetotrigs )

	# Append the veto segments
	segl = SegmentListDict()
	segl[ifo] = vetosegs
	lwsegs = ligolw_utils.segments.LigolwSegments( xmldoc )
	lwsegs.insert_from_segmentlistdict( segl, "hveto" )
	lwsegs.finalize(procrow)
	
	return xmldoc

def lalseg_from_seglist( segl ):
	"""
	Convert a glue.segments segmentlist to a LALSegList structure.
	"""
	lalsegl = lal.SegList()
	lal.SegListInit( lalsegl )
	for i, seg in enumerate(segl):
		lalseg = lal.Seg()
		lal.SegSet( lalseg,
			lal.LIGOTimeGPS( float(seg[0]) ),
			lal.LIGOTimeGPS( float(seg[1]) ),
			i
		)
		lal.SegListAppend( lalsegl, lalseg )
	return lalsegl

def seglist_from_lalseg( lalsegl ):
	"""
	Convert a LALSegList structure to a glue.segmentlist.
	"""
	segl = segmentlist()
	for i in range(lalsegl.length):
		lalseg = lal.SegListGet( lalsegl, i )
		segl.append( segment( lalseg.start, lalseg.end ) )
	return segl

def ghash_to_dict( ghash ):
	"""
	Convert a GHashTable into a python dictionary. Note that all the GHashTables in use by laldetchar map a string to a fixed type (e.g. double or string), so the dtype argument should match that.
	"""
	i = 0
	keys = []
	while True:
		key = laldetchar.GetGHashTableKey( ghash, i )
		if key is None: 
			break
		i += 1
		keys.append( key )

	dtype = laldetchar.GetGHashTableType( ghash )
	if dtype == laldetchar.LALGTYPE_INT:
		rfunc = laldetchar.GetGHashTblInt
	elif dtype == laldetchar.LALGTYPE_DBL:
		rfunc = laldetchar.GetGHashTblDbl
	elif dtype == laldetchar.LALGTYPE_STR:
		rfunc = laldetchar.GetGHashTblStr
	else:
		raise TypeError( "Don't have right function to retrieve key for LALGType=%i" % dtype )
	return dict( [ (key, rfunc( ghash, key )) for key in keys ] )

def lalburst_sb_to_glue_sb( sb_in, desired_columns ):
	"""
	Convert a lalburst SnglBurst structure to a SnglBurst row from glue.
	"""
	sb = lsctables.SnglBurstTable.RowType()
	for att in desired_columns:
		if att == "start_time":
			start_time = float( sb_in.start_time )
			sb.start_time = int(start_time) 
			sb.start_time_ns = int(1e9*(start_time - int(start_time) ))
			continue
		elif att == "peak_time":
			peak_time = float( sb_in.peak_time )
			sb.peak_time = int(peak_time) 
			sb.peak_time_ns = int(1e9*(peak_time - int(peak_time) ))
			continue
		elif att == "process_id":
			sb.process_id = ilwd.ilwdchar( "process:process_id:%d" % getattr( sb_in, att ) )
			continue
		elif att == "event_id":
			sb.event_id = ilwd.ilwdchar( "sngl_burst:sngl_burst_id:%d" % getattr( sb_in, att ) )
			continue

		try:
			setattr( sb, att, getattr( sb_in, att ) )
		except AttributeError:
			pass
	return sb

def seq_to_sbtable( sblist, get_col=None ):
	"""
	Convert a GSequence of SnglBurst to a SnglBurstTable.
	"""
        desired_columns = ["peak_time_ns", "start_time_ns", "channel",
	  "ifo", "peak_time", "start_time",
	  "duration", "central_freq", "search",
	  "bandwidth",  "snr"]
	sbtable = lsctables.New(lsctables.SnglBurstTable,
	  desired_columns)

	sblistitr = laldetchar.GSequenceBegin( sblist )

	col_ar = []
	while sblistitr is not None:
		next_sb = laldetchar.GetGSeqSnglBurst( sblistitr )
		if next_sb is None:
			break
		if get_col is not None:
			col_ar.append( getattr( next_sb, get_col ) )
		sbtable.append( lalburst_sb_to_glue_sb(next_sb, desired_columns) )
		sblistitr = laldetchar.GSequenceNext( sblistitr )

	if get_col is not None:
		return col_ar
	return sbtable

def get_chan_list( gseq ):
	"""
	Get a listing of the channels within a trigger sequence.
	"""
	# This is necessarily complicated because there is no "set" type in glib
	# thus the closest thing is a hash. This retrieves the hash, then converts
	# its keys into a proper set.
	return set(ghash_to_dict( laldetchar.GetChannelList( gseq ) ).keys())

def run_hveto(tab, trig_seq, twind, snr_thresh, sig_stop_thresh, verbose = False):
    """
    Actual hierarchical processing of trigger to make vetoes. Should
    be spin-off with helper functions into a module or something.
    """
    # Type of coincidence:
    # 0 -- Record all coincidences
    # 1 -- Record one or no coincidence per reference trigger
    # NOTE: I don't provide this as an option yet. The original algorithm only used
    # type 1, and type 0 may not even work after all the tooling I've done. Fair
    # warning.
    coinc_type = 1



    rnd = 1
    livesegs = tab.segments
    runseg = Segment(tab.start_time, tab.end_time)
    runlive = livetime = float(abs(livesegs))
    lalsegl = lalseg_from_seglist( livesegs )
    ref_trigs = 0

    sigdrop = None
    round_winner, efficiency, deadtime = [], [], []
    winner_docs = []

    # Round loop
    while True:
            subround_winner = {}
            subround_count = {}
            subround_coinc = {}
            print "Round %d" % rnd
            for snr_t in snr_thresh:

                # Mark triggers not meeting threshold criteria
                laldetchar.DetCharPruneTrigs( trig_seq, lalsegl, snr_t, tab.mainchannel )

                if laldetchar.CountUnmarkedSnglBurst( trig_seq ) <= 0:
                    if verbose:
                        print "No triggers remaining at current threshold. Moving on."
                    continue

                for wind in twind:
                        if verbose:
                                print "SNR threshold %f, time window %f" % (snr_t, wind)
                        chancount = laldetchar.CreateGHashTable(laldetchar.LALGTYPE_INT)
                        chancoinc = laldetchar.CreateGHashTable(laldetchar.LALGTYPE_INT)
                        # TODO: Wrap this better
                        laldetchar.DetCharScanTrigs( chancount, chancoinc, trig_seq, tab.mainchannel, wind, coinc_type )
                        t_ratio = wind / livetime
                        rnd_sig, winner = laldetchar.DetCharVetoRound( chancount, chancoinc, tab.mainchannel, t_ratio )
                        if verbose:
                                print "Subround winner %s (SNR: %f, wind: %f): %f" % (winner, snr_t, wind, rnd_sig)

                        chancount = ghash_to_dict( chancount )
                        subround_count[(snr_t, wind)] = chancount
                        chancoinc = ghash_to_dict( chancoinc )
                        subround_coinc[(snr_t, wind)] = chancoinc

                        if chancoinc.has_key(winner):
                                mu = t_ratio * chancount[winner] * chancount[tab.mainchannel]
                                subround_winner[(snr_t, wind)] = (winner, rnd_sig, mu)
                        #print chancount, chancoinc
                        if verbose:
                            print "Popping from window loop."
                if verbose:
                    print "Popping from threshold loop."

            if verbose:
                print  "Determining subround winner"
            subround_winner = sorted(subround_winner.iteritems(), key=lambda tup: tup[1][1] )

            if len(subround_winner) == 0:
                if verbose:
                    print "No triggers remain, exiting."
                break
            round_winner = subround_winner[-1]
            snr_win, wind_win = round_winner[0]
            winner, sig, mu = round_winner[1]

            # Create veto segments
            start = lal.LIGOTimeGPS(-wind_win/2.0)
            stop = lal.LIGOTimeGPS(wind_win/2.0)
            veto = lal.SegCreate( start, stop, rnd )

            # We need a list of the reference channel triggers for plotting later
            # TODO: Reenable for breakout
            #all_ref_trigs = filter(lambda sb: sb.channel == tab.mainchannel, seq_to_sbtable( trig_seq ) )

            print "Retrieving vetoed triggers from list."
            vetoed_trigs = laldetchar.DetCharRemoveTrigs( trig_seq, veto, winner, tab.mainchannel, snr_win )
            
            print "Converting trigger format."
            vetoed_trigs = seq_to_sbtable( vetoed_trigs )
            # Number of reference channel triggers vetoed
            n_trig_vetoed = len( filter(lambda sb: sb.channel == tab.mainchannel, vetoed_trigs ) )

            print "Calculating veto segments."
            # Remove vetoed winner triggers from livetime
            vetosegs = SegmentList([])
            fvetosegs = SegmentList([])
            vetoed_h_trigs = lsctables.New(lsctables.SnglBurstTable)
            vetoing_aux_trigs = lsctables.New(lsctables.SnglBurstTable)
            for trig in vetoed_trigs:
                    if trig.channel != winner and trig.channel != tab.mainchannel: 
                        continue
                    if trig.channel == tab.mainchannel:
                        vetoed_h_trigs.append(trig)
                        continue
                    if trig.channel == winner:
                        vetoing_aux_trigs.append(trig)
                    vs = Segment( trig.get_peak() - wind_win/2.0, trig.get_peak() + wind_win/2.0 )
                    fvs = Segment( float(trig.get_peak() - wind_win/2.0), float(trig.get_peak() + wind_win/2.0 ))
                    vetosegs.append( vs )
                    fvetosegs.append(fvs)
            vetosegs.coalesce()
            fvetosegs.coalesce()

            # Remove vetoed time from livetime
            deadt = float(abs(livesegs))
            livesegs -= vetosegs
            deadt -= float(abs(livesegs))

            # Other information
            if rnd == 1:
                    # Lowest SNR and window, for the total reference channel count
                    snr_l, wind_l = snr_thresh[0], twind[0]
                    ref_trigs = subround_count[(snr_l, wind_l)][tab.mainchannel]
                    efficiency.append( float(n_trig_vetoed)/ref_trigs*100 )
                    inc_eff = efficiency[-1]
                    deadtime.append( deadt/runlive*100 )
            else:
                    inc_eff = float(n_trig_vetoed)/ref_trigs*100
                    efficiency.append( inc_eff )
                    deadtime.append( deadt/runlive*100 )

            print """
    Round statistics:
    \treference channel / winner: %s / %s
    \tsnr threshold / time window: %f / %f
    \tmu / significance: %f / %f
    \tN_ref / vetoed = %d / %d
    \tN_aux = %d
    \tefficiency %% (rnd/cum): %f / %f
    \tdeadtime %% (rnd/cum) %f / %f""" % ( tab.mainchannel, winner, snr_win, wind_win, mu, sig, ref_trigs, n_trig_vetoed, subround_count[(snr_l, wind_l)][winner], inc_eff, sum(efficiency), deadtime[-1], sum(deadtime) )


            # We've dropped below the useful threshold, bail.
            if sig < sig_stop_thresh:
                print "Last round below significance threshold, we're done here."
                break

            # Write out vetoes and vetoed triggers
            xmldoc = write_round_xml(vetosegs, vetoed_trigs, winner, tab.ifo)
            fname = "%s/%s-HVETO_ROUND_%d-%d-%d.xml.gz" % (tab.directory, tab.ifo, rnd, int(runseg[0]), int(abs(runseg)))
            ligolw_utils.write_filename( xmldoc, fname, gz=True, verbose=verbose )

            # save round statistics
            tab.rounds[rnd] = HVetoRound()
            tab.rounds[rnd].channel = winner
            tab.rounds[rnd].etg = tab.etg
            tab.rounds[rnd].significance = sig
            tab.rounds[rnd].dt = wind_win
            tab.rounds[rnd].snr = snr_win
            tab.rounds[rnd].efficiency = [inc_eff]
            tab.rounds[rnd].deadtime = [deadtime[-1]]
            tab.rounds[rnd].safter = [numpy.nan]
            tab.rounds[rnd].veto_file = fname
            tab.rounds[rnd].veto_segments = fvetosegs
            tab.rounds[rnd].use_percentage = n_trig_vetoed/subround_count[(snr_win, wind_win)][winner]
            tab.rounds[rnd].vetoing_aux_trigs = vetoing_aux_trigs
            tab.rounds[rnd].vetoed_h_trigs = vetoed_h_trigs

            # strip channel frequency range from winner channel name and record
            tmp = winner.split(':')
            tab.winnerchannels.append(tmp[0] + ':' + tmp[1]) 


            # put segments into global holder for other parts of summary pages
            hvetoflag='G1:HVETO:1'
            if G_SEGMENTS.has_key(hvetoflag):
                if abs(SegmentList([runseg]) & G_SEGMENT_SUMMARY[hvetoflag]) \
                        != abs(SegmentList([runseg])) or \
                        abs(SegmentList([runseg])) != abs(G_SEGMENT_SUMMARY[hvetoflag]):
                    raise ValueError("Unconsistent processed time span for different rounds of hveto")
                G_SEGMENTS[hvetoflag] |= fvetosegs
            else:
                G_SEGMENT_SUMMARY[hvetoflag] = SegmentList([runseg])
                G_SEGMENTS[hvetoflag] = SegmentList([])
                G_SEGMENTS[hvetoflag] |= fvetosegs

            # Plots and the like
            chansig = defaultdict(dict)

            # Plot the vetoed reference channel tringgers
            if vetoed_trigs is not None or len(vetoed_trigs) != 0:
                    aux_trigs = filter(lambda sb: sb.channel == winner, vetoed_trigs)
                    pname = "hveto_round_%d_summary.png" % rnd 
                    #breakout.breakout_plot( all_ref_trigs, aux_trigs, tab.mainchannel, winner, livesegs, vetosegs, pname )

            # Calculate the significance of each of the channels against the reference
            # channel for each of the subrounds for the significance drop plot
            for subrnd, coinctbl in subround_coinc.iteritems():
                    ref_cnt = subround_count[subrnd][tab.mainchannel]
                    snr_t_sub, twind_sub = subrnd
                    for chan, coinc in coinctbl.iteritems():
                            cnt = subround_count[subrnd][chan]
                            mu = twind_sub*cnt*ref_cnt/livetime
                            chansig[subrnd][chan] = laldetchar.DetCharHvetoSignificance( mu, coinc )


            livetime = float(abs(livesegs))
            print "Live time after round %d: %f" % (rnd, livetime)

            #break
            #if rnd_sig < sig_stop_thresh:
            if sig < sig_stop_thresh:
                break
            else:
                rnd +=  1
    return 

def process_hveto(tab, cp, cpsec, verbose=False, doplots=True,\
                               htmlonly=False):
    """
    Process some auxiliary channel triggers relative to an analysis channel.

    @param tab
        hveto object to process
    @param cp
        configuration file object
    @param cpsec
        name of INI section configuring this Tab
    @param verbose
        print verbose output, default: False
    @param doplots
        generate plots, default: True
    @param htmlonly
        generate enough data for HTML only, default: False
    """
    tab.etg = cp.has_option(cpsec, "trigger-generator")\
              and cp.get(cpsec, "trigger-generator")\
              or cpsec[12:].split()[0]
    print_verbose("\n----------\nProcessing hveto using %s auxiliary channel triggers for "\
                  "the %s state...\n" % (tab.etg, tab.state.name),\
                  verbose=verbose, profile=False)

    #
    # setup
    #

    if not os.path.isdir(tab.directory):
        os.makedirs(tab.directory)

    etgsec = re_cchar.sub("-", tab.etg.lower())
    tab.cluster = ((cp.has_option(cpsec, "cluster-window") and
                    cp.getfloat(cpsec, "cluster-window")) or
                   (cp.has_option(etgsec, "cluster-window") and
                    cp.getfloat(etgsec, "cluster-window")) or
                   False)

    # get SNR threshold
    snr = cp.has_option(cpsec, "snr-threshold")\
          and cp.getfloat(cpsec, "snr-threshold") or 0

    # get channels
    tab.mainchannel = (cp.has_option(cpsec, "main-channel") and
                       cp.get(cpsec, "main-channel")) or None
    unsafechannels = cp.has_option(cpsec, 'unsafe-channels') and \
                     [c for i,c in cp.items(cp.get(cpsec, 'unsafe-channels'))] or []
    auxsec = cp.has_option(cpsec, "auxiliary-channels")\
             and cp.get(cpsec, "auxiliary-channels")\
             or "%s-channels" % tab.etg.lower()
    channels = [c for i,c in cp.items(auxsec) if c != tab.mainchannel and c not in unsafechannels]
    channels.sort()
    if tab.mainchannel:
        allchannels = [tab.mainchannel] + channels

    # get coinc params
    if cp.has_option(cpsec, "coincidence-windows"):
        twind = map(float, cp.get(cpsec, "coincidence-windows").split(',') or None)
    if cp.has_option(cpsec, "snr-aux-thresholds"):
        snr_thresh = map(float, cp.get(cpsec, "snr-aux-thresholds").split(',') or None)
    if cp.has_option(cpsec, "significance-threshold"):
        sig_stop_thresh = cp.getfloat(cpsec, "significance-threshold")
    freqbands = cp.has_option(cpsec, 'frequency-bands')\
        and sorted(map(float, cp.get(cpsec, 'frequency-bands').split(',')))\
        or False


    #
    # get triggers
    #

    # get trigfind args
    trigfind_args = dict()
    etgsec = re_cchar.sub("-", tab.etg.lower())
    for opt,val in cp.items(etgsec)+cp.items(cpsec):
        if opt.startswith("trigfind-"):
            trigfind_args[opt[9:]] = val



    #
    # process hveto
    #
    # for long summaries pick-up hveto results form daily pages, otherwise run hveto
    if tab.mode == summary.SUMMARY_MODE_MONTH:
        start_date = datetime.datetime(*lal.GPSToUTC(int(tab.start_time))[:6])
        globstr = os.path.join("archive_daily",
                               "%d%.2d*"
                               % (start_date.year, start_date.month),
                               "analysis/hveto", "full_results.txt")
        hveto_files = glob.glob(globstr)

        tab.rounds[1] = HVetoRound()
        tab.rounds[1].channel = 'N/A'
        for filename in hveto_files:
            print filename
            rounds = load_hveto_results(filename)
            for i in range(1,1+len(rounds)):
                tab.rounds[1].veto_segments = tab.rounds[1].veto_segments or rounds[i].veto_segments
        tab.rounds[1].use_percentage = numpy.nan
        tab.rounds[1].efficiency = [numpy.nan]
        tab.rounds[1].deadtime = [numpy.nan]
        trigtable = get_triggers(cp, tab.mainchannel, tab.etg, tab.segments,
                                 cluster=tab.cluster,
                                 minsnr=snr, **trigfind_args)
        tab.add_triggers(tab.mainchannel, trigtable)

    else:
        # get triggers
        if not htmlonly:
            trigtables = get_multi_triggers(cp, allchannels, tab.etg, tab.segments,
                                            cluster=tab.cluster,
                                            minsnr=snr, **trigfind_args)
            trig_seq = None
            swigtriglist = []
            for chan,trigtable in zip(allchannels, trigtables):
                tab.add_triggers(chan, trigtable)
                # convert from glue SnglBurst to swig/lal SnglBurst
                for trig in trigtable:
                    swigtrig = lalburst.CreateSnglBurst()
                    swigtrig.ifo = trig.ifo
                    swigtrig.search = trig.search
                    swigtrig.start_time = lal.LIGOTimeGPS(trig.start_time + 1e-9*trig.start_time_ns)
                    swigtrig.peak_time = lal.LIGOTimeGPS(trig.peak_time + 1e-9*trig.peak_time_ns)
                    swigtrig.central_freq = trig.peak_frequency
                    if trig.channel != tab.mainchannel and freqbands:
                        freq_extension = ''
                        for iF in range(len(freqbands)-1):
                            if trig.peak_frequency >= freqbands[iF] \
                                    and trig.peak_frequency < freqbands[iF+1]:
                                freq_extension = ':%g-%gHz' % (freqbands[iF], freqbands[iF+1])
                        swigtrig.channel = trig.channel + freq_extension
                    else:
                        swigtrig.channel = trig.channel
                    swigtrig.snr = trig.snr
                    swigtriglist.append(swigtrig)
                    trig_seq = laldetchar.PopulateTrigSequenceFromTrigList(trig_seq, swigtriglist[-1])
                    
        print_verbose("All triggers loaded.\n", verbose=verbose)

        run_hveto(tab, trig_seq, twind, snr_thresh, sig_stop_thresh)

    #
    # plot triggers
    #
    # get plot sections
    plots = [p for p in cp.items(cpsec) if p[0].startswith("plot-")]

    # if we got no plots, pick up the defaults
    if len(plots) == 0:
        plots = [("plot-time-frequency-snr", "0")]

    # sort plots by their given order
    plots.sort(key=lambda o: o[1].isdigit() and (int(o[1])+1) or 1000)

    #
    # process and plot auxiliary
    #

    sigmaplot = None
    # make plots for each winner channel (each round)
    for chan, rnd in zip(["combined"] + tab.winnerchannels,  range(1+len(tab.winnerchannels))):
        tab.auxplots[rnd] = list()
        for plot,_ in plots:
            # format plot
            plottag = re_cchar.sub("_", plot[5:].upper())
            outfile = os.path.join(tab.directory, "%s-HVETO_ROUND_%d_%s_%s_%s-%d-%d.png"\
                                     % (tab.ifo, rnd,\
                                         re_cchar.sub("_", chan[3:].upper()),\
                                        tab.state.tag, plottag,\
                                        tab.start_time, abs(tab.span)))
            plotparams = parse_plot_section(cp, cpsec,\
                                          plotutils.parse_plot_config(cp, plot))

            # skip if htmlonly
            if htmlonly or not doplots:
                if re.search("slide", plot, re.I):
                    sigmaplot = plot
                tab.auxplots[rnd].append((outfile,\
                                           plotparams.pop("description", None)))
                continue

            # process data
            if re.search("coinc", plot, re.I):
                # find coincs
                tab.plotcoincs(outfile, rnd, **plotparams)
            else:
                tab.plothistogram(outfile, rnd, **plotparams)        
        print_verbose("%d plots saved for %s.\n" % (len(plots), chan),\
                      verbose=verbose)

    #
    # make summary plot
    #
    tab.plots.append((tab.auxplots[0][0][0], None))
    

    #
    # write results
    #

    hveto_file = '%s/full_results.txt' % tab.directory
    write_hveto_results(hveto_file, tab.rounds)


    if not htmlonly:
        tab.finalize()
        tab.frametohtml()

    return

# =============================================================================
# Acquire data
# =============================================================================





# =============================================================================
# Process data
# =============================================================================

def process_data(tab, cp, cpsec, verbose=False, doplots=True,\
                 dosubplots=True, htmlonly=False):
    """
    Process data from a channel in the frames.

    @param tab
        DataSummaryTab object to process
    @param cp
         configuration file object
    @param cpsec
        name of INI section configuring this Tab
    @param verbose
        print verbose output, default: False
    @param doplots
        generate plots, default: True
    @param dosubplots
        generate subplots, default: True
    @param htmlonly
        generate enough data for HTML only, default: False
    """
    if cp.has_option(cpsec, "channel"):
        channels = cp.get(cpsec, "channel").split(',')
    elif cp.has_option(cpsec, "channels"):
        channels = cp.get(cpsec, "channels").split(',')
    else:
        raise ValueError("No channels configured in [%s]." % cpsec)
    print_verbose("\n----------\nProcessing data for %s for the %s state...\n"\
                  % (tab.name, tab.state.name),\
                  verbose=verbose, profile=False)
    
    # skip adding to summary if requested
    if cp.has_option(cpsec, "skip-summary"):
        tab.skip_summary = cp.get(cpsec, "skip-summary")

    #
    # setup
    #

    if not os.path.isdir(tab.directory):
        os.makedirs(tab.directory)

    #
    # get plots
    #

    # get plot sections
    plots = [p for p in cp.items(cpsec) if p[0].startswith("plot-")]

    # if we got no plots, pick up the defaults
    if len(plots) == 0:
        plots = [("plot-timeseries", "0")]

    # sort plots by their given order
    plots.sort(key=lambda o: o[1].isdigit() and (int(o[1])+1) or 1000)

    dataplot        = any(re.search("timeseries", p[0], re.I) for p in plots)
    spectrumplot    = any(re.search("spectrum", p[0], re.I) for p in plots)
    spectrogramplot = any(re.search("spectrogram", p[0], re.I)\
                          for p in plots)

    #
    # load archived data
    #

    if cp.has_option(cpsec, "use-archive") \
    and (cp.get(cpsec, "use-archive").lower() in ["", "true"]):
        for channel in channels:
            if dataplot or spectrogramplot or spectrumplot:
                load_archive_data(cp, channel, tab.segments, mode=tab.mode,\
                                  verbose=verbose)
            if spectrogramplot or spectrumplot:
                load_archive_spectrogram(cp, channel, tab.segments,\
                                         mode=tab.mode, verbose=verbose)

    #
    # load data
    #

    if not htmlonly:

        channelparams = parse_data_config(cp, cpsec, channels)
        for channel, params in zip(channels, channelparams):
            if dataplot:
                serieslist = get_data(cp, channel, tab.segments,\
                                      frametype=params["frametype"],\
                                      calibration=params["calibration"],\
                                      resample=params["resample"])
                series = PyLALConcatenateTimeSeries(*serieslist)
                tab.add_timeseries(params["label"], series)
            if spectrogramplot:
                sequencelist =\
                    get_spectrogram(cp, channel, params["time-step"],\
                                    params["segment-length"],\
                                    params["segment-overlap"], tab.segments,\
                                    frametype=params["frametype"],\
                                    calibration=params["calibration"],\
                                    resample=params["resample"],\
                                    fresponse=params["frequency-response"])
                tab.add_spectrogram(params["label"], *sequencelist)
            if spectrumplot:
                spectrumlist =\
                    get_spectrum(cp, channel, params["time-step"],\
                                 params["segment-length"],\
                                 params["segment-overlap"], tab.segments,\
                                 frametype=params["frametype"],\
                                 calibration=params["calibration"],\
                                 resample=params["resample"],\
                                 fresponse=params["frequency-response"])
                for spectrum in spectrumlist:
                    try:
                        spectrum.name = params["label"]
                    except TypeError as e:
                        warnings.warn("Cannot set spectrum name to '%s'"
                                      % params['label'])
                tab.add_median_spectrum(params["label"], spectrumlist[0])
                tab.add_min_spectrum(params["label"], spectrumlist[1])
                tab.add_max_spectrum(params["label"], spectrumlist[2])
            if not dataplot and channel in G_DATA.keys():
                if len(G_DATA[channel].keys()):
                    sampling = 1 /\
                               G_DATA[channel][G_DATA[channel].keys()[0]].deltaT
                else:
                    sampling = "N/A"
                tab.sampling[params["label"]] = sampling

    #
    # load references
    #

    if not htmlonly and (spectrumplot or spectrogramplot):
        if cp.has_option(cpsec, "design-curve"):
            series = get_reference_spectrum(cp.get(cpsec, "design-curve"),\
                                            channels[0], name="Design")
            tab.add_design_spectrum(tab.channels[0], series)
        if cp.has_option(cpsec, "reference-curve"):
            if cp.has_option(cpsec, "reference-epoch"):
                gps_range = SegmentList([Segment(map(float,cp.get(cpsec,\
                                                         "reference-epoch").split(',')))])
                name = "Reference (%s)" % cp.get(cpsec,"reference-epoch")
                spectrumlist =\
                    get_spectrum(cp, channel, params["time-step"],\
                                     params["segment-length"],\
                                     params["segment-overlap"], gps_range,\
                                     frametype=params["frametype"],\
                                     calibration=params["calibration"],\
                                     resample=params["resample"],\
                                     fresponse=params["frequency-response"])
                series = spectrumlist[0]
            else:
                gps = lal.LIGOTimeGPS(0)
                name = "Reference"
                series = get_reference_spectrum(cp.get(cpsec, "design-curve"),\
                                                    channels[0], name=name, gps=gps)
            tab.add_reference_spectrum(tab.channels[0], series)

    #
    # plot
    #

    for plot,_ in plots:
        plotparams = parse_plot_section(cp, cpsec,\
                                        plotutils.parse_plot_config(cp, plot))
        # build a spectrogram plot for each channel
        if re.search('spectrogram', plot, re.I):
            ratioregex = re.compile("\w+-ratio", re.I)
            match = re.search(ratioregex, plot)
            ratio = match and match.group()[:-6] or None
            for channel in tab.channels:
                plottag = "%s_%s" % (re_cchar.sub('_', channel.upper()),
                                     re_cchar.sub("_", plot[5:].upper()))
                outfile = os.path.join(
                              tab.directory,
                              "%s-%s_%s_%s-%d-%d.png"\
                              % (ifo, re_cchar.sub("_", tab.name.upper()),\
                              tab.state.tag, plottag, tab.start_time,\
                                  tab.end_time-tab.start_time))
                if htmlonly or not doplots:
                    tab.plots.append((outfile,
                                      plotparams.pop('description', None)))
                else:
                    tab.plotspectrogram(outfile, ratio=ratio, channel=channel,
                                        **plotparams)
                    print_verbose("%s saved.\n" % outfile, verbose=verbose)
        # otherwise build one plot with all channels
        else:
            plottag = re_cchar.sub("_", plot[5:].upper())
            outfile = os.path.join(
                          tab.directory,
                          "%s-%s_%s_%s-%d-%d.png"
                              % (ifo, re_cchar.sub("_", tab.name.upper()),
                          tab.state.tag, plottag, tab.start_time,
                          tab.end_time-tab.start_time))
            if htmlonly or not doplots:
                tab.plots.append((outfile, plotparams.pop("description", None)))
            else:
                if re.search("hist", plot, re.I):
                    tab.plothistogram(outfile, **plotparams)
                elif re.search("spectrum", plot, re.I):
                    psd = re.search("(power|psd)", plot, re.I)
                    tab.plotspectrum(outfile, psd=psd, **plotparams)
                else:
                    tab.plottimeseries(outfile, **plotparams)
                print_verbose("%s saved.\n" % outfile, verbose=verbose)

    #
    # subplots
    #

    if cp.has_option(cpsec, "sub-plot"):
        subplot = cp.get(cpsec, "sub-plot")
        subs   = tab.start_time
        substart = tab.start_time
        deltas = subplot_durations(tab.start_time, tab.end_time, tab.mode)

        for dt in deltas:
            plottag = re_cchar.sub("_", subplot[5:].upper())
            basefile = "%s-%s_%s_%s-%d-%d.png"\
                       % (ifo, re_cchar.sub("_", tab.name.upper()),
                          tab.state.tag, plottag, substart, dt)
            plotparams = plotutils.parse_plot_config(cp, subplot)
            plotparams = parse_plot_section(cp, cpsec, plotparams)
            d = datetime.datetime(*lal.GPSToUTC(int(substart))[:6])
            # test if plot exists somewhere else
            if tab.mode == summary.SUMMARY_MODE_WEEK\
            or tab.mode == summary.SUMMARY_MODE_MONTH:
                outdir = os.path.join("archive_daily", d.strftime("%Y%m%d"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            elif tab.mode == summary.SUMMARY_MODE_YEAR:
                outdir = os.path.join("archive_monthly", d.strftime("%Y%m"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            else:
                outfile = os.path.join(tab.directory, basefile)

            # otherwise make it as normal
            if htmlonly or not dosubplots:
                tab.subplots.append((outfile, plotparams.pop("description",\
                                     None)))
            else:
                if re.search("hist", subplot, re.I):
                    tab.plothistogram(outfile, subplot=True, **plotparams)
                elif re.search("spectrum", subplot, re.I):
                    psd = re.search("(power|psd)", subplot, re.I)
                    tab.plotspectrum(outfile, subplot=True, psd=psd,\
                                     **plotparams)
                elif re.search("spectrogram", subplot, re.I):
                    ratioregex = re.compile("\w+-ratio", re.I)
                    match = re.search(ratioregex, subplot)
                    ratio = match and match.group()[:-6] or None
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plotspectrogram(outfile, subplot=True, ratio=ratio,\
                                        **plotparams)
                else:
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plottimeseries(outfile, subplot=True, **plotparams)
            substart += dt
            print_verbose(".",verbose=verbose)
        print_verbose("%d subplots saved.\n" % len(tab.subplots),\
                      verbose=verbose)

    if not htmlonly:
        tab.finalize()
        tab.frametohtml()


def get_reference_spectrum(descriptor, channel, name="Design", gps=0):
    """
    Load a design or reference spectrum from file or function.

    @param descriptor: reference descriptor, either filename to read,
        type of PSD recognised as a lalsimulation.SimNoisePSD... function,
        or a string to be evaluated as a lambda function.
    @param channel
        name of channel whose reference is being loaded
    @param name: name to give to returned array
    @param gps: GPS epoch for this reference

    @return: FrequencySeries containing reference/design spectrum
    @rtype:  C{lal.XXXFrequencySeries}
    """
    descriptor = str(descriptor)
    if os.path.isfile(descriptor):
        # load file
        f,S = numpy.loadtxt(descriptor, usecols=[0,1], unpack=True)
        f0 = f[0]
        deltaF = f[1]-f[0]
        spectrum = seriesutils.fromarray(S, name, lal.LIGOTimeGPS(gps),\
                                         f0, deltaF, lal.lalStrainUnit, True)
        spectrum.f_array = f
        return spectrum


    spec     = G_SPECTRUM[channel][0]
    if spec.f_array is None:
        f = numpy.arange(spec.data.length) * spec.deltaF + spec.f0
    else:
        f = spec.f_array.astype(float)
    datatype = seriesutils.typecode(type(spec))
    TYPESTR  = seriesutils._typestr[datatype]
    Create   = getattr(lal, "Create%sFrequencySeries" % TYPESTR)
    spectrum = Create(name, lal.LIGOTimeGPS(gps), spec.f0, spec.deltaF,\
                      spec.sampleUnits, f.size)
    spectrum.f_array = f

    # otherwise we can try to use a function
    if hasattr(lalsimulation, "SimNoisePSD%s" % descriptor):
        SimNoisePSD = getattr(lalsimulation, "SimNoisePSD%s" % descriptor)
        asd = lambda f: f and SimNoisePSD(f)**(1/2) or 0

    elif descriptor.startswith("lambda"):
        asd = eval(descriptor)

    for i,freq in enumerate(f):
        try:
            spectrum.data.data[i] = float(asd(freq))
        except OverflowError:
            print freq
            raise

    return spectrum

# =============================================================================
# Process range
# =============================================================================

def process_range(tab, cp, cpsec, verbose=False, doplots=True,\
                  dosubplots=True, htmlonly=False):
    """
    Calculate the range information from a data channel in the frames

    @param tab
        RangeSummaryTab object to process
    @param cp
         configuration file object
    @param cpsec
        name of INI section configuring this Tab
    @param verbose
        print verbose output, default: False
    @param doplots
        generate plots, default: True
    @param dosubplots
        generate subplots, default: True
    @param htmlonly
        generate enough data for HTML only, default: False
    """
    if cp.has_option(cpsec, "channel"):
        rangechannels = cp.get(cpsec, "channel").split(',')
    elif cp.has_option(cpsec, "channels"):
        rangechannels = cp.get(cpsec, "channels").split(',')
    else:
        rangechannels = []
    if cp.has_option(cpsec, "strain-channel"):
        strainchannel = cp.get(cpsec, "strain-channel")
    else:
        strainchannel = None
    if len(rangechannels) + int(strainchannel!=None) == 0:
        raise ValueError("No channels configured in [%s]." % cpsec)
    print_verbose("\n----------\nProcessing range for %s for the %s state...\n"\
                  % (tab.name, tab.state.name), verbose=verbose, profile=False)

    etg = cp.has_option(cpsec, "trigger-generator")\
              and cp.get(cpsec, "trigger-generator") or cpsec[9:]
    snr = cp.has_option(cpsec, "snr-threshold")\
          and cp.getfloat(cpsec, "snr-threshold") or 0
    tab.cluster = ((cp.has_option(cpsec, "cluster-window") and
                    cp.getfloat(cpsec, "cluster-window")) or
                   False)

    #
    # setup
    #

    if not os.path.isdir(tab.directory):
        os.makedirs(tab.directory)

    #
    # load range directly from frames
    #

    if rangechannels and not htmlonly:

        channelparams = parse_data_config(cp, cpsec, rangechannels)

        for channel,params in zip(rangechannels, channelparams):
            serieslist = get_data(cp, channel, tab.segments,\
                                  frametype=params["frametype"],\
                                  calibration=params["calibration"],\
                                  resample=params["resample"])
            series = PyLALConcatenateTimeSeries(*serieslist)
            tab.add_timeseries(params["label"], series)

    #
    # load archived data
    #

    params = parse_data_config(cp, cpsec, [strainchannel],\
                               prefix="strain")[0]
    sources = parse_range_config(cp, cpsec)
    numburst = sum([True for s in sources if s["type"] == "burst"])

    # FIXME: archive loading for range data is buggy
    #runsegs2 = tab.segments
    #if cp.has_option(cpsec, "use-archive") \
    #and (cp.get(cpsec, "use-archive").lower() in ["", "true"]):
    #    for source in sources:
    #        load_archive_data(cp, source["name"], tab.segments, mode=tab.mode,\
    #                         verbose=verbose)
    #        if source["name"] in G_DATA.keys():
    #            runsegs2 -= SegmentList(G_DATA[source["name"]].keys())

    #
    # calculate range from strain channel
    #

    if strainchannel and not htmlonly:
        print_verbose("Loaded range parameters for %d source(s):\n"\
                      % (len(sources)), verbose=verbose)
        for s in sources:
            print_verbose("%s (%s)\n" % (s["name"], s["type"]), verbose=verbose)
            tab.add_source(s)

        # get data
        sequencelist = get_spectrogram(cp, strainchannel, params["time-step"],
                                       params["segment-length"],
                                       params["segment-overlap"], tab.segments,
                                       frametype=params["frametype"],
                                       calibration=params["calibration"],
                                       resample=params["resample"],
                                       fresponse=params["frequency-response"])

        for i,spectrogram in enumerate(sequencelist):
            S = spectrogram['data']
            epoch = spectrogram['epoch']
            deltaT = spectrogram['deltaT']
            f0 = spectrogram['f0']
            deltaF = spectrogram['deltaF']
            f_array = spectrogram['f_array']
            print_verbose("Calculating range from %d spectra...    "\
                          % S.length, verbose=verbose)
            seg = Segment(float(epoch), float(epoch + deltaT * S.length))
            data = dict((s["name"], numpy.zeros(S.length)) for s in sources)

            # get triggers for burst range
            if numburst:
                trigs = get_triggers(cp, strainchannel, etg,\
                                     SegmentList([seg]) & tab.trigsegments,\
                                     cluster=tab.cluster, minsnr=snr)
                trigtime = numpy.asarray(trigs.getColumnByName("peak_time")) + \
                       numpy.asarray(trigs.getColumnByName("peak_time_ns"))*1e-9
                trigsnr  = trigs.getColumnByName("snr")
                trigfreq = trigs.getColumnByName("peak_frequency")

            # initialize triggers iterators to select slices of them (time sub sets)
            trigStart = {}
            trigStop = {}
            for k,s in enumerate(sources):
                trigStart[k] = 0
                trigStop[k] = 0

            for j in range(S.length):
                spectrum = S.data[j,:]**2
                if f_array is None:
                    f = numpy.arange(spectrum.size) * deltaF + f0
                else:
                    f = f_array
                span = Segment(seg[0]+j*deltaT, seg[0]+(j+1)*deltaT)

                # loop over sources
                for k,s in enumerate(sources):
                    # calculate inspiral range
                    if s["type"] == "inspiral":
                        data[s["name"]][j] =\
                            rangeutils.inspiralrange(f, spectrum,\
                                                     snr=s["snr"], m1=s["m1"],\
                                                     m2=s["m2"])
                    # calculate burst range
                    elif s["type"] == "burst":
                        sspan = Segment(span[1]-s['dt'], span[1])
                        while trigStop[k]<len(trigs)\
                              and trigtime[trigStop[k]] < sspan[1]:
                            trigStop[k] = trigStop[k] + 1
                        while trigStart[k]<len(trigs)\
                              and trigtime[trigStart[k]] < sspan[0]:
                            trigStart[k] = trigStart[k] + 1
                        snrs = [trigsnr[itrig] for itrig in numpy.arange(trigStart[k],trigStop[k])\
                                    if s['fmin'] <= trigfreq[itrig] < s['fmax']\
                                    and trigsnr[itrig] > s['snr']]
                        rho = get_limiting_snr(snrs, s["far"], abs(SegmentList([sspan])&tab.segments),\
                                               s["snr"])
                        data[s["name"]][j] =\
                            rangeutils.burstrange(f, spectrum, rho,\
                                                  s["energy"], fmin=s["fmin"],\
                                                  fmax=s["fmax"], unit=s["unit"])

                print_verbose("\b\b\b%.2d%%" % int((j+1)/S.length*100),\
                              verbose=verbose)
            print_verbose("\n", verbose=verbose)

            # store data
            for s in sources:
                series = seriesutils.fromarray(data[s["name"]], name=s["name"],\
                                               epoch=epoch, deltaT=deltaT,\
                                               f0=f0)
                add_timeseries(series)

        for s in sources:
            serieslist = get_data(cp, s["name"], tab.segments, query=False)
            series = PyLALConcatenateTimeSeries(*serieslist, name=s["name"],\
                                          epoch=lal.LIGOTimeGPS(tab.start_time))
            tab.add_timeseries(s["name"], series)

        #
        # calculate frequency dependent burst range
        #

        # FIXME

    #
    # plot
    #

    # get plot sections
    plots = [p for p in cp.items(cpsec) if p[0].startswith("plot-")]

    # if we got no plots, pick up the defaults
    if len(plots) == 0:
        plots = [("plot-timeseries", "0")]

    # sort plots by their given order
    plots.sort(key=lambda o: o[1].isdigit() and (int(o[1])+1) or 1000)

    # sort plots by their given order
    for plot,_ in plots:
        plottag = re_cchar.sub("_", plot[5:].upper())
        outfile = os.path.join(tab.directory, "%s-%s_%s_%s-%d-%d.png"\
                                     % (tab.ifo,\
                                         re_cchar.sub("_", tab.name.upper()),\
                                        tab.state.tag, plottag,\
                                        tab.start_time, abs(tab.span)))
        plotparams = parse_plot_section(cp, cpsec,\
                                        plotutils.parse_plot_config(cp, plot))
        if htmlonly or not doplots:
            tab.plots.append((outfile, plotparams.pop("description", None)))
        else:
            if re.search("hist", plot, re.I):
                tab.plothistogram(outfile, **plotparams)
            elif re.search("spectrum", plot, re.I):
                psd = re.search("(power|psd)", plot, re.I)
                tab.plotspectrum(outfile, psd=psd, **plotparams)
            elif re.search("spectrogram", plot, re.I):
                ratioregex = re.compile("\w+-ratio", re.I)
                match = re.search(ratioregex, plot)
                ratio = match and match.group()[:-6] or None
                tab.plotspectrogram(outfile, ratio=ratio, **plotparams)
            else:
                tab.plottimeseries(outfile, **plotparams)
            print_verbose("%s saved.\n" % outfile, verbose=verbose)

    #
    # subplots
    #

    if cp.has_option(cpsec, "sub-plot"):
        subplot = cp.get(cpsec, "sub-plot")
        subs   = tab.start_time
        substart = tab.start_time
        deltas = subplot_durations(tab.start_time, tab.end_time, tab.mode)

        for dt in deltas:
            plottag = re_cchar.sub("_", subplot[5:].upper())
            basefile = "%s-%s_%s_%s-%d-%d.png"\
                       % (tab.ifo, re_cchar.sub("_", tab.name.upper()),\
                          tab.state.tag, plottag, substart, dt)
            plotparams = plotutils.parse_plot_config(cp, subplot)
            plotparams = parse_plot_section(cp, cpsec, plotparams)
            d = datetime.datetime(*lal.GPSToUTC(int(substart))[:6])
            # test if plot exists somewhere else
            if tab.mode == summary.SUMMARY_MODE_WEEK\
            or tab.mode == summary.SUMMARY_MODE_MONTH:
                outdir = os.path.join("archive_daily", d.strftime("%Y%m%d"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            elif tab.mode == summary.SUMMARY_MODE_YEAR:
                outdir = os.path.join("archive_monthly", d.strftime("%Y%m"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            else:
                outfile = os.path.join(tab.directory, basefile)

            # otherwise make it as normal
            if htmlonly or not dosubplots:
                tab.subplots.append((outfile, plotparams.pop("description", None)))
            else:
                if re.search("hist", subplot, re.I):
                    tab.plothistogram(outfile, subplot=True, **plotparams)
                elif re.search("spectrum", subplot, re.I):
                    psd = re.search("(power|psd)", subplot, re.I)
                    tab.plotspectrum(outfile, subplot=True, psd=psd, **plotparams)
                elif re.search("spectrogram", subplot, re.I):
                    ratioregex = re.compile("\w+-ratio", re.I)
                    match = re.search(ratioregex, subplot)
                    ratio = match and match.group()[:-6] or None
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plotspectrogram(outfile, subplot=True, ratio=ratio,\
                                        **plotparams)
                else:
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plottimeseries(outfile, subplot=True, **plotparams)
            substart += dt
        print_verbose("%d subplots saved.\n" % len(tab.subplots), verbose=verbose)

    if not htmlonly:
        tab.finalize()
        tab.frametohtml()

def parse_range_config(cp, cpsec, typelist=None):
    """
    Parse the given configparser.configparser section cpsec for range
    source definitions, and load their parameters.

    @param cp
         configuration file object
    @param cpsec
        name of INI section configuring this RangeSummaryTab.
    @param typelist
        list of source types to load (OPTIONAL)

    @return: list of parameter dicts for each range source
    @rtype: C{list} of C{dict}s.
    """
    if cp.has_option(cpsec, "sources"):
        sections = ["source-%s"%s for s in cp.get(cpsec, "sources").split(',')]
    else:
        sections = [s for s in cp.sections() if s.startswith("source-")]

    out = []

    for cpsec in sections:
        source = dict()
        if cp.has_option(cpsec, "name"):
            source["name"] = cp.get(cpsec, "name")
        else:
            source["name"] = cpsec[7:]
        source["type"] = type = cp.get(cpsec, "type").lower()
        if typelist:
            if type not in typelist: continue
        if cp.has_option(cpsec, "unit"):
            source["unit"] = cp.get(cpsec, "unit")
        if type == "inspiral":
            options = ["m1", "m2", "snr"]
        elif type == "burst":
            options = ["fmin", "fmax", "far", "dt", "energy", "snr"]
        elif type == "squeezing":
            options = ["fmin", "fmax"]
        for opt in options:
             source[opt] = cp.getfloat(cpsec, opt)
        out.append(source)

    return out

def get_limiting_snr(snr, far, dt, threshold=6):
    """
    Calculate limiting SNR for the given false-alarm rate threshold.

    @param snr: array of SNR values from Burst trigger table
    @param far: false-alarm rate threshold
    @param dt: duration of search (seconds)
    @param threshold: minimum threshold on SNR

    @return: limiting SNR giving required false-alarm rate
    @rtype: C{float}
    """
    snr.sort()
    snr = snr[::-1]
    faridx = int(far*dt)-1
    if len(snr) <= faridx:
        # if we don't have enough triggers use the trigger
        # reading snr threshold, it will give us a lower limit
        # on the range
        rho = threshold
    elif faridx >= 0:
        rho = snr[faridx]
    else:
        rho = numpy.infty

    return rho


# =============================================================================
# Process squeezing
# =============================================================================

def process_squeezing(tab, cp, cpsec, verbose=False, doplots=True,\
                          dosubplots=True, htmlonly=False):
    """
    Calculate the range information from a data channel in the frames

    @param tab
        RangeSummaryTab object to process
    @param cp
         configuration file object
    @param cpsec
        name of INI section configuring this Tab
    @param verbose
        print verbose output, default: False
    @param doplots
        generate plots, default: True
    @param dosubplots
        generate subplots, default: True
    @param htmlonly
        generate enough data for HTML only, default: False
    """
    if cp.has_option(cpsec, "channel"):
        rangechannels = cp.get(cpsec, "channel").split(',')
    elif cp.has_option(cpsec, "channels"):
        rangechannels = cp.get(cpsec, "channels").split(',')
    else:
        rangechannels = []
    if len(rangechannels)  != 4:
        raise ValueError("Wrong number of channels in [%s]." % cpsec)
    print_verbose("\n----------\nProcessing range for %s for the %s state...\n"\
                  % (tab.name, tab.state.name), verbose=verbose, profile=False)


    #
    # setup
    #

    if not os.path.isdir(tab.directory):
        os.makedirs(tab.directory)


    #
    # load archived data
    #

    params = parse_data_config(cp, cpsec, rangechannels)
    sources = parse_range_config(cp, cpsec, typelist=["squeezing"])

    #
    # calculate squeezing from data
    #

    print_verbose("Loaded range parameters for %d source(s):\n"\
                  % (len(sources)), verbose=verbose)
    for s in sources:
        print_verbose("%s (%s)\n" % (s["name"], s["type"]), verbose=verbose)
        tab.add_source(s)

    # get data
    sequencelist = get_spectrogram(cp, rangechannels[0], params[0]["time-step"],
                                   params[0]["segment-length"],
                                   params[0]["segment-overlap"], tab.segments,
                                   frametype=params[0]["frametype"],
                                   calibration=params[0]["calibration"],
                                   resample=params[0]["resample"],
                                   fresponse=params[0]["frequency-response"])

    serieslist = []
    for i in numpy.arange(1,4):
        serieslist.append(get_data(cp, rangechannels[i], tab.segments, frametype=params[i]["frametype"],
                                   calibration=params[i]['calibration']))


    for i,spectrogram in enumerate(sequencelist):
        S = spectrogram['data']
        epoch = spectrogram['epoch']
        deltaT = spectrogram['deltaT']
        f0 = spectrogram['f0']
        deltaF = spectrogram['deltaF']
        f_array = spectrogram['f_array']
        print_verbose("Calculating range from %d spectra...    "\
                      % S.length, verbose=verbose)
        seg = Segment(float(epoch), float(epoch + deltaT * S.length))
        data = dict((s["name"], numpy.zeros(S.length)) for s in sources)


        for j in range(S.length):
            spectrum = S.data[j,:]**2
            if f_array is None:
                f = numpy.arange(spectrum.size) * deltaF + f0
            else:
                f = f_array
            span = Segment(seg[0]+j*deltaT, seg[0]+(j+1)*deltaT)

            # compute squeezing parameters for this step
            mediandata = []
            for iParam in range(len(serieslist)):
                if i < len(serieslist[iParam]):
                    tParam = (numpy.arange(serieslist[iParam][i].data.length) * serieslist[iParam][i].deltaT +
                              float(serieslist[iParam][i].epoch))
                    mediandata.append(numpy.median(serieslist[iParam][i].data.data[(tParam>=seg[0]) & (tParam<seg[1])]))
                else:
                    mediandata.append(numpy.nan)

            # loop over sources
            for k,s in enumerate(sources):
                if s["type"] == "squeezing":
                    data[s["name"]][j] =\
                        rangeutils.squeezing(f, spectrum,\
                                                 dc=mediandata[0],\
                                                 opgain=mediandata[1],\
                                                 cavpf=mediandata[2],\
                                                 fmin=s["fmin"],\
                                                 fmax=s["fmax"])

            print_verbose("\b\b\b%.2d%%" % int((j+1)/S.length*100),\
                          verbose=verbose)
        print_verbose("\n", verbose=verbose)

        # store data
        for s in sources:
            series = seriesutils.fromarray(data[s["name"]], name=s["name"],\
                                           epoch=epoch, deltaT=deltaT,\
                                           f0=f0)
            add_timeseries(series)

    for s in sources:
        serieslist = get_data(cp, s["name"], tab.segments, query=False)
        series = PyLALConcatenateTimeSeries(*serieslist, name=s["name"],\
                                      epoch=lal.LIGOTimeGPS(tab.start_time))
        tab.add_timeseries(s["name"], series)


    #
    # plot
    #

    # get plot sections
    plots = [p for p in cp.items(cpsec) if p[0].startswith("plot-")]

    # if we got no plots, pick up the defaults
    if len(plots) == 0:
        plots = [("plot-timeseries", "0")]

    # sort plots by their given order
    plots.sort(key=lambda o: o[1].isdigit() and (int(o[1])+1) or 1000)

    # sort plots by their given order
    for plot,_ in plots:
        plottag = re_cchar.sub("_", plot[5:].upper())
        outfile = os.path.join(tab.directory, "%s-%s_%s_%s-%d-%d.png"\
                                     % (tab.ifo,\
                                         re_cchar.sub("_", tab.name.upper()),\
                                        tab.state.tag, plottag,\
                                        tab.start_time, abs(tab.span)))
        plotparams = parse_plot_section(cp, cpsec,\
                                        plotutils.parse_plot_config(cp, plot))
        if htmlonly or not doplots:
            tab.plots.append((outfile, plotparams.pop("description", None)))
        else:
            if re.search("hist", plot, re.I):
                tab.plothistogram(outfile, **plotparams)
            elif re.search("spectrum", plot, re.I):
                psd = re.search("(power|psd)", plot, re.I)
                tab.plotspectrum(outfile, psd=psd, **plotparams)
            elif re.search("spectrogram", plot, re.I):
                ratioregex = re.compile("\w+-ratio", re.I)
                match = re.search(ratioregex, plot)
                ratio = match and match.group()[:-6] or None
                tab.plotspectrogram(outfile, ratio=ratio, **plotparams)
            else:
                tab.plottimeseries(outfile, **plotparams)
            print_verbose("%s saved.\n" % outfile, verbose=verbose)

    #
    # subplots
    #

    if cp.has_option(cpsec, "sub-plot"):
        subplot = cp.get(cpsec, "sub-plot")
        subs   = tab.start_time
        substart = tab.start_time
        deltas = subplot_durations(tab.start_time, tab.end_time, tab.mode)

        for dt in deltas:
            plottag = re_cchar.sub("_", subplot[5:].upper())
            basefile = "%s-%s_%s_%s-%d-%d.png"\
                       % (tab.ifo, re_cchar.sub("_", tab.name.upper()),\
                          tab.state.tag, plottag, substart, dt)
            plotparams = plotutils.parse_plot_config(cp, subplot)
            plotparams = parse_plot_section(cp, cpsec, plotparams)
            d = datetime.datetime(*lal.GPSToUTC(int(substart))[:6])
            # test if plot exists somewhere else
            if tab.mode == summary.SUMMARY_MODE_WEEK\
            or tab.mode == summary.SUMMARY_MODE_MONTH:
                outdir = os.path.join("archive_daily", d.strftime("%Y%m%d"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            elif tab.mode == summary.SUMMARY_MODE_YEAR:
                outdir = os.path.join("archive_monthly", d.strftime("%Y%m"),\
                                      *tab.directory.split(os.path.sep)[2:])
                outfile = os.path.join(outdir, basefile)
                if os.path.isfile(outfile):
                    substart += dt
                    tab.subplots.append((outfile,\
                                         plotparams.pop("description", None)))
                    continue
                else:
                    outfile = os.path.join(tab.directory, basefile)
            else:
                outfile = os.path.join(tab.directory, basefile)

            # otherwise make it as normal
            if htmlonly or not dosubplots:
                tab.subplots.append((outfile, plotparams.pop("description", None)))
            else:
                if re.search("hist", subplot, re.I):
                    tab.plothistogram(outfile, subplot=True, **plotparams)
                elif re.search("spectrum", subplot, re.I):
                    psd = re.search("(power|psd)", subplot, re.I)
                    tab.plotspectrum(outfile, subplot=True, psd=psd, **plotparams)
                elif re.search("spectrogram", subplot, re.I):
                    ratioregex = re.compile("\w+-ratio", re.I)
                    match = re.search(ratioregex, subplot)
                    ratio = match and match.group()[:-6] or None
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plotspectrogram(outfile, subplot=True, ratio=ratio,\
                                        **plotparams)
                else:
                    plotparams.setdefault("xlim", [substart, substart+dt])
                    tab.plottimeseries(outfile, subplot=True, **plotparams)
            substart += dt
        print_verbose("%d subplots saved.\n" % len(tab.subplots), verbose=verbose)

    if not htmlonly:
        tab.finalize()
        tab.frametohtml()


# =============================================================================
# Parse command line
# =============================================================================

def parse_command_line():
    """
    Parser command line arguments, and perform sanity checks.
    """

    #
    # set parser and add default options
    #

    prog   = os.path.basename(sys.argv[0])
    usage  = "%s --config-file ${CONFIG_FILE} --day ${YYYYMMDD} --ifo ${IFO}"\
             "[OPTIONS]" % prog
    epilog = "If you're having trouble, e-mail detchar+code@ligo.org. "+\
             "To report a bug, please visit "+\
             "https://bugs.ligo.org/redmine/projects/detchar and submit "+\
             "an Issue."
    parser = optparse.OptionParser(usage=usage, description=__doc__[1:],\
                                   formatter=optparse.IndentedHelpFormatter(4),\
                                   epilog=epilog)
    parser.add_option("-p", "--profile", action="store_true", default=False,\
                      help="show second timer with verbose statements, "+\
                           "default: %default")
    parser.add_option("-v", "--verbose", action="store_true", default=False,\
                      help="show verbose output, default: %default")
    parser.add_option("-V", "--version", action="version",\
                      help="show program's version number and exit")
    parser.version = __version__

    #
    # set time options
    #

    topts = optparse.OptionGroup(parser, "Time mode options",\
                                 description="Choose to run on a day, month"\
                                             ", or GPS start/stop period.")
    topts.add_option("--day", action="store", type="string",\
                     metavar="YYYYMMDD", help="day to process.")
    topts.add_option("--week", action="store", type="string",\
                     metavar="YYYYMMDD",\
                     help="week to process (by starting day).")
    topts.add_option("--month", action="store", type="string",\
                     metavar="YYYYMM", help="month to process.")
    topts.add_option("--year", action="store", type="string",\
                     metavar="YYYY", help="year to process.")
    topts.add_option("-s", "--gps-start-time", action="store", type="int",\
                     metavar="GPSSTART", help="GPS start time")
    topts.add_option("-e", "--gps-end-time", action="store", type="int",\
                     metavar="GPSEND", help="GPS end time")

    #
    # required options
    #

    ropts = optparse.OptionGroup(parser, "Required options",\
                                  description="All options in this section "+\
                                              "are required.")
    ropts.add_option("-i", "--ifo", action="store", default=None,\
                     metavar="IFO", help="interferometer")
    ropts.add_option("-f", "--config-file", action="store", type="string",\
                     metavar="FILE",
                     help="ini file for analysis, default: %default")

    #
    # data access options
    #

    dopts = optparse.OptionGroup(parser, "Data access options",\
                                 description="Specify cache files and NDS "+\
                                             "preferences.")
    dopts.add_option("-c", "--trigger-cache", action="store", type="string",\
                     help="LAL cache for trigger files. Files are matched "+\
                          "by the description parameter in the cache "+\
                          "file which must be of the form "+\
                          "CHANNEL_NAME_ETG, e.g. DER_DATA_H_OMEGA")
    dopts.add_option("-d", "--data-cache", action="store", type="string",\
                     help="LAL cache for data frames. Files are matched "+\
                          "by the description parameter in the cache file, "+\
                          "which must include only the relevant frame "+\
                          "type, e.g. G1_RDS_C01_L3")
    dopts.add_option("-a", "--segment-cache", action="store", type="string",\
                     help="LAL cache for segment files. Files are matched "+\
                          "by the description parameter in the cache "+\
                          "file, which must be underscore delimited of "+\
                          "the form FLAG_NAME, e.g. GEO_SCIENCE")
    dopts.add_option("--nds", action="store_true", default=False,\
                     help="use NDS for frame data acquisition, "+\
                          "default: %default")

    #
    # output options
    #

    oopts = optparse.OptionGroup(parser, "Output options",\
                                 description="Specify output directory and "+\
                                             "format preferences here.")
    oopts.add_option("-o", "--output-dir", action="store", type="string",\
                     default=os.getcwd(),\
                     help="output directory, default: %default")
    oopts.add_option("-x", "--write-triggers", action="store_true",\
                     default=False,\
                     help="write the triggers used to XML, default: %default")
    oopts.add_option("-y", "--write-segments", action="store_true",\
                     default=False,\
                     help="write the segments used to segwizard files, "+\
                          "default: %default")
    oopts.add_option("-z", "--write-data", action="store_true",\
                     default=False,\
                     help="write the data generated to gwf files, "+\
                          "default: %default")
    oopts.add_option("-M", "--html-only", action="store_true", default=False,\
                     help="rerun HTML generation only, default: %default")


    #
    # section options
    #

    sopts = optparse.OptionGroup(parser, "Section options",\
                                 description="Choose to skip various "\
                                             "components of the summary "\
                                             "output.")
    # skip plots
    sopts.add_option("--skip-all-plots", action="store_false",\
                     default=True, dest="run_plots",\
                     help="skip plotting, default: False")
    sopts.add_option("--skip-sub-plots", action="store_false",\
                     default=True, dest="run_sub_plots",\
                     help="skip sub-interval plots, default: False")

    # skip sections
    sopts.add_option("-S", "--skip-segments", action="store_false",\
                     default=True, dest="run_segments",\
                     help="skip segment summary, default: False")
    sopts.add_option("-R", "--skip-range", action="store_false",\
                     default=True, dest="run_range",\
                     help="skip range summary, default: False")
    sopts.add_option("-Q", "--skip-spectrum", action="store_false",\
                     default=True, dest="run_spectrum",\
                     help="skip spectrum summary, default: False")
    sopts.add_option("-D", "--skip-state-vector", action="store_false",\
                     default=True, dest="run_state_vector",\
                     help="skip state vector summary, default: False")
    sopts.add_option("-U", "--skip-squeezing", action="store_false",\
                     default=True, dest="run_squeezing",\
                     help="skip squeezing summary, default: False")
    sopts.add_option("-T", "--skip-triggers", action="store_false",\
                     default=True, dest="run_triggers",\
                     help="skip GW trigger summary, default: False")
    sopts.add_option("-A", "--skip-auxiliary-triggers", action="store_false",\
                     default=True, dest="run_auxiliary_triggers",\
                     help="skip auxiliary channel trigger summary, "+\
                          "default: False")
    sopts.add_option("-Z", "--skip-daily-omega-scans", action="store_false",\
                     default=True, dest="run_omega_scans",\
                     help="skip daily omega scan summary: default: False")
    sopts.add_option("-B", "--skip-veto-stats", action="store_false",\
                     default=True, dest="run_vetoes",\
                     help="skip all vetoes, default: False")
    sopts.add_option("-H", "--skip-hveto", action="store_false", default=True,\
                     dest="run_hveto", help="skip HVeto, default: False")
    sopts.add_option("-E", "--skip-data", action="store_false", default=True,\
                     dest="run_data", help="skip frame data, default: False")


    #
    # collect groups, parse, and return
    #

    parser.add_option_group(topts)
    parser.add_option_group(ropts)
    parser.add_option_group(dopts)
    parser.add_option_group(oopts)
    parser.add_option_group(sopts)

    # get args and begin sanity check
    opts, args = parser.parse_args()

    # format options for ligolw output
    param_dict = dict()
    for key,val in opts.__dict__.items():
        if key.startswith("run_"):
            param_dict["skip_%s" % key.split("_", 1)[1]] = not val
        else:
            param_dict[key] = val

    global PROFILE
    PROFILE = opts.profile
    global VERBOSE
    VERBOSE = opts.verbose

    # assert all required options
    _required_options = ["ifo", "config_file"]
    for opt in _required_options:
        if not getattr(opts, opt):
            raise optparse.OptionValueError("--%s is a required option"\
                                            % re.sub("_", "-", opt))

    # check ifo
    _prefixlist = [d.frDetector.prefix for d in lal.lalCachedDetectors]+["C1"]
    if opts.ifo.upper() not in _prefixlist:
        raise optparse.OptionValueError("--ifo=%s not recognised." % opts.ifo)
    else:
        opts.ifo = opts.ifo.upper()

    # check config file
    if not os.path.isfile(opts.config_file):
        raise optparse.OptionValueError("Cannot find %s" % opts.config_file)
    # read configuration file
    config_file = opts.config_file
    opts.config_file = ConfigParser.ConfigParser()
    opts.config_file.optionxform = str
    opts.config_file.read(config_file)
    opts.config_file.filename = os.path.abspath(config_file)

    # auto-disable based on configuration
    if len([s for s in opts.config_file.sections()\
                            if re.match("hveto[- ]", s)]) == 0:
        opts.run_hveto = False
        sys.stderr.write("Auto-applying --skip-hveto based on configuration.\n")
        sys.stderr.flush()
    if not opts.config_file.has_section("daily-omega-scans"):
        opts.run_omega_scans = False
        sys.stderr.write("Auto-applying --skip-daily-omega-scans based on"+\
                         " configuration.\n")
        sys.stderr.flush()

    # check time options
    N = sum([opts.day != None, opts.month != None, opts.gps_start_time != None,\
             opts.gps_end_time != None])
    if N > 1 and not (opts.gps_start_time and opts.gps_end_time):
        raise optparse.OptionValueError("Please give only one of --day, "+\
                                        "--month, or --gps-start-time and "+\
                                        "--gps-end-time.")
    if opts.day:
        if len(opts.day) == 8:
            opts.day = datetime.datetime.strptime(opts.day, "%Y%m%d")
            opts.gps_start_time = lal.UTCToGPS(opts.day.timetuple())
            opts.gps_end_time   = lal.UTCToGPS((opts.day +\
                                      datetime.timedelta(days=1)).timetuple())
        else:
            raise optparse.OptionValueError("--day malformed. Please format "+\
                                            "as YYYYMMDD")
    elif opts.week:
        if len(opts.week) == 8:
            week = opts.week
            opts.week = datetime.datetime.strptime(opts.week, "%Y%m%d")
            if opts.config_file.has_option("calendar", "start-of-week"):
                weekday = getattr(calendar,\
                                  opts.config_file.get("calendar",\
                                                      "start-of-week").upper())
                if weekday != opts.week.timetuple().tm_wday:
                    msg = "Cannot process week starting on %s. The "\
                          "'start-of-week' option in the [calendar] section "\
                          "of the INI file specifies weeks start on %ss."\
                          % (week,\
                             opts.config_file.get("calendar", "start-of-week"))
                    raise optparse.OptionValueError(msg)
            opts.gps_start_time = lal.UTCToGPS(opts.week.timetuple())
            opts.gps_end_time   = lal.UTCToGPS((opts.week +\
                                        datetime.timedelta(days=7)).timetuple())
        else:
            raise optparse.OptionValueError("--week malformed. Please format"+\
                                            " as YYYYMMDD")
    elif opts.month:
        if len(opts.month) == 6:
            opts.month = datetime.datetime.strptime(opts.month, "%Y%m")
            opts.gps_start_time = lal.UTCToGPS(opts.month.timetuple())
            opts.gps_end_time   = lal.UTCToGPS((opts.month +\
                                           relativedelta(months=1)).timetuple())
        else:
            raise optparse.OptionValueError("--month malformed. Please format"+\
                                            " as YYYYMM")
    elif opts.year:
        if len(opts.year) == 4:
            opts.year = datetime.datetime.strptime(opts.year, "%Y")
            opts.gps_start_time = lal.UTCToGPS(opts.year.timetuple())
            opts.gps_end_time = lal.UTCToGPS((opts.year +\
                                              relativedelta(years=1))
                                                  .timetuple())
        else:
            raise optparse.OptionValueError("--year malformed. Please format"+\
                                            " as YYYY")

    elif opts.gps_start_time or opts.gps_end_time:
        if not (opts.gps_start_time and opts.gps_end_time):
            raise optparse.OptionValueError("Please give both --gps-start-"+\
                                            "time and --gps-end-time.")
    else:
        opts.day = datetime.date(*datetime.datetime.utcnow().timetuple()[:3])
        opts.gps_start_time = int(lal.UTCToGPS(opts.day.timetuple()))
        opts.gps_end_time   = int(lal.UTCToGPS((opts.day +\
                                      datetime.timedelta(days=1)).timetuple()))
    opts.gps_start_time = float(opts.gps_start_time)
    opts.gps_end_time = float(opts.gps_end_time)

    span  = Segment(opts.gps_start_time, opts.gps_end_time)

    print_verbose("Command line read.\n", opts.verbose)

    # check data access options
    if opts.data_cache and opts.nds:
        raise optparse.OptionConflictError("--data-cache and --nds are "+\
                                           "exclusive options.")
    elif opts.data_cache and not opts.html_only:
        if not os.path.isfile(opts.data_cache):
            raise optparse.OptionValueError("Cannot find %s."\
                                            % opts.data_cache)
        opts.data_cache = os.path.abspath(opts.data_cache)
        cache = Cache.fromfile(open(opts.data_cache, "r"))
        cache = cache.sieve(segment=span)
        if not opts.config_file.has_section("datafind"):
            opts.config_file.add_section("datafind")
        opts.config_file.set("datafind", "data-cache", cache)
    if opts.trigger_cache and not opts.html_only:
        if not os.path.isfile(opts.trigger_cache):
            raise optparse.OptionValueError("Cannot find %s."\
                                            % opts.trigger_cache)
        opts.trigger_cache = os.path.abspath(opts.trigger_cache)
        cache = Cache.fromfile(open(opts.trigger_cache, "r"))
        cache = cache.sieve(segment=span)
        if not opts.config_file.has_section("trigfind"):
            opts.config_file.add_section("trigfind")
        opts.config_file.set("trigfind", "trigger-cache", cache)
    if opts.segment_cache and not opts.html_only:
        if not os.path.isfile(opts.segment_cache):
            raise optparse.OptionValueError("Cannot find %s."\
                                            % opts.segment_cache)
        opts.segment_cache = os.path.abspath(opts.segment_cache)
        cache = Cache.fromfile(open(opts.segment_cache, "r"))
        cache = cache.sieve(segment=span)
        if not opts.config_file.has_section("trigfind"):
            opts.config_file.add_section("trigfind")
        opts.config_file.set("segfind", "segment-cache", cache)

    if not opts.html_only\
    and (opts.data_cache or opts.trigger_cache or opts.segment_cache):
        print_verbose("All cache files read.\n", opts.verbose)

    # check output options
    opts.output_dir = os.path.abspath(opts.output_dir)

    return opts, args, param_dict

# =============================================================================
# Run main function if command line
# =============================================================================

if __name__=="__main__":
    start_job_timer()

    #
    # parse command line and generate some helper variables
    #

    # get input options
    opts, args, param_dict = parse_command_line()

    # extract to useful variable names
    run_opts = dict((opt[4:], getattr(opts, opt)) for opt in opts.__dict__\
                    if opt.startswith("run_"))
    cp    = opts.config_file
    outdir = opts.output_dir
    start_time = opts.gps_start_time
    end_time   = opts.gps_end_time
    ifo        = opts.ifo
    verbose    = opts.verbose

    span = Segment(start_time, end_time)

    print_verbose("-----------------------------------------\n"\
                  "Welcome to the IFO summary page generator\n"\
                  "-----------------------------------------\n",\
                  verbose=opts.verbose, profile=False)

    # get now time
    global NOW
    NOW = float(LIGOTimeGPS(min(end_time, int(lal.GPSTimeNow())))) # FIXME

    # set mode
    if opts.day:
        mode = summary.SUMMARY_MODE_DAY
    elif opts.week:
        mode = summary.SUMMARY_MODE_WEEK
    elif opts.month:
        mode = summary.SUMMARY_MODE_MONTH
    elif opts.year:
        mode = summary.SUMMARY_MODE_YEAR
    else:
        mode = summary.SUMMARY_MODE_GPS
    print_verbose("You have chosen %s mode. The GPS interval is\n%d-%d\n"\
                  % (summary.MODE_NAME[mode], int(start_time), int(end_time)),\
                  verbose=opts.verbose, profile=False)
    summary.SummaryTab.mode = mode

    summary.SummaryTab.ifo = ifo

    #
    # setup shared resources
    #

    # shared segment resources
    global G_SEGMENTS, G_SEGMENT_SUMMARY
    G_SEGMENTS        = SegmentListDict()
    G_SEGMENT_SUMMARY = SegmentListDict()

    # shared trigger resources
    global G_TRIGGERS, G_CLUSTERED_TRIGGERS
    G_TRIGGERS                = dict()
    G_CLUSTERED_TRIGGERS      = dict()

    # shared data resoures
    global G_DATA, G_SPECTRUM, G_SPECTROGRAM
    G_DATA                 = dict()
    G_SPECTRUM             = dict()
    G_SPECTROGRAM          = dict()

    #
    # set directories
    #

    # move to output directory
    if not os.path.isdir(outdir):
        os.makedirs(outdir)
    os.chdir(outdir)

    # set HTML parent directory
    day = datetime.datetime(*lal.GPSToUTC(int(start_time))[:6])
    if mode == summary.SUMMARY_MODE_GPS:
        gpsdir = "%d-%d" % (start_time, end_time)
        if not os.path.isdir(gpsdir):
            os.mkdir(gpsdir)
        os.chdir(gpsdir)
        jobdir = os.curdir
    elif mode == summary.SUMMARY_MODE_DAY:
        jobdir = os.path.join("archive_daily",
                              "%d%.2d%.2d" % (day.year, day.month, day.day))
    elif mode == summary.SUMMARY_MODE_WEEK:
        jobdir = os.path.join("archive_weekly",
                              "%d%.2d%.2d" % (day.year, day.month, day.day))
    elif mode == summary.SUMMARY_MODE_MONTH:
        jobdir = os.path.join("archive_monthly",
                              "%d%.2d" % (day.year, day.month))
    elif mode == summary.SUMMARY_MODE_YEAR:
        jobdir = os.path.join("archive_yearly", str(day.year))
    else:
        raise ValueError("Invalid summary mode, ask Duncan.")

    if not os.path.isdir(jobdir):
        os.makedirs(jobdir)

    #
    # load run states
    #

    statelist = load_run_states(start_time, end_time, cp, sec="states")

    #
    # load veto definer files and set run states
    #

    # make segments directory
    segdir = os.path.join(jobdir, "segments")
    if not os.path.isdir(segdir):
        os.mkdir(segdir)

    # load veto definer table
    veto_def_table = dict()
    if opts.run_vetoes and not opts.html_only:
        # KLUDGE, hardcode hveto usage for "Cleaned" states
        veto_def_table["Cleaned"] = lsctables.New(lsctables.VetoDefTable, ["ifo","name","version"])
        tmp = lsctables.VetoDef()
        tmp.ifo = "G1"
        tmp.name = "HVETO"
        tmp.version = 1
        veto_def_table["Cleaned"].append(tmp)

        sections = [s for s in cp.sections() if s.startswith("vetoes")]
        for sec in sections:
            # load veto definer file
            veto_def_file = scp(cp.get(sec, "veto-def-file"), segdir)
            _tmp          = VetoDefUtils.ReadVetoDefFromFiles([veto_def_file])
            cats          = map(int, cp.get(sec, "veto-categories").split(","))
            veto_def_table[sec] = VetoDefUtils.ParseVetoDef(_tmp, ifo=ifo,\
                                                            start_time_time=start_time,\
                                                            end_time=end,\
                                                            category=cats)

            # FIXME: add HVeto to the veto definer files
            #if opts.run_hveto:
            #    hveto_cat = cp.has_option(sec, "hveto-category") and\
            #                cp.getint(sec, "hveto-category") or 3
            #    hveto_def = HVetoDef(cp, ifo, hveto_cat, start_time, end)
            #    add = sum([re.search("HVETO", v.name)!=None\
            #               for v in veto_def_table[sec]])==0
            #    if add:
            #        veto_def_table[sec].append(hveto_def)

            print_verbose("Loaded %s with %d flags at %d.\n"\
                          % (os.path.basename(veto_def_file),\
                             len(veto_def_table[sec]), jobtime()))

        # FIXME: uncomment when HVeto is ready
        #if len(sections) == 0 and opts.run_hveto:
        #    veto_def_table["_"] = lsctables.VetoDefTable()
        #    veto_def_table["_"].append(HVetoDef(cp,ifo,3,start_time,end_time))

    #
    # get state defining segments
    #

    #
    # generate summary tabs
    #

    valid_systems = ["ASC", "CPU", "DAQ", "HPI", "IFO", "IOO", "IOP", "ISI",\
                     "LSC", "OMC", "PEM", "PSL", "SEI", "SUS", "SQZ", "IMC"]
    valid_parents = ["Summary", "Sensitivity", "Segments", "Triggers",\
                     "Vetoes", "Analysis", "Seismic", "Environment",
                     "Misc."] + valid_systems
    valid_parents = natsort(valid_parents)

    # generate section summaries for each valid parent
    section_summary = dict()
    for parent in valid_parents:
        tab = summary.SectionSummaryTab(parent)
        tab.span = span
        tab.directory = os.path.join(jobdir, re_cchar.sub("_", parent.lower()))
        section_summary[parent] = tab

    #
    # process segments
    #

    if opts.run_segments:
        sections = natsort([s for s in cp.sections()\
                            if s.startswith("segments-")])
        for cpsec in sections:
            run_states = parse_tab_states(statelist, cp, cpsec)
            parent = find_tab_parent(cp, cpsec, valid_parents, default="Segments")
            name = cp.has_option(cpsec, "name") and cp.get(cpsec, "name")\
                   or cpsec[9:]
            metatab = summary.MetaStateTab(name)
            metatab.span = span
            metatab.parent = section_summary[parent]
            for state in run_states:
                if state in veto_def_table.keys():
                    vetotable = veto_def_table[state]
                else:
                    vetotable = None
                if not opts.html_only:
                    get_state_segments(cp, state, vetotable, mode=mode)
                tab = summary.SegmentSummaryTab(metatab.name)
                tab.span = span
                tab.parent = metatab
                tab.directory = metatab.directory
                tab.state = state
                tab.datadirectory = segdir
                tab.segments = state.segments
                tab.add_information(cp, cpsec)
                process_segments(tab, cp, cpsec, verbose=verbose,\
                                 htmlonly=opts.html_only,\
                                 doplots=opts.run_plots,\
                                 dosubplots=opts.run_sub_plots)
                metatab.add_child(tab)
            section_summary[parent].add_child(metatab)

    #
    # process state vectors
    #

    if opts.run_state_vector:
        sections = natsort([s for s in cp.sections()\
                            if s.startswith("statevector-")])
        for cpsec in sections:
            run_states = parse_tab_states(statelist, cp, cpsec)
            parent = find_tab_parent(cp, cpsec, valid_parents, default="Segments")
            name = cp.has_option(cpsec, "name") and cp.get(cpsec, "name")\
                   or cpsec[9:]
            metatab = summary.MetaStateTab(name)
            metatab.span = (start_time, end_time)
            metatab.parent = section_summary[parent]
            for state in run_states:
                if state in veto_def_table.keys():
                    vetotable = veto_def_table[state]
                else:
                    vetotable = None
                if not opts.html_only:
                    get_state_segments(cp, state, vetotable, mode=mode)
                tab = summary.StateVectorSummaryTab(metatab.name)
                tab.span = span
                tab.parent = metatab
                tab.directory = metatab.directory
                tab.state = state
                tab.datadirectory = segdir
                tab.segments = state.segments
                tab.add_information(cp, cpsec)
                process_state_vector(tab, cp, cpsec,\
                                    verbose=verbose,\
                                    htmlonly=opts.html_only,\
                                    doplots=opts.run_plots,\
                                    dosubplots=opts.run_sub_plots)
                metatab.add_child(tab)
            section_summary[parent].add_child(metatab)

    #
    # process hveto
    #
    if opts.run_hveto:
        sections = natsort([s for s in cp.sections()\
                            if re.match("hveto[- ]", s)])
        print sections
        for cpsec in sections:
            run_states = parse_tab_states(statelist, cp, cpsec)
            parent = find_tab_parent(cp, cpsec, valid_parents, default="Triggers")
            name = cp.has_option(cpsec, "name") and cp.get(cpsec, "name")\
                   or cpsec[12:]
            metatab = summary.MetaStateTab(name)
            metatab.span = span
            metatab.parent = section_summary[parent]
            for state in run_states:
                if state in veto_def_table.keys():
                    vetotable = veto_def_table[run]
                else:
                    vetotable = None
                if not opts.html_only:
                    get_state_segments(cp, state, vetotable, mode=mode)
                tab = summary.HvetoSummaryTab(metatab.name,\
                                                   start_time=start_time,\
                                                   end_time=end_time,\
                                                   parent=metatab)
                tab.directory = metatab.directory
                tab.state = state
                tab.segments = state.segments
                tab.segments.contract(state.event_buffer)
                tab.add_information(cp, cpsec)
                process_hveto(tab, cp, cpsec,\
                                  verbose=False,\
                                  htmlonly=opts.html_only,\
                                  doplots=opts.run_plots)
                metatab.add_child(tab)
            section_summary[parent].add_child(metatab)

    #
    # process triggers
    #

    if opts.run_triggers:
        sections = natsort([s for s in cp.sections()\
                            if re.match("triggers[- ]", s)])
        for cpsec in sections:
            run_states = parse_tab_states(statelist, cp, cpsec)
            parent = find_tab_parent(cp, cpsec, valid_parents, default="Triggers")
            name = cp.has_option(cpsec, "name") and cp.get(cpsec, "name")\
                   or cpsec[9:]
            metatab = summary.MetaStateTab(name)
            metatab.span = span
            metatab.parent = section_summary[parent]
            for state in run_states:
                if state.name in veto_def_table.keys():
                    vetotable = veto_def_table[state.name]
                else:
                    vetotable = None
                if not opts.html_only:
                    get_state_segments(cp, state, vetotable, mode=mode)
                tab = summary.TriggerSummaryTab(metatab.name,\
                                                start_time=start_time,\
                                                end_time=end_time,\
                                                parent=metatab)
                tab.directory = metatab.directory
                tab.state = state
                tab.segments = state.segments
                tab.segments.contract(state.event_buffer)
                tab.add_information(cp, cpsec)
                process_triggers(tab, cp, cpsec,\
                                 verbose=verbose,\
                                 htmlonly=opts.html_only,\
                                 doplots=opts.run_plots,\
                                 dosubplots=opts.run_sub_plots)
                metatab.add_child(tab)
            section_summary[parent].add_child(metatab)

    #
    # process auxiliary channel triggers
    #

    if opts.run_auxiliary_triggers:
        sections = natsort([s for s in cp.sections()\
                            if re.match("auxtriggers[- ]", s)])
        for cpsec in sections:
            run_states = parse_tab_states(statelist, cp, cpsec)
            parent = find_tab_parent(cp, cpsec, valid_parents, default="Triggers")
            name = cp.has_option(cpsec, "name") and cp.get(cpsec, "name")\
                   or cpsec[12:]
            metatab = summary.MetaStateTab(name)
            metatab.span = span
            metatab.parent = section_summary[parent]
            for state in run_states:
                if state in veto_def_table.keys():
                    vetotable = veto_def_table[run]
                else:
                    vetotable = None
                if not opts.html_only:
                    get_state_segments(cp, state, vetotable, mode=mode)
                tab = summary.AuxTriggerSummaryTab(metatab.name,\
                                                   start_time=start_time,\
                                                   end_time=end_time,\
                                                   parent=metatab)
                tab.directory = metatab.directory
                tab.state = state
                tab.segments = state.segments
                tab.segments.contract(state.event_buffer)
                tab.add_information(cp, cpsec)
                process_auxiliary_triggers(tab, cp, cpsec,\
                                           verbose=verbose,\
                                           htmlonly=opts.html_only,\
                                           doplots=opts.run_plots)
                metatab.add_child(tab)
            section_summary[parent].add_child(metatab)

    #
    # process data
    #

    if opts.run_data:
        sections = natsort([s for s in cp.sections() if s.startswith("data-")])
        for cpsec in sections:
            run_states = parse_tab_states(statelist, cp, cpsec)
            parent = find_tab_parent(cp, cpsec, valid_parents, default="Misc.")
            if cp.has_option(cpsec, "name"):
                name = cp.get(cpsec, "name")
            elif re.match("data-\w\w\w-", cpsec):
                name = cpsec[9:]
            else:
                name = cpsec[5:]
            metatab = summary.MetaStateTab(name)
            metatab.span = span
            metatab.parent = section_summary[parent]
            for state in run_states:
                if state in veto_def_table.keys():
                    vetotable = veto_def_table[state]
                else:
                    vetotable = None
                if not opts.html_only:
                    get_state_segments(cp, state, vetotable, mode=mode)
                tab = summary.DataSummaryTab(metatab.name)
                tab.span = span
                tab.parent = metatab
                tab.directory = metatab.directory
                tab.state = state
                tab.segments = state.segments
                tab.add_information(cp, cpsec)
                process_data(tab, cp, cpsec, verbose=verbose,\
                             htmlonly=opts.html_only,\
                             doplots=opts.run_plots,\
                             dosubplots=opts.run_sub_plots)
                metatab.add_child(tab)
                metatab.skip_summary = tab.skip_summary
            section_summary[parent].add_child(metatab)

    #
    # process range
    #

    if opts.run_range:
        sections = natsort([s for s in cp.sections() if s.startswith("range-")])
        for cpsec in sections:
            run_states = parse_tab_states(statelist, cp, cpsec)
            parent = find_tab_parent(cp, cpsec, valid_parents,
                                     default="Sensitivity")
            metatab = summary.MetaStateTab(cpsec[6:])
            metatab.span = span
            metatab.parent = section_summary[parent]
            for state in run_states:
                if not opts.html_only:
                    get_state_segments(cp, state, None, mode=mode)
                tab = summary.RangeSummaryTab(metatab.name)
                tab.span = span
                tab.parent = metatab
                tab.directory = metatab.directory
                tab.state = state
                tab.segments = state.segments
                # for range type, need to split segments and vetoes,
                # segments are used for spectra and segment-vetoes for
                # triggers only
                if state.name in veto_def_table.keys():
                    vetotable = veto_def_table[state.name]
                else:
                    vetotable = None
                if not opts.html_only:
                    get_state_segments(cp, state, vetotable, mode=mode)
                tab.trigsegments = state.segments
                tab.add_information(cp, cpsec)
                process_range(tab, cp, cpsec, verbose=verbose,\
                              htmlonly=opts.html_only,\
                              doplots=opts.run_plots,\
                              dosubplots=opts.run_sub_plots)
                metatab.add_child(tab)
            section_summary[parent].add_child(metatab)

    #
    # process squeezing
    #
    if opts.run_squeezing:
        sections = natsort([s for s in cp.sections() if s.startswith("squeezing")])
        for cpsec in sections:
            run_states = parse_tab_states(statelist, cp, cpsec)
            parent = find_tab_parent(cp, cpsec, valid_parents,
                                     default="SQZ")
            metatab = summary.MetaStateTab(cpsec)
            metatab.span = span
            metatab.parent = section_summary[parent]
            for state in run_states:
                if state in veto_def_table.keys():
                    vetotable = veto_def_table[state]
                else:
                    vetotable = None
                if not opts.html_only:
                    get_state_segments(cp, state, vetotable, mode=mode)
                tab = summary.RangeSummaryTab(metatab.name)
                tab.span = span
                tab.parent = metatab
                tab.directory = metatab.directory
                tab.state = state
                tab.segments = state.segments
                tab.add_information(cp, cpsec)
                process_squeezing(tab, cp, cpsec, verbose=verbose,\
                              htmlonly=opts.html_only,\
                              doplots=opts.run_plots,\
                              dosubplots=opts.run_sub_plots)
                metatab.add_child(tab)
            section_summary[parent].add_child(metatab)

    #
    # archive data
    #

    # archive frame data
    if ((opts.run_data or opts.run_range) and
            opts.write_data and not opts.html_only):
        print_verbose("\n----------\nWriting data to file...\n",\
                      verbose=verbose)

        # generate output file
        datadir = os.path.join(jobdir, "data")
        if not os.path.isdir(datadir):
            os.mkdir(datadir)
        h5filename = os.path.join(datadir, "%s-SUMMARY_PAGE_DATA-%d-%d.hdf"
                                           % (ifo, start_time, abs(span)))
        archive_data(h5filename)

    # archive segments
    if ((opts.run_segments or opts.run_state_vector) and
            opts.write_segments and not opts.html_only):
        filename = os.path.join(segdir, "%s-SUMMARY_PAGE_SEGMENTS-%d-%d.xml.gz"\
                                         % (ifo, start_time,\
                                            end_time-start_time))
        archive_segments(filename, param_dict)
        print_verbose("Segments archived to %s.\n" % filename, verbose=verbose)

    # archive triggers
    trigdir = os.path.join(jobdir, "triggers")
    if not os.path.isdir(trigdir):
        os.makedirs(trigdir)
    if ((opts.run_triggers or opts.run_auxiliary_triggers) and
            opts.write_triggers and not opts.html_only):
        archive_triggers(trigdir, start_time, end_time, param_dict, False)
        archive_triggers(trigdir, start_time, end_time, param_dict, True)
        print_verbose("Triggers archived in %s/.\n" % trigdir, verbose=verbose)

    #
    # setup HTML
    #

    print_verbose("\n----------\nGenerating HTML...\n", verbose=verbose,\
                  profile=False)

    # set html extension
    ext = cp.has_option("html", "file-type") and cp.get("html", "file-type")\
          or "html"

    if not os.path.isdir("html"):
        os.mkdir("html")

    # get html files
    css = cp.get("html", "style-sheet").split(",")
    for i,stylesheet in enumerate(css):
        css[i] = scp(stylesheet,\
                     os.path.join("html", os.path.basename(stylesheet)))

    javascript = cp.get("html", "javascript").split(",")
    for i,fp in enumerate(javascript):
        javascript[i] = scp(fp, os.path.join("html", os.path.basename(fp)))

    # get base and root
    if mode == summary.SUMMARY_MODE_GPS:
        base = build_html_base(cp, ifo, outdir,
                               "%d-%d" % (start_time, end_time))
    else:
        base = build_html_base(cp, ifo, outdir)
    root = urlparse.urlparse(base).path

    if mode > summary.SUMMARY_MODE_GPS:
        startdate = cp.has_option("calendar", "start-date")\
                    and datetime.datetime.strptime(cp.get("calendar",\
                                                       "start-date"), "%Y%m%d")\
                    or datetime.datetime(*lal.GPSToUTC(int(start_time))[:6])

        weekday   = cp.has_option("calendar", "start-of-week")\
                    and getattr(calendar,\
                                cp.get("calendar","start-of-week").upper())
    else:
        startdate = None
        weekday   = None

    baselist = [(key[:2].upper(),val) for (key,val) in cp.items("html")\
                if re.match("\w\d-base\Z", key)]


    #
    # write auxiliary HTML
    #

    auxiliary_tabs = dict()

    # ABOUT
    aboutdir = os.path.join(jobdir, "about")
    if not os.path.isdir(aboutdir):
        os.mkdir(aboutdir)
    tab = summary.AboutTab("About", directory=aboutdir, span=span,\
                           version=__version__, mode=mode)
    tab.add_file("Configuration file", cp.filename)
    tab.write_menu(jobdir, startdate=startdate, weekday=weekday)
    tab.finalize()
    tab.write_ifo_links(baselist, ifo, tab.index)
    tab.build(base=base, css=css, script=javascript)

    # CALENDAR
    if mode > summary.SUMMARY_MODE_GPS:
        caldir = "calendar"
        if not os.path.isdir(caldir):
            os.mkdir(caldir)
        tab = summary.CalendarTab("Calendar", directory=caldir, span=span,\
                                  mode=mode, start_date=startdate,\
                                  end_date=int(end_time-1))
        tab.weekday = weekday
        tab.write_menu(jobdir, startdate=startdate, weekday=weekday)
        tab.finalize()
        tab.write_ifo_links(baselist, ifo, tab.index)
        tab.build(base=base, css=css, script=javascript)
        print_verbose("Calendar HTML written.\n", verbose=verbose)

    # GLOSSARY
    glosdir = "glossary"
    if not os.path.isdir(glosdir):
        os.mkdir(glosdir)
    tab = summary.GlossaryTab("Glossary", span=span, directory=glosdir,\
                              mode=mode)
    if cp.has_section("glossary"):
        tab.add_entries(*cp.items("glossary"))
    tab.write_menu(jobdir, startdate=startdate, weekday=weekday)
    tab.finalize()
    tab.write_ifo_links(baselist, ifo, tab.index)
    tab.build(base=base, css=css, script=javascript)
    print_verbose("Glossary HTML written.\n", verbose=verbose)

    # ONLINE
    if mode > summary.SUMMARY_MODE_GPS:
        refresh  = cp.has_option("html", "online-refresh")\
                   and cp.getfloat("html", "online-refresh") or None
        ondir    = "online"
        if not os.path.isdir(ondir):
            os.mkdir(ondir)
        sections = [s for s in cp.sections() if re.match("online[-_ ]", s)]
        plots    = dict((s, cp.items(s)) for s in sections)
        if len(sections) > 1:
            sections = ["online All times"] + sections
            plots["online All times"] = []
            for s in sections[1:][::-1]:
                plots["online All times"].extend(cp.items(s))
        tab = summary.OnlineSummaryTab("Online", span=span, directory=".")
        for s in sections:
            tab.add_plots(s[7:], plots[s])
        tab.refresh = refresh
        tab.write_menu(jobdir, startdate=startdate, weekday=weekday)
        tab.finalize()
        tab.write_ifo_links(baselist, ifo, tab.index)
        tab.build(base=base, css=css, script=javascript)
        print_verbose("Online HTML written.\n", verbose=verbose)

    #
    # collate tab pages
    #

    tabs = summary.SectionSummaryTab("Summary", span=(start_time, end_time))
    tabs.directory = jobdir
    tabs.mode    = mode
    tabs.write_menu(jobdir, startdate=startdate, weekday=weekday)
    for p in valid_parents:
        if len(section_summary[p].children) == 0:
            continue
        if cp.has_section('summary-tab') and cp.has_option('summary-tab', p):
            summlist = cp.get('summary-tab', p).split(',')
            for summname in summlist:
                summtab = section_summary[p].get_child(summname)
                tabs.add_child(summtab)
        else:
            summtab = section_summary[p].children[0]
            if not summtab.skip_summary:
                tabs.add_child(summtab)

    section_summary_tabs = [p for p in section_summary.values() if
                            len(p.children)!=0]
    summaries = [tabs] + section_summary_tabs
    tabs.write_tabs(summaries)
    tabs.finalize()
    tabs.write_ifo_links(baselist, ifo, tabs.index)
    tabs.build(base=base, css=css, script=javascript)
    print_verbose("Summary HTML written.\n", verbose=verbose)
    for tab in section_summary_tabs:
        tab.mode    = mode
        tab.write_tabs(summaries)
        tab.write_menu(jobdir, startdate=startdate, weekday=weekday)
        tab.finalize()
        tab.write_ifo_links(baselist, ifo, tab.index)
        tab.build(base=base, css=css, script=javascript)
        for subtab in tab.children:
            subtab.mode = mode
            subtab.write_tabs(summaries)
            subtab.write_menu(jobdir, startdate=startdate, weekday=weekday)
            subtab.finalize()
            subtab.write_ifo_links(baselist, ifo, subtab.index)
            subtab.build(base=base, css=css, script=javascript)
    print_verbose("All tabs HTML written.\n", verbose=verbose)
    print_verbose("\n----------\nFinished\n----------\n", verbose=verbose,\
                  profile=False)
