#! /usr/bin/env python3
# -*- coding: utf-8 -*-

# pylint: disable=global-statement

import argparse
import math
import multiprocessing
import os
import re
import subprocess
import sys
import time
import timeit
import warnings

import vcsn


def git_describe():
    b = subprocess.check_output(
        ['git', 'describe', '--first-parent', '--long', '--dirty'])
    res = b.decode('utf-8').strip()
    # v2.9-41-gf6bb8af2 -> v2.9-041-gf6bb8af2.
    res = re.sub(r'-(\d+)-g',
                 lambda m: '-{:03}-g'.format(int(m.group(1))),
                 res)
    return res


def getargs():
    p = argparse.ArgumentParser(description='Bench some algorithms.')
    opt = p.add_argument
    opt('-O', '--only', metavar='RE', type=re.compile, default='.*',
        help='run only benches whose title is matched by RE')
    opt('-j', '--jobs', metavar='NUM', type=int, default='0',
        help='''number of benches to run concurrently.  Pass 0 to
        avoid using any Thread.''')
    opt('-r', '--repeat', metavar='NUM', type=int, default='3',
        help='''number runs to perform, the best of which being kept.  The higher,
        the more stable the results are expected to be.  Pass 0 to run
        only the set-up parts; this is useful to warm up the plugin
        cache (ccache).''')
    opt('-v', '--verbose', action='store_true', help='be verbose')
    opt('--check', action='store_true',
        help='''run scaled down versions of the benchmarks.  Implies -r 1.  Very
        useful with `-j` to warn the algorithm cache.  Do not use it
        to record actual benchmarks.''')
    opt('-l', '--list', action='store_true', help='list of benches cases')
    opt('-s', '--sort', action='store_true', help='sort benches by name')
    opt('-o', '--output', metavar='FILE',
        help='''also save the result in `FILE`''')
    opt('-d', '--dir', metavar='DIR',
        help='''also print the result in `DIR/DESC` where DESC is the
        result of `git describe`''')
    res = p.parse_args()

    if res.dir is not None and res.output is not None:
        raise RuntimeError('--dir and --output cannot be used together')

    # Implement --dir/--output.
    res.out = None
    if res.dir is not None:
        res.output = os.path.join(res.dir, git_describe())
    if res.output is not None:
        res.out = open(res.output, 'w')

    return res

args = getargs()

if '-DNDEBUG' not in ' '.join([vcsn.config('configuration.cppflags'),
                               vcsn.config('configuration.cxxflags')]):
    warnings.warn('not compiled with -DNDEBUG, benches are irrelevant')
if '-O3' not in vcsn.config('configuration.cxxflags'):
    warnings.warn('not compiled with -O3, benches are irrelevant')
if args.check:
    warnings.warn('run with --check, benches are irrelevant')

# The number of failures
nfail = 0


def ws_signature(ws):
    if ws.endswith('min'):
        return ws[:-3].upper() + 'min'
    elif ws.endswith('mp'):
        return ws[:-2].upper() + 'min'
    else:
        return ws.upper()


def ctx_signature(ctx):
    '''A short, nice looking, description of a context.  To replace eventually
    with `ctx.format()`.'''
    ctx = re.sub(r'lal(?:_char)?\((.*?)\)',
                 r'[\1]',
                 ctx)
    ctx = re.sub(r'lan(?:_char)?\((.*?)\)',
                 r'[\1]?',
                 ctx)
    ctx = re.sub(r'law(?:_char)?\((.*?)\)',
                 r'[\1]*',
                 ctx)
    ctx = re.sub(r'lat<(.*?)>',
                 lambda m: 'x'.join([x.strip()
                                     for x in m.group(1).split(',')]),
                 ctx)
    ctx = re.sub(r'^(.*), *(.*)$',
                 lambda m: '{} -> {}'.format(m.group(1),
                                             ws_signature(m.group(2))),
                 ctx)
    return ctx


def downsize(statement):
    'Replace size identifiers with 1.'
    # Examples of numbers not to match:
    # - lal, f2
    # - lal(A-Z0-9), ...
    # - a2 = ...
    return re.sub(r'(?<![-a-zA-Z])\d+', '1', statement)


def log(*logs):
    'Print logs to stderr.'
    if args.verbose:
        print(*logs, file=sys.stderr)


def comment(title, lines):
    'Output metadata to the output file.'
    lines = re.sub('^', '# {}: '.format(title), lines.rstrip(), flags=re.M)
    print(lines, flush=True, file=args.out)


# Number of the next bench report.  Because we are using a process
# pool (not a thread pool), we can't just use some simple global.
#
# We use processes, not threads, because the GIL is locked when we go
# into C++ code, so there is no parallelism at all with threads.
#
# We could have a `Bench.number` counter, but we are interested in a
# "progress bar", so what matters is the complete benches, not the
# order in which they are run.
count = multiprocessing.Value('i', 1)

# List of benches to run.
benches = []


def output(s):
    'Print to stdout and possibly output file.'
    global count
    with count.get_lock():
        c = count.value
        count.value += 1
    nbenches = len(benches)
    width = int(math.log10(nbenches) + 1) if benches else 1
    print('{:{}}/{:{}}'.format(c, width, nbenches, width),
          s, flush=True)
    if args.out is not None:
        print(s, flush=True, file=args.out)


def bench(stmt, comment, title=None, setup=None, number=1):
    'Report the best timing of three batches of number runs of cmd.'
    if title is None:
        title = stmt
    if setup is None:
        setup = ['pass']

    if not isinstance(setup, list):
        setup = [setup]

    if args.only.search(title + ' # ' + comment):
        benches.append(Bench(stmt, comment, title, setup, number))


class Bench:
    # pylint: disable=too-many-arguments

    def __init__(self, stmt, comment, title, setup, number):
        self.__stmt_orig = stmt
        self.__setup_orig = setup
        if args.check:
            number = 1
            args.repeat = 1
            stmt = downsize(stmt)
            comment = downsize(comment)
            title = downsize(title)
            setup = [downsize(s) for s in setup]

        self.__stmt = stmt
        self.__comment = comment
        self.__title = title
        self.__setup = setup
        self.__number = number

        if 1 < self.__number:
            self.__comment += ', {}x'.format(self.__number)

    def log(self, cmd, ocmd):
        'Report that we run `cmd` (originally `ocmd`).'
        if args.check and cmd != ocmd:
            log('run: {:27s} (was: {})'.format(cmd, ocmd))
        else:
            log('run:', cmd)

    @property
    def title(self):
        return '{:23s} # {}'.format(self.__title, self.__comment)

    def run(self):
        log('Bench.run: {}'.format(self.title))
        env = {
            'vcsn': vcsn,
            'b': vcsn.context('lal(abc), b'),
        }
        try:
            for i, s in enumerate(self.__setup):
                self.log(s, self.__setup_orig[i])
                exec(s, globals(), env)  # pylint: disable=exec-used

            self.log(self.__stmt, self.__stmt_orig)
            if args.repeat:
                t = min(timeit.repeat(self.__stmt, number=self.__number,
                                      repeat=args.repeat, globals=env))
                t = '{:5.2f}s'.format(t)
            else:
                # Just run the set up, the user is just warming ccache
                # by running the setups.
                t = 'SKIP'
            err = None
        except RuntimeError as exc:
            t = ' FAIL'
            err = exc
            global nfail
            nfail += 1
        output('{:7s}: {}'.format(t, self.title))
        if err:
            print(err, file=sys.stderr)


# Check the cost of dyn calls.
#
# We used to check a call to "b.format('text')", but that makes us too
# sensitive to the cost of the formatting itself.  Any operation can
# hardly be simpler than `automaton.is_proper`, which just returns
# "true" for an automaton on a free labelset.  So this does measure
# the pure speed of our interface with Python, and dispatch.
#
# To check the cost of the dispatch, it must be done in C++.
#
# FWIW:
#
# %timeit a.is_empty()
# 1000000 loops, best of 3: 1.03 µs per loop
#
# %timeit a.proper()
# 10000 loops, best of 3: 62.8 µs per loop
#
# %timeit a.is_proper()
# 1000000 loops, best of 3: 1 µs per loop
#
# So really, is_proper looks the right tool.
bench('a.is_proper()',
      'a = ""',
      setup='''a = vcsn.automaton('', 'daut')''',
      number=1000000)

# Check formatting.  was used to check dyn:: round-trip.
bench('b.format("text")',
      'b = [abc] -> B',
      number=100000)

# I/O on expressions.
e = '[ab]{20000}'
bench('b.expression(e)',
      'e = {}'.format(e),
      setup='e = "{}"'.format(e),
      number=1000)

e = r'"(\e+a)" * 500'
bench('b.expression(e)',
      'e = {}'.format(e),
      setup='e = {}'.format(e),
      number=100)

e = r'"(\e+a)" * 500'
bench('r.format("text")',
      'r = b.expression({})'.format(e),
      setup=['e = {}'.format(e),
             'r = b.expression(e)'],
      number=1000)

# Output should be fast.
r = 'a?{500}'
for fmt in ['daut', 'dot', 'efsm', 'fado', 'grail', 'tikz']:
    bench('a.format("{}")'.format(fmt),
          'a = std({})'.format(r),
          setup=['r = "{}"'.format(r),
                 'a = b.expression(r).standard()'],
          number=5)

# Input should be too.
r = 'a?{500}'
for fmt, number in [('daut', 5), ('dot', 1), ('efsm', 5), ('fado', 5)]:
    bench('vcsn.automaton(a, "{}")'.format(fmt),
          's = {}(std({}))'.format(fmt, r),
          'read(s)',
          setup=['r = "{}"'.format(r),
                 'a = b.expression(r).standard().format("{}")'.format(fmt)],
          number=number)

## -------------- ##
## derived_term.  ##
## -------------- ##


def bench_derived_term(alphabet, exp, algo, number):
    ctx = "lal({}), z".format(alphabet)
    cmd = 'r.derived_term("{}")'.format(algo)
    if 'lazy' in algo:
        # Accessible resolves the lazy states.
        cmd = cmd + '.accessible()'
    bench(cmd,
          'r = {}, c = {}'.format(exp, ctx_signature(ctx)),
          setup=['ctx = "{}"'.format(ctx),
                 'e = "{}"'.format(exp),
                 'c = vcsn.context(ctx)',
                 'r = c.expression(e)'],
          number=number)


def bench_dt(alphabet, size, algo, number):
    bench_derived_term(alphabet,
                       "(a+b)*b(<2>a+<2>b){{{}}}".format(size),
                       algo,
                       number=number)

bench_dt('ab',  150, 'derivation', 50)
bench_dt('a-z', 150, 'derivation', 50)
bench_dt('a-z', 150, 'expansion',  50)
bench_dt('ab',  300, 'derivation', 20)
bench_dt('a-z', 300, 'derivation', 20)
bench_dt('a-z', 300, 'expansion',  20)
bench_dt('a-z', 300, 'lazy,expansion',  20)

# Measure the cost of a lazy construction.
bench_dt('a-z', 3, 'expansion', 10000)
bench_dt('a-z', 3, 'lazy,expansion', 10000)

bench_derived_term('a', 'a?{150}', 'derivation', 2)
bench_derived_term('a', 'a?{150}', 'expansion',  2)
bench_derived_term('a', 'a?{150}', 'lazy,expansion',  2)

# standard
ctx = 'lal(a-z), z'
e = "(a+b)*b(<2>a+<2>b){20000}"
bench('r.standard()',
      'r = {}, c = {}'.format(e, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'e = "{}"'.format(e),
             'r = vcsn.context(ctx).expression(e)'],
      number=10)

# thompson
ctx = 'lan(a-z), z'
bench('r.thompson()',
      'r = {}, c = {}'.format(e, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'e = "{}"'.format(e),
             'r = vcsn.context(ctx).expression(e)'],
      number=10)


## ------------- ##
## Determinize.  ##
## ------------- ##

# These are the well known worst cases.  21 is too long, slowly moving
# to use 18 as reference.
for n in [18, 21]:
    bench('a.determinize()',
          'a = ladybird({})'.format(n),
          setup=['n = {}'.format(n),
                 'c = vcsn.context("lal(abc), b")',
                 'a = c.ladybird(n)'])

# In the case of 18, check that we scale well with the size of the
# (context's) alphabet.
n = 18
ctx = 'lal(a-zA-Z0-9), b'
bench('a.determinize()',
      'a = ladybird({}), c = {}'.format(n, ctx_signature(ctx)),
      setup=['n = {}'.format(n),
             'c = vcsn.context("{}")'.format(ctx),
             'a = c.ladybird(n)'])

# See how boolean vs. weighted determinization goes.
for n, algo, number in [(13, '', 20), (13, '"weighted"', 5),
                        (14, '', 10), (14, '"weighted"', 5),
                        (16, '', 2),  (16, '"weighted"', 1)]:
    for ws in ['b', 'f2']:
        ctx = 'lal(abc), ' + ws
        bench('a.determinize({})'.format(algo),
              'a = de_bruijn({}), c = {}'.format(n, ctx_signature(ctx)),
              setup=['c = vcsn.context("{}")'.format(ctx),
                     'n = {}'.format(n),
                     'a = c.de_bruijn(n)'],
              number=number)

# Check the cost of laziness.
n = 13
bench('a.determinize("lazy,weighted").accessible()',
      'a = de_bruijn({}), c = {}'.format(n, ctx_signature(ctx)),
      setup=['c = vcsn.context("{}")'.format(ctx),
             'n = {}'.format(n),
             'a = c.de_bruijn(n)'],
      number=5)

# Something more realistic: the previous automata explode in an
# exponential number of states, and half of them end up being final.
# This exagerates the importance of the handling of the final states.
#
# The following bench tries to be more realistic (i.e., more NLP
# like): the automata are almost deterministic (and easy to
# determinize), are "wide", and have few final states.
n = 100
ctx = 'lal(a-zA-Z0-9), b'
r = '([^]+a){{{}}}'.format(n)
for algo in ['', '"weighted"', '"lazy,weighted"']:
    cmd = 'a.determinize({})'.format(algo)
    if 'lazy' in algo:
        # Accessible resolves the lazy states.
        cmd = cmd + '.accessible()'
    bench(cmd,
          'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
          setup=['r = "{}"'.format(r),
                 'c = vcsn.context("{}")'.format(ctx),
                 'a = c.expression(r).standard()'],
          number=2)


## ---------- ##
## evaluate.  ##
## ---------- ##

n = 150
# Many many states active concurrently.
# too slow: a = b.expression('(a+b)*a(a+b){{{n}}}).derived_term()
bench('a.evaluate("a"*{})'.format(n + 1),
      'a = de_bruijn({})'.format(n),
      setup=['n = {}'.format(n),
             'a = b.de_bruijn(n)'],
      number=10000)
# Something less wide.
for ctx, number in [('lal(a-z), z', 20000),
                    ('lan(a-z), z', 10000),
                    ('law(a-z), z', 10000)]:
    n = 25
    r = '[a-z]*'
    bench('a.evaluate("abcxyz"*{})'.format(n),
          'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
          setup=['c = vcsn.context("{}")'.format(ctx),
                 'r = c.expression("{}")'.format(r),
                 'a = r.standard()'],
          number=number)

    bench('a.evaluate(wc.polynomial("abcxyz"*{}))'.format(n),
          'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
          setup=['c = vcsn.context("{}")'.format(ctx),
                 'r = c.expression("{}")'.format(r),
                 'a = r.standard()',
                 'exp = "abcxyz"*{}'.format(n),
                 'wc = c.word_context()'],
          number=number)

## ---------- ##
## shortest.  ##
## ---------- ##

n = 9
# too slow: a = b.expression('(a+b)*a(a+b){{{n}}}).derived_term()
bench('a.shortest(5)',
      'a = de_bruijn({})'.format(n),
      setup=['n = {}'.format(n),
             'a = b.de_bruijn(n)'],
      number=10)

bench('a.shortest(5000)',
      'a = de_bruijn({})'.format(n),
      setup=['n = {}'.format(n),
             'a = b.de_bruijn(n)'],
      number=10)

ctx = 'lal(a-e), z'
r = "[a-e]?{600}"
bench('a.shortest(5)',
      'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'c = vcsn.context(ctx)',
             'r = "{}"'.format(r),
             'a = c.expression(r).standard()'],
      number=5)

ctx = 'lat<lan(a), lan(x)>, q'
r = r'(\e|x + a|\e)*'
bench('a.shortest(5000)',
      'a = derived_term({}), c = {}'.format(r, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'c = vcsn.context(ctx)',
             'r = "{}"'.format(r),
             'a = c.expression(r).derived_term().strip()'],
      number=10)

# FIXME: restore once issue #99 is fixed.
# ctx = 'lal(a-e), z'
# r = "[a-e]{10000}"
# bench('a.shortest()',
#       'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
#       setup=['c = vcsn.context(ctx)',
#              'r = "{}"'.format(r),
#              'a = c.expression(r).standard()'])

## ---------- ##
## lightest.  ##
## ---------- ##

# lightest
ctx = 'lal(a-e), nmin'
r = "[a-e]?{150}"
for algo, number in [('auto', 10), ('yen', 500), ('eppstein', 500)]:
    bench('a.lightest(5, "{}")'.format(algo),
          'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
          setup=['ctx = "{}"'.format(ctx),
                 'c = vcsn.context(ctx)',
                 'r = "{}"'.format(r),
                 'a = c.expression(r).standard()'],
          number=number)

# Building [a-z]?{300} is really time-consuming.  Make more iterations
# instead.
ctx = 'lal(a-e), nmin'
bench('a.lightest()',
      'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'c = vcsn.context(ctx)',
             'r = "{}"'.format(r),
             'a = c.expression(r).standard()'],
      number=500)

ctx = 'lat<lan(a), lan(x)>, q'
r = r'(\e|x + a|\e)*'
bench('a.lightest(5000)',
      'a = derived_term({}), c = {}'.format(r, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'c = vcsn.context(ctx)',
             'r = "{}"'.format(r),
             'a = c.expression(r).derived_term().strip()'],
      number=10)

# lightest-automaton.
ctx = 'lal(a-e), nmin'
r = "[a-e]?{150}"
for algo, number in [('a-star', 20), ('bellman-ford', 1), ('dijkstra', 500)]:
    bench('a.lightest_automaton(1, "{}")'.format(algo),
          'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
          setup=['ctx = "{}"'.format(ctx),
                 'c = vcsn.context(ctx)',
                 'r = "{}"'.format(r),
                 'a = c.expression(r).standard()'],
          number=number)

# sort.
ctx = "lal(a-e), z"
r = "[a-e]?{700}"
bench('a.sort()',
      'a = std({})'.format(r),
      setup=['ctx = "{}"'.format(ctx),
             'c = vcsn.context(ctx)',
             'r = "{}"'.format(r),
             'a = c.expression(r).standard()'])

# split.
ctx = "lal(a-e), q"
e = "[a-e]?{15}"
bench('e.split()',
      'e = {}'.format(e),
      setup=['ctx = "{}"'.format(ctx),
             'c = vcsn.context(ctx)',
             'e = c.expression("{}")'.format(e)])

# proper.
r = "a?{1200}"
bench('a.proper()',
      'a = thompson({})'.format(r),
      setup=['r = "{}"'.format(r),
             'a = vcsn.context("lan(a), b").expression(r).thompson()'])

# to-expression.
bench('a.expression("associative", "delgado")',
      'a = ladybird(2000)',
      setup=['a = vcsn.context("lal(abc), b").ladybird(2000)'],
      number=10)

r = '[a-d]?{100}'
bench('a.expression("associative", "naive")',
      'a = std({})'.format(r),
      setup=['r = "{}"'.format(r),
             'a = vcsn.context("lal(a-d), b").expression(r).standard()'])

bench('a.expression("associative", "naive")',
      'a = ladybird(8000)',
      setup=['a = vcsn.context("lal(abc), b").ladybird(8000)'],
      number=10)

r = '[a-d]?{15}'
bench('a.expression("linear", "delgado")',
      'a = std({})'.format(r),
      setup=['r = "{}"'.format(r),
             'a = vcsn.context("lal(a-d), b").expression(r).standard()'],
      number=5)

bench('a.expression("linear", "delgado")',
      'a = ladybird(2000)',
      setup=['a = vcsn.context("lal(abc), b").ladybird(2000)'],
      number=5)

r = '[a-d]?{9}'
bench('a.expression("linear", "naive")',
      'a = std({})'.format(r),
      setup=['r = "{}"'.format(r),
             'a = vcsn.context("lal(a-d), b").expression(r).standard()'],
      number=5)


bench('a.expression("linear", "naive")',
      'a = ladybird(4000)',
      setup=['a = vcsn.context("lal(abc), b").ladybird(4000)'],
      number=2)

# Other conjunction and power testcases, with more outgoing transitions
# per state.  This stresses much better the new conjunction algorithm.
# The very narrow automaton, a{200000}, highlights the cost of the
# possible spontaneous transitions for lan.
for r in ['[a-e]?{50}', 'a{200000}']:
    bench('a.conjunction(a)',
          'a = std({})'.format(r),
          setup=['r = "{}"'.format(r),
                 'a = vcsn.context("lal(a-e), z").expression(r).standard()'])

r = '[a-e]?{50}'
bench('a.shuffle(a)',
      'a = std({})'.format(r),
      setup=['r = "{}"'.format(r),
             'a = vcsn.context("lal(a-e), z").expression(r).standard()'])

# infiltrate.
r = "[a-e]?{30}"
bench('a.infiltrate(a)',
      'a = std({})'.format(r),
      setup=['r = "{}"'.format(r),
             'a = vcsn.context("lal(a-e), z").expression(r).standard()'])

# Conjunction with spontaneous transitions.
r = "[a-e]?{80}"
bench('a.conjunction(a)',
      'a = thompson({})'.format(r),
      setup=['r = "{}"'.format(r),
             'a = vcsn.context("lan(a-e), z").expression(r).thompson()'])

r = "[a-e]?{5}"
bench('a.conjunction(a, a, a)',
      'a = thompson({})'.format(r),
      setup=['r = "{}"'.format(r),
             'a = vcsn.context("lan(a-e), z").expression(r).thompson()'])

# power.
n = 12
r = "[a-e]*b(<2>[a-e])*"
bench('a & {}'.format(n),
      'a = std({})'.format(r),
      setup=['r = "{}"'.format(r),
             'a = vcsn.context("lal(a-e), z").expression(r).standard()'])

# compose.
ctx = "lat<lan<char(a-z)>, lan<char(a-z)>>, b"
r = "([a-z]|[a-z])?{20}"
bench('a.compose(a2)',
      'a2 = std({}), a = std(a|a)'.format(r),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r),
             'c = vcsn.context(ctx)',
             'a = c.expression("a|a").standard()',
             'a2 = c.expression(r).automaton()'],
      number=50)


def bench_compose(ls, r, a, number=50):
    ctx = "lat<{ls}(a-z), {ls}(a-z)>, b".format(ls=ls)
    algo = {'std': 'standard', 'thm': 'thompson'}[a]
    bench('a.compose(a)',
          'a = {}({}), c = {}'.format(a, r, ctx_signature(ctx)),
          setup=['ctx = "{}"'.format(ctx),
                 'r = "{}"'.format(r),
                 'c = vcsn.context(ctx)',
                 'a = c.expression(r).{}()'.format(algo)],
          number=number)

# Those two are obsolete, remove them in some future.  They are
# irrelevant (this pattern does not exist in "real life", and
# extremely costly to run.  Besides, commit f893b2f1 changed the
# meaning of this expression when in LAN (see #119).
r = "['(a,a)'-'(i,z)']{4}"
bench_compose('lal', r, 'std', 1)
# bench_compose('lan', r, 'thm', 2)

r = '((a|b+b|c+c|d+d|a)(w|x+x|y+y|z+z|w)){300}'
bench_compose('lal', r, 'std')
bench_compose('lan', r, 'std')
bench_compose('lan', r, 'thm', 20)
r = r'((\e|a+a|b+b|c+c|d+d|a)(w|x+x|y+y|z+z|w+z|\e)){300}'
bench_compose('lan', r, 'std', 20)
bench_compose('lan', r, 'thm', 10)

# automaton.has_bounded_lag.
ctx = "lat<lan(abc), lan(xyz)>, b"
r1 = "['(a,x)'-'(b,y)']{1000}*"
r2 = "(a|x+a|y+a|z+b|x+b|y){1000}*"
bench('a.has_bounded_lag()',
      'a = std({})'.format(r1),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r2),
             'c = vcsn.context(ctx)',
             'a = c.expression(r).standard()'],
      number=10)
ctx = "lat<lan(abc), lan(xyz)>, b"
r1 = "['(a,x)'-'(b,y)']*{600}"
r2 = "(a|x+a|y+a|z+b|x+b|y)*{600}"
bench('a.has_bounded_lag()',
      'a = std({})'.format(r1),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r2),
             'c = vcsn.context(ctx)',
             'a = c.expression(r).standard()'],
      number=20)

# Minimize.
def bench_minimize(ws, algo, number=1):
    ctx = 'lal(a-k), ' + ws
    r = '[a-g]{300}'
    bench('a.minimize("{}")'.format(algo),
          'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
          setup=['c = vcsn.context("{}")'.format(ctx),
                 'r = c.expression("{}")'.format(r),
                 'a = r.standard()'],
          number=number)
bench_minimize('b', 'brzozowski', 200)
bench_minimize('b', 'hopcroft',  2)
bench_minimize('b', 'moore',    40)
bench_minimize('b', 'signature', 2)
bench_minimize('b', 'weighted')
bench_minimize('z', 'weighted')


# reduce.
ctx = "lal(a-k), z"
r = "[a-g]{300}"
bench('a.reduce()',
      'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'c = vcsn.context(ctx)',
             'r = "{}"'.format(r),
             'a = c.expression(r).standard()'])
ctx = "lal(a-k), q"
bench('a.reduce()',
      'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r),
             'a = vcsn.context(ctx).expression(r).standard()'])

# synchronizing_word.
bench('a.synchronizing_word()',
      'a = de_bruijn(6)',
      setup='a = vcsn.context("lal(abc), b").de_bruijn(6).determinize()')

# has_twins_property.
ctx = "lal(abc), zmin"
r = "[a-c]{200}*+[a-c]{200}*"
bench('a.has_twins_property()',
      'a = std({}, "associative"), c = {}'.format(r, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r),
             'a = vcsn.context(ctx).expression(r, "associative").standard()'],
      number=20)

# is_cycle_ambiguous.
ctx = "lal(abc), z"
r = "[a-c]{2000}(<2>ab+a<3>b)"
bench('a.is_cycle_ambiguous()',
      'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r),
             'a = vcsn.context(ctx).expression(r).standard()'],
      number=20)

# is_ambiguous.
ctx = "lal(abc), z"
r = "[a-c]{2000}(<2>ab+a<3>b)"
bench('a.is_ambiguous()',
      'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r),
             'a = vcsn.context(ctx).expression(r).standard()'],
      number=20)

# is_functional.
ctx = "lat<lal(ab), lal(xy)>, b"
r = "(a|x){2000}(b|y)"
bench('a.is_functional()',
      'a = std({})'.format(r),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r),
             'a = vcsn.context(ctx).expression(r).standard()'],
      number=100)

# accessible.
ctx = "lan(a-z), b"
r = "[a-m]{20000}"
bench('a.accessible()',
      'a = thompson({}).proper(prune=False)'.format(r),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r),
             'a = vcsn.context(ctx).expression(r).thompson().proper(prune=False)'],
      number=4)

# scc.
ctx = "lal(abc), b"
r = '(abc)*{1000}'
for algo in ['dijkstra', 'kosaraju', 'tarjan_iterative', 'tarjan_recursive']:
    bench('a.scc("{}")'.format(algo),
          'a = std({})'.format(r),
          setup=['ctx = "{}"'.format(ctx),
                 'r = "{}"'.format(r),
                 'c = vcsn.context(ctx)',
                 'a = c.expression(r).standard()'],
          number=20)

# polynomial.trie.
r = '[a-j]{6}'
bench('p.trie()',
      'p = [a-j]{6}',
      setup=['c = vcsn.context("law(a-j), b")',
             'm = c.polynomial("[a-j]")',
             'p = m ** 6'])

# synchronize
ctx = "lat<law, law>, b"
r = r"""(abc|de)((f|\e)+((g|h)(i|j)*(\e|k)))(l|mn){2000}"""
bench('a.synchronize()',
      'a = std({})'.format(r),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r),
             'c = vcsn.context(ctx)',
             'a = c.expression(r).standard()'])

# is_synchronized
ctx = "lat<law, law>, b"
r = r"""(abc|de)((f|\e)+((g|h)(i|j)*(\e|k)))(l|mn){500000}"""
bench('a.is_synchronized()',
      'a = std({})'.format(r),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r),
             'c = vcsn.context(ctx)',
             'a = c.expression(r).standard()'])

# weight_series
ctx = "lal(a-z), nmin"
r = "a{12000}+<1>[b-z]{12000}"
bench('a.weight_series()',
      'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r),
             'a = vcsn.context(ctx).expression(r).standard()'],
      number=200)

ctx = "lal(a-z), z"
r = "[a-z]{200}"
bench('a.weight_series()',
      'a = std({}), c = {}'.format(r, ctx_signature(ctx)),
      setup=['ctx = "{}"'.format(ctx),
             'r = "{}"'.format(r),
             'a = vcsn.context(ctx).expression(r).standard()'],
      number=10)


## ------ ##
## main.  ##
## ------ ##

if args.sort:
    benches = sorted(benches, key=lambda b: b.title)

if args.list:
    for b in benches:
        output(b.title)
    exit(0)

print('vcsn version: {}'.format(vcsn.version))
if args.out is not None:
    cxx = vcsn.config('configuration.cxx')
    comment('vcsn', vcsn.version)
    comment('date', time.strftime("%Y-%m-%d %H:%M:%S"))
    comment('compiler', cxx)
    comment('compiler', subprocess.check_output([cxx, '--version'],
                                                universal_newlines=True))
    uname = os.uname()
    comment('sysname', uname.sysname)
    comment('nodename', uname.nodename)
    comment('release', uname.release)
    comment('machine', uname.machine)
if args.jobs == 0:
    for b in benches:
        Bench.run(b)
else:
    with multiprocessing.Pool(args.jobs) as pool:
        pool.map(Bench.run, benches)

if nfail:
    print("vcsn-score: error: there were {} failures".format(nfail),
          file=sys.stderr)
    exit(1)
