numpy>=1.17
pyarrow>=0.17.1
dill
pandas
requests>=2.19.0
tqdm<4.50.0,>=4.27
xxhash
multiprocess
fsspec
huggingface_hub==0.0.2

[:python_version < "3.7"]
dataclasses

[:python_version < "3.8"]
importlib_metadata

[apache-beam]
apache-beam

[benchmarks]
numpy==1.18.5
tensorflow==2.3.0
torch==1.6.0
transformers==3.0.2

[dev]
absl-py
pytest
pytest-xdist
apache-beam>=2.24.0
elasticsearch
boto3==1.16.43
botocore==1.19.43
faiss-cpu
fsspec[s3]
moto[s3]==1.3.16
rarfile>=4.0
tensorflow>=2.3
torch
transformers
bs4
conllu
langdetect
lxml
mwparserfromhell
nltk
openpyxl
py7zr
tldextract
zstandard
bert_score>=0.3.6
rouge_score
sacrebleu
scipy
seqeval
sklearn
jiwer
toml>=0.10.1
requests_file>=1.5.1
tldextract>=3.1.0
texttable>=1.6.3
s3fs>=0.4.2
Werkzeug>=1.0.1
wget>=3.2
pytorch-nlp==0.5.0
pytorch_lightning
fastBPE==0.1.0
fairseq
black
isort
flake8==3.7.9

[docs]
recommonmark
sphinx==3.1.2
sphinx-markdown-tables
sphinx-rtd-theme==0.4.3
sphinx-copybutton
fsspec[s3]

[quality]
black
isort
flake8==3.7.9

[s3]
fsspec[s3]
boto3==1.16.43
botocore==1.19.43

[tensorflow]
tensorflow>=2.2.0

[tensorflow_gpu]
tensorflow-gpu>=2.2.0

[tests]
absl-py
pytest
pytest-xdist
apache-beam>=2.24.0
elasticsearch
boto3==1.16.43
botocore==1.19.43
faiss-cpu
fsspec[s3]
moto[s3]==1.3.16
rarfile>=4.0
tensorflow>=2.3
torch
transformers
bs4
conllu
langdetect
lxml
mwparserfromhell
nltk
openpyxl
py7zr
tldextract
zstandard
bert_score>=0.3.6
rouge_score
sacrebleu
scipy
seqeval
sklearn
jiwer
toml>=0.10.1
requests_file>=1.5.1
tldextract>=3.1.0
texttable>=1.6.3
s3fs>=0.4.2
Werkzeug>=1.0.1
wget>=3.2
pytorch-nlp==0.5.0
pytorch_lightning
fastBPE==0.1.0
fairseq

[torch]
torch
