pyproject.toml

[tool.poetry]
name = "ScandEval"
version = "12.10.4"
description = "Evaluation of pretrained language models on mono- or multilingual language tasks."
authors = ["Dan Saattrup Nielsen <dan.nielsen@alexandra.dk>"]
maintainers = [
    "Dan Saattrup Nielsen <dan.nielsen@alexandra.dk>",
    "Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>",
]
readme = "README.md"
license = "MIT"
homepage = "https://scandeval.github.io"
repository = "https://github.com/ScandEval/ScandEval"

[tool.poetry.dependencies]
python = ">=3.10,<3.12"

# Core packages
torch = "^2.3.0"
pandas = "^2.2.0"
numpy = "^1.23.0"
transformers = "~4.40.0"
accelerate = "^0.26.0"
evaluate = "^0.4.1"
datasets = "^2.15.0"
click = "^8.1.3"
pydantic = "^2.6.0"
termcolor = "^2.0.0"
seqeval = "^1.2.2"
python-dotenv = "^1.0.1"
pyinfer = "^0.0.3"  # Used for speed evaluation

# Needed for some tokenisers
sentencepiece = "^0.1.96"
protobuf = "~3.20.0"
sacremoses = "^0.1.1"

# Needed for evaluation of generative models
rouge-score = { version = "^0.1.2", optional = true }
bert-score = { version = "^0.3.13", optional = true }
demjson3 = { version = "^3.0.6", optional = true }
bitsandbytes = { markers = "sys_platform != 'darwin' or platform_machine != 'arm64'", version = "^0.43.1", optional = true }
vllm = { markers = "sys_platform != 'darwin'", version = "^0.4.2", optional = true }
outlines = { version = "^0.0.34", optional = true }

# Needed for loading JAX based encoder models
jax = { version = "^0.4.24", optional = true }
jaxlib = { version = "^0.4.24", optional = true }
flax = { version = "^0.8.1", optional = true }

# Needed for evaluating OpenAI models
openai = { version = "^1.11.1", optional = true }
tiktoken = {version = "^0.6.0", optional = true}
levenshtein = { version = "^0.24.0", optional = true }  # This is also used for human evaluation

# Needed for quantised models
auto-gptq = { version = "^0.7.1", optional = true }
optimum = { version = "^1.19.1", optional = true }
autoawq = { version = "^0.2.5", optional = true }

# Needed for human evaluation
gradio = { version = "^4.26.0", optional = true }

[tool.poetry.group.dev.dependencies]
pytest = ">=8.1.1"
pre-commit = ">=3.6.2"
lxml = ">=5.1.0"
pytest-cov = ">=4.1.0"
readme-coverage-badger = ">=0.1.2"
ruff = ">=0.3.2"
mypy = ">=1.9.0"
nbstripout = ">=0.7.1"

[tool.poetry.extras]
jax = [
    "jax",
    "jaxlib",
    "flax",
]
generative = [
    "demjson3",
    "outlines",
    "bitsandbytes",
    "vllm",
    "rouge-score",
    "bert-score",
]
openai = [
    "rouge-score",
    "bert-score",
    "openai",
    "tiktoken",
    "levenshtein",
]
quantization = [
    "optimum",
    "auto-gptq",
    "autoawq",
]
human_evaluation = [
    "gradio",
    "levenshtein",
]
all = [
    "jax",
    "jaxlib",
    "flax",
    "rouge-score",
    "bert-score",
    "demjson3",
    "outlines",
    "bitsandbytes",
    "vllm",
    "openai",
    "tiktoken",
    "levenshtein",
    "optimum",
    "auto-gptq",
    "autoawq",
    "gradio",
]
cpu_all = [
    "jax",
    "jaxlib",
    "flax",
    "rouge-score",
    "bert-score",
    "demjson3",
    "outlines",
    "openai",
    "tiktoken",
    "levenshtein",
    "gradio",
]

[tool.poetry.scripts]
scandeval = "scandeval.cli:benchmark"
human_evaluate = "scandeval.human_evaluation:main"

[tool.pytest.ini_options]
minversion = "6.0"
addopts = [
    '--verbose',
    '--durations=10',
    '--color=no',
    '-s',
    '-vv',
    '--doctest-modules',
    '--cov=src/scandeval',
]
xfail_strict = true
filterwarnings = [
    "error",
    "ignore::UserWarning",
    "ignore::DeprecationWarning",
    "ignore::PendingDeprecationWarning",
    "ignore::ImportWarning",
]
log_cli_level = "info"
testpaths = ["tests", "src/scandeval"]

[tool.ruff]
line-length = 88
target-version = "py310"
exclude = [
    ".git",
    ".mypy_cache",
    ".pytest_cache",
    ".ruff_cache",
    ".scandeval_cache",
    ".venv",
]

[tool.ruff.format]
quote-style = "double"
indent-style = "space"
docstring-code-format = true
skip-magic-trailing-comma = true

[tool.ruff.lint]
extend-select = [
    "I",
    "D",
]

[tool.ruff.lint.extend-per-file-ignores]
"__init__.py" = ["F401"]

[tool.ruff.lint.isort]
split-on-trailing-comma = false

[tool.ruff.lint.pydocstyle]
convention = "google"

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"