From 887d43a8d9b41b0cc6c22bb5527a21558ea41853 Mon Sep 17 00:00:00 2001 From: Philipp Emanuel Weidmann Date: Wed, 1 Apr 2026 14:27:43 +0530 Subject: [PATCH] fix: set batch size on HFLM object --- src/heretic/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/heretic/main.py b/src/heretic/main.py index fcc7e3d..1e5cad0 100644 --- a/src/heretic/main.py +++ b/src/heretic/main.py @@ -966,6 +966,7 @@ def run(): hflm = HFLM( pretrained=model.model, # ty:ignore[invalid-argument-type] tokenizer=model.tokenizer, # ty:ignore[invalid-argument-type] + batch_size="auto", ) table = Table() @@ -989,7 +990,6 @@ def run(): results = lm_eval.simple_evaluate( model=hflm, tasks=[benchmark.task], - batch_size="auto", ) return results["results"][benchmark.task]