eval: Add eval unit tests as a CI job (#32152)

We run the unit evals once a day in the middle of the night, and trigger
a Slack post if it fails.


Release Notes:

- N/A

---------

Co-authored-by: Oleksiy Syvokon <oleksiy.syvokon@gmail.com>
This commit is contained in:
Ben Brandt
2025-06-05 13:16:27 +00:00
committed by GitHub
co-authored by Oleksiy Syvokon
parent fa9da6ad5b
commit dda614091a
2 changed files with 86 additions and 1 deletions
@@ -1351,7 +1351,7 @@ fn eval(iterations: usize, expected_pass_ratio: f32, mut eval: EvalInput) {
let mismatched_tag_ratio =
cumulative_parser_metrics.mismatched_tags as f32 / cumulative_parser_metrics.tags as f32;
if mismatched_tag_ratio > 0.05 {
if mismatched_tag_ratio > 0.10 {
for eval_output in eval_outputs {
println!("{}", eval_output);
}