eval: Add eval unit tests as a CI job (#32152)
We run the unit evals once a day in the middle of the night, and trigger a Slack post if it fails. Release Notes: - N/A --------- Co-authored-by: Oleksiy Syvokon <oleksiy.syvokon@gmail.com>
This commit is contained in:
co-authored by
Oleksiy Syvokon
parent
fa9da6ad5b
commit
dda614091a
@@ -1351,7 +1351,7 @@ fn eval(iterations: usize, expected_pass_ratio: f32, mut eval: EvalInput) {
|
||||
|
||||
let mismatched_tag_ratio =
|
||||
cumulative_parser_metrics.mismatched_tags as f32 / cumulative_parser_metrics.tags as f32;
|
||||
if mismatched_tag_ratio > 0.05 {
|
||||
if mismatched_tag_ratio > 0.10 {
|
||||
for eval_output in eval_outputs {
|
||||
println!("{}", eval_output);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user