{"id":"https://openalex.org/W7162465549","doi":"https://doi.org/10.48550/arxiv.2605.24213","title":"Towards Evaluation Engineering: An Empirical Study of ML Evaluation Harnesses in the Wild","display_name":"Towards Evaluation Engineering: An Empirical Study of ML Evaluation Harnesses in the Wild","publication_year":2026,"publication_date":"2026-05-22","ids":{"openalex":"https://openalex.org/W7162465549","doi":"https://doi.org/10.48550/arxiv.2605.24213"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.24213","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.24213","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.24213","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100576294","display_name":"Zhimin Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Zhimin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137074878","display_name":"Zehao Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zehao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063403084","display_name":"Abdul Ali Bangash","orcid":"https://orcid.org/0000-0002-5311-6061"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bangash, Abdul Ali","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137071830","display_name":"Bram Adams","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Adams, Bram","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136995097","display_name":"Ahmed E. Hassan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hassan, Ahmed E.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.42160001397132874,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.42160001397132874,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.17069999873638153,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10639","display_name":"Advanced Software Engineering Methodologies","score":0.08540000021457672,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dependency","display_name":"Dependency (UML)","score":0.6539999842643738},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6478999853134155},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.5727999806404114},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.49790000915527344},{"id":"https://openalex.org/keywords/documentation","display_name":"Documentation","score":0.4754999876022339},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4408999979496002},{"id":"https://openalex.org/keywords/root-cause","display_name":"Root cause","score":0.4065000116825104},{"id":"https://openalex.org/keywords/root-cause-analysis","display_name":"Root cause analysis","score":0.39989998936653137},{"id":"https://openalex.org/keywords/provisioning","display_name":"Provisioning","score":0.375}],"concepts":[{"id":"https://openalex.org/C19768560","wikidata":"https://www.wikidata.org/wiki/Q320727","display_name":"Dependency (UML)","level":2,"score":0.6539999842643738},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6478999853134155},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5935999751091003},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.5727999806404114},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.49790000915527344},{"id":"https://openalex.org/C56666940","wikidata":"https://www.wikidata.org/wiki/Q788790","display_name":"Documentation","level":2,"score":0.4754999876022339},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4408999979496002},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.40860000252723694},{"id":"https://openalex.org/C84945661","wikidata":"https://www.wikidata.org/wiki/Q7366567","display_name":"Root cause","level":2,"score":0.4065000116825104},{"id":"https://openalex.org/C130963320","wikidata":"https://www.wikidata.org/wiki/Q1401207","display_name":"Root cause analysis","level":2,"score":0.39989998936653137},{"id":"https://openalex.org/C172191483","wikidata":"https://www.wikidata.org/wiki/Q1071806","display_name":"Provisioning","level":2,"score":0.375},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35569998621940613},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.35499998927116394},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3537999987602234},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.3495999872684479},{"id":"https://openalex.org/C207850805","wikidata":"https://www.wikidata.org/wiki/Q269608","display_name":"Reverse engineering","level":2,"score":0.32519999146461487},{"id":"https://openalex.org/C171078966","wikidata":"https://www.wikidata.org/wiki/Q111029","display_name":"Root (linguistics)","level":2,"score":0.3237999975681305},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.3125},{"id":"https://openalex.org/C86610423","wikidata":"https://www.wikidata.org/wiki/Q1925081","display_name":"Metamodeling","level":2,"score":0.3093999922275543},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2939000129699707},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2935999929904938},{"id":"https://openalex.org/C92446256","wikidata":"https://www.wikidata.org/wiki/Q3306762","display_name":"Data validation","level":2,"score":0.2906999886035919},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.28450000286102295},{"id":"https://openalex.org/C133199616","wikidata":"https://www.wikidata.org/wiki/Q25386885","display_name":"Empirical modelling","level":2,"score":0.28130000829696655},{"id":"https://openalex.org/C23725684","wikidata":"https://www.wikidata.org/wiki/Q616377","display_name":"Maintenance engineering","level":2,"score":0.2791999876499176},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.27309998869895935},{"id":"https://openalex.org/C33762810","wikidata":"https://www.wikidata.org/wiki/Q461671","display_name":"Data integrity","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.2563000023365021},{"id":"https://openalex.org/C48002344","wikidata":"https://www.wikidata.org/wiki/Q2919644","display_name":"Verification and validation","level":2,"score":0.25619998574256897},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.2540999948978424}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.24213","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.24213","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.24213","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.24213","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.650054931640625,"id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Evaluation":[0],"harnesses":[1,77],"are":[2,94],"software":[3,171],"systems":[4],"that":[5,123],"orchestrate":[6],"model":[7,11,53],"evaluation":[8,47,166],"by":[9,58,131],"managing":[10],"invocation,":[12],"data":[13],"loading,":[14],"metric":[15],"computation,":[16],"and":[17,31,54,61,82,101,120,136,150],"result":[18],"reporting.":[19],"Despite":[20],"their":[21,28],"critical":[22],"role":[23],"in":[24,69,117],"machine":[25],"learning":[26],"infrastructure,":[27],"operational":[29,66,92],"challenges":[30,67,93],"engineering":[32,167,172],"concerns":[33],"have":[34],"received":[35],"limited":[36],"attention":[37],"so":[38],"far.":[39],"We":[40],"present":[41],"an":[42,161],"empirical":[43,162],"study":[44],"of":[45,74,91,111,143],"57":[46],"harnesses,":[48],"deriving":[49],"a":[50,169],"five-stage":[51],"harness":[52,65],"classifying":[55],"16,560":[56],"issues":[57],"workflow":[59,132],"stage":[60,72],"root":[62,89],"cause.":[63],"Most":[64],"concentrate":[68],"the":[70],"Specification":[71],"(41.4%":[73],"issues),":[75],"where":[76],"integrate":[78],"external":[79,137],"models,":[80],"datasets,":[81],"scoring":[83],"judges.":[84],"The":[85],"three":[86],"most":[87],"frequent":[88],"causes":[90,128],"unimplemented":[95],"features":[96],"(24.3%),":[97],"documentation":[98],"gaps":[99,122],"(20.3%),":[100],"missing":[102],"input":[103],"validation":[104,151],"(17.2%),":[105],"which":[106],"together":[107],"account":[108,140],"for":[109,141,164],"61.7%":[110],"classified":[112],"issues,":[113,145],"spanning":[114],"both":[115],"defects":[116],"existing":[118],"functionality":[119],"capability":[121],"block":[124],"intended":[125],"workflows.":[126],"Root":[127],"also":[129],"vary":[130],"stage:":[133],"environment":[134],"incompatibility":[135],"dependency":[138],"breakage":[139],"36.2%":[142],"provisioning":[144],"whereas":[146],"algorithmic":[147],"error":[148],"(25.9%)":[149],"gap":[152],"(22.5%)":[153],"dominate":[154],"assessment":[155],"issues.":[156],"Together,":[157],"these":[158],"contributions":[159],"establish":[160],"foundation":[163],"treating":[165],"as":[168],"distinct":[170],"concern.":[173]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-27T00:00:00"}