{"id":"https://openalex.org/W7165621234","doi":"https://doi.org/10.48550/arxiv.2606.23403","title":"Litmus: Zero-Label, Code-Driven Metric Specification for Evaluating AI Systems","display_name":"Litmus: Zero-Label, Code-Driven Metric Specification for Evaluating AI Systems","publication_year":2026,"publication_date":"2026-06-22","ids":{"openalex":"https://openalex.org/W7165621234","doi":"https://doi.org/10.48550/arxiv.2606.23403"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.23403","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.23403","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.23403","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002077403","display_name":"Prajjwal Gupta","orcid":"https://orcid.org/0000-0002-0704-5127"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gupta, Prajjwal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041268877","display_name":"Prasang Gupta","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gupta, Prasang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011914192","display_name":"Vishal Bhutani","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bhutani, Vishal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039768487","display_name":"A Sharma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sharma, Apoorva","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139142850","display_name":"Sumanth Chundru","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chundru, Sumanth","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139215894","display_name":"Waqar Sarguroh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sarguroh, Waqar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5123047053","display_name":"Kevin Paul","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paul, Kevin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.2093999981880188,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.2093999981880188,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.17520000040531158,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.1728000044822693,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.8187000155448914},{"id":"https://openalex.org/keywords/litmus","display_name":"Litmus","score":0.7842000126838684},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6284000277519226},{"id":"https://openalex.org/keywords/metric-system","display_name":"Metric system","score":0.5981000065803528},{"id":"https://openalex.org/keywords/relation","display_name":"Relation (database)","score":0.5047000050544739},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.487199991941452},{"id":"https://openalex.org/keywords/pipeline-transport","display_name":"Pipeline transport","score":0.478300005197525},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4350000023841858}],"concepts":[{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.8187000155448914},{"id":"https://openalex.org/C157329479","wikidata":"https://www.wikidata.org/wiki/Q185082","display_name":"Litmus","level":2,"score":0.7842000126838684},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6334999799728394},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6284000277519226},{"id":"https://openalex.org/C193429382","wikidata":"https://www.wikidata.org/wiki/Q232405","display_name":"Metric system","level":2,"score":0.5981000065803528},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.5047000050544739},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.487199991941452},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.478300005197525},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4350000023841858},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.37720000743865967},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.3626999855041504},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.36169999837875366},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3504999876022339},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3319999873638153},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.3093999922275543},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3046000003814697},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2985000014305115},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.288100004196167},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C112758219","wikidata":"https://www.wikidata.org/wiki/Q16038819","display_name":"Duration (music)","level":2,"score":0.26739999651908875},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2639000117778778},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.26269999146461487},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.25850000977516174},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.25690001249313354}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.23403","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.23403","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.23403","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.23403","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"agentic":[1],"LLM":[2],"systems":[3,238],"move":[4],"from":[5,90,223],"prototypes":[6],"to":[7,52,66,204,227,235],"deployment":[8],"across":[9],"increasingly":[10],"diverse":[11],"domains,":[12],"evaluating":[13],"them":[14],"has":[15],"become":[16,64],"both":[17],"more":[18,21,163],"important":[19],"and":[20,58,80,93,113,143,150,170,197,245],"difficult.":[22],"The":[23],"challenge":[24],"is":[25,50,103],"not":[26],"only":[27],"that":[28,35,77,99],"individual":[29],"metrics":[30,82],"may":[31],"be":[32,111,243],"unreliable,":[33],"but":[34],"evaluation":[36,79,88,101,237],"goals":[37],"are":[38],"often":[39],"left":[40],"implicit.":[41],"Without":[42],"a":[43,48,74,123,167,221],"clear":[44],"account":[45,139],"of":[46,97,207],"what":[47,109,241],"system":[49,76],"expected":[51],"do,":[53],"how":[54],"it":[55],"can":[56],"fail,":[57],"which":[59,233],"failures":[60],"matter,":[61],"metric":[62,126,216,225,229,234],"choices":[63],"difficult":[65],"justify,":[67],"interpret,":[68],"or":[69,158],"validate.":[70],"We":[71,128],"present":[72],"Litmus,":[73],"zero-label":[75],"designs":[78],"monitoring":[81],"for":[83,121,194],"AI":[84,135],"pipelines":[85,136,182],"by":[86],"eliciting":[87],"intent":[89],"source":[91],"code":[92],"targeted":[94],"interrogation.":[95],"Instead":[96],"assuming":[98],"the":[100,156,208],"target":[102],"already":[104],"known,":[105],"Litmus":[106,130,154],"first":[107,172],"identifies":[108],"must":[110,242],"measured":[112,244],"why,":[114],"then":[115],"converts":[116],"those":[117],"answers":[118],"into":[119],"constraints":[120],"constructing":[122],"justified,":[124],"per-stage":[125],"portfolio.":[127],"evaluate":[129],"on":[131,179,185],"three":[132,151,181],"real,":[133],"code-defined":[134],"-":[137,147,183],"financial":[138],"grouping,":[140],"scientific":[141,186],"QA,":[142],"inherent":[144],"risk":[145],"assessment":[146],"against":[148,175],"AutoMetrics":[149],"DynamicRubric":[152],"baselines.":[153],"achieves":[155],"broadest":[157],"tied-broadest":[159],"concern":[160],"coverage,":[161],"spans":[162],"pipeline":[164],"stages,":[165],"produces":[166],"near-zero-redundancy":[168],"portfolio,":[169],"ranks":[171],"in":[173,202],"validity":[174],"per-row":[176],"quality":[177],"labels":[178,214],"all":[180],"decisively":[184],"QA":[187],"(Spearman":[188],"$\u03c1=0.72$":[189],"vs.":[190],"less":[191],"than":[192],"$0.47$":[193],"every":[195],"baseline),":[196],"within":[198],"overlapping":[199],"confidence":[200],"intervals":[201],"relation":[203],"two":[205],"components":[206],"audit":[209],"framework":[210],"despite":[211],"using":[212],"no":[213],"during":[215],"design.":[217],"Our":[218],"results":[219],"support":[220],"shift":[222],"automatic":[224,228],"implementation":[226],"specification:":[230],"before":[231],"asking":[232],"compute,":[236],"should":[239],"ask":[240],"why.":[246]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-24T00:00:00"}