{"id":"https://openalex.org/W7154081539","doi":"https://doi.org/10.48550/arxiv.2604.08970","title":"Litmus (Re)Agent: A Benchmark and Agentic System for Predictive Evaluation of Multilingual Models","display_name":"Litmus (Re)Agent: A Benchmark and Agentic System for Predictive Evaluation of Multilingual Models","publication_year":2026,"publication_date":"2026-04-10","ids":{"openalex":"https://openalex.org/W7154081539","doi":"https://doi.org/10.48550/arxiv.2604.08970"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.08970","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08970","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.08970","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133499322","display_name":"Avni Mittal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mittal, Avni","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133477620","display_name":"Shanu Kumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kumar, Shanu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006779728","display_name":"Sandipan Dandapat","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dandapat, Sandipan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133494819","display_name":"Monojit Choudhury","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choudhury, Monojit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6152999997138977,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6152999997138977,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.19979999959468842,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0625,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/litmus","display_name":"Litmus","score":0.9656000137329102},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.9014000296592712},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6766999959945679},{"id":"https://openalex.org/keywords/missing-data","display_name":"Missing data","score":0.3580999970436096},{"id":"https://openalex.org/keywords/calibration","display_name":"Calibration","score":0.33889999985694885},{"id":"https://openalex.org/keywords/task-analysis","display_name":"Task analysis","score":0.3052999973297119}],"concepts":[{"id":"https://openalex.org/C157329479","wikidata":"https://www.wikidata.org/wiki/Q185082","display_name":"Litmus","level":2,"score":0.9656000137329102},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.9014000296592712},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6766999959945679},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6761999726295471},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5849999785423279},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5228000283241272},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3898000121116638},{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.3580999970436096},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.33889999985694885},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3127000033855438},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3052999973297119},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3034000098705292},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.2890999913215637},{"id":"https://openalex.org/C2781170535","wikidata":"https://www.wikidata.org/wiki/Q30587856","display_name":"Noisy data","level":2,"score":0.26170000433921814},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.2597000002861023}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.08970","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08970","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.08970","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.08970","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7346929907798767}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,48,84],"study":[1],"predictive":[2],"multilingual":[3,30,142],"evaluation:":[4],"estimating":[5],"how":[6],"well":[7],"a":[8,13,16,50,89,138],"model":[9,46],"will":[10],"perform":[11],"on":[12],"task":[14],"in":[15,29,120],"target":[17],"language":[18],"when":[19],"direct":[20,124],"benchmark":[21,52,64],"results":[22,79,131],"are":[23],"missing.":[24],"This":[25],"problem":[26],"is":[27,35,40,126,137],"common":[28],"deployment,":[31],"where":[32,123],"evaluation":[33,72],"coverage":[34],"sparse":[36],"and":[37,45,59,100],"published":[38],"evidence":[39,61,67,125],"uneven":[41],"across":[42],"languages,":[43],"tasks,":[44],"families.":[47],"introduce":[49],"controlled":[51],"of":[53,73],"1,500":[54],"questions":[55],"spanning":[56],"six":[57,107],"tasks":[58],"five":[60],"scenarios.":[62],"The":[63],"separates":[65],"accessible":[66],"from":[68,80],"ground":[69],"truth,":[70],"enabling":[71],"systems":[74],"that":[75,93,133],"must":[76],"infer":[77],"missing":[78],"incomplete":[81,146],"literature":[82],"evidence.":[83,147],"also":[85],"present":[86],"Litmus":[87,109],"(Re)Agent,":[88],"DAG-orchestrated":[90],"agentic":[91,135],"system":[92],"decomposes":[94],"queries":[95],"into":[96],"hypotheses,":[97],"retrieves":[98],"evidence,":[99],"synthesises":[101],"predictions":[102],"through":[103],"feature-aware":[104],"aggregation.":[105],"Across":[106],"systems,":[108],"(Re)Agent":[110],"achieves":[111],"the":[112,117],"best":[113],"overall":[114],"performance,":[115],"with":[116],"largest":[118],"gains":[119],"transfer-heavy":[121],"scenarios":[122],"weak":[127],"or":[128],"absent.":[129],"These":[130],"show":[132],"structured":[134],"reasoning":[136],"promising":[139],"approach":[140],"to":[141],"performance":[143],"estimation":[144],"under":[145]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-14T00:00:00"}