{"id":"https://openalex.org/W7147088641","doi":"https://doi.org/10.48550/arxiv.2603.29231","title":"Beyond pass@1: A Reliability Science Framework for Long-Horizon LLM Agents","display_name":"Beyond pass@1: A Reliability Science Framework for Long-Horizon LLM Agents","publication_year":2026,"publication_date":"2026-03-31","ids":{"openalex":"https://openalex.org/W7147088641","doi":"https://doi.org/10.48550/arxiv.2603.29231"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2603.29231","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.29231","pdf_url":"https://arxiv.org/pdf/2603.29231","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2603.29231","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129705637","display_name":"Aaditya Khanal","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I168416876","display_name":"Northern Kentucky University","ror":"https://ror.org/01k44g025","country_code":"US","type":"education","lineage":["https://openalex.org/I168416876"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Khanal, Aaditya","raw_affiliation_strings":["School of Computing and Analytics Northern Kentucky University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Analytics Northern Kentucky University","institution_ids":["https://openalex.org/I168416876"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132564122","display_name":"Yangyang Tao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I168416876","display_name":"Northern Kentucky University","ror":"https://ror.org/01k44g025","country_code":"US","type":"education","lineage":["https://openalex.org/I168416876"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tao, Yangyang","raw_affiliation_strings":["School of Computing and Analytics Northern Kentucky University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Analytics Northern Kentucky University","institution_ids":["https://openalex.org/I168416876"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5132565158","display_name":"Junxiu Zhou","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I168416876","display_name":"Northern Kentucky University","ror":"https://ror.org/01k44g025","country_code":"US","type":"education","lineage":["https://openalex.org/I168416876"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhou, Junxiu","raw_affiliation_strings":["School of Computing and Analytics Northern Kentucky University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Analytics Northern Kentucky University","institution_ids":["https://openalex.org/I168416876"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I168416876"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.36570000648498535,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.36570000648498535,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.20010000467300415,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.11999999731779099,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.7940000295639038},{"id":"https://openalex.org/keywords/reliability","display_name":"reliability","score":0.5189999938011169},{"id":"https://openalex.org/keywords/graceful-degradation","display_name":"graceful degradation","score":0.8330000042915344},{"id":"https://openalex.org/keywords/task-duration","display_name":"task duration","score":0.828000009059906}],"concepts":[{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.781499981880188},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.6371999979019165},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.569100022315979},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5651999711990356},{"id":"https://openalex.org/C112758219","wikidata":"https://www.wikidata.org/wiki/Q16038819","display_name":"Duration (music)","level":2,"score":0.5267999768257141},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.505299985408783},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.47839999198913574},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.4742000102996826},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4510999917984009},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.41519999504089355},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.3862999975681305},{"id":"https://openalex.org/C51485801","wikidata":"https://www.wikidata.org/wiki/Q16966861","display_name":"Efficient frontier","level":3,"score":0.32919999957084656},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.3255000114440918},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.28760001063346863},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2799000144004822},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.2797999978065491},{"id":"https://openalex.org/C34778210","wikidata":"https://www.wikidata.org/wiki/Q376791","display_name":"Production\u2013possibility frontier","level":3,"score":0.27880001068115234},{"id":"https://openalex.org/C137668524","wikidata":"https://www.wikidata.org/wiki/Q189813","display_name":"Censoring (clinical trials)","level":2,"score":0.27459999918937683},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.27309998869895935},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.2711000144481659},{"id":"https://openalex.org/C201729545","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability theory","level":3,"score":0.25589999556541443},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.2500999867916107}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2603.29231","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.29231","pdf_url":"https://arxiv.org/pdf/2603.29231","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2603.29231","doi":"https://doi.org/10.48550/arxiv.2603.29231","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.29231","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2603.29231","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.29231","pdf_url":"https://arxiv.org/pdf/2603.29231","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7147088641.pdf","grobid_xml":"https://content.openalex.org/works/W7147088641.grobid-xml"},"referenced_works_count":7,"referenced_works":["https://openalex.org/W4399836817","https://openalex.org/W4412877164","https://openalex.org/W4415972402","https://openalex.org/W4416870030","https://openalex.org/W6967362843","https://openalex.org/W7123695262","https://openalex.org/W7130615426"],"related_works":["https://openalex.org/W7207803844","https://openalex.org/W7165473354","https://openalex.org/W7123695262","https://openalex.org/W7202118689","https://openalex.org/W7223171185","https://openalex.org/W7131376626","https://openalex.org/W7167913969","https://openalex.org/W7162948904","https://openalex.org/W7221314292","https://openalex.org/W7213298332"],"abstract_inverted_index":{"Existing":[0],"benchmarks":[1],"measure":[2],"capability":[3,128,135,142],"--":[4,13,19,107,130],"whether":[5],"a":[6,10,54,90,134,191],"model":[7],"succeeds":[8],"on":[9,25,43,89],"single":[11],"attempt":[12,167],"but":[14],"production":[15],"deployments":[16],"require":[17],"reliability":[18,55,103,144,189],"consistent":[20],"success":[21],"across":[22,86,182],"repeated":[23],"attempts":[24],"tasks":[26,45],"of":[27],"varying":[28],"duration.":[29],"We":[30,52,82],"show":[31],"these":[32],"properties":[33],"diverge":[34,146],"systematically":[35],"as":[36,190],"task":[37],"duration":[38,95],"grows,":[39],"and":[40,77,97,143,174],"that":[41,171],"pass@1":[42],"short":[44],"is":[46,105,118,133],"structurally":[47],"blind":[48],"to":[49,113,122,163],"this":[50],"divergence.":[51],"introduce":[53],"science":[56],"framework":[57],"for":[58],"long-horizon":[59,180],"LLM":[60],"agents":[61],"with":[62,148],"four":[63,94],"metrics:":[64],"Reliability":[65],"Decay":[66],"Curve":[67],"(RDC),":[68],"Variance":[69],"Amplification":[70],"Factor":[71],"(VAF),":[72],"Graceful":[73],"Degradation":[74],"Score":[75],"(GDS),":[76],"Meltdown":[78],"Onset":[79],"Point":[80],"(MOP).":[81],"evaluate":[83],"10":[84,184],"models":[85,156],"23,392":[87],"episodes":[88],"396-task":[91],"benchmark":[92],"spanning":[93],"buckets":[96],"three":[98],"domains.":[99],"Key":[100],"findings:":[101],"(1)":[102],"decay":[104],"domain-stratified":[106],"SE":[108],"GDS":[109],"drops":[110],"from":[111],"0.90":[112],"0.44":[114],"while":[115],"document":[116],"processing":[117],"nearly":[119],"flat":[120],"(0.74":[121],"0.71);":[123],"(2)":[124],"VAF":[125,132],"bifurcates":[126],"by":[127],"tier":[129],"high":[131],"signature,":[136],"not":[137],"an":[138],"instability":[139],"signal;":[140],"(3)":[141],"rankings":[145],"substantially,":[147],"multi-rank":[149],"inversions":[150],"at":[151],"long":[152],"horizons;":[153],"(4)":[154],"frontier":[155],"have":[157],"the":[158],"highest":[159],"meltdown":[160],"rates":[161],"(up":[162],"19%)":[164],"because":[165],"they":[166],"ambitious":[168],"multi-step":[169],"strategies":[170],"sometimes":[172],"spiral;":[173],"(5)":[175],"memory":[176],"scaffolds":[177],"universally":[178],"hurt":[179],"performance":[181],"all":[183],"models.":[185],"These":[186],"results":[187],"motivate":[188],"first-class":[192],"evaluation":[193],"dimension":[194],"alongside":[195],"capability.":[196]},"counts_by_year":[{"year":2026,"cited_by_count":10}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2026-04-02T00:00:00"}