{"id":"https://openalex.org/W4414835304","doi":"https://doi.org/10.48550/arxiv.2502.15840","title":"Vending-Bench: A Benchmark for Long-Term Coherence of Autonomous Agents","display_name":"Vending-Bench: A Benchmark for Long-Term Coherence of Autonomous Agents","publication_year":2025,"publication_date":"2025-02-20","ids":{"openalex":"https://openalex.org/W4414835304","doi":"https://doi.org/10.48550/arxiv.2502.15840"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.15840","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.15840","pdf_url":"https://arxiv.org/pdf/2502.15840","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.15840","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119852631","display_name":"Axel Backlund","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Backlund, Axel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5119852632","display_name":"Lukas Petersson","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Petersson, Lukas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":105,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.26269999146461487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.26269999146461487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.24300000071525574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1298000067472458,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-based-agents","display_name":"LLM agents","score":0.6759999990463257},{"id":"https://openalex.org/keywords/autonomous-agents","display_name":"autonomous agents","score":0.6549999713897705},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.44200000166893005},{"id":"https://openalex.org/keywords/simulation-environment","display_name":"simulation environment","score":0.5659999847412109},{"id":"https://openalex.org/keywords/inventory-management","display_name":"inventory management","score":0.7160000205039978},{"id":"https://openalex.org/keywords/long-horizon-decision-making","display_name":"long-horizon decision-making","score":0.7250000238418579},{"id":"https://openalex.org/keywords/context-window-limits","display_name":"context window limits","score":0.574999988079071},{"id":"https://openalex.org/keywords/capital-acquisition","display_name":"capital acquisition","score":0.7649999856948853},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.5090000033378601}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7116000056266785},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6351000070571899},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5978000164031982},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5471000075340271},{"id":"https://openalex.org/C2781181686","wikidata":"https://www.wikidata.org/wiki/Q4226068","display_name":"Coherence (philosophical gambling strategy)","level":2,"score":0.534600019454956},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5303000211715698},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.5001000165939331},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4474000036716461},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41130000352859497},{"id":"https://openalex.org/C7149132","wikidata":"https://www.wikidata.org/wiki/Q1377840","display_name":"Forgetting","level":2,"score":0.4099000096321106},{"id":"https://openalex.org/C2778751112","wikidata":"https://www.wikidata.org/wiki/Q835016","display_name":"Window (computing)","level":2,"score":0.40639999508857727},{"id":"https://openalex.org/C2778775528","wikidata":"https://www.wikidata.org/wiki/Q5135432","display_name":"Closing (real estate)","level":2,"score":0.36340001225471497},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.32109999656677246},{"id":"https://openalex.org/C2781020372","wikidata":"https://www.wikidata.org/wiki/Q533093","display_name":"On the fly","level":2,"score":0.313400000333786},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.30959999561309814},{"id":"https://openalex.org/C204983608","wikidata":"https://www.wikidata.org/wiki/Q2111958","display_name":"Productivity","level":2,"score":0.2752000093460083},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.27090001106262207}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.15840","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.15840","pdf_url":"https://arxiv.org/pdf/2502.15840","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.15840","doi":"https://doi.org/10.48550/arxiv.2502.15840","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.15840","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.15840","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.15840","pdf_url":"https://arxiv.org/pdf/2502.15840","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414835304.pdf","grobid_xml":"https://content.openalex.org/works/W4414835304.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7148248081","https://openalex.org/W7138948018","https://openalex.org/W7164921373","https://openalex.org/W7172344510","https://openalex.org/W7164882062","https://openalex.org/W7126113874","https://openalex.org/W7204931996","https://openalex.org/W7162089146","https://openalex.org/W7131825876","https://openalex.org/W4385682136"],"abstract_inverted_index":{"While":[0],"Large":[1],"Language":[2],"Models":[3],"(LLMs)":[4],"can":[5,198],"exhibit":[6],"impressive":[7],"proficiency":[8],"in":[9,92,106,172,188,200],"isolated,":[10],"short-term":[11],"tasks,":[12],"they":[13,135],"often":[14],"fail":[15],"to":[16,34,41,183],"maintain":[17],"coherent":[18,85],"performance":[19,93,173],"over":[20,72,174],"longer":[21],"time":[22,176],"horizons.":[23],"In":[24],"this":[25],"paper,":[26],"we":[27],"present":[28],"Vending-Bench,":[29],"a":[30,43,49,111,186],"simulated":[31],"environment":[32],"designed":[33],"specifically":[35],"test":[36],"an":[37,80],"LLM-based":[38],"agent's":[39],"ability":[40,182],"manage":[42,102],"straightforward,":[44],"long-running":[45],"business":[46],"scenario:":[47],"operating":[48],"vending":[50],"machine.":[51],"Agents":[52],"must":[53],"balance":[54],"inventories,":[55],"place":[56],"orders,":[57,126],"set":[58],"prices,":[59],"and":[60,100,109,145],"handle":[61],"daily":[62],"fees":[63],"-":[64],"tasks":[65],"that":[66,118,157],"are":[67],"each":[68],"simple":[69],"but":[70,113],"collectively,":[71],"long":[73,175],"horizons":[74],"(>20M":[75],"tokens":[76],"per":[77],"run)":[78],"stress":[79],"LLM's":[81],"capacity":[82],"for":[83,202],"sustained,":[84],"decision-making.":[86],"Our":[87],"experiments":[88],"reveal":[89],"high":[90,170],"variance":[91,171],"across":[94],"multiple":[95],"LLMs:":[96],"Claude":[97],"3.5":[98],"Sonnet":[99],"o3-mini":[101],"the":[103,146,150,169,196,203],"machine":[104],"well":[105],"most":[107],"runs":[108,117],"turn":[110],"profit,":[112],"all":[114],"models":[115],"have":[116],"derail,":[119],"either":[120],"through":[121],"misinterpreting":[122],"delivery":[123],"schedules,":[124],"forgetting":[125],"or":[127],"descending":[128],"into":[129],"tangential":[130],"\"meltdown\"":[131],"loops":[132],"from":[133,163,167],"which":[134,149],"rarely":[136],"recover.":[137],"We":[138,194],"find":[139],"no":[140],"clear":[141],"correlation":[142],"between":[143],"failures":[144],"point":[147],"at":[148],"model's":[151],"context":[152],"window":[153],"becomes":[154],"full,":[155],"suggesting":[156],"these":[158],"breakdowns":[159],"do":[160],"not":[161],"stem":[162],"memory":[164],"limits.":[165],"Apart":[166],"highlighting":[168],"horizons,":[177],"Vending-Bench":[178],"also":[179],"tests":[180],"models'":[181],"acquire":[184],"capital,":[185],"necessity":[187],"many":[189],"hypothetical":[190],"dangerous":[191],"AI":[192,207],"scenarios.":[193],"hope":[195],"benchmark":[197],"help":[199],"preparing":[201],"advent":[204],"of":[205],"stronger":[206],"systems.":[208]},"counts_by_year":[{"year":2026,"cited_by_count":89},{"year":2025,"cited_by_count":16}],"updated_date":"2026-11-12T12:28:57.059296","created_date":"2025-10-10T00:00:00"}