{"id":"https://openalex.org/W7161599925","doi":"https://doi.org/10.48550/arxiv.2605.15846","title":"RoadmapBench: Evaluating Long-Horizon Agentic Software Development Across Version Upgrades","display_name":"RoadmapBench: Evaluating Long-Horizon Agentic Software Development Across Version Upgrades","publication_year":2026,"publication_date":"2026-05-15","ids":{"openalex":"https://openalex.org/W7161599925","doi":"https://doi.org/10.48550/arxiv.2605.15846"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.15846","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.15846","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.15846","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114206967","display_name":"Xinbo Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Xinbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018136443","display_name":"Ruihan Yang","orcid":"https://orcid.org/0000-0003-0635-2894"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Ruihan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136380680","display_name":"Haiyang Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Haiyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136452921","display_name":"Wendong Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Wendong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002310167","display_name":"Bofei Gao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Bofei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136367005","display_name":"Ruoyu Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Ruoyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020080521","display_name":"Kean Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Kean","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136355786","display_name":"Weichu Xie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Weichu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136429742","display_name":"Xuanzhong Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Xuanzhong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136375358","display_name":"Ming Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Ming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136409463","display_name":"Jason Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Jason","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136415042","display_name":"Michael Heinrich","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Heinrich, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136397547","display_name":"Elvis Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Elvis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136412744","display_name":"Liang Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Liang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100723030","display_name":"Kuan Li","orcid":"https://orcid.org/0000-0002-3272-4084"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Kuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5021459300","display_name":"Baobao Chang","orcid":"https://orcid.org/0000-0003-2824-6750"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chang, Baobao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":16,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.5203999876976013,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.5203999876976013,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10639","display_name":"Advanced Software Engineering Methodologies","score":0.14339999854564667,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.06109999865293503,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.57669997215271},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.5408999919891357},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.5058000087738037},{"id":"https://openalex.org/keywords/snapshot","display_name":"Snapshot (computer storage)","score":0.5055000185966492},{"id":"https://openalex.org/keywords/source-lines-of-code","display_name":"Source lines of code","score":0.49050000309944153},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4627000093460083},{"id":"https://openalex.org/keywords/software-development-process","display_name":"Software development process","score":0.414900004863739},{"id":"https://openalex.org/keywords/distributed-development","display_name":"Distributed development","score":0.359499990940094},{"id":"https://openalex.org/keywords/software-bug","display_name":"Software bug","score":0.34389999508857727}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7134000062942505},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.6758000254631042},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.57669997215271},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.5408999919891357},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.5058000087738037},{"id":"https://openalex.org/C55282118","wikidata":"https://www.wikidata.org/wiki/Q252683","display_name":"Snapshot (computer storage)","level":2,"score":0.5055000185966492},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.49050000309944153},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4627000093460083},{"id":"https://openalex.org/C180152950","wikidata":"https://www.wikidata.org/wiki/Q2904257","display_name":"Software development process","level":4,"score":0.414900004863739},{"id":"https://openalex.org/C2781421246","wikidata":"https://www.wikidata.org/wiki/Q5283148","display_name":"Distributed development","level":4,"score":0.359499990940094},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.34389999508857727},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33980000019073486},{"id":"https://openalex.org/C10272871","wikidata":"https://www.wikidata.org/wiki/Q929972","display_name":"Software inspection","level":5,"score":0.32440000772476196},{"id":"https://openalex.org/C139968098","wikidata":"https://www.wikidata.org/wiki/Q3055454","display_name":"Development environment","level":2,"score":0.3122999966144562},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.29989999532699585},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2996000051498413},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.2955000102519989},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2842999994754791},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.28029999136924744},{"id":"https://openalex.org/C174063052","wikidata":"https://www.wikidata.org/wiki/Q607013","display_name":"Pair programming","level":4,"score":0.272599995136261},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.2718000113964081},{"id":"https://openalex.org/C76518257","wikidata":"https://www.wikidata.org/wiki/Q271680","display_name":"Software framework","level":5,"score":0.2694000005722046},{"id":"https://openalex.org/C39890963","wikidata":"https://www.wikidata.org/wiki/Q1702721","display_name":"Personal software process","level":5,"score":0.26899999380111694},{"id":"https://openalex.org/C202105479","wikidata":"https://www.wikidata.org/wiki/Q265013","display_name":"Software evolution","level":5,"score":0.2612000107765198},{"id":"https://openalex.org/C143587482","wikidata":"https://www.wikidata.org/wiki/Q1543216","display_name":"Iterative and incremental development","level":2,"score":0.26109999418258667},{"id":"https://openalex.org/C32810761","wikidata":"https://www.wikidata.org/wiki/Q17082146","display_name":"Crowdsourcing software development","level":5,"score":0.2549999952316284},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2531000077724457}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.15846","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.15846","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.15846","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.15846","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.49289804697036743,"id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Coding":[0],"agents":[1],"are":[2],"increasingly":[3],"deployed":[4],"in":[5,67,102,143],"real":[6,49,68],"software":[7,153],"development,":[8],"where":[9],"a":[10,59,85,91,107,118,156],"single":[11],"version":[12,70],"iteration":[13],"requires":[14],"months":[15],"of":[16,61,110,135],"coordinated":[17],"work":[18],"across":[19,72,113],"many":[20],"files.":[21,115],"However,":[22],"most":[23],"existing":[24,147],"benchmarks":[25],"focus":[26],"predominantly":[27],"on":[28,84,121],"single-issue":[29],"bug":[30],"fixes":[31],"from":[32],"Python":[33],"repositories,":[34],"with":[35,106],"coarse":[36],"pass/fail":[37],"evaluation":[38,120],"outcomes,":[39],"and":[40,75,89,125],"thus":[41],"fail":[42],"to":[43,97,146],"capture":[44],"long-horizon,":[45],"multi-target":[46,92],"development":[47,154],"at":[48],"engineering":[50],"scale.":[51],"To":[52],"address":[53],"this":[54],"gap,":[55],"we":[56],"present":[57],"RoadmapBench,":[58],"benchmark":[60],"115":[62],"long-horizon":[63,152],"coding":[64],"tasks":[65],"grounded":[66],"open-source":[69],"upgrades":[71],"17":[73],"repositories":[74],"5":[76],"programming":[77],"languages.":[78],"Each":[79],"task":[80],"places":[81],"the":[82,99,103,129,138],"agent":[83],"source-version":[86],"code":[87],"snapshot":[88],"provides":[90],"roadmap":[93],"instruction":[94],"requiring":[95],"it":[96],"implement":[98],"functionality":[100],"introduced":[101],"target":[104],"version,":[105],"median":[108],"modification":[109],"3,700":[111],"lines":[112],"51":[114],"We":[116],"conduct":[117],"systematic":[119],"thirteen":[122],"frontier":[123],"models":[124],"find":[126],"that":[127,151],"even":[128],"strongest,":[130],"Claude-Opus-4.7,":[131],"resolves":[132],"only":[133],"39.1%":[134],"tasks,":[136],"while":[137],"weakest":[139],"achieves":[140],"merely":[141],"5.2%,":[142],"stark":[144],"contrast":[145],"bug-fix":[148],"benchmarks,":[149],"suggesting":[150],"remains":[155],"largely":[157],"unsolved":[158],"problem.":[159]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-05-19T00:00:00"}