{"id":"https://openalex.org/W7162784677","doi":"https://doi.org/10.48550/arxiv.2605.30314","title":"SpecBench: Evaluating Specification-Level Reasoning for Software Engineering LLM Agents","display_name":"SpecBench: Evaluating Specification-Level Reasoning for Software Engineering LLM Agents","publication_year":2026,"publication_date":"2026-05-28","ids":{"openalex":"https://openalex.org/W7162784677","doi":"https://doi.org/10.48550/arxiv.2605.30314"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.30314","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.30314","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.30314","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137369097","display_name":"Grant Hamblin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hamblin, Grant","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137373937","display_name":"Kevin Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Kevin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101154638","display_name":"Zhanda Zhu","orcid":"https://orcid.org/0000-0001-7433-7384"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Zhanda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022583753","display_name":"Anand Jayarajan","orcid":"https://orcid.org/0000-0002-2118-5935"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jayarajan, Anand","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101571781","display_name":"Sihang Liu","orcid":"https://orcid.org/0000-0001-5711-8462"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Sihang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080873211","display_name":"Nandita Vijaykumar","orcid":"https://orcid.org/0000-0003-3315-9336"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vijaykumar, Nandita","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5007585346","display_name":"Gennady Pekhimenko","orcid":"https://orcid.org/0000-0002-3839-0919"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pekhimenko, Gennady","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.3154999911785126,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.3154999911785126,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.09009999781847,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.08839999884366989,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4611999988555908},{"id":"https://openalex.org/keywords/software-requirements-specification","display_name":"Software requirements specification","score":0.4449999928474426},{"id":"https://openalex.org/keywords/software-development-process","display_name":"Software development process","score":0.4422999918460846},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.42730000615119934},{"id":"https://openalex.org/keywords/expert-system","display_name":"Expert system","score":0.40950000286102295},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4009999930858612},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3961000144481659},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.38019999861717224},{"id":"https://openalex.org/keywords/system-lifecycle","display_name":"System lifecycle","score":0.3727000057697296}],"concepts":[{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.7045999765396118},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6642000079154968},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4611999988555908},{"id":"https://openalex.org/C102780508","wikidata":"https://www.wikidata.org/wiki/Q1761598","display_name":"Software requirements specification","level":5,"score":0.4449999928474426},{"id":"https://openalex.org/C180152950","wikidata":"https://www.wikidata.org/wiki/Q2904257","display_name":"Software development process","level":4,"score":0.4422999918460846},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.42730000615119934},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.4163999855518341},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.40950000286102295},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4009999930858612},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3961000144481659},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.38019999861717224},{"id":"https://openalex.org/C35280785","wikidata":"https://www.wikidata.org/wiki/Q559486","display_name":"System lifecycle","level":4,"score":0.3727000057697296},{"id":"https://openalex.org/C116253237","wikidata":"https://www.wikidata.org/wiki/Q1437424","display_name":"Formal specification","level":2,"score":0.328900009393692},{"id":"https://openalex.org/C52913732","wikidata":"https://www.wikidata.org/wiki/Q857102","display_name":"Software design","level":4,"score":0.32679998874664307},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.32120001316070557},{"id":"https://openalex.org/C34972735","wikidata":"https://www.wikidata.org/wiki/Q2920267","display_name":"Engineering design process","level":2,"score":0.31040000915527344},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3061999976634979},{"id":"https://openalex.org/C54534927","wikidata":"https://www.wikidata.org/wiki/Q4462275","display_name":"Software requirements","level":5,"score":0.29980000853538513},{"id":"https://openalex.org/C48002344","wikidata":"https://www.wikidata.org/wiki/Q2919644","display_name":"Verification and validation","level":2,"score":0.2962999939918518},{"id":"https://openalex.org/C2778956278","wikidata":"https://www.wikidata.org/wiki/Q392813","display_name":"Agent-oriented software engineering","level":4,"score":0.28459998965263367},{"id":"https://openalex.org/C84651959","wikidata":"https://www.wikidata.org/wiki/Q17052506","display_name":"System requirements specification","level":2,"score":0.2793999910354614},{"id":"https://openalex.org/C59488412","wikidata":"https://www.wikidata.org/wiki/Q187147","display_name":"Requirements analysis","level":3,"score":0.2791999876499176},{"id":"https://openalex.org/C10272871","wikidata":"https://www.wikidata.org/wiki/Q929972","display_name":"Software inspection","level":5,"score":0.2782000005245209},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2752000093460083},{"id":"https://openalex.org/C139143892","wikidata":"https://www.wikidata.org/wiki/Q7441615","display_name":"Search-based software engineering","level":5,"score":0.2635999917984009},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.25859999656677246},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.2567000091075897},{"id":"https://openalex.org/C202105479","wikidata":"https://www.wikidata.org/wiki/Q265013","display_name":"Software evolution","level":5,"score":0.25529998540878296},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.2533999979496002},{"id":"https://openalex.org/C57371142","wikidata":"https://www.wikidata.org/wiki/Q16914225","display_name":"Functional specification","level":5,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.30314","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.30314","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.30314","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.30314","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/12","display_name":"Responsible consumption and production","score":0.4020729064941406}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Software":[0],"engineering":[1],"(SWE)":[2],"agents":[3,193],"are":[4,39,58,71,111],"transitioning":[5],"from":[6,113,179],"code":[7,48],"generation":[8],"to":[9,46,94,100,199],"full":[10],"software":[11,67],"development":[12],"lifecycle":[13,20],"automation.":[14],"A":[15],"critical":[16,66],"phase":[17],"in":[18,159],"this":[19,89],"is":[21,130,147],"specification":[22,151],"design:":[23],"transforming":[24],"initial":[25,69,133,161],"proposals":[26],"into":[27],"carefully":[28],"considered":[29],"requirements":[30],"through":[31],"expert":[32,78,170],"review.":[33],"Existing":[34],"benchmarks":[35],"such":[36],"as":[37],"SWE-Bench":[38],"implementation-focused":[40],"by":[41,121,169],"measuring":[42],"the":[43,98,114,136,160],"agent's":[44],"ability":[45,99],"generate":[47,101],"given":[49,131],"fixed,":[50],"precise":[51],"design":[52,134,203],"requirements.":[53],"This":[54],"formulation":[55],"assumes":[56],"specifications":[57,70],"correct":[59,106],"and":[60,65,74,80,105,139,187],"complete.":[61],"In":[62],"real-world":[63],"complex":[64],"systems,":[68],"often":[72],"incomplete":[73],"flawed,":[75],"requiring":[76],"extensive":[77],"reviews":[79],"revisions":[81],"before":[82],"being":[83],"accepted":[84],"for":[85,116],"implementation.":[86],"To":[87],"fill":[88],"gap,":[90],"we":[91],"introduce":[92],"SpecBench":[93,109,176],"evaluate":[95,164,190],"specification-level":[96],"reasoning:":[97],"complete,":[102],"unambiguous,":[103],"consistent,":[104],"system":[107,202],"specifications.":[108],"tasks":[110,178],"derived":[112],"Request":[115],"Comments":[117],"(RFC)":[118],"process":[119],"used":[120],"mature":[122],"open-source":[123],"projects.":[124],"For":[125],"each":[126],"task,":[127],"an":[128,132],"agent":[129,146],"proposal,":[135],"project":[137,142],"codebase,":[138],"all":[140],"past":[141],"RFC":[143,174],"discussions.":[144],"The":[145,207],"tasked":[148],"with":[149],"identifying":[150],"deficiencies:":[152],"omissions,":[153],"ambiguities,":[154],"inconsistencies,":[155],"or":[156],"incorrect":[157],"assumptions":[158],"proposal.":[162],"We":[163,189],"predictions":[165],"against":[166],"critiques":[167],"raised":[168],"maintainers":[171],"during":[172],"historical":[173],"reviews.":[175],"contains":[177],"5":[180],"diverse":[181],"repositories:":[182],"Kubernetes,":[183],"React,":[184],"Rust,":[185],"TVM,":[186],"vLLM.":[188],"state-of-the-art":[191],"SWE":[192],"on":[194],"SpecBench,":[195],"analyzing":[196],"their":[197],"capacity":[198],"reason":[200],"about":[201],"without":[204],"execution":[205],"feedback.":[206],"best":[208],"performing":[209],"agent,":[210],"GPT-5.4,":[211],"achieves":[212],"44.4%":[213],"accuracy.":[214]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-30T00:00:00"}