{"id":"https://openalex.org/W7208838833","doi":null,"title":"SWE-Gate: Passing Functional Tests Is Not Enough for Software Engineering Agents","display_name":"SWE-Gate: Passing Functional Tests Is Not Enough for Software Engineering Agents","publication_year":2026,"publication_date":"2026-09-03","ids":{"openalex":"https://openalex.org/W7208838833"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.04167","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.04167","pdf_url":"https://arxiv.org/pdf/2609.04167","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.04167","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5143425845","display_name":"Xin He","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"He, Xin","raw_affiliation_strings":["School of Software Engineering , Sun Yat-sen University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Software Engineering , Sun Yat-sen University","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122287264","display_name":"Yanlin Wang","orcid":"https://orcid.org/0000-0001-7761-7269","observed_orcids":["https://orcid.org/0000-0001-7761-7269"]},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Yanlin","raw_affiliation_strings":["School of Software Engineering , Sun Yat-sen University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Software Engineering , Sun Yat-sen University","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139827205","display_name":"Mingwei Liu","orcid":"https://orcid.org/0000-0002-3462-997X","observed_orcids":["https://orcid.org/0000-0002-3462-997X"]},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Mingwei","raw_affiliation_strings":["School of Software Engineering , Sun Yat-sen University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Software Engineering , Sun Yat-sen University","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143418223","display_name":"Jiachi Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen, Jiachi","raw_affiliation_strings":["College of Computer Science and Technology , Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology , Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5150065303","display_name":"Hongyu Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I158842170","display_name":"Chongqing University","ror":"https://ror.org/023rhb549","country_code":"CN","type":"education","lineage":["https://openalex.org/I158842170"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, Hongyu","raw_affiliation_strings":["School of Big Data and Software Engineering , Chongqing University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Big Data and Software Engineering , Chongqing University","institution_ids":["https://openalex.org/I158842170"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5143464211","display_name":"Guanbin Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Guanbin","raw_affiliation_strings":["School of Computer Science and Engineering , Sun Yat-sen University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering , Sun Yat-sen University","institution_ids":["https://openalex.org/I157773358"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.004000000189989805,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.0005000000237487257,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/software-engineering-agents","display_name":"software engineering agents","score":0.8939999938011169},{"id":"https://openalex.org/keywords/ai-coding-agents","display_name":"coding agents","score":0.7419999837875366},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.5630000233650208},{"id":"https://openalex.org/keywords/pull-request-review","display_name":"pull request review","score":0.8019999861717224},{"id":"https://openalex.org/keywords/functional-correctness","display_name":"functional correctness","score":0.5590000152587891},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7070000171661377},{"id":"https://openalex.org/keywords/patch-evaluation","display_name":"patch evaluation","score":0.5350000262260437}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7059000134468079},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5712000131607056},{"id":"https://openalex.org/C139143892","wikidata":"https://www.wikidata.org/wiki/Q7441615","display_name":"Search-based software engineering","level":5,"score":0.5012999773025513},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4975999891757965},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.49639999866485596},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.492900013923645},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.49000000953674316},{"id":"https://openalex.org/C62235348","wikidata":"https://www.wikidata.org/wiki/Q3264234","display_name":"Functional requirement","level":2,"score":0.48890000581741333},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.41269999742507935},{"id":"https://openalex.org/C199747065","wikidata":"https://www.wikidata.org/wiki/Q3254666","display_name":"Non-functional requirement","level":5,"score":0.4018000066280365},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.373199999332428},{"id":"https://openalex.org/C57371142","wikidata":"https://www.wikidata.org/wiki/Q16914225","display_name":"Functional specification","level":5,"score":0.36649999022483826},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.33250001072883606},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.32710000872612},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.32679998874664307},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.30660000443458557},{"id":"https://openalex.org/C174683762","wikidata":"https://www.wikidata.org/wiki/Q609588","display_name":"Component-based software engineering","level":4,"score":0.29409998655319214},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.29179999232292175},{"id":"https://openalex.org/C10272871","wikidata":"https://www.wikidata.org/wiki/Q929972","display_name":"Software inspection","level":5,"score":0.29010000824928284},{"id":"https://openalex.org/C207850805","wikidata":"https://www.wikidata.org/wiki/Q269608","display_name":"Reverse engineering","level":2,"score":0.27900001406669617},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.27140000462532043},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.2703000009059906},{"id":"https://openalex.org/C202105479","wikidata":"https://www.wikidata.org/wiki/Q265013","display_name":"Software evolution","level":5,"score":0.2676999866962433},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.2662999927997589},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.25760000944137573},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.25099998712539673}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:arXiv.org:2609.04167","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.04167","pdf_url":"https://arxiv.org/pdf/2609.04167","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.04167","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.04167","pdf_url":"https://arxiv.org/pdf/2609.04167","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1292579918","display_name":null,"funder_award_id":"92582202","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4425133490","display_name":null,"funder_award_id":"62302534","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7208838833.pdf","grobid_xml":"https://content.openalex.org/works/W7208838833.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7220678928","https://openalex.org/W7214300551","https://openalex.org/W4410553253","https://openalex.org/W7143317080","https://openalex.org/W7212059813","https://openalex.org/W4415273522","https://openalex.org/W7221505180","https://openalex.org/W7151650802","https://openalex.org/W7223025953","https://openalex.org/W7202166648"],"abstract_inverted_index":{"Repository-level":[0],"software":[1,40,49,119],"engineering":[2,50],"benchmarks":[3,14],"have":[4],"significantly":[5],"advanced":[6],"the":[7,145,154,161,176],"evaluation":[8,170],"of":[9,179],"coding":[10],"agents,":[11],"but":[12],"existing":[13],"primarily":[15],"measure":[16],"whether":[17,33],"generated":[18],"patches":[19],"pass":[20,153],"functional":[21,59,83,140,155],"tests":[22],"and":[23,71,84,90,100,142,189],"overlook":[24],"review-derived":[25],"acceptance":[26],"constraints":[27,64],"(review":[28],"constraints)":[29],"that":[30,52,152,168],"often":[31],"influence":[32],"a":[34,45,131,136],"patch":[35],"is":[36,192],"acceptable":[37],"in":[38],"real-world":[39],"development.":[41],"We":[42,104],"introduce":[43],"SWE-Gate,":[44],"repository-level":[46,73,109,180],"benchmark":[47],"for":[48],"agents":[51],"explicitly":[53],"evaluates":[54],"review":[55,63,69,101,163],"constraint":[56,85,102],"compliance":[57],"alongside":[58],"correctness.":[60],"SWE-Gate":[61,106],"derives":[62],"from":[65],"real":[66],"pull":[67],"request":[68],"comments":[70],"synthesizes":[72],"repair":[74,110,147,181],"instances":[75,111],"around":[76],"these":[77],"constraints.":[78,164],"Each":[79],"instance":[80],"provides":[81],"separate":[82],"tests,":[86,156],"together":[87],"with":[88,107,122],"non-compliant":[89],"gold":[91],"patches,":[92],"enabling":[93],"explicit":[94],"separation":[95],"between":[96,139],"issue":[97],"resolution":[98],"capability":[99,128],"compliance.":[103],"construct":[105],"303":[108],"spanning":[112,126],"75":[113],"open-source":[114],"Python":[115],"repositories":[116],"across":[117],"diverse":[118],"domains.":[120],"Experiments":[121],"four":[123],"LLM":[124],"backends":[125],"different":[127],"levels":[129],"under":[130,144],"common":[132],"coding-agent":[133],"scaffold":[134],"reveal":[135],"substantial":[137],"gap":[138],"success":[141,143],"complete":[146],"specification:":[148],"among":[149],"644":[150],"repairs":[151],"221":[157],"fail":[158],"to":[159,174],"satisfy":[160,175],"provided":[162],"These":[165],"findings":[166],"show":[167],"functional-only":[169],"overestimates":[171],"agents'":[172],"ability":[173],"full":[177],"requirements":[178],"tasks.":[182],"The":[183],"replication":[184],"package":[185],"including":[186],"code,":[187],"data,":[188],"experimental":[190],"results":[191],"available":[193],"at":[194],"https://github.com/DeepSoftwareAnalytics/SWE-Gate.":[195]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-06T00:00:00"}