{"id":"https://openalex.org/W7204154703","doi":"https://doi.org/10.17605/osf.io/7c5zs","title":"Beyond SWE-Bench: A Systematic Review of Reliability, Security, and Real-World Evaluation in Autonomous Coding Agents","display_name":"Beyond SWE-Bench: A Systematic Review of Reliability, Security, and Real-World Evaluation in Autonomous Coding Agents","publication_year":2026,"publication_date":"2026-08-25","ids":{"openalex":"https://openalex.org/W7204154703","doi":"https://doi.org/10.17605/osf.io/7c5zs"},"language":"en","primary_location":{"id":"pmh:doi:10.17605/osf.io/7c5zs","doi":null,"is_oa":true,"landing_page_url":"https://osf.io/7c5zs","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"StudyRegistration"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://osf.io/7c5zs","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136549234","display_name":"Mahmoud M. Selim","orcid":"https://orcid.org/0000-0002-5530-6432","observed_orcids":["https://orcid.org/0000-0002-5530-6432"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Mahmoud M. Selim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5136549234"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9375,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9375,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.029400000348687172,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.019200000911951065,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/autonomous-coding-agents","display_name":"autonomous coding agents","score":0.9959999918937683},{"id":"https://openalex.org/keywords/swe-bench","display_name":"SWE-bench","score":0.7839999794960022},{"id":"https://openalex.org/keywords/benchmark-validity","display_name":"benchmark validity","score":0.625},{"id":"https://openalex.org/keywords/contamination","display_name":"contamination","score":0.5989999771118164},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.8510000109672546},{"id":"https://openalex.org/keywords/robustness","display_name":"robustness","score":0.8709999918937683},{"id":"https://openalex.org/keywords/security-assessment","display_name":"security analysis","score":0.6610000133514404},{"id":"https://openalex.org/keywords/safety","display_name":"safety","score":0.49300000071525574},{"id":"https://openalex.org/keywords/real-world-evaluation","display_name":"real-world evaluation","score":0.6669999957084656},{"id":"https://openalex.org/keywords/human-involvement","display_name":"human involvement","score":0.39100000262260437},{"id":"https://openalex.org/keywords/maintainability","display_name":"maintainability","score":0.652999997138977}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6233000159263611},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.5318999886512756},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5224999785423279},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.5138999819755554},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.47859999537467957},{"id":"https://openalex.org/C2780428219","wikidata":"https://www.wikidata.org/wiki/Q16952335","display_name":"Cover (algebra)","level":2,"score":0.37880000472068787},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.365200012922287},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3375999927520752},{"id":"https://openalex.org/C2778805511","wikidata":"https://www.wikidata.org/wiki/Q1713","display_name":"Citation","level":2,"score":0.32440000772476196},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.29679998755455017},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.2700999975204468},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.2590999901294708},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2563999891281128}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.17605/osf.io/7c5zs","doi":null,"is_oa":true,"landing_page_url":"https://osf.io/7c5zs","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"StudyRegistration"},{"id":"doi:10.17605/osf.io/7c5zs","doi":"https://doi.org/10.17605/osf.io/7c5zs","is_oa":true,"landing_page_url":"https://doi.org/10.17605/osf.io/7c5zs","pdf_url":null,"source":{"id":"https://openalex.org/S7407050956","display_name":"Open Science Framework","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"StudyRegistration"}],"best_oa_location":{"id":"pmh:doi:10.17605/osf.io/7c5zs","doi":null,"is_oa":true,"landing_page_url":"https://osf.io/7c5zs","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"StudyRegistration"},"sustainable_development_goals":[],"study_designs":[{"id":"https://openalex.org/study-designs/systematic-review","display_name":"Systematic Review"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7203621245","https://openalex.org/W7134884971","https://openalex.org/W7212059813","https://openalex.org/W7203661215","https://openalex.org/W7161138927","https://openalex.org/W7221021903","https://openalex.org/W7221539421","https://openalex.org/W7214207320","https://openalex.org/W7223019374","https://openalex.org/W7165662610"],"abstract_inverted_index":{"This":[0],"systematic":[1,95],"review":[2,19,76],"will":[3,20,42,77,93],"critically":[4],"synthesize":[5],"evidence":[6,106],"on":[7],"how":[8],"autonomous":[9],"coding":[10],"agents":[11],"are":[12],"evaluated":[13],"beyond":[14],"conventional":[15],"task-completion":[16],"scores.":[17],"The":[18,75],"examine":[21],"functional":[22],"reliability,":[23],"robustness":[24],"and":[25,28,32,50,69,88,92,104],"reproducibility,":[26],"security":[27,63],"safety,":[29],"benchmark":[30],"validity,":[31],"performance":[33],"under":[34],"real-world":[35],"or":[36,58],"deployment-relevant":[37],"software-engineering":[38],"conditions.":[39],"Particular":[40],"attention":[41],"be":[43],"given":[44],"to":[45,71],"evaluation":[46],"practices":[47],"surrounding":[48],"SWE-bench":[49],"related":[51],"repository-level":[52],"benchmarks,":[53],"including":[54],"test-oracle":[55],"adequacy,":[56],"contamination":[57],"solution":[59],"leakage,":[60],"repeated-run":[61],"consistency,":[62],"assessment,":[64,103],"human":[65],"involvement,":[66],"cost,":[67],"maintainability,":[68],"transfer":[70],"realistic":[72],"development":[73],"workflows.":[74],"cover":[78],"eligible":[79],"literature":[80],"first":[81],"publicly":[82],"available":[83],"between":[84],"1":[85],"January":[86],"2015":[87],"31":[89],"July":[90],"2026":[91],"use":[94],"database":[96],"searching,":[97],"citation":[98],"chasing,":[99],"structured":[100],"screening,":[101],"quality":[102],"critical":[105],"synthesis.":[107]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-26T00:00:00"}