{"id":"https://openalex.org/W7213230374","doi":"https://doi.org/10.3390/electronics15204663","title":"SEBGen: A Self-Evolving Benchmark Generation Framework for Software Engineering Agents","display_name":"SEBGen: A Self-Evolving Benchmark Generation Framework for Software Engineering Agents","alternate_titles":[],"publication_year":2026,"publication_date":"2026-10-12","ids":{"openalex":"https://openalex.org/W7213230374","doi":"https://doi.org/10.3390/electronics15204663"},"language":"en","primary_location":{"id":"doi:10.3390/electronics15204663","doi":"https://doi.org/10.3390/electronics15204663","is_oa":true,"landing_page_url":"https://doi.org/10.3390/electronics15204663","pdf_url":null,"source":{"id":"https://openalex.org/S4210202905","display_name":"Electronics","issn_l":"2079-9292","issn":["2079-9292"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","norway-1","poland-100","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Electronics","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W7213230374"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.3390/electronics15204663","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5151840690","display_name":"Jingjing Huang","orcid":"https://orcid.org/0009-0006-3232-4239","observed_orcids":["https://orcid.org/0009-0006-3232-4239"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Jingjing Huang","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0009-0006-3232-4239","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042327362","display_name":"Zhaowen Feng","orcid":"https://orcid.org/0000-0002-4471-1977","observed_orcids":["https://orcid.org/0000-0002-4471-1977"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhaowen Feng","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-4471-1977","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039012652","display_name":"Daode Xiong","orcid":"https://orcid.org/0009-0003-8739-0514","observed_orcids":["https://orcid.org/0009-0003-8739-0514"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Daode Xiong","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0009-0003-8739-0514","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5151733481","display_name":"Changwei Ren","orcid":"https://orcid.org/0009-0004-3437-7604","observed_orcids":["https://orcid.org/0009-0004-3437-7604"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Changwei Ren","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0009-0004-3437-7604","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5151838261","display_name":"Yunxiao Geng","orcid":"https://orcid.org/0009-0008-1618-6475","observed_orcids":["https://orcid.org/0009-0008-1618-6475"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yunxiao Geng","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0009-0008-1618-6475","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5151840690"],"corresponding_institution_ids":[],"apc_list":{"value":2400,"currency":"CHF","value_usd":2896},"apc_paid":{"value":2400,"currency":"CHF","value_usd":2896},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"15","issue":"20","first_page":"4663","last_page":"4663"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11975","display_name":"Evolutionary Algorithms and Applications","score":0.003000000026077032,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.0013000000035390258,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark-generation","display_name":"benchmark generation","score":0.8709999918937683},{"id":"https://openalex.org/keywords/software-engineering-agents","display_name":"software engineering agents","score":0.8759999871253967},{"id":"https://openalex.org/keywords/source-code-repositories","display_name":"software repositories","score":0.7979999780654907},{"id":"https://openalex.org/keywords/parental-selection","display_name":"parent selection","score":0.49399998784065247},{"id":"https://openalex.org/keywords/swe-bench","display_name":"SWE-bench","score":0.753000020980835}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7936000227928162},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6523000001907349},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6358000040054321},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5776000022888184},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.48730000853538513},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4740000069141388},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4542999863624573},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.43869999051094055},{"id":"https://openalex.org/C139143892","wikidata":"https://www.wikidata.org/wiki/Q7441615","display_name":"Search-based software engineering","level":5,"score":0.41909998655319214},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4059999883174896},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.35920000076293945},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3133000135421753},{"id":"https://openalex.org/C202105479","wikidata":"https://www.wikidata.org/wiki/Q265013","display_name":"Software evolution","level":5,"score":0.30000001192092896},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2971000075340271},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.2888999879360199},{"id":"https://openalex.org/C509989072","wikidata":"https://www.wikidata.org/wiki/Q15188241","display_name":"Model-driven architecture","level":4,"score":0.2856999933719635},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2842999994754791},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.2736999988555908},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.267300009727478},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.26499998569488525},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.26170000433921814},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.2615000009536743},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.25929999351501465},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.25850000977516174},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.257999986410141},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2549000084400177},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.2524000108242035},{"id":"https://openalex.org/C13736549","wikidata":"https://www.wikidata.org/wiki/Q4489420","display_name":"Industrial engineering","level":1,"score":0.2502000033855438}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3390/electronics15204663","doi":"https://doi.org/10.3390/electronics15204663","is_oa":true,"landing_page_url":"https://doi.org/10.3390/electronics15204663","pdf_url":null,"source":{"id":"https://openalex.org/S4210202905","display_name":"Electronics","issn_l":"2079-9292","issn":["2079-9292"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","norway-1","poland-100","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Electronics","raw_type":"journal-article"},{"id":"doi:10.20944/preprints202609.1250.v1","doi":"https://doi.org/10.20944/preprints202609.1250.v1","is_oa":true,"landing_page_url":"https://doi.org/10.20944/preprints202609.1250.v1","pdf_url":null,"source":{"id":"https://openalex.org/S6309402219","display_name":"Preprints.org","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.3390/electronics15204663","doi":"https://doi.org/10.3390/electronics15204663","is_oa":true,"landing_page_url":"https://doi.org/10.3390/electronics15204663","pdf_url":null,"source":{"id":"https://openalex.org/S4210202905","display_name":"Electronics","issn_l":"2079-9292","issn":["2079-9292"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","norway-1","poland-100","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Electronics","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W1497256448","https://openalex.org/W2146950091","https://openalex.org/W2156723666","https://openalex.org/W2296073425","https://openalex.org/W2970641574","https://openalex.org/W3177813494","https://openalex.org/W3195140297","https://openalex.org/W4226287673","https://openalex.org/W4376167329","https://openalex.org/W4378505261","https://openalex.org/W4385572634","https://openalex.org/W4385682136","https://openalex.org/W4402665833","https://openalex.org/W4412505633","https://openalex.org/W7077063733","https://openalex.org/W7103754783","https://openalex.org/W7107871452","https://openalex.org/W7133198585","https://openalex.org/W7133214447","https://openalex.org/W7133222210","https://openalex.org/W7133239970","https://openalex.org/W7203670101"],"related_works":["https://openalex.org/W7116905337","https://openalex.org/W4415273522","https://openalex.org/W7127418923","https://openalex.org/W4414565512","https://openalex.org/W4417439098","https://openalex.org/W7167250826","https://openalex.org/W7223019374","https://openalex.org/W7117153355","https://openalex.org/W7210280953","https://openalex.org/W7133943242"],"abstract_inverted_index":{"Static":[0],"software":[1],"engineering":[2],"benchmarks":[3],"such":[4],"as":[5,19],"SWE-bench":[6,206],"suffer":[7],"from":[8,51,172,214],"three":[9,66,99],"fundamental":[10],"limitations:":[11],"task":[12],"sets":[13],"are":[14],"fixed":[15],"and":[16,28,30,54,87,109,113,124,145,181,188,291],"become":[17],"saturated":[18],"model":[20,37,217],"capabilities":[21],"rapidly":[22],"improve;":[23],"expert":[24],"construction":[25],"is":[26],"costly":[27],"slow;":[29],"difficulty":[31,83],"levels":[32],"cannot":[33],"adapt":[34],"to":[35,244],"evolving":[36],"abilities.":[38],"We":[39],"present":[40],"SEBGen,":[41],"a":[42,94,110,114,129,134,159,164,215],"self-evolving":[43,115],"benchmark":[44,170],"generation":[45,58,78,147],"framework":[46,278],"that":[47,75,177,258],"synthesizes":[48],"evaluation":[49],"tasks":[50,171,197,224,227,281],"code":[52],"repositories":[53,247],"continuously":[55],"updates":[56],"its":[57],"strategies":[59,79],"through":[60,149],"an":[61,69,105,211],"evolution":[62,88,151],"loop.":[63],"SEBGen":[64,132],"introduces":[65],"key":[67],"innovations:":[68],"explicit":[70],"five-dimension":[71],"exploration":[72],"value":[73],"evaluator":[74],"quantitatively":[76],"scores":[77],"across":[80,253],"quality,":[81],"novelty,":[82],"coverage,":[84],"skill":[85],"diversity,":[86],"potential,":[89],"replacing":[90],"implicit":[91],"LLM":[92],"judgments;":[93],"hybrid":[95],"validation":[96,138],"pipeline":[97],"combining":[98],"mechanical":[100],"checks":[101],"(schema,":[102],"compile,":[103],"functional),":[104],"LLM-based":[106,182],"semantic":[107],"assessment,":[108],"repeated-judgment":[111],"gate;":[112],"loop":[116,152],"with":[117],"archive-based":[118],"experience":[119],"accumulation,":[120],"sigmoid-scaled":[121],"parent":[122],"selection,":[123],"LLM-driven":[125],"strategy":[126],"mutation.":[127],"On":[128,163],"synthetic":[130],"corpus,":[131],"achieves":[133,250],"96":[135],"percent":[136],"stage-level":[137],"pass":[139],"rate":[140],"(E1;":[141],"48/50":[142],"stage":[143],"observations)":[144],"preserves":[146],"quality":[148],"the":[150,179,186,259,267,277],"while":[153,283],"showing":[154],"no":[155,199],"measurable":[156],"improvement":[157],"over":[158],"budget-matched":[160],"non-evolution":[161],"baseline.":[162],"real":[165,222],"repository,":[166],"it":[167],"generates":[168,279],"candidate":[169,280],"all":[173],"six":[174],"tqdm":[175],"commits":[176],"passed":[178],"schema":[180],"screening":[183,255],"stages,":[184,256],"although":[185],"compile":[187],"functional":[189],"stages":[190],"could":[191],"not":[192,262],"be":[193],"executed":[194,254],"because":[195],"these":[196],"carry":[198],"test":[200],"patch.":[201],"A":[202],"judge-level":[203],"comparison":[204],"against":[205],"Lite,":[207],"partially":[208],"corroborated":[209],"by":[210],"independent":[212],"judge":[213],"different":[216],"family":[218],"(Qwen),":[219],"places":[220],"generated":[221],"repository":[223],"below":[225],"expert-constructed":[226],"(0.48":[228],"vs.":[229,234],"0.83":[230],"under":[231,236],"deepseek-chat;":[232],"0.55":[233],"0.78":[235],"qwen-plus,":[237],"mean":[238],"of":[239,275,286],"two":[240,245],"passes).":[241],"An":[242],"extension":[243],"further":[246],"(requests,":[248],"flask)":[249],"0/4":[251],"acceptance":[252],"indicating":[257],"system":[260],"does":[261],"yet":[263],"transfer":[264,293],"reliably":[265],"beyond":[266],"original":[268],"repository.":[269],"These":[270],"results":[271],"provide":[272],"proof-of-concept":[273],"evidence":[274],"feasibility:":[276],"automatically,":[282],"executable":[284],"verification":[285],"real-repository":[287],"tasks,":[288],"reliable":[289],"self-improvement,":[290],"cross-repository":[292],"remain":[294],"open":[295],"problems.":[296]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7213230374","counts_by_year":[],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2026-09-16T00:00:00"}
