{"id":"https://openalex.org/W7123362087","doi":"https://doi.org/10.1109/esem64174.2025.00057","title":"How Do Programmers Evaluate AI-Generated Code?","display_name":"How Do Programmers Evaluate AI-Generated Code?","publication_year":2025,"publication_date":"2025-10-02","ids":{"openalex":"https://openalex.org/W7123362087","doi":"https://doi.org/10.1109/esem64174.2025.00057"},"language":"en","primary_location":{"id":"doi:10.1109/esem64174.2025.00057","doi":"https://doi.org/10.1109/esem64174.2025.00057","is_oa":false,"landing_page_url":"https://doi.org/10.1109/esem64174.2025.00057","pdf_url":null,"source":{"id":"https://openalex.org/S4210197744","display_name":"Proceedings of the ... ACM-IEEE International Symposium on Empirical Software Engineering and Measurement/International Symposium on Empirical Software Engineering and Measurement","issn_l":"1938-6451","issn":["1938-6451","1949-3770","1949-3789"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 ACM/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5122880845","display_name":"Samuli M\u00e4\u00e4tt\u00e4","orcid":"https://orcid.org/0009-0006-0209-4057","observed_orcids":["https://orcid.org/0009-0006-0209-4057"]},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":true,"raw_author_name":"Samuli M\u00e4\u00e4tt\u00e4","raw_affiliation_strings":["M3S/ITEE, University of Oulu,Oulu,Finland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"M3S/ITEE, University of Oulu,Oulu,Finland","institution_ids":["https://openalex.org/I98381234"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5122880845"],"corresponding_institution_ids":["https://openalex.org/I98381234"],"apc_list":null,"apc_paid":null,"fwci":1.0631,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.77163468,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"522","last_page":"527"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.011900000274181366,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ai-generated-code","display_name":"AI-generated code","score":0.9340000152587891},{"id":"https://openalex.org/keywords/code-evaluation","display_name":"code evaluation","score":0.7459999918937683},{"id":"https://openalex.org/keywords/program-comprehension","display_name":"program comprehension","score":0.7260000109672546},{"id":"https://openalex.org/keywords/code-repair","display_name":"code repair","score":0.9049999713897705},{"id":"https://openalex.org/keywords/code-reading","display_name":"code reading","score":0.5289999842643738},{"id":"https://openalex.org/keywords/generative-ai-tools","display_name":"generative AI tools","score":0.6650000214576721}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7440000176429749},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.6492000222206116},{"id":"https://openalex.org/C2777526511","wikidata":"https://www.wikidata.org/wiki/Q691543","display_name":"Pace","level":2,"score":0.5501999855041504},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.4984000027179718},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.48510000109672546},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.48170000314712524},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.45489999651908875},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.44760000705718994},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.3813000023365021},{"id":"https://openalex.org/C46934059","wikidata":"https://www.wikidata.org/wiki/Q61515","display_name":"Outsourcing","level":2,"score":0.3659000098705292},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3278999924659729},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3278000056743622},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.32089999318122864},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.2946000099182129},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26499998569488525},{"id":"https://openalex.org/C2778035492","wikidata":"https://www.wikidata.org/wiki/Q7315176","display_name":"Research program","level":2,"score":0.26409998536109924}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/esem64174.2025.00057","doi":"https://doi.org/10.1109/esem64174.2025.00057","is_oa":false,"landing_page_url":"https://doi.org/10.1109/esem64174.2025.00057","pdf_url":null,"source":{"id":"https://openalex.org/S4210197744","display_name":"Proceedings of the ... ACM-IEEE International Symposium on Empirical Software Engineering and Measurement/International Symposium on Empirical Software Engineering and Measurement","issn_l":"1938-6451","issn":["1938-6451","1949-3770","1949-3789"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 ACM/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[{"id":"https://openalex.org/study-designs/study-protocol","display_name":"Study Protocol"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320321408","display_name":"Ministry of Education","ror":"https://ror.org/01p262204"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7213947806","https://openalex.org/W7168138576","https://openalex.org/W4399361983","https://openalex.org/W7166259573","https://openalex.org/W7220878329","https://openalex.org/W4411552879","https://openalex.org/W4400442250","https://openalex.org/W7168181450","https://openalex.org/W7214003585","https://openalex.org/W4415965954"],"abstract_inverted_index":{"Background]":[0],"Applications":[1],"of":[2,23,88,110,116,145,163,168,196,225,237,244],"generative":[3,245],"artificial":[4],"intelligence":[5],"are":[6,31],"being":[7],"proposed":[8],"at":[9],"a":[10,183,209],"rapid":[11],"pace":[12],"to":[13,35,41,47,62,92,102,120,126,130,160],"support":[14,107],"various":[15],"software":[16,111],"engineering":[17],"tasks.":[18],"Although":[19],"versatile,":[20],"the":[21,24,36,51,58,94,105,108,143,197,226,230,234,242],"performance":[22],"tools":[25,71,101,106],"depends":[26],"on":[27,211,241],"multiple":[28],"factors":[29],"that":[30,55],"not":[32],"always":[33],"visible":[34],"users,":[37],"and":[38,66,98,129,138,165,171,181,186,208,216],"they":[39,56,220],"tend":[40],"camouflage":[42],"their":[43],"failure":[44],"points":[45],"(i.e.,":[46],"\u201challucinate\u201d).":[48],"In":[49],"programming,":[50],"existing":[52],"literature":[53],"suggests":[54],"shift":[57],"effort":[59],"from":[60],"writing":[61],"reading,":[63],"comprehending,":[64],"evaluating,":[65],"repairing":[67],"generated":[68],"code.":[69],"The":[70,85,113,191],"can":[72],"also":[73],"enable":[74],"outsourcing":[75],"these":[76,151],"efforts,":[77],"even":[78],"when":[79],"it":[80],"might":[81],"be":[82],"unwise.":[83],"[Aims]":[84],"broad":[86],"aim":[87],"this":[89,117],"research":[90,118,240],"is":[91,119,158],"investigate":[93],"interaction":[95],"between":[96],"programmers":[97,133,201],"code":[99,204],"generation":[100],"understand":[103],"how":[104,150,212],"needs":[109],"practitioners.":[112],"specific":[114],"approach":[115],"examine":[121],"how,":[122],"when,":[123],"for":[124,148],"what,":[125],"what":[127,131],"extent,":[128],"effect":[132],"read,":[134],"comprehend,":[135],"evaluate,":[136],"repair,":[137],"trust":[139,172,217],"AI-generated":[140,203],"code,":[141],"with":[142,233],"goal":[144,236],"theory":[146,210],"building":[147],"predicting":[149],"will":[152],"change":[153,218],"over":[154],"time.":[155],"[Method]":[156],"Research":[157,176],"expected":[159,192],"consist":[161],"mainly":[162],"qualitative":[164],"quantitative":[166],"analyses":[167],"programmers'":[169,213],"behaviour":[170,215],"in":[173,199,205],"realistic":[174],"contexts.":[175],"methods":[177],"include":[178,194],"practitioner":[179],"surveys":[180],"interviews,":[182],"scoping":[184],"review,":[185],"observational":[187],"field":[188],"studies.":[189],"[Results]":[190],"results":[193],"descriptions":[195],"ways":[198],"which":[200],"evaluate":[202],"different":[206],"scenarios,":[207],"evaluation-related":[214],"as":[219,229],"become":[221],"more":[222],"skilled":[223],"users":[224],"technology":[227,231],"and/or":[228],"matures,":[232],"broader":[235],"supporting":[238],"future":[239],"nature":[243],"AI":[246],"assistance.":[247]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2026-01-14T00:00:00"}