{"id":"https://openalex.org/W7128798868","doi":"https://doi.org/10.1145/3770762.3772551","title":"Benchmarking AI Tools for Software Engineering Education: Insights into Design, Implementation, and Testing","display_name":"Benchmarking AI Tools for Software Engineering Education: Insights into Design, Implementation, and Testing","publication_year":2026,"publication_date":"2026-02-13","ids":{"openalex":"https://openalex.org/W7128798868","doi":"https://doi.org/10.1145/3770762.3772551"},"language":null,"primary_location":{"id":"doi:10.1145/3770762.3772551","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770762.3772551","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.1","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3770762.3772551","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082689282","display_name":"Nimisha Roy","orcid":"https://orcid.org/0000-0003-2480-9974"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nimisha Roy","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA"],"raw_orcid":"https://orcid.org/0000-0003-2480-9974","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116304979","display_name":"Oleksandr Horielko","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Oleksandr Horielko","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA"],"raw_orcid":"https://orcid.org/0009-0001-3283-6094","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5110005166","display_name":"Olufisayo Omojokun","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Olufisayo Omojokun","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA"],"raw_orcid":"https://orcid.org/0000-0001-5692-2045","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I130701444"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.15019676,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"957","last_page":"963"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.29019999504089355,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.29019999504089355,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.19429999589920044,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.12780000269412994,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6912000179290771},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4668999910354614},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.40869998931884766},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.39980000257492065},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.36970001459121704},{"id":"https://openalex.org/keywords/task-analysis","display_name":"Task analysis","score":0.35580000281333923}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7085999846458435},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6912000179290771},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5561000108718872},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4668999910354614},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.40869998931884766},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4016000032424927},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.39980000257492065},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.36970001459121704},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.35580000281333923},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.3012000024318695},{"id":"https://openalex.org/C145644426","wikidata":"https://www.wikidata.org/wiki/Q169411","display_name":"Unified Modeling Language","level":3,"score":0.28850001096725464},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.288100004196167},{"id":"https://openalex.org/C9354725","wikidata":"https://www.wikidata.org/wiki/Q286017","display_name":"Operationalization","level":2,"score":0.2867000102996826},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.2818000018596649},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.28119999170303345},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.27790001034736633},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.2718999981880188},{"id":"https://openalex.org/C509989072","wikidata":"https://www.wikidata.org/wiki/Q15188241","display_name":"Model-driven architecture","level":4,"score":0.27129998803138733}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3770762.3772551","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770762.3772551","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.1","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3770762.3772551","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770762.3772551","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.1","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":13,"referenced_works":["https://openalex.org/W4206029481","https://openalex.org/W4293227750","https://openalex.org/W4323033775","https://openalex.org/W4323033785","https://openalex.org/W4384302557","https://openalex.org/W4388092696","https://openalex.org/W4390471695","https://openalex.org/W4396788609","https://openalex.org/W4399357162","https://openalex.org/W4403330425","https://openalex.org/W4405276448","https://openalex.org/W4406328239","https://openalex.org/W4407771444"],"related_works":[],"abstract_inverted_index":{"As":[0],"generative":[1],"AI":[2,32,52,134,194],"(Gen":[3],"AI)":[4],"tools":[5,33,135,168],"reshape":[6],"software":[7],"engineering":[8],"(SE)":[9],"workflows,":[10],"educators":[11],"are":[12],"exploring":[13],"how":[14],"to":[15,125,204],"meaningfully":[16],"integrate":[17],"them":[18],"into":[19],"computing":[20],"education.":[21],"This":[22],"experience":[23],"report":[24],"presents":[25],"a":[26,102,190],"structured":[27],"benchmarking":[28],"of":[29,104],"widely":[30],"used":[31],"--":[34,53,107,124],"such":[35,138],"as":[36,139],"GitHub":[37],"Copilot,":[38],"GPT-4,":[39],"Codeium,":[40],"Claude":[41],"3.5,":[42],"Gemini":[43],"1.5,":[44],"Supermaven,":[45],"TabNine,":[46],"Testim,":[47],"Postman,":[48],"Eraser.io,":[49],"and":[50,61,73,101,121,130,142,154,172,177,189,207],"Lucidchart":[51],"across":[54],"key":[55,202],"SE":[56,77,197],"phases:":[57],"design,":[58],"implementation,":[59],"debugging,":[60],"testing.":[62],"Tools":[63],"were":[64],"selected":[65],"based":[66],"on":[67],"industry":[68],"relevance,":[69],"accessibility":[70],"for":[71,185,192],"students,":[72],"alignment":[74],"with":[75,87,169],"common":[76],"tasks.":[78],"Through":[79],"controlled":[80],"experiments":[81],"conducted":[82],"by":[83],"five":[84],"AI-experienced":[85],"evaluators":[86],"matched":[88],"exposure":[89],"levels,":[90],"we":[91],"assessed":[92],"tool":[93,131],"performance":[94],"using":[95],"standardized":[96],"prompts,":[97],"counterbalanced":[98],"task":[99,111,170],"roles,":[100],"range":[103],"proxy":[105],"metrics":[106],"including":[108,163],"prompt":[109,175],"iterations,":[110],"completion":[112],"time,":[113],"human":[114],"correction":[115],"burden,":[116],"hallucination":[117],"frequency,":[118],"output":[119],"accuracy,":[120],"cross-file":[122,152],"consistency":[123],"capture":[126],"both":[127],"cognitive":[128,165],"load":[129],"limitations.":[132],"While":[133],"accelerated":[136],"tasks":[137],"boilerplate":[140],"generation":[141],"UML":[143],"sketching,":[144],"they":[145],"exhibited":[146],"challenges":[147],"in":[148,196],"test":[149],"coverage":[150],"quality,":[151],"coherence,":[153],"reliability":[155],"under":[156],"complex":[157],"prompts.":[158],"We":[159],"discuss":[160],"educational":[161],"implications,":[162],"managing":[164],"load,":[166],"aligning":[167],"types,":[171],"explicitly":[173],"teaching":[174],"refinement":[176],"verification":[178],"strategies.":[179],"The":[180],"paper":[181],"offers":[182],"actionable":[183],"guidance":[184],"instructors,":[186],"curriculum-ready":[187],"artifacts,":[188],"roadmap":[191],"scaling":[193],"integration":[195],"classrooms,":[198],"while":[199],"also":[200],"noting":[201],"limitations":[203],"support":[205],"replication":[206],"contextual":[208],"adoption.":[209]},"counts_by_year":[],"updated_date":"2026-06-26T08:34:08.712188","created_date":"2026-02-14T00:00:00"}