{"id":"https://openalex.org/W4394745320","doi":"https://doi.org/10.1145/3597503.3639176","title":"Are Prompt Engineering and TODO Comments Friends or Foes? An Evaluation on GitHub Copilot","display_name":"Are Prompt Engineering and TODO Comments Friends or Foes? An Evaluation on GitHub Copilot","publication_year":2024,"publication_date":"2024-04-12","ids":{"openalex":"https://openalex.org/W4394745320","doi":"https://doi.org/10.1145/3597503.3639176"},"language":"en","primary_location":{"id":"doi:10.1145/3597503.3639176","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3639176","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639176","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639176","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101794271","display_name":"David O'Brien","orcid":"https://orcid.org/0000-0001-9730-9220"},"institutions":[{"id":"https://openalex.org/I173911158","display_name":"Iowa State University","ror":"https://ror.org/04rswrd78","country_code":"US","type":"education","lineage":["https://openalex.org/I173911158"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"David OBrien","raw_affiliation_strings":["Iowa State University, Ames, Iowa, USA"],"raw_orcid":"https://orcid.org/0000-0001-9730-9220","affiliations":[{"raw_affiliation_string":"Iowa State University, Ames, Iowa, USA","institution_ids":["https://openalex.org/I173911158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090690054","display_name":"Sumon Biswas","orcid":"https://orcid.org/0000-0001-7074-1953"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sumon Biswas","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, Pennsylvania, USA"],"raw_orcid":"https://orcid.org/0000-0001-7074-1953","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, Pennsylvania, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059057746","display_name":"Sayem Mohammad Imtiaz","orcid":"https://orcid.org/0000-0002-0357-0098"},"institutions":[{"id":"https://openalex.org/I173911158","display_name":"Iowa State University","ror":"https://ror.org/04rswrd78","country_code":"US","type":"education","lineage":["https://openalex.org/I173911158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sayem Mohammad Imtiaz","raw_affiliation_strings":["Iowa State University, Ames, Iowa, USA"],"raw_orcid":"https://orcid.org/0000-0002-0357-0098","affiliations":[{"raw_affiliation_string":"Iowa State University, Ames, Iowa, USA","institution_ids":["https://openalex.org/I173911158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036111839","display_name":"Rabe Abdalkareem","orcid":"https://orcid.org/0000-0001-9914-5434"},"institutions":[{"id":"https://openalex.org/I299799659","display_name":"Omar Al-Mukhtar University","ror":"https://ror.org/01wykm490","country_code":"LY","type":"education","lineage":["https://openalex.org/I299799659"]}],"countries":["LY"],"is_corresponding":false,"raw_author_name":"Rabe Abdalkareem","raw_affiliation_strings":["Omar Al-Mukhtar University, Bayda, Libya"],"raw_orcid":"https://orcid.org/0000-0001-9914-5434","affiliations":[{"raw_affiliation_string":"Omar Al-Mukhtar University, Bayda, Libya","institution_ids":["https://openalex.org/I299799659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049727493","display_name":"Emad Shihab","orcid":"https://orcid.org/0000-0003-1285-9878"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Emad Shihab","raw_affiliation_strings":["Concordia University, Montreal, Canada"],"raw_orcid":"https://orcid.org/0000-0003-1285-9878","affiliations":[{"raw_affiliation_string":"Concordia University, Montreal, Canada","institution_ids":["https://openalex.org/I60158472"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5059626072","display_name":"Hridesh Rajan","orcid":"https://orcid.org/0000-0002-9410-9562"},"institutions":[{"id":"https://openalex.org/I173911158","display_name":"Iowa State University","ror":"https://ror.org/04rswrd78","country_code":"US","type":"education","lineage":["https://openalex.org/I173911158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hridesh Rajan","raw_affiliation_strings":["Iowa State University, Ames, Iowa, USA"],"raw_orcid":"https://orcid.org/0000-0002-9410-9562","affiliations":[{"raw_affiliation_string":"Iowa State University, Ames, Iowa, USA","institution_ids":["https://openalex.org/I173911158"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5101794271"],"corresponding_institution_ids":["https://openalex.org/I173911158"],"apc_list":null,"apc_paid":null,"fwci":5.2053,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.95419211,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"13"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9894999861717224,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8121405839920044},{"id":"https://openalex.org/keywords/technical-debt","display_name":"Technical debt","score":0.6721539497375488},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.6336764097213745},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.5784181952476501},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.5746402144432068},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5602120161056519},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5006740093231201},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.4875180423259735},{"id":"https://openalex.org/keywords/code-review","display_name":"Code review","score":0.45395344495773315},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4206421375274658},{"id":"https://openalex.org/keywords/static-program-analysis","display_name":"Static program analysis","score":0.2644481658935547},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.19004735350608826},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.17872408032417297},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.0949486792087555}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8121405839920044},{"id":"https://openalex.org/C159198006","wikidata":"https://www.wikidata.org/wiki/Q1532172","display_name":"Technical debt","level":4,"score":0.6721539497375488},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6336764097213745},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.5784181952476501},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5746402144432068},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5602120161056519},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5006740093231201},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.4875180423259735},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.45395344495773315},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4206421375274658},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.2644481658935547},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.19004735350608826},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.17872408032417297},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0949486792087555},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3597503.3639176","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3639176","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639176","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3597503.3639176","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3639176","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639176","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1433996340","display_name":null,"funder_award_id":"CCF-15-18897","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2154516011","display_name":null,"funder_award_id":"CCF-22-23812","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G537839615","display_name":null,"funder_award_id":"CCF-19-34884","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7770872176","display_name":null,"funder_award_id":"CNS-15-13263","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8872482365","display_name":null,"funder_award_id":"CNS-21-20448","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4394745320.pdf","grobid_xml":"https://content.openalex.org/works/W4394745320.grobid-xml"},"referenced_works_count":54,"referenced_works":["https://openalex.org/W140915724","https://openalex.org/W2015187825","https://openalex.org/W2028836139","https://openalex.org/W2045336717","https://openalex.org/W2057433090","https://openalex.org/W2082792969","https://openalex.org/W2104577574","https://openalex.org/W2108025693","https://openalex.org/W2130602377","https://openalex.org/W2152874840","https://openalex.org/W2202602917","https://openalex.org/W2395122565","https://openalex.org/W2579161546","https://openalex.org/W2579539894","https://openalex.org/W2612705982","https://openalex.org/W2767729231","https://openalex.org/W2805357019","https://openalex.org/W2809626883","https://openalex.org/W2954876572","https://openalex.org/W2966158888","https://openalex.org/W2968594320","https://openalex.org/W2998678832","https://openalex.org/W3015174650","https://openalex.org/W3089767927","https://openalex.org/W3093959562","https://openalex.org/W3099112256","https://openalex.org/W3149778010","https://openalex.org/W3161997752","https://openalex.org/W3174210392","https://openalex.org/W3185244049","https://openalex.org/W3191096151","https://openalex.org/W3193393431","https://openalex.org/W3195505171","https://openalex.org/W3204491293","https://openalex.org/W3211801722","https://openalex.org/W3215668407","https://openalex.org/W4210411691","https://openalex.org/W4210729440","https://openalex.org/W4226203778","https://openalex.org/W4229003063","https://openalex.org/W4284677512","https://openalex.org/W4284709233","https://openalex.org/W4285204876","https://openalex.org/W4288057765","https://openalex.org/W4297632510","https://openalex.org/W4307926351","https://openalex.org/W4308643031","https://openalex.org/W4308643048","https://openalex.org/W4312438588","https://openalex.org/W4312763760","https://openalex.org/W4313547604","https://openalex.org/W4384345639","https://openalex.org/W4388502424","https://openalex.org/W4394745320"],"related_works":["https://openalex.org/W2904997879","https://openalex.org/W2344367508","https://openalex.org/W3081644756","https://openalex.org/W1997548934","https://openalex.org/W2097696338","https://openalex.org/W2967990473","https://openalex.org/W20188161","https://openalex.org/W4389600167","https://openalex.org/W2969773072","https://openalex.org/W2909969119"],"abstract_inverted_index":{"Code":[0],"intelligence":[1,124,232],"tools":[2,91],"such":[3,54,85],"as":[4,55],"GitHub":[5,173,179,222],"Copilot":[6,180],"have":[7,43,129],"begun":[8],"to":[9,46,75,92,131,199,234],"bridge":[10],"the":[11,25,64,77,82,94,102,110,119,153,185,196,209],"gap":[12],"between":[13],"natural":[14],"language":[15],"and":[16,118,143,166,191,206,213,225],"programming":[17],"language.":[18],"A":[19],"frequent":[20],"software":[21,40,52,237],"development":[22],"task":[23],"is":[24,60],"management":[26],"of":[27,84,97,104,122,148,157,187,211],"technical":[28,48,99,116],"debts,":[29],"which":[30,37,163,221],"are":[31],"suboptimal":[32],"solutions":[33],"or":[34],"unaddressed":[35],"issues":[36],"hinder":[38],"future":[39,226],"development.":[41],"Developers":[42],"been":[44],"found":[45],"\"self-admit\"":[47],"debts":[49],"(SATD)":[50],"in":[51,67,152,220],"artifacts":[53],"source":[56],"code":[57,72,89,123,170,183,231],"comments.":[58,215],"Thus,":[59],"it":[61],"possible":[62],"that":[63,178],"information":[65],"present":[66],"these":[68],"comments":[69,86,151],"can":[70,181,228],"enhance":[71],"generative":[73,90],"prompts":[74],"repay":[76,201],"described":[78,98],"SATD?":[79],"Or,":[80],"does":[81],"inclusion":[83],"instead":[87],"cause":[88],"reproduce":[93],"harmful":[95],"symptoms":[96,186],"debt?":[100],"Does":[101],"modification":[103],"SATD":[105,133,202],"impact":[106],"this":[107,140],"reaction?":[108],"Despite":[109],"heavy":[111],"maintenance":[112],"costs":[113],"caused":[114],"by":[115,138],"debt":[117],"recent":[120],"improvements":[121],"tools,":[125],"no":[126],"prior":[127],"works":[128],"sought":[130],"incorporate":[132],"towards":[134],"prompt":[135],"engineering.":[136],"Inspired":[137],"this,":[139],"paper":[141],"contributes":[142],"analyzes":[144],"a":[145],"dataset":[146],"consisting":[147],"36,381":[149],"TODO":[150],"latest":[154],"available":[155],"revisions":[156],"their":[158],"respective":[159],"102,424":[160],"repositories,":[161],"from":[162],"we":[164,194,217],"sample":[165],"manually":[167],"generate":[168,182],"1,140":[169],"bodies":[171],"using":[172],"Copilot.":[174],"Our":[175],"experiments":[176],"show":[177],"with":[184],"SATD,":[188],"both":[189],"prompted":[190],"unprompted.":[192],"Moreover,":[193],"demonstrate":[195],"tool's":[197],"ability":[198],"automatically":[200],"under":[203],"different":[204],"circumstances":[205],"qualitatively":[207],"investigate":[208],"characteristics":[210],"successful":[212],"unsuccessful":[214],"Finally,":[216],"discuss":[218],"gaps":[219],"Copilot's":[223],"successors":[224],"researchers":[227],"improve":[229],"upon":[230],"tasks":[233],"facilitate":[235],"AI-assisted":[236],"maintenance.":[238]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3}],"updated_date":"2026-04-26T08:31:28.666265","created_date":"2025-10-10T00:00:00"}