{"id":"https://openalex.org/W4308627320","doi":"https://doi.org/10.1145/3558489.3559072","title":"Assessing the quality of GitHub copilot\u2019s code generation","display_name":"Assessing the quality of GitHub copilot\u2019s code generation","alternate_titles":[],"publication_year":2022,"publication_date":"2022-11-07","ids":{"openalex":"https://openalex.org/W4308627320","doi":"https://doi.org/10.1145/3558489.3559072"},"language":"en","primary_location":{"id":"doi:10.1145/3558489.3559072","doi":"https://doi.org/10.1145/3558489.3559072","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3558489.3559072","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3558489.3559072","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th International Conference on Predictive Models and Data Analytics in Software Engineering","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4308627320"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3558489.3559072","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060296981","display_name":"Burak Yeti\u015ftiren","orcid":"https://orcid.org/0000-0002-4505-9423","observed_orcids":["https://orcid.org/0000-0002-4505-9423"]},"institutions":[{"id":"https://openalex.org/I168864056","display_name":"Bilkent University","ror":"https://ror.org/02vh8a032","country_code":"TR","type":"education","lineage":["https://openalex.org/I168864056"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Burak Yetistiren","raw_affiliation_strings":["Bilkent University, Turkey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bilkent University, Turkey","institution_ids":["https://openalex.org/I168864056"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017726773","display_name":"Isik Ozsoy","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I168864056","display_name":"Bilkent University","ror":"https://ror.org/02vh8a032","country_code":"TR","type":"education","lineage":["https://openalex.org/I168864056"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Isik Ozsoy","raw_affiliation_strings":["Bilkent University, Turkey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bilkent University, Turkey","institution_ids":["https://openalex.org/I168864056"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027949836","display_name":"Eray T\u00fcz\u00fcn","orcid":"https://orcid.org/0000-0002-5550-7816","observed_orcids":["https://orcid.org/0000-0002-5550-7816"]},"institutions":[{"id":"https://openalex.org/I168864056","display_name":"Bilkent University","ror":"https://ror.org/02vh8a032","country_code":"TR","type":"education","lineage":["https://openalex.org/I168864056"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Eray Tuzun","raw_affiliation_strings":["Bilkent University, Turkey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Bilkent University, Turkey","institution_ids":["https://openalex.org/I168864056"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I168864056"],"apc_list":null,"apc_paid":null,"fwci":24.7216,"has_fulltext":true,"cited_by_count":239,"citation_normalized_percentile":{"value":0.99787794,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"62","last_page":"71"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/github-copilot","display_name":"GitHub Copilot","score":1.0},{"id":"https://openalex.org/keywords/ai-assisted-code-generation","display_name":"AI-assisted code generation","score":0.7540000081062317},{"id":"https://openalex.org/keywords/code-quality-assessment","display_name":"code quality assessment","score":0.5910000205039978},{"id":"https://openalex.org/keywords/program-correctness","display_name":"program correctness","score":0.7310000061988831},{"id":"https://openalex.org/keywords/code-efficiency","display_name":"code efficiency","score":0.6710000038146973},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7319999933242798},{"id":"https://openalex.org/keywords/open-source-code","display_name":"open-source code","score":0.5339999794960022}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7704432010650635},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5780078768730164},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.570101797580719},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2256128191947937},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3558489.3559072","doi":"https://doi.org/10.1145/3558489.3559072","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3558489.3559072","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3558489.3559072","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th International Conference on Predictive Models and Data Analytics in Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:repository.bilkent.edu.tr:11693/111572","doi":null,"is_oa":false,"landing_page_url":"http://hdl.handle.net/11693/111572","pdf_url":null,"source":{"id":"https://openalex.org/S4306400079","display_name":"Bilkent University Institutional Repository (Bilkent University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I168864056","host_organization_name":"Bilkent University","host_organization_lineage":["https://openalex.org/I168864056"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"International Conference on Software Engineering","raw_type":"Conference Paper"}],"best_oa_location":{"id":"doi:10.1145/3558489.3559072","doi":"https://doi.org/10.1145/3558489.3559072","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3558489.3559072","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3558489.3559072","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th International Conference on Predictive Models and Data Analytics in Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4308627320.pdf","grobid_xml":"https://content.openalex.org/works/W4308627320.grobid-xml"},"referenced_works_count":11,"referenced_works":["https://openalex.org/W2949734169","https://openalex.org/W2997847174","https://openalex.org/W3123221944","https://openalex.org/W3142154623","https://openalex.org/W3177813494","https://openalex.org/W4213241976","https://openalex.org/W4225108562","https://openalex.org/W4285734663","https://openalex.org/W4288057765","https://openalex.org/W4312438588","https://openalex.org/W4391991925"],"related_works":["https://openalex.org/W7084054875","https://openalex.org/W4312438588","https://openalex.org/W4399795041","https://openalex.org/W4366851162","https://openalex.org/W4406866173","https://openalex.org/W7008700857","https://openalex.org/W4407392582","https://openalex.org/W4401611058","https://openalex.org/W7015086753","https://openalex.org/W4402751425"],"abstract_inverted_index":{"The":[0,70],"introduction":[1],"of":[2,17,29,67,73,81,95,100,123,147,151],"GitHub\u2019s":[3],"new":[4],"code":[5,38,45,83,120,139,148],"generation":[6,46],"tool,":[7],"GitHub":[8,21,68,86,105,132,175],"Copilot,":[9],"seems":[10],"to":[11,25,34,61,77,91,104,136],"be":[12],"the":[13,49,63,79,93,96,118,183,196],"first":[14],"well-established":[15],"instance":[16],"an":[18,113],"AI":[19],"pair-programmer.":[20],"Copilot":[22,133,176],"has":[23],"access":[24],"a":[26,56,141,178],"large":[27],"number":[28],"open-source":[30],"projects,":[31],"enabling":[32],"it":[33],"utilize":[35],"more":[36,190],"extensive":[37],"in":[39,121,195],"various":[40],"programming":[41],"languages":[42],"than":[43],"other":[44],"tools.":[47],"Although":[48],"initial":[50],"and":[51,65,98,126,164,189],"informal":[52],"assessments":[53],"are":[54],"promising,":[55],"systematic":[57],"evaluation":[58],"is":[59,76,177,193],"needed":[60,194],"explore":[62],"limits":[64],"benefits":[66],"Copilot.":[69,87,106],"main":[71],"objective":[72],"this":[74,109],"study":[75],"assess":[78],"quality":[80,97],"generated":[82,119],"provided":[84],"by":[85],"We":[88],"also":[89],"aim":[90],"evaluate":[92],"impact":[94],"variety":[99],"input":[101],"parameters":[102],"fed":[103],"To":[107],"achieve":[108],"aim,":[110],"we":[111,185],"created":[112],"experimental":[114],"setup":[115],"for":[116],"evaluating":[117],"terms":[122,146],"validity,":[124],"correctness,":[125,149],"efficiency.":[127],"Our":[128,170],"results":[129,184],"suggest":[130],"that":[131,174],"was":[134],"able":[135],"generate":[137],"valid":[138],"with":[140],"91.5%":[142],"success":[143],"rate.":[144],"In":[145],"out":[150],"164":[152],"problems,":[153],"47":[154],"(28.7%)":[155],"were":[156,161,167],"correctly,":[157,163],"while":[158],"84":[159],"(51.2%)":[160],"partially":[162],"33":[165],"(20.1%)":[166],"incorrectly":[168],"generated.":[169],"empirical":[171],"analysis":[172],"shows":[173],"promising":[179],"tool":[180],"based":[181],"on":[182],"obtained,":[186],"however":[187],"further":[188],"comprehensive":[191],"assessment":[192],"future.":[197]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4308627320","counts_by_year":[{"year":2026,"cited_by_count":66},{"year":2025,"cited_by_count":66},{"year":2024,"cited_by_count":84},{"year":2023,"cited_by_count":23}],"updated_date":"2026-11-19T14:11:59.513116","created_date":"2022-11-13T00:00:00"}
