{"id":"https://openalex.org/W4400242271","doi":"https://doi.org/10.1145/3643991.3644917","title":"Exploring the Effect of Multiple Natural Languages on Code Suggestion Using GitHub Copilot","display_name":"Exploring the Effect of Multiple Natural Languages on Code Suggestion Using GitHub Copilot","publication_year":2024,"publication_date":"2024-04-15","ids":{"openalex":"https://openalex.org/W4400242271","doi":"https://doi.org/10.1145/3643991.3644917"},"language":"en","primary_location":{"id":"doi:10.1145/3643991.3644917","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3643991.3644917","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st International Conference on Mining Software Repositories","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://pure.tue.nl/ws/files/350760550/3643991.3644917.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5022418783","display_name":"Kei Koyanagi","orcid":"https://orcid.org/0009-0000-3880-5152"},"institutions":[{"id":"https://openalex.org/I135598925","display_name":"Kyushu University","ror":"https://ror.org/00p4k0j84","country_code":"JP","type":"education","lineage":["https://openalex.org/I135598925"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Kei Koyanagi","raw_affiliation_strings":["Kyushu University, Fukuoka, Japan"],"affiliations":[{"raw_affiliation_string":"Kyushu University, Fukuoka, Japan","institution_ids":["https://openalex.org/I135598925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062199738","display_name":"Dong Wang","orcid":"https://orcid.org/0000-0002-2004-0902"},"institutions":[{"id":"https://openalex.org/I135598925","display_name":"Kyushu University","ror":"https://ror.org/00p4k0j84","country_code":"JP","type":"education","lineage":["https://openalex.org/I135598925"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Dong Wang","raw_affiliation_strings":["Kyushu University, Fukuoka, Japan"],"affiliations":[{"raw_affiliation_string":"Kyushu University, Fukuoka, Japan","institution_ids":["https://openalex.org/I135598925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070814622","display_name":"Kotaro Noguchi","orcid":"https://orcid.org/0009-0004-7942-1162"},"institutions":[{"id":"https://openalex.org/I135598925","display_name":"Kyushu University","ror":"https://ror.org/00p4k0j84","country_code":"JP","type":"education","lineage":["https://openalex.org/I135598925"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Kotaro Noguchi","raw_affiliation_strings":["Kyushu University, Fukuoka, Japan"],"affiliations":[{"raw_affiliation_string":"Kyushu University, Fukuoka, Japan","institution_ids":["https://openalex.org/I135598925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070395037","display_name":"Masanari Kondo","orcid":"https://orcid.org/0000-0002-6317-7001"},"institutions":[{"id":"https://openalex.org/I135598925","display_name":"Kyushu University","ror":"https://ror.org/00p4k0j84","country_code":"JP","type":"education","lineage":["https://openalex.org/I135598925"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Masanari Kondo","raw_affiliation_strings":["Kyushu University, Fukuoka, Japan"],"affiliations":[{"raw_affiliation_string":"Kyushu University, Fukuoka, Japan","institution_ids":["https://openalex.org/I135598925"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054753279","display_name":"Alexander Serebrenik","orcid":"https://orcid.org/0000-0002-1418-0095"},"institutions":[{"id":"https://openalex.org/I83019370","display_name":"Eindhoven University of Technology","ror":"https://ror.org/02c2kyt77","country_code":"NL","type":"education","lineage":["https://openalex.org/I83019370"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Alexander Serebrenik","raw_affiliation_strings":["Eindhoven University of Technology, Eindhoven, Netherlands"],"affiliations":[{"raw_affiliation_string":"Eindhoven University of Technology, Eindhoven, Netherlands","institution_ids":["https://openalex.org/I83019370"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045097606","display_name":"Yasutaka Kamei","orcid":"https://orcid.org/0000-0002-7058-1045"},"institutions":[{"id":"https://openalex.org/I135598925","display_name":"Kyushu University","ror":"https://ror.org/00p4k0j84","country_code":"JP","type":"education","lineage":["https://openalex.org/I135598925"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Yasutaka Kamei","raw_affiliation_strings":["Kyushu University, Fukuoka, Japan"],"affiliations":[{"raw_affiliation_string":"Kyushu University, Fukuoka, Japan","institution_ids":["https://openalex.org/I135598925"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5003574134","display_name":"Naoyasu Ubayashi","orcid":"https://orcid.org/0000-0003-1578-7195"},"institutions":[{"id":"https://openalex.org/I135598925","display_name":"Kyushu University","ror":"https://ror.org/00p4k0j84","country_code":"JP","type":"education","lineage":["https://openalex.org/I135598925"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Naoyasu Ubayashi","raw_affiliation_strings":["Kyushu University, Fukuoka, Japan"],"affiliations":[{"raw_affiliation_string":"Kyushu University, Fukuoka, Japan","institution_ids":["https://openalex.org/I135598925"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5022418783"],"corresponding_institution_ids":["https://openalex.org/I135598925"],"apc_list":null,"apc_paid":null,"fwci":2.3591,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.90445136,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"481","last_page":"486"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9825999736785889,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.973800003528595,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.697516143321991},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5942180752754211},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.49425530433654785},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.48567473888397217},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.41391226649284363},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.23345062136650085},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.06374135613441467}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.697516143321991},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5942180752754211},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.49425530433654785},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.48567473888397217},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.41391226649284363},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.23345062136650085},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.06374135613441467},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3643991.3644917","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3643991.3644917","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st International Conference on Mining Software Repositories","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.tue.nl:openaire_cris_publications/e102d415-3796-4c6d-b73d-7a7d44a7f359","is_oa":true,"landing_page_url":"https://research.tue.nl/en/publications/e102d415-3796-4c6d-b73d-7a7d44a7f359","pdf_url":"https://pure.tue.nl/ws/files/350760550/3643991.3644917.pdf","source":{"id":"https://openalex.org/S4406922641","display_name":"TU/e Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Koyanagi, K, Wang, D, Noguchi, K, Kondo, M, Serebrenik, A, Kamei, Y & Ubayashi, N 2024, Exploring the Effect of Multiple Natural Languages on Code Suggestion Using GitHub Copilot. in MSR '24 : Proceedings of the 21st International Conference on Mining Software Repositories. Association for Computing Machinery, Inc., New York, pp. 481-486, 21st International Conference on Mining Software Repositories, MSR 2024, Lisbon, Portugal, 15/04/24. https://doi.org/10.1145/3643991.3644917","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"pmh:oai:pure.tue.nl:openaire_cris_publications/e102d415-3796-4c6d-b73d-7a7d44a7f359","is_oa":true,"landing_page_url":"https://research.tue.nl/en/publications/e102d415-3796-4c6d-b73d-7a7d44a7f359","pdf_url":"https://pure.tue.nl/ws/files/350760550/3643991.3644917.pdf","source":{"id":"https://openalex.org/S4406922641","display_name":"TU/e Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Koyanagi, K, Wang, D, Noguchi, K, Kondo, M, Serebrenik, A, Kamei, Y & Ubayashi, N 2024, Exploring the Effect of Multiple Natural Languages on Code Suggestion Using GitHub Copilot. in MSR '24 : Proceedings of the 21st International Conference on Mining Software Repositories. Association for Computing Machinery, Inc., New York, pp. 481-486, 21st International Conference on Mining Software Repositories, MSR 2024, Lisbon, Portugal, 15/04/24. https://doi.org/10.1145/3643991.3644917","raw_type":"info:eu-repo/semantics/publishedVersion"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.44999998807907104}],"awards":[{"id":"https://openalex.org/G4354030512","display_name":null,"funder_award_id":"JP21H04877","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G4954916424","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G4967554400","display_name":null,"funder_award_id":"JP23K16864","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G5506463476","display_name":null,"funder_award_id":"JP22K1","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G5880565589","display_name":"Just-in-Time Automated Bug Repair Enabling Discovery and Repair of Latent Bugs","funder_award_id":"22K17874","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G6237854213","display_name":null,"funder_award_id":"JPJSBP1","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G6333623221","display_name":"The Era of Machine-Driven Bug Repair: Advancing Pseudo-Oracle Generation and Automated Bug Fixing Technologies","funder_award_id":"21H04877","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G6429992933","display_name":"Development of Automated Bug Fixing and Generation Techniques to Support Novice Programmers","funder_award_id":"22K18630","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G7079439939","display_name":null,"funder_award_id":"JPJSBP120239929","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G7146533649","display_name":null,"funder_award_id":"JPJSBP120","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G7299751915","display_name":"Towards Efficient Code Review: Automatic Recommendation of Needed Information","funder_award_id":"23K16864","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G7345842084","display_name":null,"funder_award_id":"JP22K17874","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G8822921597","display_name":null,"funder_award_id":"JP22K18630","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"},{"id":"https://openalex.org/G887466300","display_name":null,"funder_award_id":"JP23K1686","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"}],"funders":[{"id":"https://openalex.org/F4320322210","display_name":"Inamori Foundation","ror":"https://ror.org/00ht8je70"},{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4400242271.pdf","grobid_xml":"https://content.openalex.org/works/W4400242271.grobid-xml"},"referenced_works_count":23,"referenced_works":["https://openalex.org/W1970034574","https://openalex.org/W2108017975","https://openalex.org/W2109099384","https://openalex.org/W2407582512","https://openalex.org/W2513969877","https://openalex.org/W2912924812","https://openalex.org/W3156470785","https://openalex.org/W3158733382","https://openalex.org/W3167900421","https://openalex.org/W3184144760","https://openalex.org/W3216925668","https://openalex.org/W4231124227","https://openalex.org/W4285734663","https://openalex.org/W4288057765","https://openalex.org/W4308627320","https://openalex.org/W4312438588","https://openalex.org/W4367672983","https://openalex.org/W4384026553","https://openalex.org/W4384345639","https://openalex.org/W4386576744","https://openalex.org/W4386982649","https://openalex.org/W4389209181","https://openalex.org/W6929963779"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"GitHub":[0],"Copilot":[1],"is":[2,39,53],"an":[3,79],"AI-enabled":[4],"tool":[5],"that":[6,114],"automates":[7],"program":[8],"synthesis.":[9],"It":[10],"has":[11],"gained":[12],"significant":[13],"attention":[14],"since":[15],"its":[16,34],"launch":[17],"in":[18,27,58,151,158],"2021.":[19],"Recent":[20],"studies":[21],"have":[22],"extensively":[23],"examined":[24],"Copilot's":[25,159],"capabilities":[26],"various":[28],"programming":[29],"tasks,":[30],"as":[31,33,139],"well":[32],"security":[35],"issues.":[36],"However,":[37],"little":[38],"known":[40],"about":[41],"the":[42,59,68,84,115,124,130,135,140,148,153],"effect":[43,85],"of":[44,61,70,86,101,129,132,142,155],"different":[45],"natural":[46,89,119,133,156],"languages":[47,90,157],"on":[48,95],"code":[49],"suggestion.":[50],"Natural":[51],"language":[52],"considered":[54],"a":[55],"social":[56],"bias":[57,65],"field":[60],"NLP,":[62],"and":[63,93,161],"this":[64,75],"could":[66],"impact":[67],"diversity":[69],"software":[71],"engineering.":[72],"To":[73],"address":[74],"gap,":[76],"we":[77],"conducted":[78],"empirical":[80],"study":[81],"to":[82],"investigate":[83],"three":[87],"popular":[88],"(English,":[91],"Japanese,":[92],"Chinese)":[94],"Copilot.":[96],"We":[97],"used":[98],"756":[99],"questions":[100,143],"varying":[102],"difficulty":[103,141],"levels":[104],"from":[105],"AtCoder":[106],"contests":[107],"for":[108,165],"evaluation":[109],"purposes.":[110],"The":[111],"results":[112],"highlight":[113],"capability":[116,160],"varies":[117],"across":[118],"languages,":[120],"with":[121],"Chinese":[122],"achieving":[123],"worst":[125],"performance.":[126],"Furthermore,":[127],"regardless":[128],"type":[131],"language,":[134],"performance":[136],"decreases":[137],"significantly":[138],"increases.":[144],"Our":[145],"work":[146],"represents":[147],"initial":[149],"step":[150],"comprehending":[152],"significance":[154],"introduces":[162],"promising":[163],"opportunities":[164],"future":[166],"endeavors.":[167]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-03-25T14:56:36.534964","created_date":"2025-10-10T00:00:00"}