{"id":"https://openalex.org/W3011564318","doi":"https://doi.org/10.1145/3377811.3380342","title":"Big code != big vocabulary","display_name":"Big code != big vocabulary","publication_year":2020,"publication_date":"2020-06-27","ids":{"openalex":"https://openalex.org/W3011564318","doi":"https://doi.org/10.1145/3377811.3380342","mag":"3011564318"},"language":"en","primary_location":{"id":"doi:10.1145/3377811.3380342","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3377811.3380342","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3377811.3380342","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM/IEEE 42nd International Conference on Software Engineering","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3377811.3380342","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109698394","display_name":"Rafael-Michael Karampatsis","orcid":null},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Rafael-Michael Karampatsis","raw_affiliation_strings":["University of Edinburgh, Edinburgh, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Edinburgh, Edinburgh, United Kingdom","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022629985","display_name":"Hlib Babii","orcid":"https://orcid.org/0000-0001-6782-2410"},"institutions":[{"id":"https://openalex.org/I171543936","display_name":"Free University of Bozen-Bolzano","ror":"https://ror.org/012ajp527","country_code":"IT","type":"education","lineage":["https://openalex.org/I171543936"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Hlib Babii","raw_affiliation_strings":["Free University of Bozen-Bolzano, Bozen-Bolzano, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Free University of Bozen-Bolzano, Bozen-Bolzano, Italy","institution_ids":["https://openalex.org/I171543936"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031712876","display_name":"Romain Robbes","orcid":"https://orcid.org/0000-0003-4569-6868"},"institutions":[{"id":"https://openalex.org/I171543936","display_name":"Free University of Bozen-Bolzano","ror":"https://ror.org/012ajp527","country_code":"IT","type":"education","lineage":["https://openalex.org/I171543936"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Romain Robbes","raw_affiliation_strings":["Free University of Bozen-Bolzano, Bozen-Bolzano, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Free University of Bozen-Bolzano, Bozen-Bolzano, Italy","institution_ids":["https://openalex.org/I171543936"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028501178","display_name":"Charles Sutton","orcid":"https://orcid.org/0000-0002-0041-3820"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Charles Sutton","raw_affiliation_strings":["Google Research and University of Edinburgh"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research and University of Edinburgh","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I98677209"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015575660","display_name":"Andrea Janes","orcid":"https://orcid.org/0000-0002-1423-6773"},"institutions":[{"id":"https://openalex.org/I171543936","display_name":"Free University of Bozen-Bolzano","ror":"https://ror.org/012ajp527","country_code":"IT","type":"education","lineage":["https://openalex.org/I171543936"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Andrea Janes","raw_affiliation_strings":["Free University of Bozen-Bolzano, Bozen-Bolzano, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Free University of Bozen-Bolzano, Bozen-Bolzano, Italy","institution_ids":["https://openalex.org/I171543936"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":25.9216,"has_fulltext":true,"cited_by_count":141,"citation_normalized_percentile":{"value":0.99807105,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1073","last_page":"1085"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.991599977016449,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8614360094070435},{"id":"https://openalex.org/keywords/readability","display_name":"Readability","score":0.7088353037834167},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.6884530782699585},{"id":"https://openalex.org/keywords/identifier","display_name":"Identifier","score":0.6285076141357422},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.5531660914421082},{"id":"https://openalex.org/keywords/rendering","display_name":"Rendering (computer graphics)","score":0.4863875210285187},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.4806981682777405},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4743719696998596},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4273502826690674},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4244532287120819},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3808959126472473},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.1440742313861847}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8614360094070435},{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.7088353037834167},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.6884530782699585},{"id":"https://openalex.org/C154504017","wikidata":"https://www.wikidata.org/wiki/Q853614","display_name":"Identifier","level":2,"score":0.6285076141357422},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.5531660914421082},{"id":"https://openalex.org/C205711294","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Rendering (computer graphics)","level":2,"score":0.4863875210285187},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.4806981682777405},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4743719696998596},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4273502826690674},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4244532287120819},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3808959126472473},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.1440742313861847},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1145/3377811.3380342","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3377811.3380342","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3377811.3380342","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM/IEEE 42nd International Conference on Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2003.07914","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2003.07914","pdf_url":"https://arxiv.org/pdf/2003.07914","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:pure.ed.ac.uk:publications/8797375e-f2f9-4c72-b722-5d899caa9f7b","is_oa":true,"landing_page_url":"https://www.research.ed.ac.uk/en/publications/8797375e-f2f9-4c72-b722-5d899caa9f7b","pdf_url":"https://www.research.ed.ac.uk/files/165188855/Big_Code_KARAMPATSIS_DOA09122019_AFV.pdf","source":{"id":"https://openalex.org/S4306400321","display_name":"Edinburgh Research Explorer (University of Edinburgh)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98677209","host_organization_name":"University of Edinburgh","host_organization_lineage":["https://openalex.org/I98677209"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Karampatsis, R-M, Babii, H, Robbes, R, Sutton, C & Janes, A 2020, Big Code != Big Vocabulary: Open-Vocabulary Models for Source Code. in ICSE '20: Proceedings of the ACM/IEEE 42nd International Conference on Software Engineering. Association for Computing Machinery (ACM), pp. 1073-1085, 42nd International Conference on Software Engineering, Online, 24/06/20. https://doi.org/10.1145/3377811.3380342","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:alma.39UBZ_INST:11292968410001241","is_oa":true,"landing_page_url":"https://dl.acm.org/doi/abs/10.1145/3377811.3380342","pdf_url":null,"source":{"id":"https://openalex.org/S4210197018","display_name":"Open Research (University of Surrey)","issn_l":"2688-268X","issn":["2688-268X","2688-3988"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Proceedings"},{"id":"pmh:oai:pure.ed.ac.uk:openaire/8797375e-f2f9-4c72-b722-5d899caa9f7b","is_oa":true,"landing_page_url":"https://hdl.handle.net/20.500.11820/8797375e-f2f9-4c72-b722-5d899caa9f7b","pdf_url":null,"source":{"id":"https://openalex.org/S4306400321","display_name":"Edinburgh Research Explorer (University of Edinburgh)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98677209","host_organization_name":"University of Edinburgh","host_organization_lineage":["https://openalex.org/I98677209"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Karampatsis, R-M, Babii, H, Robbes, R, Sutton, C & Janes, A 2020, Big Code != Big Vocabulary: Open-Vocabulary Models for Source Code. in ICSE '20: Proceedings of the ACM/IEEE 42nd International Conference on Software Engineering. Association for Computing Machinery (ACM), pp. 1073-1085, 42nd International Conference on Software Engineering, Online, 24/06/20. https://doi.org/10.1145/3377811.3380342","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.1145/3377811.3380342","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3377811.3380342","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3377811.3380342","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM/IEEE 42nd International Conference on Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4437068315","display_name":"EPSRC Centre for Doctoral Training in Data Science","funder_award_id":"EP/L016427/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320320332","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90"},{"id":"https://openalex.org/F4320325400","display_name":"Libera Universit\u00e0 di Bolzano","ror":"https://ror.org/012ajp527"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3011564318.pdf","grobid_xml":"https://content.openalex.org/works/W3011564318.grobid-xml"},"referenced_works_count":94,"referenced_works":["https://openalex.org/W46679369","https://openalex.org/W179875071","https://openalex.org/W1501139663","https://openalex.org/W1655078475","https://openalex.org/W1771830246","https://openalex.org/W1860267373","https://openalex.org/W1924770834","https://openalex.org/W1938755728","https://openalex.org/W1970607969","https://openalex.org/W1972141422","https://openalex.org/W1973681806","https://openalex.org/W1974020522","https://openalex.org/W1993318811","https://openalex.org/W1994573369","https://openalex.org/W2010608861","https://openalex.org/W2018389835","https://openalex.org/W2032942114","https://openalex.org/W2060384944","https://openalex.org/W2064675550","https://openalex.org/W2074032109","https://openalex.org/W2077537588","https://openalex.org/W2095705004","https://openalex.org/W2100664567","https://openalex.org/W2128737833","https://openalex.org/W2140609933","https://openalex.org/W2142403498","https://openalex.org/W2143861926","https://openalex.org/W2143960295","https://openalex.org/W2148190602","https://openalex.org/W2153579005","https://openalex.org/W2156723666","https://openalex.org/W2157331557","https://openalex.org/W2158195707","https://openalex.org/W2165747537","https://openalex.org/W2175297521","https://openalex.org/W2251012068","https://openalex.org/W2259472270","https://openalex.org/W2308618763","https://openalex.org/W2402619042","https://openalex.org/W2444132761","https://openalex.org/W2497764072","https://openalex.org/W2507756961","https://openalex.org/W2511803001","https://openalex.org/W2516621648","https://openalex.org/W2525332836","https://openalex.org/W2547880329","https://openalex.org/W2553397501","https://openalex.org/W2571859396","https://openalex.org/W2579161546","https://openalex.org/W2605202003","https://openalex.org/W2740130862","https://openalex.org/W2740220421","https://openalex.org/W2768572539","https://openalex.org/W2782547877","https://openalex.org/W2795516572","https://openalex.org/W2795753518","https://openalex.org/W2806551457","https://openalex.org/W2806718802","https://openalex.org/W2884276923","https://openalex.org/W2884681705","https://openalex.org/W2887364112","https://openalex.org/W2890560993","https://openalex.org/W2907705732","https://openalex.org/W2921792613","https://openalex.org/W2927177344","https://openalex.org/W2954149564","https://openalex.org/W2954274464","https://openalex.org/W2954451301","https://openalex.org/W2954823997","https://openalex.org/W2954950681","https://openalex.org/W2962739339","https://openalex.org/W2962784628","https://openalex.org/W2962894772","https://openalex.org/W2962995178","https://openalex.org/W2963026768","https://openalex.org/W2963099225","https://openalex.org/W2963341956","https://openalex.org/W2963403868","https://openalex.org/W2963935794","https://openalex.org/W2963951265","https://openalex.org/W2964315653","https://openalex.org/W2976890614","https://openalex.org/W2979792666","https://openalex.org/W3099302725","https://openalex.org/W3104874136","https://openalex.org/W3146720657","https://openalex.org/W3149750418","https://openalex.org/W4229675450","https://openalex.org/W4232728046","https://openalex.org/W4245415816","https://openalex.org/W4285719527","https://openalex.org/W6601894380","https://openalex.org/W6604199136","https://openalex.org/W6604896550"],"related_works":["https://openalex.org/W1964661231","https://openalex.org/W4254960163","https://openalex.org/W3110264473","https://openalex.org/W2999103397","https://openalex.org/W2370831213","https://openalex.org/W4315704968","https://openalex.org/W4285478380","https://openalex.org/W3120493416","https://openalex.org/W2098192829","https://openalex.org/W3029691491"],"abstract_inverted_index":{"Statistical":[0],"language":[1],"modeling":[2],"techniques":[3,37],"have":[4],"successfully":[5],"been":[6],"applied":[7],"to":[8,79],"large":[9,58],"source":[10,70],"code":[11,25,40],"corpora,":[12],"yielding":[13],"a":[14,45],"variety":[15],"of":[16,69],"new":[17,42,53],"software":[18],"development":[19],"tools,":[20],"such":[21],"as":[22,52],"tools":[23],"for":[24],"suggestion,":[26],"improving":[27],"readability,":[28],"and":[29,60,75],"API":[30],"migration.":[31],"A":[32],"major":[33],"issue":[34],"with":[35],"these":[36],"is":[38],"that":[39],"introduces":[41],"vocabulary":[43],"at":[44],"far":[46],"higher":[47],"rate":[48],"than":[49],"natural":[50],"language,":[51],"identifier":[54],"names":[55],"proliferate.":[56],"Both":[57],"vocabularies":[59],"out-of-vocabulary":[61],"issues":[62],"severely":[63],"affect":[64],"Neural":[65],"Language":[66],"Models":[67],"(NLMs)":[68],"code,":[71],"degrading":[72],"their":[73],"performance":[74],"rendering":[76],"them":[77],"unable":[78],"scale.":[80]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":25},{"year":2022,"cited_by_count":45},{"year":2021,"cited_by_count":31},{"year":2020,"cited_by_count":12}],"updated_date":"2026-08-08T01:25:22.217667","created_date":"2020-03-23T00:00:00"}