{"id":"https://openalex.org/W4398766359","doi":"https://doi.org/10.1145/3639476.3639774","title":"Naturalness of Attention: Revisiting Attention in Code Language Models","display_name":"Naturalness of Attention: Revisiting Attention in Code Language Models","publication_year":2024,"publication_date":"2024-04-14","ids":{"openalex":"https://openalex.org/W4398766359","doi":"https://doi.org/10.1145/3639476.3639774"},"language":"en","primary_location":{"id":"doi:10.1145/3639476.3639774","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639476.3639774","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639476.3639774","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 ACM/IEEE 44th International Conference on Software Engineering: New Ideas and Emerging Results","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3639476.3639774","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004400004","display_name":"Mootez Saad","orcid":"https://orcid.org/0009-0008-8159-3632"},"institutions":[{"id":"https://openalex.org/I129902397","display_name":"Dalhousie University","ror":"https://ror.org/01e6qks80","country_code":"CA","type":"education","lineage":["https://openalex.org/I129902397"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Mootez Saad","raw_affiliation_strings":["Dalhousie University, Halifax, Canada"],"raw_orcid":"https://orcid.org/0009-0008-8159-3632","affiliations":[{"raw_affiliation_string":"Dalhousie University, Halifax, Canada","institution_ids":["https://openalex.org/I129902397"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023044082","display_name":"Tushar Sharma","orcid":"https://orcid.org/0000-0002-0538-052X"},"institutions":[{"id":"https://openalex.org/I129902397","display_name":"Dalhousie University","ror":"https://ror.org/01e6qks80","country_code":"CA","type":"education","lineage":["https://openalex.org/I129902397"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Tushar Sharma","raw_affiliation_strings":["Dalhousie University, Halifax, Canada"],"raw_orcid":"https://orcid.org/0000-0002-0538-052X","affiliations":[{"raw_affiliation_string":"Dalhousie University, Halifax, Canada","institution_ids":["https://openalex.org/I129902397"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5004400004"],"corresponding_institution_ids":["https://openalex.org/I129902397"],"apc_list":null,"apc_paid":null,"fwci":1.492,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.85295782,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"107","last_page":"111"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9836999773979187,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9743000268936157,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8130331039428711},{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.7452270984649658},{"id":"https://openalex.org/keywords/naturalness","display_name":"Naturalness","score":0.6225124001502991},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.5555887818336487},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.46783578395843506},{"id":"https://openalex.org/keywords/java","display_name":"Java","score":0.46491557359695435},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.46167895197868347},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4513574540615082},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4337933659553528},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4038304090499878}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8130331039428711},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.7452270984649658},{"id":"https://openalex.org/C134537474","wikidata":"https://www.wikidata.org/wiki/Q17144832","display_name":"Naturalness","level":2,"score":0.6225124001502991},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.5555887818336487},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.46783578395843506},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.46491557359695435},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.46167895197868347},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4513574540615082},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4337933659553528},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4038304090499878},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3639476.3639774","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639476.3639774","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639476.3639774","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 ACM/IEEE 44th International Conference on Software Engineering: New Ideas and Emerging Results","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3639476.3639774","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639476.3639774","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639476.3639774","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 ACM/IEEE 44th International Conference on Software Engineering: New Ideas and Emerging Results","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.4099999964237213,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4398766359.pdf","grobid_xml":"https://content.openalex.org/works/W4398766359.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W2887364112","https://openalex.org/W2963341956","https://openalex.org/W2964150020","https://openalex.org/W2972324944","https://openalex.org/W3098605233","https://openalex.org/W3099143320","https://openalex.org/W3146720657","https://openalex.org/W4284710241","https://openalex.org/W4313137049"],"related_works":["https://openalex.org/W2029561777","https://openalex.org/W2905433371","https://openalex.org/W172797710","https://openalex.org/W3165080709","https://openalex.org/W2945105049","https://openalex.org/W2626699140","https://openalex.org/W2909357361","https://openalex.org/W4387098302","https://openalex.org/W4308641625","https://openalex.org/W2789551765"],"abstract_inverted_index":{"Language":[0],"models":[1,147],"for":[2,133,143],"code":[3,14,119,137],"such":[4],"as":[5],"CodeBERT":[6,116],"offer":[7],"the":[8,43,57,62,66,94,99,124,141],"capability":[9],"to":[10,21,52,106],"learn":[11],"advanced":[12],"source":[13],"representation,":[15],"but":[16],"their":[17],"opacity":[18],"poses":[19],"barriers":[20],"understanding":[22,135],"of":[23,47,61,98,114,126,151],"captured":[24],"properties.":[25,120],"Recent":[26],"attention":[27,38,63,67,77,107,131,152],"analysis":[28,111],"studies":[29],"provide":[30],"initial":[31,72],"interpretability":[32],"insights":[33],"by":[34],"focusing":[35],"solely":[36],"on":[37,56],"weights":[39,108,132],"rather":[40],"than":[41],"considering":[42],"wider":[44],"context":[45],"modeling":[46],"Transformers.":[48],"This":[49,139],"study":[50,74],"aims":[51],"shed":[53],"some":[54],"light":[55],"previously":[58],"ignored":[59],"factors":[60,128],"mechanism":[64],"beyond":[65,129],"weights.":[68],"We":[69],"conduct":[70],"an":[71],"empirical":[73],"analyzing":[75],"both":[76],"distributions":[78],"and":[79,89,148],"transformed":[80],"representations":[81],"in":[82,154],"CodeBERT.":[83],"Across":[84],"two":[85],"programming":[86],"languages,":[87],"Java":[88],"Python,":[90],"we":[91],"find":[92],"that":[93],"scaled":[95],"transformation":[96],"norms":[97],"input":[100],"better":[101],"capture":[102],"syntactic":[103,118],"structure":[104],"compared":[105],"alone.":[109],"Our":[110],"reveals":[112],"characterization":[113],"how":[115],"embeds":[117],"The":[121],"findings":[122],"demonstrate":[123],"importance":[125],"incorporating":[127],"just":[130],"rigorously":[134],"neural":[136],"models.":[138],"lays":[140],"groundwork":[142],"developing":[144],"more":[145],"interpretable":[146],"effective":[149],"uses":[150],"mechanisms":[153],"program":[155],"analysis.":[156]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}