{"id":"https://openalex.org/W4392120462","doi":"https://doi.org/10.1145/3643916.3644434","title":"Do Machines and Humans Focus on Similar Code? Exploring Explainability of Large Language Models in Code Summarization","display_name":"Do Machines and Humans Focus on Similar Code? Exploring Explainability of Large Language Models in Code Summarization","publication_year":2024,"publication_date":"2024-04-15","ids":{"openalex":"https://openalex.org/W4392120462","doi":"https://doi.org/10.1145/3643916.3644434"},"language":"en","primary_location":{"id":"doi:10.1145/3643916.3644434","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3643916.3644434","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3643916.3644434","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd IEEE/ACM International Conference on Program Comprehension","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3643916.3644434","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058677546","display_name":"Jiliang Li","orcid":"https://orcid.org/0009-0004-2852-0865"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jiliang Li","raw_affiliation_strings":["Vanderbilt University, Nashville, Tennessee, USA"],"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, Tennessee, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100376917","display_name":"Yifan Zhang","orcid":"https://orcid.org/0000-0001-5719-772X"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yifan Zhang","raw_affiliation_strings":["Vanderbilt University, Nashville, Tennessee, USA"],"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, Tennessee, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057662526","display_name":"Zachary Karas","orcid":"https://orcid.org/0000-0002-5721-8794"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zachary Karas","raw_affiliation_strings":["Vanderbilt University, Nashville, Tennessee, USA"],"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, Tennessee, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084874990","display_name":"Collin McMillan","orcid":"https://orcid.org/0009-0005-0887-1083"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Collin McMillan","raw_affiliation_strings":["University of Notre Dame, Notre Dame, Indiana, USA"],"affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, Indiana, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030030910","display_name":"Kevin Leach","orcid":"https://orcid.org/0000-0002-4001-3442"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kevin Leach","raw_affiliation_strings":["Vanderbilt University, Nashville, Tennessee, USA"],"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, Tennessee, USA","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5055069752","display_name":"Yu Huang","orcid":"https://orcid.org/0000-0003-2730-5077"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yu Huang","raw_affiliation_strings":["Vanderbilt University, Nashville, Tennessee, USA"],"affiliations":[{"raw_affiliation_string":"Vanderbilt University, Nashville, Tennessee, USA","institution_ids":["https://openalex.org/I200719446"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5058677546"],"corresponding_institution_ids":["https://openalex.org/I200719446"],"apc_list":null,"apc_paid":null,"fwci":8.5737,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":{"value":0.9747271,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"47","last_page":"51"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9836000204086304,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8387921452522278},{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.809027373790741},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.6908882856369019},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5776705741882324},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5394424200057983},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5208804607391357},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5006468296051025},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.47427257895469666},{"id":"https://openalex.org/keywords/source-lines-of-code","display_name":"Source lines of code","score":0.4330001771450043},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.36511683464050293},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3556289076805115}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8387921452522278},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.809027373790741},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.6908882856369019},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5776705741882324},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5394424200057983},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5208804607391357},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5006468296051025},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.47427257895469666},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.4330001771450043},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.36511683464050293},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3556289076805115},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3643916.3644434","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3643916.3644434","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3643916.3644434","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd IEEE/ACM International Conference on Program Comprehension","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2402.14182","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.14182","pdf_url":"https://arxiv.org/pdf/2402.14182","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3643916.3644434","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3643916.3644434","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3643916.3644434","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd IEEE/ACM International Conference on Program Comprehension","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.75,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1089092415","display_name":null,"funder_award_id":"2100035","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G1887447798","display_name":null,"funder_award_id":"2211428","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G399585375","display_name":null,"funder_award_id":"2312057","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6648286727","display_name":null,"funder_award_id":"2211429","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4392120462.pdf"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W1901145490","https://openalex.org/W1981276685","https://openalex.org/W2079887492","https://openalex.org/W2240536489","https://openalex.org/W2362779523","https://openalex.org/W2898936689","https://openalex.org/W2947430838","https://openalex.org/W2949547800","https://openalex.org/W3031696893","https://openalex.org/W3092785544","https://openalex.org/W4210440021","https://openalex.org/W4281763794","https://openalex.org/W4283751459","https://openalex.org/W4285490465","https://openalex.org/W6600339963","https://openalex.org/W6600424091","https://openalex.org/W6601158483","https://openalex.org/W6627919676","https://openalex.org/W6752934852","https://openalex.org/W6761280018"],"related_works":["https://openalex.org/W3081644756","https://openalex.org/W2031796104","https://openalex.org/W4312763760","https://openalex.org/W2033235985","https://openalex.org/W2763289028","https://openalex.org/W4282830668","https://openalex.org/W4213113901","https://openalex.org/W2149635184","https://openalex.org/W4361792201","https://openalex.org/W4308641625"],"abstract_inverted_index":{"Recent":[0],"language":[1,19,45,90,122,157,211,227,251],"models":[2,20,39,46,54,91,212,228],"have":[3],"demonstrated":[4],"proficiency":[5],"in":[6,12,64,92,116,171,220],"summarizing":[7],"source":[8],"code.":[9,42],"However,":[10],"as":[11,52,71,111],"many":[13],"other":[14,255],"domains":[15],"of":[16,21,35,44,87,89,98,145,181,205,226,250],"machine":[17],"learning,":[18],"code":[22,59,68,93,106,117,140,214],"lack":[23,29],"sufficient":[24],"explainability":[25,88],"---":[26],"informally,":[27],"we":[28,80,125,150],"a":[30,127],"formulaic":[31],"or":[32],"intuitive":[33],"understanding":[34],"what":[36,254],"and":[37,114,160,168,216,238,253],"how":[38],"learn":[40,55],"from":[41,84],"Explainability":[43],"can":[47,246],"be":[48],"partially":[49],"provided":[50],"if,":[51],"the":[53,66,96,179,182,223,248],"to":[56,137,177,190],"produce":[57],"higher-quality":[58],"summaries,":[60],"they":[61],"also":[62],"align":[63,191],"deeming":[65],"same":[67],"parts":[69],"important":[70],"those":[72],"identified":[73],"by":[74],"human":[75,99,103,161,169,192,237,244],"programmers.":[76],"In":[77],"this":[78,172],"paper,":[79],"report":[81],"negative":[82],"results":[83],"our":[85],"investigation":[86,204],"summarization":[94,118,215],"through":[95],"lens":[97],"comprehension.":[100],"We":[101],"measure":[102],"focus":[104,159,193,197,257],"on":[105,241],"using":[107],"eye-tracking":[108],"metrics":[109],"such":[110],"fixation":[112],"counts":[113],"duration":[115],"tasks.":[119],"To":[120],"approximate":[121],"model":[123,167,196,239,256],"focus,":[124],"employ":[126],"state-of-the-art":[128],"model-agnostic,":[129],"black-box,":[130],"perturbation-based":[131],"approach,":[132],"SHAP":[133],"(SHapley":[134],"Additive":[135],"exPlanations),":[136],"identify":[138],"which":[139],"tokens":[141],"influence":[142],"that":[143],"generation":[144],"summaries.":[146,184],"Using":[147],"these":[148],"settings,":[149],"find":[151],"no":[152],"statistically":[153],"significant":[154],"relationship":[155],"between":[156,166,236],"models'":[158],"programmers'":[162],"attention.":[163],"Furthermore,":[164],"alignment":[165,235],"foci":[170],"setting":[173],"does":[174],"not":[175],"seem":[176],"dictate":[178],"quality":[180],"LLM-generated":[183],"Our":[185],"study":[186],"highlights":[187],"an":[188,234],"inability":[189],"with":[194],"SHAP-based":[195],"measures.":[198],"This":[199],"result":[200],"calls":[201],"for":[202,209,213,229,261],"future":[203],"multiple":[206],"open":[207],"questions":[208],"explainable":[210],"software":[217],"engineering":[218],"tasks":[219],"general,":[221],"including":[222],"training":[224],"mechanisms":[225],"code,":[230,242],"whether":[231,243],"there":[232],"is":[233],"attention":[240,245],"improve":[247],"development":[249],"models,":[252],"measures":[258],"are":[259],"appropriate":[260],"improving":[262],"explainability.":[263]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":3}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}