{"id":"https://openalex.org/W4388412504","doi":"https://doi.org/10.1145/3631975","title":"Deep Is Better? An Empirical Comparison of Information Retrieval and Deep Learning Approaches to Code Summarization","display_name":"Deep Is Better? An Empirical Comparison of Information Retrieval and Deep Learning Approaches to Code Summarization","publication_year":2023,"publication_date":"2023-11-06","ids":{"openalex":"https://openalex.org/W4388412504","doi":"https://doi.org/10.1145/3631975"},"language":"en","primary_location":{"id":"doi:10.1145/3631975","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3631975","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063786973","display_name":"Tingwei Zhu","orcid":"https://orcid.org/0000-0002-6962-6330"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tingwei Zhu","raw_affiliation_strings":["State Key Lab for Novel Software Technology and the Department of Computer Science and Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-6962-6330","affiliations":[{"raw_affiliation_string":"State Key Lab for Novel Software Technology and the Department of Computer Science and Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100664779","display_name":"Zhong Li","orcid":"https://orcid.org/0000-0002-3849-3416"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhong Li","raw_affiliation_strings":["State Key Lab for Novel Software Technology and the Department of Computer Science and Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-3849-3416","affiliations":[{"raw_affiliation_string":"State Key Lab for Novel Software Technology and the Department of Computer Science and Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002720603","display_name":"Minxue Pan","orcid":"https://orcid.org/0000-0002-4011-5350"},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]},{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Minxue Pan","raw_affiliation_strings":["State Key Lab for Novel Software Technology and the Software Institute, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-4011-5350","affiliations":[{"raw_affiliation_string":"State Key Lab for Novel Software Technology and the Software Institute, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103039956","display_name":"Chaoxuan Shi","orcid":"https://orcid.org/0009-0001-9037-6698"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chaoxuan Shi","raw_affiliation_strings":["State Key Lab for Novel Software Technology and the Department of Computer Science and Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0001-9037-6698","affiliations":[{"raw_affiliation_string":"State Key Lab for Novel Software Technology and the Department of Computer Science and Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078890349","display_name":"Tian Zhang","orcid":"https://orcid.org/0000-0003-0104-2731"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tian Zhang","raw_affiliation_strings":["State Key Lab for Novel Software Technology and the Department of Computer Science and Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0003-0104-2731","affiliations":[{"raw_affiliation_string":"State Key Lab for Novel Software Technology and the Department of Computer Science and Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028795031","display_name":"Yu Pei","orcid":"https://orcid.org/0000-0001-6065-6958"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yu Pei","raw_affiliation_strings":["Department of Computing, The Hong Kong Polytechnic University, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-6065-6958","affiliations":[{"raw_affiliation_string":"Department of Computing, The Hong Kong Polytechnic University, Hong Kong, China","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5090810072","display_name":"Xuandong Li","orcid":"https://orcid.org/0000-0003-3090-9568"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuandong Li","raw_affiliation_strings":["State Key Lab for Novel Software Technology and the Department of Computer Science and Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0003-3090-9568","affiliations":[{"raw_affiliation_string":"State Key Lab for Novel Software Technology and the Department of Computer Science and Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.2208,"has_fulltext":false,"cited_by_count":14,"citation_normalized_percentile":{"value":0.89896775,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"33","issue":"3","first_page":"1","last_page":"37"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8692538142204285},{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.8511471748352051},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.666252076625824},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5433131456375122},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5116408467292786},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4723314642906189},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4203769564628601},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3449104428291321},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.13110843300819397}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8692538142204285},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.8511471748352051},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.666252076625824},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5433131456375122},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5116408467292786},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4723314642906189},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4203769564628601},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3449104428291321},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.13110843300819397},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3631975","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3631975","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.4099999964237213}],"awards":[{"id":"https://openalex.org/G1828253250","display_name":null,"funder_award_id":"62232014","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4958647104","display_name":null,"funder_award_id":"No. 62232014 and No. 61972193","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5471745240","display_name":"\u9762\u5411\u667a\u80fd\u5236\u9020\u590d\u6742\u573a\u666f\u7684\u8f6f\u4ef6\u884c\u4e3a\u5efa\u6a21\u4e0e\u9a8c\u8bc1\u6280\u672f\u7814\u7a76","funder_award_id":"61972193","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":99,"referenced_works":["https://openalex.org/W1416271685","https://openalex.org/W1593271688","https://openalex.org/W1603598191","https://openalex.org/W1970407057","https://openalex.org/W1978394996","https://openalex.org/W1984561845","https://openalex.org/W1995423397","https://openalex.org/W1997358723","https://openalex.org/W2018844270","https://openalex.org/W2023925487","https://openalex.org/W2034209539","https://openalex.org/W2070620842","https://openalex.org/W2081749632","https://openalex.org/W2082160726","https://openalex.org/W2109125971","https://openalex.org/W2113610359","https://openalex.org/W2118593414","https://openalex.org/W2133333349","https://openalex.org/W2136296681","https://openalex.org/W2138756793","https://openalex.org/W2140609933","https://openalex.org/W2165747537","https://openalex.org/W2516621648","https://openalex.org/W2593675739","https://openalex.org/W2600463316","https://openalex.org/W2606974598","https://openalex.org/W2741705590","https://openalex.org/W2796167946","https://openalex.org/W2807964941","https://openalex.org/W2884276923","https://openalex.org/W2887364112","https://openalex.org/W2888312537","https://openalex.org/W2888539709","https://openalex.org/W2888557792","https://openalex.org/W2899132877","https://openalex.org/W2899171197","https://openalex.org/W2927392967","https://openalex.org/W2949297108","https://openalex.org/W2955426500","https://openalex.org/W2963935794","https://openalex.org/W2964110616","https://openalex.org/W2964194820","https://openalex.org/W2964199361","https://openalex.org/W2964268484","https://openalex.org/W2964270303","https://openalex.org/W2973529529","https://openalex.org/W2979271470","https://openalex.org/W2979486033","https://openalex.org/W2988394319","https://openalex.org/W3008733198","https://openalex.org/W3016234956","https://openalex.org/W3034689979","https://openalex.org/W3034999214","https://openalex.org/W3039512627","https://openalex.org/W3086449553","https://openalex.org/W3091730360","https://openalex.org/W3099636232","https://openalex.org/W3099640733","https://openalex.org/W3101118213","https://openalex.org/W3103748122","https://openalex.org/W3105247453","https://openalex.org/W3123545166","https://openalex.org/W3125488228","https://openalex.org/W3131328028","https://openalex.org/W3159616622","https://openalex.org/W3170092793","https://openalex.org/W3172977323","https://openalex.org/W3176248591","https://openalex.org/W3176776234","https://openalex.org/W3176913510","https://openalex.org/W3183859557","https://openalex.org/W3195727321","https://openalex.org/W3198685994","https://openalex.org/W3211962263","https://openalex.org/W3217290366","https://openalex.org/W4206238733","https://openalex.org/W4221166942","https://openalex.org/W4226485558","https://openalex.org/W4249122235","https://openalex.org/W4282045446","https://openalex.org/W4284667247","https://openalex.org/W4284696121","https://openalex.org/W4284704125","https://openalex.org/W4285428896","https://openalex.org/W4287262333","https://openalex.org/W4287328196","https://openalex.org/W4288028629","https://openalex.org/W4288804596","https://openalex.org/W4294214983","https://openalex.org/W4308641647","https://openalex.org/W4313563640","https://openalex.org/W4361866100","https://openalex.org/W4384302770","https://openalex.org/W4385245566","https://openalex.org/W4394638297","https://openalex.org/W6739901393","https://openalex.org/W6767047860","https://openalex.org/W6850195851","https://openalex.org/W7042967642"],"related_works":["https://openalex.org/W2366403280","https://openalex.org/W1495108544","https://openalex.org/W2091301346","https://openalex.org/W3148229873","https://openalex.org/W4389760904","https://openalex.org/W2150160875","https://openalex.org/W4242223894","https://openalex.org/W4306886878","https://openalex.org/W4380075502","https://openalex.org/W4366307084"],"abstract_inverted_index":{"Code":[0],"summarization":[1,114,140],"aims":[2],"to":[3,11,41,116,223],"generate":[4],"short":[5],"functional":[6],"descriptions":[7],"for":[8,104,119,179,266],"source":[9],"code":[10,13,23,46,113,139,230,268],"facilitate":[12],"comprehension.":[14],"While":[15],"Information":[16],"Retrieval":[17],"(IR)":[18],"approaches":[19,36,59,74,83,115,141,152,169,178,196,221,242,250],"that":[20,37,57,191,201,239],"leverage":[21],"similar":[22],"snippets":[24],"and":[25,47,80,86,92,111,137,159,164,167,173,186,213,238],"corresponding":[26],"summaries":[27,48],"have":[28],"led":[29],"the":[30,72,88,109,128,181,192,227],"early":[31],"research,":[32],"Deep":[33],"Learning":[34],"(DL)":[35],"use":[38],"neural":[39],"models":[40,204],"capture":[42],"statistical":[43],"properties":[44],"between":[45],"are":[49,60],"now":[50],"mainstream.":[51],"Although":[52],"some":[53,64],"preliminary":[54],"studies":[55],"suggest":[56],"IR":[58,91,110,166,185,195,220],"more":[61],"effective":[62,71],"in":[63,77,122,207,217,226,251],"cases,":[65],"it":[66],"is":[67,100],"currently":[68],"unclear":[69],"how":[70],"existing":[73,240],"can":[75,94],"be":[76,199],"general,":[78],"where":[79,163],"why":[81,165],"IR/DL":[82],"perform":[84,170,205,224,245],"better,":[85,171],"whether":[87],"integration":[89],"of":[90,108,133,151,162,183,194,229,255],"DL":[93,112,168,203],"achieve":[95],"better":[96,206,225,267],"performance.":[97],"Consequently,":[98],"there":[99],"an":[101],"urgent":[102],"need":[103],"a":[105],"comprehensive":[106],"study":[107,132,176,189],"provide":[117],"guidance":[118],"future":[120,263],"development":[121],"this":[123],"area.":[124],"This":[125],"article":[126],"presents":[127],"first":[129],"large-scale":[130],"empirical":[131],"18":[134],"IR,":[135],"DL,":[136],"hybrid":[138,177,241],"on":[142,258],"five":[143],"benchmark":[144],"datasets.":[145],"We":[146],"extensively":[147],"compare":[148],"different":[149],"types":[150],"using":[153],"automatic":[154],"metrics,":[155],"we":[156,174,261],"conduct":[157],"quantitative":[158],"qualitative":[160],"analyses":[161],"respectively,":[172],"also":[175],"assessing":[180],"effectiveness":[182],"integrating":[184],"DL.":[187],"The":[188],"shows":[190],"performance":[193],"should":[197],"not":[198,244],"underestimated,":[200],"while":[202],"predicting":[208],"tokens":[209],"from":[210],"method":[211],"signatures":[212],"capturing":[214],"structural":[215],"similarities":[216],"code,":[218],"simple":[219],"tend":[222],"presence":[228],"with":[231],"high":[232],"similarity":[233],"or":[234],"long":[235],"reference":[236],"summaries,":[237],"do":[243],"as":[246,248],"well":[247],"individual":[249],"their":[252],"respective":[253],"areas":[254],"strength.":[256],"Based":[257],"our":[259],"findings,":[260],"discuss":[262],"research":[264],"directions":[265],"summarization.":[269]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":4}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}