{"id":"https://openalex.org/W4284667247","doi":"https://doi.org/10.1145/3510003.3510060","title":"On the evaluation of neural code summarization","display_name":"On the evaluation of neural code summarization","publication_year":2022,"publication_date":"2022-05-21","ids":{"openalex":"https://openalex.org/W4284667247","doi":"https://doi.org/10.1145/3510003.3510060"},"language":"en","primary_location":{"id":"doi:10.1145/3510003.3510060","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3510003.3510060","pdf_url":null,"source":{"id":"https://openalex.org/S4363608872","display_name":"Proceedings of the 44th International Conference on Software Engineering","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International Conference on Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2107.07112","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028655969","display_name":"Ensheng Shi","orcid":"https://orcid.org/0000-0002-5543-2025"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Ensheng Shi","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100350708","display_name":"Yanlin Wang","orcid":"https://orcid.org/0000-0001-7761-7269"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yanlin Wang","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008387608","display_name":"Lun Du","orcid":"https://orcid.org/0000-0002-7625-0650"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Lun Du","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100365536","display_name":"Junjie Chen","orcid":"https://orcid.org/0000-0002-0483-303X"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junjie Chen","raw_affiliation_strings":["Tianjin University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tianjin University","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006300825","display_name":"Shi Han","orcid":"https://orcid.org/0000-0002-0360-6089"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Shi Han","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100412598","display_name":"Hongyu Zhang","orcid":"https://orcid.org/0000-0002-3063-9425"},"institutions":[{"id":"https://openalex.org/I78757542","display_name":"University of Newcastle Australia","ror":"https://ror.org/00eae9z71","country_code":"AU","type":"education","lineage":["https://openalex.org/I78757542"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Hongyu Zhang","raw_affiliation_strings":["The University of Newcastle"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Newcastle","institution_ids":["https://openalex.org/I78757542"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100331488","display_name":"Dongmei Zhang","orcid":"https://orcid.org/0000-0002-9230-2799"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Dongmei Zhang","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100706029","display_name":"Hongbin Sun","orcid":"https://orcid.org/0000-0003-2153-2906"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongbin Sun","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5028655969"],"corresponding_institution_ids":["https://openalex.org/I87445476"],"apc_list":null,"apc_paid":null,"fwci":7.3777,"has_fulltext":false,"cited_by_count":76,"citation_normalized_percentile":{"value":0.97906508,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1597","last_page":"1608"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.9389532804489136},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.862169086933136},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6521585583686829},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6147362589836121},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.6097031831741333},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.5613260865211487},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.525301992893219},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.49224796891212463},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.48245126008987427},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4394811689853668},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3809334635734558},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3366231918334961},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.07514053583145142}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.9389532804489136},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.862169086933136},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6521585583686829},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6147362589836121},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6097031831741333},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.5613260865211487},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.525301992893219},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.49224796891212463},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.48245126008987427},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4394811689853668},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3809334635734558},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3366231918334961},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.07514053583145142},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3510003.3510060","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3510003.3510060","pdf_url":null,"source":{"id":"https://openalex.org/S4363608872","display_name":"Proceedings of the 44th International Conference on Software Engineering","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International Conference on Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2107.07112","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2107.07112","pdf_url":"https://arxiv.org/pdf/2107.07112","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:uon:47307","is_oa":false,"landing_page_url":"http://hdl.handle.net/1959.13/1465586","pdf_url":null,"source":{"id":"https://openalex.org/S4377196471","display_name":"NOVA (University of Newcastle Australia)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I78757542","host_organization_name":"University of Newcastle Australia","host_organization_lineage":["https://openalex.org/I78757542"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"conference paper"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2107.07112","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2107.07112","pdf_url":"https://arxiv.org/pdf/2107.07112","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W1956340063","https://openalex.org/W1972978214","https://openalex.org/W1979354511","https://openalex.org/W2081749632","https://openalex.org/W2082160726","https://openalex.org/W2101105183","https://openalex.org/W2104561662","https://openalex.org/W2109125971","https://openalex.org/W2132944054","https://openalex.org/W2133333349","https://openalex.org/W2136296681","https://openalex.org/W2517332594","https://openalex.org/W2762844179","https://openalex.org/W2807964941","https://openalex.org/W2887364112","https://openalex.org/W2888651608","https://openalex.org/W2949297108","https://openalex.org/W2955426500","https://openalex.org/W2963381560","https://openalex.org/W2964270303","https://openalex.org/W2979792666","https://openalex.org/W2999553660","https://openalex.org/W3008733198","https://openalex.org/W3011564318","https://openalex.org/W3011632945","https://openalex.org/W3016234956","https://openalex.org/W3034689979","https://openalex.org/W3085939759","https://openalex.org/W3086449553","https://openalex.org/W3091730360","https://openalex.org/W3099636232","https://openalex.org/W3103748122","https://openalex.org/W3174885139","https://openalex.org/W3176776234","https://openalex.org/W3176913510","https://openalex.org/W4248905411","https://openalex.org/W4288601300","https://openalex.org/W4292355384"],"related_works":["https://openalex.org/W2366403280","https://openalex.org/W1495108544","https://openalex.org/W2091301346","https://openalex.org/W3148229873","https://openalex.org/W402673672","https://openalex.org/W51364034","https://openalex.org/W2141938446","https://openalex.org/W2898073868","https://openalex.org/W4284663758","https://openalex.org/W3081644756"],"abstract_inverted_index":{"Source":[0],"code":[1,35,71,138,196,263,278],"summaries":[2,32],"are":[3,12,46],"important":[4,97,165,233],"for":[5,33,261],"program":[6],"comprehension":[7],"and":[8,51,64,83,86,113,162,166,183,212,225,242,265],"maintenance.":[9],"However,":[10,119],"there":[11],"plenty":[13],"of":[14,42,67,111,136,155,222,229,235],"programs":[15],"with":[16],"missing,":[17],"outdated,":[18],"or":[19],"mismatched":[20],"summaries.":[21],"Recently,":[22],"deep":[23],"learning":[24],"techniques":[25],"have":[26,99,200,245],"been":[27],"exploited":[28],"to":[29,54,192,205,281],"automatically":[30],"generate":[31],"given":[34],"snippets.":[36],"To":[37],"achieve":[38],"a":[39,62,100,174,201,246,276],"profound":[40],"understanding":[41],"how":[43],"far":[44],"we":[45,60,160,179,257],"from":[47],"solving":[48],"this":[49,58],"problem":[50],"provide":[52],"suggestions":[53,260],"future":[55,283],"research,":[56],"in":[57,133,170,173,270],"paper,":[59],"conduct":[61,180],"systematic":[63],"in-depth":[65],"analysis":[66],"5":[68],"state-of-the-art":[69],"neural":[70],"summarization":[72,139,210,264,279],"models":[73,112,140],"on":[74,103,108,208,249,253],"6":[75],"widely":[76,88,131],"used":[77,89,132],"BLEU":[78,129,171],"variants,":[79],"4":[80],"pre-processing":[81,223],"operations":[82],"their":[84],"combinations,":[85],"3":[87],"datasets.":[90],"The":[91],"evaluation":[92],"results":[93],"show":[94],"that":[95,185],"some":[96,232],"factors":[98,121],"great":[101],"influence":[102],"the":[104,109,114,117,128,145,153,156,186,209,220,227,254,267],"model":[105,250],"evaluation,":[106],"especially":[107],"performance":[110,211,228],"ranking":[115],"among":[116,147],"models.":[118],"these":[120,148],"might":[122],"be":[123,215],"easily":[124],"overlooked.":[125],"Specifically,":[126],"(1)":[127],"metric":[130,187],"existing":[134],"work":[135],"evaluating":[137,262],"has":[141],"many":[142],"variants.":[143],"Ignoring":[144],"differences":[146],"variants":[149],"could":[150],"greatly":[151],"affect":[152],"validity":[154],"claimed":[157],"results.":[158],"Besides,":[159],"discover":[161],"resolve":[163],"an":[164],"previously":[167],"unknown":[168],"bug":[169],"calculation":[172],"commonly-used":[175],"software":[176],"package.":[177],"Furthermore,":[178],"human":[181,193],"evaluations":[182],"find":[184],"BLEU-DC":[188],"is":[189],"most":[190],"correlated":[191],"perception;":[194],"(2)":[195],"preprocessing":[197],"choices":[198],"can":[199],"large":[202],"(from":[203],"-18%":[204],"+25%)":[206],"impact":[207,248],"should":[213],"not":[214],"neglected.":[216],"We":[217,273],"also":[218,274],"explore":[219],"aggregation":[221],"combinations":[224],"boost":[226],"models;":[230],"(3)":[231],"characteristics":[234],"datasets":[236],"(corpus":[237],"sizes,":[238],"data":[239],"splitting":[240],"methods,":[241],"duplication":[243],"ratios)":[244],"significant":[247],"evaluation.":[251],"Based":[252],"experimental":[255],"results,":[256],"give":[258],"actionable":[259],"choosing":[266],"best":[268],"method":[269],"different":[271],"scenarios.":[272],"build":[275],"shared":[277],"toolbox":[280],"facilitate":[282],"research.":[284]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":25},{"year":2024,"cited_by_count":20},{"year":2023,"cited_by_count":24},{"year":2022,"cited_by_count":2}],"updated_date":"2026-05-17T08:19:37.847499","created_date":"2025-10-10T00:00:00"}