{"id":"https://openalex.org/W4386212383","doi":"https://doi.org/10.1109/tse.2023.3308952","title":"Do Pretrained Language Models Indeed Understand Software Engineering Tasks?","display_name":"Do Pretrained Language Models Indeed Understand Software Engineering Tasks?","publication_year":2023,"publication_date":"2023-08-28","ids":{"openalex":"https://openalex.org/W4386212383","doi":"https://doi.org/10.1109/tse.2023.3308952"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2023.3308952","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2023.3308952","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100411221","display_name":"Yao Li","orcid":"https://orcid.org/0000-0002-0474-0159"},"institutions":[{"id":"https://openalex.org/I111950717","display_name":"Macau University of Science and Technology","ror":"https://ror.org/03jqs2n27","country_code":"MO","type":"education","lineage":["https://openalex.org/I111950717","https://openalex.org/I4391767947"]}],"countries":["MO"],"is_corresponding":true,"raw_author_name":"Yao Li","raw_affiliation_strings":["School of Computer Science and Engineering, Macau University of Science and Technology, Macao, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Macau University of Science and Technology, Macao, China","institution_ids":["https://openalex.org/I111950717"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100375823","display_name":"Tao Zhang","orcid":"https://orcid.org/0000-0002-6272-4069"},"institutions":[{"id":"https://openalex.org/I111950717","display_name":"Macau University of Science and Technology","ror":"https://ror.org/03jqs2n27","country_code":"MO","type":"education","lineage":["https://openalex.org/I111950717","https://openalex.org/I4391767947"]}],"countries":["MO"],"is_corresponding":false,"raw_author_name":"Tao Zhang","raw_affiliation_strings":["School of Computer Science and Engineering, Macau University of Science and Technology, Macao, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Macau University of Science and Technology, Macao, China","institution_ids":["https://openalex.org/I111950717"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100400376","display_name":"Xiapu Luo","orcid":"https://orcid.org/0000-0002-9082-3208"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xiapu Luo","raw_affiliation_strings":["Department of Computing, Hong Kong Polytechnic University, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Department of Computing, Hong Kong Polytechnic University, Hong Kong, China","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076081056","display_name":"Haipeng Cai","orcid":"https://orcid.org/0000-0002-5224-9970"},"institutions":[{"id":"https://openalex.org/I72951846","display_name":"Washington State University","ror":"https://ror.org/05dk0ce17","country_code":"US","type":"education","lineage":["https://openalex.org/I72951846"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haipeng Cai","raw_affiliation_strings":["School of Electrical Engineering and Computer Science, Washington State University Pullman, WA, USA"],"affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Computer Science, Washington State University Pullman, WA, USA","institution_ids":["https://openalex.org/I72951846"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025207624","display_name":"Sen Fang","orcid":"https://orcid.org/0000-0002-9918-7180"},"institutions":[{"id":"https://openalex.org/I111950717","display_name":"Macau University of Science and Technology","ror":"https://ror.org/03jqs2n27","country_code":"MO","type":"education","lineage":["https://openalex.org/I111950717","https://openalex.org/I4391767947"]}],"countries":["MO"],"is_corresponding":false,"raw_author_name":"Sen Fang","raw_affiliation_strings":["School of Computer Science and Engineering, Macau University of Science and Technology, Macao, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Macau University of Science and Technology, Macao, China","institution_ids":["https://openalex.org/I111950717"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101509302","display_name":"Dawei Yuan","orcid":"https://orcid.org/0000-0002-9469-5864"},"institutions":[{"id":"https://openalex.org/I111950717","display_name":"Macau University of Science and Technology","ror":"https://ror.org/03jqs2n27","country_code":"MO","type":"education","lineage":["https://openalex.org/I111950717","https://openalex.org/I4391767947"]}],"countries":["MO"],"is_corresponding":false,"raw_author_name":"Dawei Yuan","raw_affiliation_strings":["School of Computer Science and Engineering, Macau University of Science and Technology, Macao, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Macau University of Science and Technology, Macao, China","institution_ids":["https://openalex.org/I111950717"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100411221"],"corresponding_institution_ids":["https://openalex.org/I111950717"],"apc_list":null,"apc_paid":null,"fwci":6.8833,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.96963614,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"49","issue":"10","first_page":"4639","last_page":"4655"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9359999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.8039630055427551},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8024949431419373},{"id":"https://openalex.org/keywords/salient","display_name":"Salient","score":0.5764050483703613},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5443264245986938},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.49935460090637207},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.488455206155777},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.48702770471572876},{"id":"https://openalex.org/keywords/phenomenon","display_name":"Phenomenon","score":0.45032280683517456},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4357432425022125},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4346250891685486},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.41597485542297363},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37669074535369873},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2243322730064392}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.8039630055427551},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8024949431419373},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.5764050483703613},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5443264245986938},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.49935460090637207},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.488455206155777},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.48702770471572876},{"id":"https://openalex.org/C50335755","wikidata":"https://www.wikidata.org/wiki/Q483247","display_name":"Phenomenon","level":2,"score":0.45032280683517456},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4357432425022125},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4346250891685486},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.41597485542297363},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37669074535369873},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2243322730064392},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2023.3308952","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2023.3308952","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5299999713897705,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[{"id":"https://openalex.org/G874560949","display_name":null,"funder_award_id":"0014/2022/A","funder_id":"https://openalex.org/F4320323893","funder_display_name":"Fundo para o Desenvolvimento das Ci\u00eancias e da Tecnologia"},{"id":"https://openalex.org/G913651422","display_name":null,"funder_award_id":"0047/2020/A1","funder_id":"https://openalex.org/F4320323893","funder_display_name":"Fundo para o Desenvolvimento das Ci\u00eancias e da Tecnologia"}],"funders":[{"id":"https://openalex.org/F4320323893","display_name":"Fundo para o Desenvolvimento das Ci\u00eancias e da Tecnologia","ror":"https://ror.org/05vna4324"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":98,"referenced_works":["https://openalex.org/W131533222","https://openalex.org/W1875990341","https://openalex.org/W1892345042","https://openalex.org/W1974655094","https://openalex.org/W2014755981","https://openalex.org/W2037843344","https://openalex.org/W2082160726","https://openalex.org/W2085030399","https://openalex.org/W2094704982","https://openalex.org/W2097001189","https://openalex.org/W2108086273","https://openalex.org/W2120575527","https://openalex.org/W2123301721","https://openalex.org/W2136922672","https://openalex.org/W2154652894","https://openalex.org/W2157943826","https://openalex.org/W2167630669","https://openalex.org/W2175297521","https://openalex.org/W2247374552","https://openalex.org/W2250539671","https://openalex.org/W2251939518","https://openalex.org/W2282821441","https://openalex.org/W2298285108","https://openalex.org/W2356325415","https://openalex.org/W2516621648","https://openalex.org/W2547932336","https://openalex.org/W2607306668","https://openalex.org/W2739351760","https://openalex.org/W2743771122","https://openalex.org/W2767852858","https://openalex.org/W2794601162","https://openalex.org/W2803732139","https://openalex.org/W2896457183","https://openalex.org/W2898496135","https://openalex.org/W2952370363","https://openalex.org/W2962739339","https://openalex.org/W2963491027","https://openalex.org/W2963748441","https://openalex.org/W2963773358","https://openalex.org/W2965373594","https://openalex.org/W2970597249","https://openalex.org/W3002842489","https://openalex.org/W3005653326","https://openalex.org/W3014451403","https://openalex.org/W3034689979","https://openalex.org/W3035010485","https://openalex.org/W3095992020","https://openalex.org/W3098605233","https://openalex.org/W3103392356","https://openalex.org/W3104033643","https://openalex.org/W3104874136","https://openalex.org/W3106099468","https://openalex.org/W3121995718","https://openalex.org/W3153574020","https://openalex.org/W3155191037","https://openalex.org/W3164087770","https://openalex.org/W3166095789","https://openalex.org/W3168488662","https://openalex.org/W3175951726","https://openalex.org/W3177116043","https://openalex.org/W3194069451","https://openalex.org/W3194346579","https://openalex.org/W4206581673","https://openalex.org/W4207063495","https://openalex.org/W4213072374","https://openalex.org/W4220836561","https://openalex.org/W4221138649","https://openalex.org/W4224214784","https://openalex.org/W4246548175","https://openalex.org/W4280517564","https://openalex.org/W4283156623","https://openalex.org/W4283261419","https://openalex.org/W4287646293","https://openalex.org/W4287825389","https://openalex.org/W4294170691","https://openalex.org/W4302343710","https://openalex.org/W4312632714","https://openalex.org/W4385245566","https://openalex.org/W4386566638","https://openalex.org/W6605323724","https://openalex.org/W6678262379","https://openalex.org/W6682631176","https://openalex.org/W6682691769","https://openalex.org/W6691459498","https://openalex.org/W6739901393","https://openalex.org/W6742080785","https://openalex.org/W6751441926","https://openalex.org/W6755207826","https://openalex.org/W6755438426","https://openalex.org/W6763701032","https://openalex.org/W6766673545","https://openalex.org/W6775082190","https://openalex.org/W6783754597","https://openalex.org/W6803607395","https://openalex.org/W6807463058","https://openalex.org/W6809741598","https://openalex.org/W6898505805","https://openalex.org/W6973432066"],"related_works":["https://openalex.org/W2366403280","https://openalex.org/W1495108544","https://openalex.org/W3148229873","https://openalex.org/W2150160875","https://openalex.org/W4242223894","https://openalex.org/W2091301346","https://openalex.org/W1517524280","https://openalex.org/W4389760904","https://openalex.org/W4306886878","https://openalex.org/W2463467089"],"abstract_inverted_index":{"Artificial":[0],"intelligence":[1],"(AI)":[2],"for":[3,49,192,198,203],"software":[4],"engineering":[5],"(SE)":[6],"tasks":[7,29,52],"has":[8],"recently":[9],"achieved":[10],"promising":[11],"performance.":[12],"In":[13],"this":[14,118,180],"article,":[15],"we":[16,172],"investigate":[17],"to":[18,101,117,155,178,183,194],"what":[19],"extent":[20],"the":[21,72,102,144,148,157,167,174,184,196,199],"pre-trained":[22,96],"language":[23,97],"model":[24,127,138],"truly":[25],"understands":[26],"those":[27],"SE":[28,50,79,113],"such":[30],"as":[31,120],"code":[32,34,82,84],"search,":[33,83],"summarization,":[35,85],"etc.":[36],"We":[37,115],"conduct":[38],"a":[39,44,126,130,137],"comprehensive":[40],"empirical":[41],"study":[42,151],"on":[43,77],"board":[45],"set":[46],"of":[47,169],"AI":[48],"(AI4SE)":[51],"by":[53],"feeding":[54],"them":[55],"with":[56,60,66],"variant":[57],"inputs:":[58],"1)":[59],"various":[61],"masking":[62],"rates":[63],"and":[64,86,147,164,201,209],"2)":[65],"sufficient":[67],"input":[68,197],"subset":[69],"method.":[70],"Then,":[71],"trained":[73],"models":[74,98,200],"are":[75,99,173],"evaluated":[76],"different":[78],"tasks,":[80],"including":[81],"duplicate":[87],"bug":[88],"report":[89],"detection.":[90],"Our":[91,150],"experimental":[92],"results":[93],"show":[94],"that":[95],"insensitive":[100],"given":[103],"input,":[104],"thus":[105],"they":[106],"achieve":[107],"similar":[108],"performance":[109],"in":[110,207],"these":[111],"three":[112],"tasks.":[114,212],"refer":[116],"phenomenon":[119,182],"<italic":[121,175],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[122,176],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">overinterpretation</i>":[123],",":[124],"where":[125,136],"confidently":[128],"makes":[129],"decision":[131,146],"without":[132],"salient":[133],"features,":[134],"or":[135],"finds":[139],"some":[140],"irrelevant":[141],"relationships":[142],"between":[143],"final":[145],"dataset.":[149],"investigates":[152],"two":[153],"approaches":[154],"mitigate":[156],"overinterpretation":[158,181],"phenomenon:":[159],"whole":[160],"word":[161],"mask":[162],"strategy":[163],"ensembling.":[165],"To":[166],"best":[168],"our":[170],"knowledge,":[171],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">first</i>":[177],"reveal":[179],"AI4SE":[185,211],"community,":[186],"which":[187],"is":[188],"an":[189],"important":[190],"reminder":[191],"researchers":[193],"design":[195],"calls":[202],"necessary":[204],"future":[205],"work":[206],"understanding":[208],"implementing":[210]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}