{"id":"https://openalex.org/W4399665376","doi":"https://doi.org/10.1145/3672455","title":"Automated Testing Linguistic Capabilities of NLP Models","display_name":"Automated Testing Linguistic Capabilities of NLP Models","publication_year":2024,"publication_date":"2024-06-14","ids":{"openalex":"https://openalex.org/W4399665376","doi":"https://doi.org/10.1145/3672455"},"language":"en","primary_location":{"id":"doi:10.1145/3672455","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3672455","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3672455","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3672455","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100740655","display_name":"Jaeseong Lee","orcid":"https://orcid.org/0009-0001-1756-5767"},"institutions":[{"id":"https://openalex.org/I162577319","display_name":"The University of Texas at Dallas","ror":"https://ror.org/049emcs32","country_code":"US","type":"education","lineage":["https://openalex.org/I162577319"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jaeseong Lee","raw_affiliation_strings":["The University of Texas at Dallas, Richardson, TX, USA","The University of Texas at Dallas, USA"],"raw_orcid":"https://orcid.org/0009-0001-1756-5767","affiliations":[{"raw_affiliation_string":"The University of Texas at Dallas, Richardson, TX, USA","institution_ids":["https://openalex.org/I162577319"]},{"raw_affiliation_string":"The University of Texas at Dallas, USA","institution_ids":["https://openalex.org/I162577319"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045987094","display_name":"S. H. Chen","orcid":"https://orcid.org/0000-0001-5035-3398"},"institutions":[{"id":"https://openalex.org/I162577319","display_name":"The University of Texas at Dallas","ror":"https://ror.org/049emcs32","country_code":"US","type":"education","lineage":["https://openalex.org/I162577319"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Simin Chen","raw_affiliation_strings":["The University of Texas at Dallas, Richardson, TX, USA","The University of Texas at Dallas, USA"],"raw_orcid":"https://orcid.org/0000-0001-5035-3398","affiliations":[{"raw_affiliation_string":"The University of Texas at Dallas, Richardson, TX, USA","institution_ids":["https://openalex.org/I162577319"]},{"raw_affiliation_string":"The University of Texas at Dallas, USA","institution_ids":["https://openalex.org/I162577319"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019029870","display_name":"Austin Mordahl","orcid":"https://orcid.org/0000-0003-3031-8848"},"institutions":[{"id":"https://openalex.org/I162577319","display_name":"The University of Texas at Dallas","ror":"https://ror.org/049emcs32","country_code":"US","type":"education","lineage":["https://openalex.org/I162577319"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Austin Mordahl","raw_affiliation_strings":["The University of Texas at Dallas, Richardson, TX, USA","The University of Texas at Dallas, USA"],"raw_orcid":"https://orcid.org/0000-0003-3031-8848","affiliations":[{"raw_affiliation_string":"The University of Texas at Dallas, Richardson, TX, USA","institution_ids":["https://openalex.org/I162577319"]},{"raw_affiliation_string":"The University of Texas at Dallas, USA","institution_ids":["https://openalex.org/I162577319"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102704908","display_name":"Cong Liu","orcid":"https://orcid.org/0009-0001-6775-1269"},"institutions":[{"id":"https://openalex.org/I103635307","display_name":"University of California, Riverside","ror":"https://ror.org/03nawhv43","country_code":"US","type":"education","lineage":["https://openalex.org/I103635307"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cong Liu","raw_affiliation_strings":["University of California, Riverside, CA, USA","University of California, Riverside, USA"],"raw_orcid":"https://orcid.org/0009-0001-6775-1269","affiliations":[{"raw_affiliation_string":"University of California, Riverside, CA, USA","institution_ids":["https://openalex.org/I103635307"]},{"raw_affiliation_string":"University of California, Riverside, USA","institution_ids":["https://openalex.org/I103635307"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100613524","display_name":"Wei Yang","orcid":"https://orcid.org/0000-0002-5338-7347"},"institutions":[{"id":"https://openalex.org/I162577319","display_name":"The University of Texas at Dallas","ror":"https://ror.org/049emcs32","country_code":"US","type":"education","lineage":["https://openalex.org/I162577319"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wei Yang","raw_affiliation_strings":["The University of Texas at Dallas, Richardson, TX, USA","The University of Texas at Dallas, USA"],"raw_orcid":"https://orcid.org/0000-0002-5338-7347","affiliations":[{"raw_affiliation_string":"The University of Texas at Dallas, Richardson, TX, USA","institution_ids":["https://openalex.org/I162577319"]},{"raw_affiliation_string":"The University of Texas at Dallas, USA","institution_ids":["https://openalex.org/I162577319"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5033317811","display_name":"Shiyi Wei","orcid":"https://orcid.org/0000-0002-2826-1857"},"institutions":[{"id":"https://openalex.org/I162577319","display_name":"The University of Texas at Dallas","ror":"https://ror.org/049emcs32","country_code":"US","type":"education","lineage":["https://openalex.org/I162577319"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shiyi Wei","raw_affiliation_strings":["The University of Texas at Dallas, Richardson, TX, USA","The University of Texas at Dallas, USA"],"raw_orcid":"https://orcid.org/0000-0002-2826-1857","affiliations":[{"raw_affiliation_string":"The University of Texas at Dallas, Richardson, TX, USA","institution_ids":["https://openalex.org/I162577319"]},{"raw_affiliation_string":"The University of Texas at Dallas, USA","institution_ids":["https://openalex.org/I162577319"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100740655"],"corresponding_institution_ids":["https://openalex.org/I162577319"],"apc_list":null,"apc_paid":null,"fwci":0.6294,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.72635534,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":"33","issue":"7","first_page":"1","last_page":"33"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9861999750137329,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8674871921539307},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.7418007850646973},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6741286516189575},{"id":"https://openalex.org/keywords/lexical-diversity","display_name":"Lexical diversity","score":0.5421617031097412},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.5384621024131775},{"id":"https://openalex.org/keywords/syntax","display_name":"Syntax","score":0.4398324191570282},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.19834411144256592},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.0727880597114563}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8674871921539307},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7418007850646973},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6741286516189575},{"id":"https://openalex.org/C2781202465","wikidata":"https://www.wikidata.org/wiki/Q18346297","display_name":"Lexical diversity","level":3,"score":0.5421617031097412},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5384621024131775},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.4398324191570282},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.19834411144256592},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.0727880597114563},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3672455","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3672455","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3672455","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3672455","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3672455","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3672455","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.6299999952316284}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4399665376.pdf"},"referenced_works_count":67,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2131774270","https://openalex.org/W2250539671","https://openalex.org/W2282821441","https://openalex.org/W2343954916","https://openalex.org/W2599674900","https://openalex.org/W2616028256","https://openalex.org/W2740168486","https://openalex.org/W2799194071","https://openalex.org/W2799640043","https://openalex.org/W2804337238","https://openalex.org/W2809701591","https://openalex.org/W2890991187","https://openalex.org/W2905526464","https://openalex.org/W2949128310","https://openalex.org/W2954629067","https://openalex.org/W2954903132","https://openalex.org/W2957066083","https://openalex.org/W2957905354","https://openalex.org/W2962718684","https://openalex.org/W2963084773","https://openalex.org/W2963327228","https://openalex.org/W2963341956","https://openalex.org/W2963456134","https://openalex.org/W2963626623","https://openalex.org/W2963643701","https://openalex.org/W2968940383","https://openalex.org/W2988194011","https://openalex.org/W2990118957","https://openalex.org/W2994987245","https://openalex.org/W3034397670","https://openalex.org/W3035507081","https://openalex.org/W3035671939","https://openalex.org/W3090988182","https://openalex.org/W3091407209","https://openalex.org/W3100726446","https://openalex.org/W3100869665","https://openalex.org/W3101449015","https://openalex.org/W3105249702","https://openalex.org/W3105507623","https://openalex.org/W3125205424","https://openalex.org/W3128630643","https://openalex.org/W3133966466","https://openalex.org/W3163644064","https://openalex.org/W3168704388","https://openalex.org/W3173380736","https://openalex.org/W3175487198","https://openalex.org/W3175663471","https://openalex.org/W3178469298","https://openalex.org/W4205221547","https://openalex.org/W4206388084","https://openalex.org/W4206908526","https://openalex.org/W4224308101","https://openalex.org/W4226216828","https://openalex.org/W4253752119","https://openalex.org/W4287780384","https://openalex.org/W4287888350","https://openalex.org/W4297965481","https://openalex.org/W4308643054","https://openalex.org/W4312648739","https://openalex.org/W4313563684","https://openalex.org/W4322718191","https://openalex.org/W4322719791","https://openalex.org/W4327810158","https://openalex.org/W6679436768","https://openalex.org/W6739901393","https://openalex.org/W6810081322"],"related_works":["https://openalex.org/W4388713123","https://openalex.org/W3194438104","https://openalex.org/W2570592793","https://openalex.org/W2753140600","https://openalex.org/W1603736412","https://openalex.org/W2354135050","https://openalex.org/W1979017339","https://openalex.org/W3204019825","https://openalex.org/W4392345938","https://openalex.org/W4378713476"],"abstract_inverted_index":{"Natural":[0],"language":[1],"processing":[2],"(NLP)":[3],"has":[4,37],"gained":[5],"widespread":[6],"adoption":[7],"in":[8,21,128,156,182],"the":[9,15,43,189],"development":[10],"of":[11,18,45,110,130,173,178,185],"real-world":[12],"applications.":[13,47,192],"However,":[14,48],"black-box":[16],"nature":[17],"neural":[19],"networks":[20],"NLP":[22,66,88,120,179,191],"applications":[23,89],"poses":[24],"a":[25,53,175],"challenge":[26],"when":[27],"evaluating":[28],"their":[29,92],"performance,":[30],"let":[31],"alone":[32],"ensuring":[33],"it.":[34],"Recent":[35],"research":[36],"proposed":[38],"testing":[39,84],"techniques":[40],"to":[41,64,162],"enhance":[42],"trustworthiness":[44],"NLP-based":[46],"most":[49],"existing":[50],"works":[51],"use":[52],"single,":[54],"aggregated":[55],"metric":[56],"(i.e.,":[57],"accuracy)":[58],"which":[59],"is":[60,171],"difficult":[61],"for":[62,86,108],"users":[63],"assess":[65],"model":[67,180],"performance":[68],"on":[69,91,116],"fine-grained":[70],"aspects,":[71],"such":[72],"as":[73,98],"LCs.":[74,93],"To":[75],"address":[76],"this":[77],"limitation,":[78],"we":[79],"present":[80],"ALiCT,":[81],"an":[82],"automated":[83],"technique":[85],"validating":[87],"based":[90],"ALiCT":[94,115,145,170],"takes":[95],"user-specified":[96],"LCs":[97,187],"inputs":[99],"and":[100,124,133,137,152,158],"produces":[101],"diverse":[102,155],"test":[103,106,147],"suite":[104],"with":[105],"oracles":[107],"each":[109],"given":[111],"LC.":[112],"We":[113],"evaluate":[114],"two":[117,190],"widely":[118],"adopted":[119],"tasks,":[121],"sentiment":[122],"analysis":[123],"hate":[125],"speech":[126],"detection,":[127],"terms":[129],"diversity,":[131],"effectiveness,":[132],"consistency.":[134],"Using":[135],"Self-BLEU":[136],"syntactic":[138],"diversity":[139],"metrics,":[140],"our":[141],"findings":[142],"reveal":[143],"that":[144,149],"generates":[146],"cases":[148],"are":[150],"190%":[151],"2213%":[153],"more":[154],"semantics":[157],"syntax,":[159],"respectively,":[160],"compared":[161],"those":[163],"produced":[164],"by":[165],"state-of-the-art":[166],"techniques.":[167],"In":[168],"addition,":[169],"capable":[172],"producing":[174],"larger":[176],"number":[177],"failures":[181],"22":[183],"out":[184],"25":[186],"over":[188]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}