{"id":"https://openalex.org/W4405205232","doi":"https://doi.org/10.1162/tacl_a_00718","title":"<tt>Holmes</tt> \u2315 A Benchmark to Assess the Linguistic Competence of Language Models","display_name":"<tt>Holmes</tt> \u2315 A Benchmark to Assess the Linguistic Competence of Language Models","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4405205232","doi":"https://doi.org/10.1162/tacl_a_00718"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00718","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00718","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1162/tacl_a_00718","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085003328","display_name":"Andreas Waldis","orcid":"https://orcid.org/0000-0002-2772-5701"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technische Universit\u00e4t Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]},{"id":"https://openalex.org/I81007117","display_name":"Lucerne University of Applied Sciences and Arts","ror":"https://ror.org/04nd0xd48","country_code":"CH","type":"education","lineage":["https://openalex.org/I81007117"]}],"countries":["CH","DE"],"is_corresponding":false,"raw_author_name":"Andreas Waldis","raw_affiliation_strings":["Information Systems Research Lab, Lucerne University of Applied Sciences and Arts, Switzerland","Ubiquitous Knowledge Processing Lab (UKP Lab), Technical University of Darmstadt, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Information Systems Research Lab, Lucerne University of Applied Sciences and Arts, Switzerland","institution_ids":["https://openalex.org/I81007117"]},{"raw_affiliation_string":"Ubiquitous Knowledge Processing Lab (UKP Lab), Technical University of Darmstadt, Germany","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085003328","display_name":"Andreas Waldis","orcid":"https://orcid.org/0000-0002-2772-5701"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technische Universit\u00e4t Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]},{"id":"https://openalex.org/I81007117","display_name":"Lucerne University of Applied Sciences and Arts","ror":"https://ror.org/04nd0xd48","country_code":"CH","type":"education","lineage":["https://openalex.org/I81007117"]}],"countries":["CH","DE"],"is_corresponding":false,"raw_author_name":"Andreas Waldis","raw_affiliation_strings":["Information Systems Research Lab, Lucerne University of Applied Sciences and Arts, Switzerland","Ubiquitous Knowledge Processing Lab (UKP Lab), Technical University of Darmstadt, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Information Systems Research Lab, Lucerne University of Applied Sciences and Arts, Switzerland","institution_ids":["https://openalex.org/I81007117"]},{"raw_affiliation_string":"Ubiquitous Knowledge Processing Lab (UKP Lab), Technical University of Darmstadt, Germany","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068497719","display_name":"Yotam Perlitz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yotam Perlitz","raw_affiliation_strings":["ABM Research AI, Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ABM Research AI, Israel","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040286212","display_name":"Leshem Choshen","orcid":"https://orcid.org/0000-0002-0085-6496"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]},{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Leshem Choshen","raw_affiliation_strings":["MIT CSAIL, USA","MIT-IBM Watson AI Lab, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT CSAIL, USA","institution_ids":["https://openalex.org/I4405258931"]},{"raw_affiliation_string":"MIT-IBM Watson AI Lab, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040286212","display_name":"Leshem Choshen","orcid":"https://orcid.org/0000-0002-0085-6496"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]},{"id":"https://openalex.org/I4405258931","display_name":"MIT Computer Science and Artificial Intelligence Laboratory","ror":"https://ror.org/01e85r662","country_code":"US","type":"facility","lineage":["https://openalex.org/I4405258931","https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Leshem Choshen","raw_affiliation_strings":["MIT CSAIL, USA","MIT-IBM Watson AI Lab, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT CSAIL, USA","institution_ids":["https://openalex.org/I4405258931"]},{"raw_affiliation_string":"MIT-IBM Watson AI Lab, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078812348","display_name":"Yufang Hou","orcid":null},"institutions":[{"id":"https://openalex.org/I4210145784","display_name":"IBM Research - Ireland","ror":"https://ror.org/04jnxr720","country_code":"IE","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210145784"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Yufang Hou","raw_affiliation_strings":["IBM Research Europe, Ireland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research Europe, Ireland","institution_ids":["https://openalex.org/I4210145784"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027450194","display_name":"Iryna Gurevych","orcid":"https://orcid.org/0000-0003-2187-7621"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technische Universit\u00e4t Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Iryna Gurevych","raw_affiliation_strings":["Ubiquitous Knowledge Processing Lab (UKP Lab), Technical University of Darmstadt, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ubiquitous Knowledge Processing Lab (UKP Lab), Technical University of Darmstadt, Germany","institution_ids":["https://openalex.org/I31512782"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.6635,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.86824936,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":"12","issue":null,"first_page":"1616","last_page":"1647"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7936133146286011},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7137943506240845},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6262567639350891},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.5755518674850464},{"id":"https://openalex.org/keywords/competence","display_name":"Competence (human resources)","score":0.508878231048584},{"id":"https://openalex.org/keywords/linguistic-competence","display_name":"Linguistic competence","score":0.48470592498779297},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.48282894492149353},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4752773344516754},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1328311562538147},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.05686849355697632},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.04797437787055969},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.043501853942871094}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7936133146286011},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7137943506240845},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6262567639350891},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.5755518674850464},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.508878231048584},{"id":"https://openalex.org/C17601614","wikidata":"https://www.wikidata.org/wiki/Q1346205","display_name":"Linguistic competence","level":2,"score":0.48470592498779297},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48282894492149353},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4752773344516754},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1328311562538147},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.05686849355697632},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.04797437787055969},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.043501853942871094},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/tacl_a_00718","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00718","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:tubiblio.ulb.tu-darmstadt.de:150808","is_oa":false,"landing_page_url":"http://tubiblio.ulb.tu-darmstadt.de/view/person/Waldis=3AAndreas=3A=3A.html>","pdf_url":null,"source":{"id":"https://openalex.org/S4377196390","display_name":"TUbilio (Technical University of Darmstadt)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I31512782","host_organization_name":"Technische Universit\u00e4t Darmstadt","host_organization_lineage":["https://openalex.org/I31512782"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Artikel"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00718","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00718","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":161,"referenced_works":["https://openalex.org/W574867956","https://openalex.org/W1541096673","https://openalex.org/W1581597064","https://openalex.org/W1967390364","https://openalex.org/W1996296896","https://openalex.org/W2043335066","https://openalex.org/W2044599851","https://openalex.org/W2081580037","https://openalex.org/W2099779943","https://openalex.org/W2108598243","https://openalex.org/W2250382531","https://openalex.org/W2250539671","https://openalex.org/W2251939518","https://openalex.org/W2264869955","https://openalex.org/W2299976354","https://openalex.org/W2488375705","https://openalex.org/W2510461687","https://openalex.org/W2524643686","https://openalex.org/W2531638282","https://openalex.org/W2549835527","https://openalex.org/W2563574619","https://openalex.org/W2567181374","https://openalex.org/W2605717780","https://openalex.org/W2753392522","https://openalex.org/W2771576526","https://openalex.org/W2771976988","https://openalex.org/W2826721128","https://openalex.org/W2882319491","https://openalex.org/W2888482885","https://openalex.org/W2915085887","https://openalex.org/W2946417913","https://openalex.org/W2951732656","https://openalex.org/W2952750383","https://openalex.org/W2963154734","https://openalex.org/W2963189761","https://openalex.org/W2964040911","https://openalex.org/W2964204621","https://openalex.org/W2964222268","https://openalex.org/W2964263366","https://openalex.org/W2965373594","https://openalex.org/W2970476646","https://openalex.org/W2970862333","https://openalex.org/W2972353900","https://openalex.org/W2996728628","https://openalex.org/W3034503989","https://openalex.org/W3034850762","https://openalex.org/W3034928924","https://openalex.org/W3034999214","https://openalex.org/W3035160860","https://openalex.org/W3035305735","https://openalex.org/W3087927280","https://openalex.org/W3092448486","https://openalex.org/W3098300729","https://openalex.org/W3099668342","https://openalex.org/W3102812725","https://openalex.org/W3104235057","https://openalex.org/W3111372685","https://openalex.org/W3118485687","https://openalex.org/W3156170450","https://openalex.org/W3159900299","https://openalex.org/W3167227435","https://openalex.org/W3167831019","https://openalex.org/W3176198948","https://openalex.org/W3177468934","https://openalex.org/W3185980407","https://openalex.org/W3202070718","https://openalex.org/W3213014097","https://openalex.org/W4205945417","https://openalex.org/W4206415992","https://openalex.org/W4210267726","https://openalex.org/W4212774754","https://openalex.org/W4212865381","https://openalex.org/W4226145103","https://openalex.org/W4281690148","https://openalex.org/W4285263440","https://openalex.org/W4285302754","https://openalex.org/W4286892945","https://openalex.org/W4288089799","https://openalex.org/W4304697829","https://openalex.org/W4307079201","https://openalex.org/W4367000491","https://openalex.org/W4378189609","https://openalex.org/W4384918448","https://openalex.org/W4385567149","https://openalex.org/W4385570682","https://openalex.org/W4385572001","https://openalex.org/W4385572404","https://openalex.org/W4385572845","https://openalex.org/W4385573083","https://openalex.org/W4385573444","https://openalex.org/W4385574363","https://openalex.org/W4386557203","https://openalex.org/W4386566590","https://openalex.org/W4386576685","https://openalex.org/W4388890950","https://openalex.org/W4388963442","https://openalex.org/W4389326025","https://openalex.org/W4389518756","https://openalex.org/W4390136566","https://openalex.org/W4392490478","https://openalex.org/W4392935324","https://openalex.org/W4394019814","https://openalex.org/W4394591516","https://openalex.org/W4395443821","https://openalex.org/W4399911213","https://openalex.org/W4401042935","https://openalex.org/W4401208776","https://openalex.org/W4402671197","https://openalex.org/W4402671818","https://openalex.org/W6622671808","https://openalex.org/W6640851869","https://openalex.org/W6677104278","https://openalex.org/W6691829196","https://openalex.org/W6721446893","https://openalex.org/W6726295259","https://openalex.org/W6732098531","https://openalex.org/W6732298534","https://openalex.org/W6750615492","https://openalex.org/W6755207826","https://openalex.org/W6755641643","https://openalex.org/W6757635932","https://openalex.org/W6757817989","https://openalex.org/W6762392948","https://openalex.org/W6766673545","https://openalex.org/W6768021236","https://openalex.org/W6769627184","https://openalex.org/W6771917389","https://openalex.org/W6776231494","https://openalex.org/W6778883912","https://openalex.org/W6779068807","https://openalex.org/W6782879696","https://openalex.org/W6792210918","https://openalex.org/W6803084059","https://openalex.org/W6803096969","https://openalex.org/W6803557570","https://openalex.org/W6810738896","https://openalex.org/W6839193947","https://openalex.org/W6847076894","https://openalex.org/W6849590751","https://openalex.org/W6851579256","https://openalex.org/W6851762504","https://openalex.org/W6852449896","https://openalex.org/W6852874933","https://openalex.org/W6853775986","https://openalex.org/W6854866820","https://openalex.org/W6855830118","https://openalex.org/W6858023062","https://openalex.org/W6858600445","https://openalex.org/W6858885809","https://openalex.org/W6858957404","https://openalex.org/W6859725859","https://openalex.org/W6860710830","https://openalex.org/W6861322272","https://openalex.org/W6861381738","https://openalex.org/W6861610311","https://openalex.org/W6862106737","https://openalex.org/W6864079567","https://openalex.org/W6864741799","https://openalex.org/W6865748782","https://openalex.org/W6869532840","https://openalex.org/W7054915730"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W4246352526","https://openalex.org/W2121910908","https://openalex.org/W915438175","https://openalex.org/W4382042711"],"abstract_inverted_index":{"Abstract":[0],"We":[1],"introduce":[2],"Holmes,":[3,61],"a":[4,38,120],"new":[5],"benchmark":[6],"designed":[7],"to":[8,25,44,74],"assess":[9,75],"language":[10],"models\u2019":[11],"(LMs\u2019)":[12],"linguistic":[13,18,32,47,94],"competence\u2014their":[14],"unconscious":[15],"understanding":[16],"of":[17],"phenomena.":[19,82],"Specifically,":[20],"we":[21,40,62,117],"use":[22],"classifier-based":[23],"probing":[24,66],"examine":[26],"LMs\u2019":[27,46],"internal":[28],"representations":[29],"regarding":[30],"distinct":[31],"phenomena":[33],"(e.g.,":[34],"part-of-speech":[35],"tagging).":[36],"As":[37],"result,":[39],"meet":[41],"recent":[42],"calls":[43],"disentangle":[45],"competence":[48,95],"from":[49],"other":[50],"cognitive":[51],"abilities,":[52],"such":[53],"as":[54],"following":[55],"instructions":[56],"in":[57,112],"prompting-based":[58],"evaluations.":[59],"Composing":[60],"review":[63],"over":[64,84],"270":[65],"studies":[67],"and":[68,80,104,114],"include":[69],"more":[70],"than":[71],"200":[72],"datasets":[73],"syntax,":[76],"morphology,":[77],"semantics,":[78],"reasoning,":[79],"discourse":[81],"Analyzing":[83],"50":[85],"LMs":[86],"reveals":[87],"that,":[88],"aligned":[89],"with":[90,97],"known":[91],"trends,":[92],"their":[93],"correlates":[96],"model":[98,102],"size.":[99],"However,":[100],"surprisingly,":[101],"architecture":[103],"instruction":[105],"tuning":[106],"also":[107],"significantly":[108],"influence":[109],"performance,":[110],"particularly":[111],"morphology":[113],"syntax.":[115],"Finally,":[116],"propose":[118],"FlashHolmes,":[119],"streamlined":[121],"version":[122],"that":[123],"reduces":[124],"the":[125],"computation":[126],"load":[127],"while":[128],"maintaining":[129],"high-ranking":[130],"precision.":[131]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4}],"updated_date":"2026-07-14T08:27:34.040176","created_date":"2024-12-10T00:00:00"}