{"id":"https://openalex.org/W7135380730","doi":"https://doi.org/10.5715/jnlp.33.51","title":"Rethinking the Evaluation Methods of LLMs' Grammatical Knowledge","display_name":"Rethinking the Evaluation Methods of LLMs' Grammatical Knowledge","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7135380730","doi":"https://doi.org/10.5715/jnlp.33.51"},"language":"en","primary_location":{"id":"doi:10.5715/jnlp.33.51","is_oa":true,"landing_page_url":"https://doi.org/10.5715/jnlp.33.51","pdf_url":"https://www.jstage.jst.go.jp/article/jnlp/33/1/33_51/_pdf","source":{"id":"https://openalex.org/S4210212357","display_name":"Journal of Natural Language Processing","issn_l":"1340-7619","issn":["1340-7619","2185-8314"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Natural Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://www.jstage.jst.go.jp/article/jnlp/33/1/33_51/_pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073233506","display_name":"Yusuke Ide","orcid":"https://orcid.org/0000-0002-6901-6954"},"institutions":[{"id":"https://openalex.org/I75917431","display_name":"Nara Institute of Science and Technology","ror":"https://ror.org/05bhada84","country_code":"JP","type":"education","lineage":["https://openalex.org/I75917431"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Yusuke Ide","raw_affiliation_strings":["Nara Institute of Science and Technology"],"affiliations":[{"raw_affiliation_string":"Nara Institute of Science and Technology","institution_ids":["https://openalex.org/I75917431"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102570223","display_name":"Yuto Nishida","orcid":null},"institutions":[{"id":"https://openalex.org/I75917431","display_name":"Nara Institute of Science and Technology","ror":"https://ror.org/05bhada84","country_code":"JP","type":"education","lineage":["https://openalex.org/I75917431"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Yuto Nishida","raw_affiliation_strings":["Nara Institute of Science and Technology"],"affiliations":[{"raw_affiliation_string":"Nara Institute of Science and Technology","institution_ids":["https://openalex.org/I75917431"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040181719","display_name":"Justin Vasselli","orcid":null},"institutions":[{"id":"https://openalex.org/I75917431","display_name":"Nara Institute of Science and Technology","ror":"https://ror.org/05bhada84","country_code":"JP","type":"education","lineage":["https://openalex.org/I75917431"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Justin Vasselli","raw_affiliation_strings":["Nara Institute of Science and Technology"],"affiliations":[{"raw_affiliation_string":"Nara Institute of Science and Technology","institution_ids":["https://openalex.org/I75917431"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104152387","display_name":"Miyu Oba","orcid":null},"institutions":[{"id":"https://openalex.org/I75917431","display_name":"Nara Institute of Science and Technology","ror":"https://ror.org/05bhada84","country_code":"JP","type":"education","lineage":["https://openalex.org/I75917431"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Miyu Oba","raw_affiliation_strings":["Nara Institute of Science and Technology"],"affiliations":[{"raw_affiliation_string":"Nara Institute of Science and Technology","institution_ids":["https://openalex.org/I75917431"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129231733","display_name":"Yusuke Sakai","orcid":null},"institutions":[{"id":"https://openalex.org/I75917431","display_name":"Nara Institute of Science and Technology","ror":"https://ror.org/05bhada84","country_code":"JP","type":"education","lineage":["https://openalex.org/I75917431"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Yusuke Sakai","raw_affiliation_strings":["Nara Institute of Science and Technology"],"affiliations":[{"raw_affiliation_string":"Nara Institute of Science and Technology","institution_ids":["https://openalex.org/I75917431"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016936747","display_name":"Hidetaka Kamigaito","orcid":"https://orcid.org/0000-0002-5249-5813"},"institutions":[{"id":"https://openalex.org/I75917431","display_name":"Nara Institute of Science and Technology","ror":"https://ror.org/05bhada84","country_code":"JP","type":"education","lineage":["https://openalex.org/I75917431"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hidetaka Kamigaito","raw_affiliation_strings":["Nara Institute of Science and Technology"],"affiliations":[{"raw_affiliation_string":"Nara Institute of Science and Technology","institution_ids":["https://openalex.org/I75917431"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5129131719","display_name":"Taro Watanabe","orcid":null},"institutions":[{"id":"https://openalex.org/I75917431","display_name":"Nara Institute of Science and Technology","ror":"https://ror.org/05bhada84","country_code":"JP","type":"education","lineage":["https://openalex.org/I75917431"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Taro Watanabe","raw_affiliation_strings":["Nara Institute of Science and Technology"],"affiliations":[{"raw_affiliation_string":"Nara Institute of Science and Technology","institution_ids":["https://openalex.org/I75917431"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5073233506"],"corresponding_institution_ids":["https://openalex.org/I75917431"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.94044444,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":"33","issue":"1","first_page":"51","last_page":"75"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11587","display_name":"Second Language Acquisition and Learning","score":0.31450000405311584,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11587","display_name":"Second Language Acquisition and Learning","score":0.31450000405311584,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.08969999849796295,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12353","display_name":"Lexicography and Language Studies","score":0.06759999692440033,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/evaluation-methods","display_name":"Evaluation methods","score":0.43209999799728394},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.28450000286102295},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.2784000039100647},{"id":"https://openalex.org/keywords/knowledge-based-systems","display_name":"Knowledge-based systems","score":0.258899986743927}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5210999846458435},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.43209999799728394},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4097000062465668},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3869999945163727},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3765999972820282},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.28450000286102295},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.2784000039100647},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.26260000467300415},{"id":"https://openalex.org/C115925183","wikidata":"https://www.wikidata.org/wiki/Q1412694","display_name":"Knowledge-based systems","level":2,"score":0.258899986743927},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2468000054359436}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5715/jnlp.33.51","is_oa":true,"landing_page_url":"https://doi.org/10.5715/jnlp.33.51","pdf_url":"https://www.jstage.jst.go.jp/article/jnlp/33/1/33_51/_pdf","source":{"id":"https://openalex.org/S4210212357","display_name":"Journal of Natural Language Processing","issn_l":"1340-7619","issn":["1340-7619","2185-8314"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Natural Language Processing","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.5715/jnlp.33.51","is_oa":true,"landing_page_url":"https://doi.org/10.5715/jnlp.33.51","pdf_url":"https://www.jstage.jst.go.jp/article/jnlp/33/1/33_51/_pdf","source":{"id":"https://openalex.org/S4210212357","display_name":"Journal of Natural Language Processing","issn_l":"1340-7619","issn":["1340-7619","2185-8314"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Natural Language Processing","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7943962812423706,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7135380730.pdf","grobid_xml":"https://content.openalex.org/works/W7135380730.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"\u8a00\u8a9e\u30e2\u30c7\u30eb\u306e\u6587\u6cd5\u77e5\u8b58\u306e\u8a55\u4fa1\u306f\uff0c\u3057\u3070\u3057\u3070\u30df\u30cb\u30de\u30eb\u30da\u30a2\u30d9\u30f3\u30c1\u30de\u30fc\u30af\u3092\u7528\u3044\u3066\u884c\u308f\u308c\u308b\uff0e\u30df\u30cb\u30de\u30eb\u30da\u30a2\u3068\u306f\uff0c\u5bb9\u8a8d\u53ef\u80fd\u306a\u6587\u3068\u5bb9\u8a8d\u4e0d\u53ef\u80fd\u306a\u6587\u304b\u3089\u306a\u308b\u6587\u5bfe\u3067\u3042\u308b\uff0e\u5bb9\u8a8d\u6027\u5224\u65ad\u306e\u65b9\u6cd5\u3068\u3057\u3066\u306f\uff0c\u30e2\u30c7\u30eb\u3092\u7528\u3044\u3066\u8a08\u7b97\u3057\u305f\u5404\u6587\u306e\u751f\u6210\u78ba\u7387\u3092\u6bd4\u8f03\u3059\u308b\u624b\u6cd5\u304c\u652f\u914d\u7684\u3067\u3042\u308b\uff0e\u3057\u304b\u3057\uff0c\u8fd1\u5e74\u306e\u5927\u898f\u6a21\u8a00\u8a9e\u30e2\u30c7\u30eb":[0],"(LLM)":[1],"\u306f\u30d7\u30ed\u30f3\u30d7\u30c8\u306b\u57fa\u3065\u3044\u3066\u30bf\u30b9\u30af\u3092\u9042\u884c\u3059\u308b\u3053\u3068\u3092\u60f3\u5b9a\u3057\u3066\u8a13\u7df4\u3055\u308c\u3066\u3044\u308b\u70b9\u3092\u8e0f\u307e\u3048\u308b\u3068\uff0cLLM":[2],"\u304c\u5272\u308a\u5f53\u3066\u308b\u78ba\u7387\u304c\u30e2\u30c7\u30eb\u306e\u77e5\u8b58\u3092\u5b8c\u5168\u306b\u53cd\u6620\u3057\u305f\u3082\u306e\u3067\u3042\u308b\u3068\u306f\u9650\u3089\u306a\u3044\uff0e\u305d\u3053\u3067\u672c\u7814\u7a76\u3067\u306f\uff0c\u30d7\u30ed\u30f3\u30d7\u30c8\u307e\u305f\u306f\u30c6\u30f3\u30d7\u30ec\u30fc\u30c8\u3092\u7528\u3044\u3066\uff0cLLM":[3],"\u304b\u3089\u3088\u308a\u6b63\u78ba\u306a\u5bb9\u8a8d\u6027\u5224\u65ad\u3092\u5f15\u304d\u51fa\u3059\u65b9\u6cd5\u3092\u63a2\u308b\uff0e\u82f1\u8a9e\u3068\u4e2d\u56fd\u8a9e\u306b\u3064\u3044\u3066\u5b9f\u9a13\u3092\u884c\u3044\uff0c9":[4],"\u3064\u306e\u5224\u65ad\u624b\u6cd5\u3092\u6bd4\u8f03\u3057\u305f\u7d50\u679c\uff0c\u30c6\u30f3\u30d7\u30ec\u30fc\u30c8\u3092\u7528\u3044\u305f\u624b\u6cd5\u3067\u3042\u308bIn-template":[5],"LP\u304a\u3088\u3073\u30d7\u30ed\u30f3\u30d7\u30c8\u306b\u57fa\u3065\u304f\u624b\u6cd5\u3067\u3042\u308b":[6],"Yes/No":[7,15],"\u78ba\u7387\u6bd4\u8f03\u304c\uff0c\u5f93\u6765\u624b\u6cd5\u3088\u308a\u3082\u9ad8\u3044\u6b63\u89e3\u7387\u3092\u9054\u6210\u3057\u305f\uff0e\u307e\u305f\uff0c\u4e21\u624b\u6cd5\u3092\u30a2\u30f3\u30b5\u30f3\u30d6\u30eb\u3059\u308b\u3053\u3068\u3067\uff0c\u4e00\u65b9\u306e\u624b\u6cd5\u306e\u307f\u3092\u7528\u3044\u308b\u5834\u5408\u3088\u308a\u9ad8\u3044\u6027\u80fd\u304c\u5f97\u3089\u308c\u308b\u3053\u3068\u3082\u78ba\u8a8d\u3055\u308c\u305f\uff0e\u4ee5\u4e0a\u306e\u7d50\u679c\u304b\u3089\uff0c\u6211\u3005\u306f\uff0cLLM":[8],"\u306e\u6587\u6cd5\u77e5\u8b58\u3092\u5341\u5168\u306b\u5f15\u304d\u51fa\u3057\u306a\u304c\u3089\u8a55\u4fa1\u3059\u308b\u305f\u3081\u306e\u624b\u6cd5\u3068\u3057\u3066\uff0cIn-template":[9],"LP":[10,13],"\u3092\u7528\u3044\u308b\u304b\uff0c\u307e\u305f\u306f":[11],"In-template":[12],"\u3068":[14],"\u78ba\u7387\u6bd4\u8f03\u306e\u30a2\u30f3\u30b5\u30f3\u30d6\u30eb\u3092\u7528\u3044\u308b\u3053\u3068\u3092\u63a8\u5968\u3059\u308b\uff0e":[16]},"counts_by_year":[],"updated_date":"2026-03-16T07:19:36.451410","created_date":"2026-03-15T00:00:00"}