{"id":"https://openalex.org/W7166846862","doi":"https://doi.org/10.18653/v1/2026.acl-srw.125","title":"LLM as a Meta-Judge: Synthetic Data for NLP Evaluation Metric Validation","display_name":"LLM as a Meta-Judge: Synthetic Data for NLP Evaluation Metric Validation","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166846862","doi":"https://doi.org/10.18653/v1/2026.acl-srw.125"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.acl-srw.125","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-srw.125","pdf_url":"https://aclanthology.org/2026.acl-srw.125.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 4: Student Research Workshop)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.acl-srw.125.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128738589","display_name":"Luk\u00e1\u0161 Eigler","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luk\u00e1\u0161 Eigler","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139716644","display_name":"Jind\u0159ich Libovick\u00fd","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jind\u0159ich Libovick\u00fd","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5086144380","display_name":"David Hurych","orcid":"https://orcid.org/0000-0002-0580-4038"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"David Hurych","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.83663156,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1417","last_page":"1435"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.25220000743865967,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.25220000743865967,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.12120000272989273,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.08209999650716782,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5947999954223633},{"id":"https://openalex.org/keywords/synthetic-data","display_name":"Synthetic data","score":0.5666999816894531},{"id":"https://openalex.org/keywords/textual-entailment","display_name":"Textual entailment","score":0.33559998869895935},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.30889999866485596},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.2987000048160553}],"concepts":[{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6703000068664551},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5947999954223633},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5795000195503235},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.5666999816894531},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.554099977016449},{"id":"https://openalex.org/C95318506","wikidata":"https://www.wikidata.org/wiki/Q6588467","display_name":"Textual entailment","level":3,"score":0.33559998869895935},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.30889999866485596},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2987000048160553},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2727000117301941},{"id":"https://openalex.org/C3019813237","wikidata":"https://www.wikidata.org/wiki/Q65089264","display_name":"Model validation","level":2,"score":0.26019999384880066}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.acl-srw.125","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-srw.125","pdf_url":"https://aclanthology.org/2026.acl-srw.125.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 4: Student Research Workshop)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.acl-srw.125","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-srw.125","pdf_url":"https://aclanthology.org/2026.acl-srw.125.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 4: Student Research Workshop)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5960638817","display_name":null,"funder_award_id":"90254","funder_id":"https://openalex.org/F4320309755","funder_display_name":"Univerzita Karlova v Praze"}],"funders":[{"id":"https://openalex.org/F4320309755","display_name":"Univerzita Karlova v Praze","ror":"https://ror.org/024d6js02"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166846862.pdf","grobid_xml":"https://content.openalex.org/works/W7166846862.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Validating":[0],"evaluation":[1,34],"metrics":[2],"for":[3,17,83],"NLG":[4],"typically":[5],"relies":[6],"on":[7],"expensive":[8,105],"and":[9,61,72,93],"time-consuming":[10],"human":[11,44,65,84,99],"annotations,":[12],"which":[13],"predominantly":[14],"exist":[15],"only":[16],"English":[18],"datasets.We":[19],"propose":[20],"LLM":[21],"as":[22,79],"a":[23,25,80,95],"Meta-Judge,":[24],"scalable":[26],"framework":[27],"that":[28,75],"utilizes":[29],"LLMs":[30],"to":[31,106],"generate":[32],"synthetic":[33,59,76],"datasets":[35],"via":[36],"controlled":[37],"semantic":[38],"degradation":[39],"of":[40],"real":[41],"data,":[42],"replacing":[43],"judgment.We":[45],"validate":[46],"our":[47],"approach":[48],"using":[49],"meta-correlation,":[50],"measuring":[51],"the":[52],"alignment":[53],"between":[54],"metric":[55],"rankings":[56],"derived":[57],"from":[58,63],"data":[60],"those":[62],"standard":[64],"benchmarks.Experiments":[66],"across":[67],"Machine":[68],"Translation,":[69],"Question":[70],"Answering,":[71],"Summarization":[73],"demonstrate":[74],"validation":[77],"serves":[78],"reliable":[81],"proxy":[82],"judgment,":[85],"achieving":[86],"meta-correlations":[87],"exceeding":[88],"0.9":[89],"in":[90],"multilingual":[91],"QA,":[92],"is":[94],"viable":[96],"alternative":[97],"when":[98],"judgments":[100],"are":[101],"unavailable":[102],"or":[103],"too":[104],"obtain.":[107]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-07-02T00:00:00"}