{"id":"https://openalex.org/W7127268979","doi":"https://doi.org/10.48550/arxiv.2602.02287","title":"Cross-Lingual Stability of LLM Judges Under Controlled Generation: Evidence from Finno-Ugric Languages","display_name":"Cross-Lingual Stability of LLM Judges Under Controlled Generation: Evidence from Finno-Ugric Languages","publication_year":2026,"publication_date":"2026-02-02","ids":{"openalex":"https://openalex.org/W7127268979","doi":"https://doi.org/10.48550/arxiv.2602.02287"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2602.02287","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.02287","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2602.02287","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042595859","display_name":"Isaac Chung","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chung, Isaac","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5042665536","display_name":"Linda Freienthal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Freienthal, Linda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.47540000081062317,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.47540000081062317,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.125,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.04039999842643738,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.767799973487854},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.6414999961853027},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.5916000008583069},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5782999992370605},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5708000063896179},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5641000270843506},{"id":"https://openalex.org/keywords/constant","display_name":"Constant (computer programming)","score":0.48669999837875366},{"id":"https://openalex.org/keywords/calibration","display_name":"Calibration","score":0.4578999876976013},{"id":"https://openalex.org/keywords/replication","display_name":"Replication (statistics)","score":0.41940000653266907}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.767799973487854},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.6414999961853027},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6376000046730042},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5916000008583069},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5782999992370605},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5708000063896179},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5641000270843506},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5315999984741211},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5148000121116638},{"id":"https://openalex.org/C2777027219","wikidata":"https://www.wikidata.org/wiki/Q1284190","display_name":"Constant (computer programming)","level":2,"score":0.48669999837875366},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.4578999876976013},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.41940000653266907},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36730000376701355},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.35030001401901245},{"id":"https://openalex.org/C2776092919","wikidata":"https://www.wikidata.org/wiki/Q9072","display_name":"Estonian","level":2,"score":0.3463999927043915},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.3352999985218048},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.3303999900817871},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.31200000643730164},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.3010999858379364},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.29840001463890076},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.29840001463890076},{"id":"https://openalex.org/C2776175482","wikidata":"https://www.wikidata.org/wiki/Q1195816","display_name":"Transfer (computing)","level":2,"score":0.2939999997615814},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.29159998893737793},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.26910001039505005},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.25119999051094055}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2602.02287","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.02287","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2602.02287","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.02287","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7709367871284485}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Cross-lingual":[0],"evaluation":[1,22,131,179],"of":[2,11,69],"large":[3],"language":[4,185],"models":[5],"(LLMs)":[6],"typically":[7],"conflates":[8],"two":[9],"sources":[10],"variance:":[12],"genuine":[13],"model":[14,56,122],"performance":[15],"differences":[16],"and":[17,44,51,88,102,178],"measurement":[18],"instability.":[19],"We":[20,170],"investigate":[21],"reliability":[23],"by":[24],"holding":[25],"generation":[26,106,140,174],"conditions":[27,141],"constant":[28],"while":[29],"varying":[30],"target":[31],"language.":[32],"Using":[33],"synthetic":[34,176],"customer-support":[35],"dialogues":[36],"generated":[37],"with":[38],"identical":[39,139],"parameters":[40],"across":[41,58,117,184],"Estonian,":[42],"Finnish,":[43],"Hungarian,":[45],"we":[46,78],"test":[47],"whether":[48],"automatic":[49],"metrics":[50,84],"LLM-as-a-judge":[52],"scoring":[53,114],"produce":[54],"stable":[55],"rankings":[57],"these":[59,109],"morphologically":[60,160],"rich,":[61],"related":[62],"Finno-Ugric":[63],"languages.":[64],"With":[65],"a":[66,75,128],"small":[67],"set":[68],"Estonian":[70],"native":[71],"speaker":[72],"annotations":[73],"as":[74],"reference":[76],"point,":[77],"find":[79],"systematic":[80],"ranking":[81],"instabilities:":[82],"surface-level":[83],"(lexical":[85],"diversity,":[86],"surface":[87],"semantic":[89],"similarity)":[90],"maintain":[91,136],"cross-language":[92],"stability,":[93],"but":[94],"pragmatic":[95],"judgments":[96],"(coherence,":[97],"instruction-following)":[98],"exhibit":[99],"rank":[100],"inversions":[101],"near-zero":[103],"correlations.":[104],"Because":[105],"is":[107,154],"controlled,":[108],"inconsistencies":[110],"reflect":[111],"how":[112],"judge":[113,152],"behaves":[115],"differently":[116],"languages":[118],"rather":[119],"than":[120],"true":[121],"differences.":[123],"This":[124],"controlled":[125,173],"design":[126],"provides":[127],"diagnostic":[129],"probe:":[130],"methods":[132],"that":[133,150],"fail":[134],"to":[135,181],"stability":[137],"under":[138],"signal":[142],"transfer":[143,153],"failure":[144],"before":[145],"deployment.":[146],"Our":[147],"findings":[148],"suggest":[149],"zero-shot":[151],"unreliable":[155],"for":[156],"discourse-level":[157],"assessment":[158],"in":[159],"rich":[161],"languages,":[162],"motivating":[163],"language-specific":[164],"calibration":[165],"against":[166],"targeted":[167],"human":[168],"baselines.":[169],"release":[171],"our":[172],"protocol,":[175],"data,":[177],"framework":[180],"enable":[182],"replication":[183],"families":[186],"at":[187],"https://github.com/isaac-chung/cross-lingual-stability-judges.":[188]},"counts_by_year":[],"updated_date":"2026-08-16T07:02:28.622633","created_date":"2026-02-04T00:00:00"}