{"id":"https://openalex.org/W4404782844","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.451","title":"PARIKSHA: A Large-Scale Investigation of Human-LLM Evaluator Agreement on Multilingual and Multi-Cultural Data","display_name":"PARIKSHA: A Large-Scale Investigation of Human-LLM Evaluator Agreement on Multilingual and Multi-Cultural Data","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4404782844","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.451"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2024.emnlp-main.451","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.451","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.emnlp-main.451","pdf_url":"https://aclanthology.org/2024.emnlp-main.451.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2024.emnlp-main.451.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052667187","display_name":"Ishaan Watts","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ishaan Watts","raw_affiliation_strings":["Microsoft Corporation"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040945700","display_name":"Varun Gumma","orcid":"https://orcid.org/0009-0002-5746-3017","observed_orcids":["https://orcid.org/0009-0002-5746-3017"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Varun Gumma","raw_affiliation_strings":["Microsoft Corporation"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051703527","display_name":"Aditya Yadavalli","orcid":"https://orcid.org/0000-0002-6882-3762","observed_orcids":["https://orcid.org/0000-0002-6882-3762"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aditya Yadavalli","raw_affiliation_strings":["Microsoft Corporation"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029415636","display_name":"Vivek Seshadri","orcid":"https://orcid.org/0009-0000-4388-4246","observed_orcids":["https://orcid.org/0009-0000-4388-4246"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vivek Seshadri","raw_affiliation_strings":["Microsoft Corporation"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107913584","display_name":"Manohar Swaminathan","orcid":"https://orcid.org/0000-0003-4857-636X","observed_orcids":["https://orcid.org/0000-0003-4857-636X","https://orcid.org/0000-0001-7879-1015"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Manohar Swaminathan","raw_affiliation_strings":["Microsoft Corporation"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5005513786","display_name":"Sunayana Sitaram","orcid":"https://orcid.org/0000-0003-4251-9719","observed_orcids":["https://orcid.org/0000-0003-4251-9719"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sunayana Sitaram","raw_affiliation_strings":["Microsoft Corporation"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":4.7358,"has_fulltext":true,"cited_by_count":30,"citation_normalized_percentile":{"value":0.96336033,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"7900","last_page":"7932"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5435000061988831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5435000061988831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.423799991607666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.002300000051036477,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multilingual-large-language-models","display_name":"multilingual large language models","score":0.7739999890327454},{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.7559999823570251},{"id":"https://openalex.org/keywords/indo-aryan-languages","display_name":"Indo-Aryan languages","score":0.9279999732971191},{"id":"https://openalex.org/keywords/cross-lingual-transfer","display_name":"cross-lingual transfer","score":0.6890000104904175},{"id":"https://openalex.org/keywords/assessment-bias","display_name":"evaluation bias","score":0.5090000033378601},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"pairwise comparison","score":0.5770000219345093},{"id":"https://openalex.org/keywords/direct-assessment","display_name":"direct assessment","score":0.8190000057220459},{"id":"https://openalex.org/keywords/benchmark-contamination","display_name":"benchmark contamination","score":0.621999979019165},{"id":"https://openalex.org/keywords/bengali-language","display_name":"Bengali language","score":0.8230000138282776},{"id":"https://openalex.org/keywords/odia","display_name":"Odia","score":0.9959999918937683}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6057354807853699},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5713987946510315},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38289177417755127},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3278721570968628},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.07905352115631104},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.07768845558166504}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2024.emnlp-main.451","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.451","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.emnlp-main.451","pdf_url":"https://aclanthology.org/2024.emnlp-main.451.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2406.15053","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.15053","pdf_url":"https://arxiv.org/pdf/2406.15053","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2406.15053","doi":"https://doi.org/10.48550/arxiv.2406.15053","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2406.15053","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2024.emnlp-main.451","doi":"https://doi.org/10.18653/v1/2024.emnlp-main.451","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.emnlp-main.451","pdf_url":"https://aclanthology.org/2024.emnlp-main.451.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404782844.pdf","grobid_xml":"https://content.openalex.org/works/W4404782844.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W3013840636","https://openalex.org/W4327810158","https://openalex.org/W4383665181","https://openalex.org/W4386365131","https://openalex.org/W4387561528","https://openalex.org/W4388555916","https://openalex.org/W4390041933","https://openalex.org/W4391336917","https://openalex.org/W4391591722","https://openalex.org/W4392088193","https://openalex.org/W4392679398","https://openalex.org/W4392822465","https://openalex.org/W4399061743","https://openalex.org/W4402670101","https://openalex.org/W4402671190","https://openalex.org/W4402671557","https://openalex.org/W4402671800","https://openalex.org/W4412945402","https://openalex.org/W4415797148","https://openalex.org/W7166495070"],"related_works":["https://openalex.org/W4386794666","https://openalex.org/W4403579647","https://openalex.org/W7138888256","https://openalex.org/W4392088193","https://openalex.org/W7162631365","https://openalex.org/W4404305023","https://openalex.org/W7155574228","https://openalex.org/W7203905314","https://openalex.org/W4416113182","https://openalex.org/W4403579978"],"abstract_inverted_index":{"Evaluation":[0],"of":[1,13,17,24,34,147,163],"multilingual":[2,161],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"is":[7],"challenging":[8],"due":[9],"to":[10],"a":[11,50,155],"variety":[12],"factors":[14],"-the":[15],"lack":[16,33],"benchmarks":[18,26],"with":[19],"sufficient":[20],"linguistic":[21],"diversity,":[22],"contamination":[23],"popular":[25],"into":[27],"LLM":[28],"pre-training":[29],"data":[30],"and":[31,46,66,70,77,95,98,104,109,132,141,144],"the":[32,100,115,119,150],"local,":[35],"cultural":[36],"nuances":[37],"in":[38,49,114,139,149],"translated":[39],"benchmarks.In":[40],"this":[41],"work,":[42],"we":[43],"study":[44],"human":[45,64,140],"LLM-based":[47],"evaluation":[48,91,125,143,162],"multilingual,":[51],"multi-cultural":[52],"setting.We":[53],"evaluate":[54],"30":[55],"models":[56,73],"across":[57],"10":[58],"Indic":[59,85],"languages":[60,128],"by":[61],"conducting":[62],"90K":[63],"evaluations":[65,69],"30K":[67],"LLMbased":[68,142],"find":[71,106,145],"that":[72,107],"such":[74,129],"as":[75,130],"GPT-4o":[76],"Llama-3":[78],"70B":[79],"consistently":[80],"perform":[81],"best":[82],"for":[83,89,122,127,136],"most":[84],"languages.We":[86],"build":[87],"leaderboards":[88],"two":[90],"settings":[92],"-pairwise":[93],"comparison":[94],"direct":[96,123],"assessment":[97,124],"analyse":[99],"agreement":[101,120],"between":[102],"humans":[103,108],"LLMs.We":[105],"LLMs":[110],"agree":[111],"fairly":[112],"well":[113],"pairwise":[116],"setting":[117],"but":[118],"drops":[121],"especially":[126],"Bengali":[131],"Odia.We":[133],"also":[134],"check":[135],"various":[137],"biases":[138],"evidence":[146],"self-bias":[148],"GPT-based":[151],"evaluator.Our":[152],"work":[153],"presents":[154],"significant":[156],"step":[157],"towards":[158],"scaling":[159],"up":[160],"LLMs.":[164],"1":[165],"few-shot":[166],"cross-lingual":[167],"transfer.In":[168]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":6}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-11-28T00:00:00"}