{"id":"https://openalex.org/W4408615073","doi":"https://doi.org/10.1145/3708359.3712091","title":"Limitations of the LLM-as-a-Judge Approach for Evaluating LLM Outputs in Expert Knowledge Tasks","display_name":"Limitations of the LLM-as-a-Judge Approach for Evaluating LLM Outputs in Expert Knowledge Tasks","publication_year":2025,"publication_date":"2025-03-19","ids":{"openalex":"https://openalex.org/W4408615073","doi":"https://doi.org/10.1145/3708359.3712091"},"language":"en","primary_location":{"id":"doi:10.1145/3708359.3712091","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3708359.3712091","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3708359.3712091","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093206754","display_name":"Annalisa Szymanski","orcid":null},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Annalisa Szymanski","raw_affiliation_strings":["Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA,"],"raw_orcid":"https://orcid.org/0009-0009-5472-282X","affiliations":[{"raw_affiliation_string":"Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA,","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032058128","display_name":"Noah Ziems","orcid":null},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Noah Ziems","raw_affiliation_strings":["Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA,"],"raw_orcid":"https://orcid.org/0009-0002-1002-1959","affiliations":[{"raw_affiliation_string":"Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA,","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016803484","display_name":"Heather A. Eicher\u2010Miller","orcid":"https://orcid.org/0000-0002-1261-4291"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Heather A. Eicher-Miller","raw_affiliation_strings":["Nutrition Science, Purdue University, West Lafayette, Indiana, USA,"],"raw_orcid":"https://orcid.org/0000-0002-1261-4291","affiliations":[{"raw_affiliation_string":"Nutrition Science, Purdue University, West Lafayette, Indiana, USA,","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007240808","display_name":"Toby Jia-Jun Li","orcid":"https://orcid.org/0000-0001-7902-7625"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Toby Jia-Jun Li","raw_affiliation_strings":["Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA,"],"raw_orcid":"https://orcid.org/0000-0001-7902-7625","affiliations":[{"raw_affiliation_string":"Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA,","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074821819","display_name":"Meng Jiang","orcid":"https://orcid.org/0000-0002-3009-519X"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Meng Jiang","raw_affiliation_strings":["Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA,"],"raw_orcid":"https://orcid.org/0000-0002-3009-519X","affiliations":[{"raw_affiliation_string":"Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA,","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5018871532","display_name":"Ronald Metoyer","orcid":"https://orcid.org/0000-0003-2206-1720"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ronald A. Metoyer","raw_affiliation_strings":["Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA,"],"raw_orcid":"https://orcid.org/0000-0003-2206-1720","affiliations":[{"raw_affiliation_string":"Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA,","institution_ids":["https://openalex.org/I107639228"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5093206754"],"corresponding_institution_ids":["https://openalex.org/I107639228"],"apc_list":null,"apc_paid":null,"fwci":332.2426,"has_fulltext":false,"cited_by_count":45,"citation_normalized_percentile":{"value":0.99995203,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"952","last_page":"966"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9334999918937683,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9334999918937683,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12339","display_name":"Medical Malpractice and Liability Issues","score":0.9175000190734863,"subfield":{"id":"https://openalex.org/subfields/3611","display_name":"Pharmacy"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6922788619995117},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.32463759183883667},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1588837206363678}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6922788619995117},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.32463759183883667},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1588837206363678}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3708359.3712091","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3708359.3712091","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3708359.3712091","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3708359.3712091","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G495881888","display_name":null,"funder_award_id":"N00014-22-1-2507","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W2013669012","https://openalex.org/W2101105183","https://openalex.org/W2115201927","https://openalex.org/W2963995027","https://openalex.org/W3090002517","https://openalex.org/W3145026862","https://openalex.org/W4225129570","https://openalex.org/W4366548345","https://openalex.org/W4379278559","https://openalex.org/W4385438855","https://openalex.org/W4386117324","https://openalex.org/W4387880441","https://openalex.org/W4387993560","https://openalex.org/W4388453260","https://openalex.org/W4389180454","https://openalex.org/W4391136507","https://openalex.org/W4391564557","https://openalex.org/W4392503764","https://openalex.org/W4393327656","https://openalex.org/W4393971165","https://openalex.org/W4396832024","https://openalex.org/W4396833396","https://openalex.org/W4396833424","https://openalex.org/W4400412362","https://openalex.org/W4403334020","https://openalex.org/W4404781205","https://openalex.org/W4404782385"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":null,"counts_by_year":[{"year":2026,"cited_by_count":17},{"year":2025,"cited_by_count":27},{"year":2024,"cited_by_count":1}],"updated_date":"2026-04-25T08:17:42.794288","created_date":"2025-10-10T00:00:00"}