{"id":"https://openalex.org/W4412377811","doi":"https://doi.org/10.1145/3726302.3730218","title":"Large Language Model Relevance Assessors Agree With One Another More Than With Human Assessors","display_name":"Large Language Model Relevance Assessors Agree With One Another More Than With Human Assessors","publication_year":2025,"publication_date":"2025-07-13","ids":{"openalex":"https://openalex.org/W4412377811","doi":"https://doi.org/10.1145/3726302.3730218"},"language":"en","primary_location":{"id":"doi:10.1145/3726302.3730218","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730218","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730218","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730218","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068616222","display_name":"Maik Fr\u00f6be","orcid":"https://orcid.org/0000-0002-1003-981X"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Maik Fr\u00f6be","raw_affiliation_strings":["Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"],"affiliations":[{"raw_affiliation_string":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075770550","display_name":"Andrew Parry","orcid":"https://orcid.org/0000-0001-5446-8328"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Andrew Parry","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020636685","display_name":"Ferdinand Schlatt","orcid":"https://orcid.org/0000-0002-6032-909X"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ferdinand Schlatt","raw_affiliation_strings":["Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"],"affiliations":[{"raw_affiliation_string":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014199889","display_name":"Sean MacAvaney","orcid":"https://orcid.org/0000-0002-8914-2659"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Sean MacAvaney","raw_affiliation_strings":["University of Glasgow, Glasgow, UK, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, UK, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027915931","display_name":"Benno Stein","orcid":"https://orcid.org/0000-0001-9033-2217"},"institutions":[{"id":"https://openalex.org/I51441396","display_name":"Bauhaus-Universit\u00e4t Weimar","ror":"https://ror.org/033bb5z47","country_code":"DE","type":"education","lineage":["https://openalex.org/I51441396"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Benno Stein","raw_affiliation_strings":["Bauhaus Universit\u00e4t Weimar, Weimar, Germany"],"affiliations":[{"raw_affiliation_string":"Bauhaus Universit\u00e4t Weimar, Weimar, Germany","institution_ids":["https://openalex.org/I51441396"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083712311","display_name":"Martin Potthast","orcid":"https://orcid.org/0000-0003-2451-0665"},"institutions":[{"id":"https://openalex.org/I106157433","display_name":"University of Kassel","ror":"https://ror.org/04zc7p361","country_code":"DE","type":"education","lineage":["https://openalex.org/I106157433"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Martin Potthast","raw_affiliation_strings":["University of Kassel, hessian.AI, and ScaDS.AI, Leipzig, Germany"],"affiliations":[{"raw_affiliation_string":"University of Kassel, hessian.AI, and ScaDS.AI, Leipzig, Germany","institution_ids":["https://openalex.org/I106157433"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014322854","display_name":"Matthias Hagen","orcid":"https://orcid.org/0000-0002-9733-2890"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matthias Hagen","raw_affiliation_strings":["Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"],"affiliations":[{"raw_affiliation_string":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5068616222"],"corresponding_institution_ids":["https://openalex.org/I76198965"],"apc_list":null,"apc_paid":null,"fwci":9.6227,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.97669169,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"2858","last_page":"2863"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9771999716758728,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7259737253189087},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7093208432197571},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.44456854462623596},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.43732312321662903},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3513662815093994}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7259737253189087},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7093208432197571},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.44456854462623596},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.43732312321662903},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3513662815093994},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3726302.3730218","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730218","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730218","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:eprints.gla.ac.uk:349482","is_oa":true,"landing_page_url":"http://eprints.gla.ac.uk/view/author/77010.html>,","pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"},{"id":"pmh:oai:eprints.gla.ac.uk:352747","is_oa":true,"landing_page_url":"https://eprints.gla.ac.uk/view/author/69236.html>,","pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"}],"best_oa_location":{"id":"doi:10.1145/3726302.3730218","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730218","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730218","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412377811.pdf","grobid_xml":"https://content.openalex.org/works/W4412377811.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W1981065051","https://openalex.org/W1983595289","https://openalex.org/W2052989395","https://openalex.org/W2053757624","https://openalex.org/W2077324407","https://openalex.org/W2095638614","https://openalex.org/W2104679989","https://openalex.org/W2138680231","https://openalex.org/W2160892561","https://openalex.org/W2954264361","https://openalex.org/W2968287414","https://openalex.org/W3133594109","https://openalex.org/W3156166379","https://openalex.org/W4251372957","https://openalex.org/W4317937814","https://openalex.org/W4384107234","https://openalex.org/W4385688511","https://openalex.org/W4392864805","https://openalex.org/W4399554477","https://openalex.org/W4400525274","https://openalex.org/W4400526284","https://openalex.org/W4400526908","https://openalex.org/W4403781006","https://openalex.org/W4404407248","https://openalex.org/W4409166856","https://openalex.org/W6948268552"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Relevance":[0],"judgments":[1],"can":[2],"differ":[3],"between":[4,28,33,44,86,91],"assessors,":[5],"but":[6],"previous":[7],"work":[8],"has":[9,47],"shown":[10],"that":[11,83,97,102,116],"such":[12],"disagreements":[13,27],"have":[14],"little":[15],"impact":[16],"on":[17,61,129],"the":[18,42,62,67,71,84,117],"effectiveness":[19],"rankings":[20,119],"of":[21,70],"retrieval":[22,68,100,113],"systems.":[23],"This":[24],"applies":[25],"to":[26],"humans":[29,94],"as":[30,32],"well":[31],"human":[34,79],"and":[35,66,77,93],"large":[36],"language":[37],"model":[38],"(LLM)":[39],"assessors.":[40,80],"However,":[41],"agreement":[43,85],"different":[45],"LLM~assessors":[46,60,87,98,122],"not":[48],"yet":[49],"been":[50],"systematically":[51],"investigated.":[52],"To":[53],"close":[54],"this":[55],"gap,":[56],"we":[57],"compare":[58],"eight":[59],"TREC":[63],"DL":[64],"tracks":[65],"task":[69],"RAG":[72],"track":[73],"with":[74,78,111],"each":[75],"other":[76],"We":[81],"find":[82],"is":[88],"higher":[89],"than":[90],"LLMs":[92,104],"and,":[95],"importantly,":[96],"favor":[99],"systems":[101,114],"use":[103],"in":[105],"their":[106],"ranking":[107],"decisions:":[108],"our":[109],"analyses":[110],"30-50":[112],"show":[115],"system":[118],"obtained":[120],"by":[121,126],"overestimate":[123],"LLM-based":[124],"re-rankers":[125],"9~to":[127],"17~positions":[128],"average.":[130]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":1}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}