{"id":"https://openalex.org/W4385688511","doi":"https://doi.org/10.1145/3578337.3605136","title":"Perspectives on Large Language Models for Relevance Judgment","display_name":"Perspectives on Large Language Models for Relevance Judgment","publication_year":2023,"publication_date":"2023-08-09","ids":{"openalex":"https://openalex.org/W4385688511","doi":"https://doi.org/10.1145/3578337.3605136"},"language":"en","primary_location":{"id":"doi:10.1145/3578337.3605136","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3578337.3605136","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3578337.3605136","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM SIGIR International Conference on Theory of Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3578337.3605136","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079246354","display_name":"Guglielmo Faggioli","orcid":"https://orcid.org/0000-0002-5070-2049"},"institutions":[{"id":"https://openalex.org/I138689650","display_name":"University of Padua","ror":"https://ror.org/00240q980","country_code":"IT","type":"education","lineage":["https://openalex.org/I138689650"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Guglielmo Faggioli","raw_affiliation_strings":["University of Padova, Padova, Italy"],"raw_orcid":"https://orcid.org/0000-0002-5070-2049","affiliations":[{"raw_affiliation_string":"University of Padova, Padova, Italy","institution_ids":["https://openalex.org/I138689650"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027260515","display_name":"Laura Dietz","orcid":"https://orcid.org/0000-0003-1624-3907"},"institutions":[{"id":"https://openalex.org/I161057412","display_name":"University of New Hampshire","ror":"https://ror.org/01rmh9n78","country_code":"US","type":"education","lineage":["https://openalex.org/I161057412"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Laura Dietz","raw_affiliation_strings":["University of New Hampshire, Durham, NH, USA"],"raw_orcid":"https://orcid.org/0000-0003-1624-3907","affiliations":[{"raw_affiliation_string":"University of New Hampshire, Durham, NH, USA","institution_ids":["https://openalex.org/I161057412"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037737168","display_name":"Charles L. A. Clarke","orcid":"https://orcid.org/0000-0001-8178-9194"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Charles L. A. Clarke","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0000-0001-8178-9194","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052565959","display_name":"Gianluca Demartini","orcid":"https://orcid.org/0000-0002-7311-3693"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Gianluca Demartini","raw_affiliation_strings":["University of Queensland, Brisbane, Australia"],"raw_orcid":"https://orcid.org/0000-0002-7311-3693","affiliations":[{"raw_affiliation_string":"University of Queensland, Brisbane, Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014322854","display_name":"Matthias Hagen","orcid":"https://orcid.org/0000-0002-9733-2890"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matthias Hagen","raw_affiliation_strings":["Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"],"raw_orcid":"https://orcid.org/0000-0002-9733-2890","affiliations":[{"raw_affiliation_string":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084915697","display_name":"Claudia Hauff","orcid":"https://orcid.org/0000-0001-9879-6470"},"institutions":[{"id":"https://openalex.org/I4210163196","display_name":"Signify (Netherlands)","ror":"https://ror.org/0532vdr17","country_code":"NL","type":"company","lineage":["https://openalex.org/I4210163196"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Claudia Hauff","raw_affiliation_strings":["Spotify, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0001-9879-6470","affiliations":[{"raw_affiliation_string":"Spotify, Delft, Netherlands","institution_ids":["https://openalex.org/I4210163196"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045875579","display_name":"Noriko Kando","orcid":"https://orcid.org/0000-0002-2133-0215"},"institutions":[{"id":"https://openalex.org/I184597095","display_name":"National Institute of Informatics","ror":"https://ror.org/04ksd4g47","country_code":"JP","type":"facility","lineage":["https://openalex.org/I1319490839","https://openalex.org/I184597095","https://openalex.org/I4210158934"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Noriko Kando","raw_affiliation_strings":["National Institute of Informatics, Tokio, Japan"],"raw_orcid":"https://orcid.org/0000-0002-2133-0215","affiliations":[{"raw_affiliation_string":"National Institute of Informatics, Tokio, Japan","institution_ids":["https://openalex.org/I184597095"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092794014","display_name":"Evangelos Kanoulas","orcid":"https://orcid.org/0000-0001-6383-6196"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Evangelos Kanoulas","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"raw_orcid":"https://orcid.org/0000-0001-6383-6196","affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083712311","display_name":"Martin Potthast","orcid":"https://orcid.org/0000-0003-2451-0665"},"institutions":[{"id":"https://openalex.org/I926574661","display_name":"Leipzig University","ror":"https://ror.org/03s7gtk40","country_code":"DE","type":"education","lineage":["https://openalex.org/I926574661"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Martin Potthast","raw_affiliation_strings":["Leipzig University, Leipzig, Germany"],"raw_orcid":"https://orcid.org/0000-0003-2451-0665","affiliations":[{"raw_affiliation_string":"Leipzig University, Leipzig, Germany","institution_ids":["https://openalex.org/I926574661"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101533129","display_name":"B. Stein","orcid":"https://orcid.org/0000-0002-0070-891X"},"institutions":[{"id":"https://openalex.org/I51441396","display_name":"Bauhaus-Universit\u00e4t Weimar","ror":"https://ror.org/033bb5z47","country_code":"DE","type":"education","lineage":["https://openalex.org/I51441396"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Benno Stein","raw_affiliation_strings":["Bauhaus-Universit\u00e4t Weimar, Weimar, Germany"],"raw_orcid":"https://orcid.org/0000-0002-0070-891X","affiliations":[{"raw_affiliation_string":"Bauhaus-Universit\u00e4t Weimar, Weimar, Germany","institution_ids":["https://openalex.org/I51441396"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014375244","display_name":"Henning Wachsmuth","orcid":"https://orcid.org/0000-0003-2792-621X"},"institutions":[{"id":"https://openalex.org/I114112103","display_name":"Leibniz University Hannover","ror":"https://ror.org/0304hq317","country_code":"DE","type":"education","lineage":["https://openalex.org/I114112103"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Henning Wachsmuth","raw_affiliation_strings":["Leibniz University Hannover, Hannover, Germany"],"raw_orcid":"https://orcid.org/0000-0003-2792-621X","affiliations":[{"raw_affiliation_string":"Leibniz University Hannover, Hannover, Germany","institution_ids":["https://openalex.org/I114112103"]}]}],"institutions":[],"countries_distinct_count":7,"institutions_distinct_count":12,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":167,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"39","last_page":"50"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9902999997138977,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.9205260276794434},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.6562550067901611},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.621330201625824},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5324790477752686},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.5032774806022644},{"id":"https://openalex.org/keywords/compromise","display_name":"Compromise","score":0.4874839782714844},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.3953555226325989},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.38720959424972534},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2756311893463135},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.1243789792060852},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.11699551343917847},{"id":"https://openalex.org/keywords/social-science","display_name":"Social science","score":0.07305249571800232}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.9205260276794434},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.6562550067901611},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.621330201625824},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5324790477752686},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.5032774806022644},{"id":"https://openalex.org/C46355384","wikidata":"https://www.wikidata.org/wiki/Q726686","display_name":"Compromise","level":2,"score":0.4874839782714844},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3953555226325989},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.38720959424972534},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2756311893463135},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.1243789792060852},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.11699551343917847},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.07305249571800232},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3578337.3605136","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3578337.3605136","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3578337.3605136","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM SIGIR International Conference on Theory of Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:www.research.unipd.it:11577/3497340","is_oa":true,"landing_page_url":"https://hdl.handle.net/11577/3497340","pdf_url":"https://www.research.unipd.it/bitstream/11577/3497340/1/3578337.3605136.pdf","source":{"id":"https://openalex.org/S4377196283","display_name":"Research Padua  Archive (University of Padua)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I138689650","host_organization_name":"University of Padua","host_organization_lineage":["https://openalex.org/I138689650"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.1145/3578337.3605136","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3578337.3605136","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3578337.3605136","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM SIGIR International Conference on Theory of Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.4699999988079071}],"awards":[{"id":"https://openalex.org/G3847272511","display_name":"CAREER: Utilizing Fine-Grained Knowledge Annotations in Text Understanding and Retrieval","funder_award_id":"1846017","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385688511.pdf","grobid_xml":"https://content.openalex.org/works/W4385688511.grobid-xml"},"referenced_works_count":52,"referenced_works":["https://openalex.org/W1543648998","https://openalex.org/W1983595289","https://openalex.org/W1999322447","https://openalex.org/W2006149654","https://openalex.org/W2008285050","https://openalex.org/W2017633929","https://openalex.org/W2047810014","https://openalex.org/W2057495142","https://openalex.org/W2058896506","https://openalex.org/W2061503648","https://openalex.org/W2110933113","https://openalex.org/W2113474780","https://openalex.org/W2115536324","https://openalex.org/W2124504084","https://openalex.org/W2139516750","https://openalex.org/W2144200917","https://openalex.org/W2146938270","https://openalex.org/W2159981039","https://openalex.org/W2166455213","https://openalex.org/W2554260549","https://openalex.org/W2559836063","https://openalex.org/W2752644629","https://openalex.org/W2780799698","https://openalex.org/W2794432940","https://openalex.org/W2799108077","https://openalex.org/W2903058329","https://openalex.org/W2914694065","https://openalex.org/W3004392977","https://openalex.org/W3132995484","https://openalex.org/W3133702157","https://openalex.org/W3142525300","https://openalex.org/W3191640844","https://openalex.org/W3208394801","https://openalex.org/W3212290861","https://openalex.org/W4234995456","https://openalex.org/W4281699561","https://openalex.org/W4283791551","https://openalex.org/W4284666093","https://openalex.org/W4284713818","https://openalex.org/W4292341621","https://openalex.org/W4293998609","https://openalex.org/W4306317650","https://openalex.org/W4308244910","https://openalex.org/W4313216063","https://openalex.org/W4316192572","https://openalex.org/W4318239801","https://openalex.org/W4318903783","https://openalex.org/W4321649502","https://openalex.org/W4361193900","https://openalex.org/W4366733439","https://openalex.org/W4367860590","https://openalex.org/W4384660953"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2053222733","https://openalex.org/W1967783546","https://openalex.org/W2122072511","https://openalex.org/W2148291855","https://openalex.org/W2365213443","https://openalex.org/W2122009727","https://openalex.org/W4254905218","https://openalex.org/W2110318293"],"abstract_inverted_index":{"When":[0],"asked,":[1],"large":[2],"language":[3],"models~(LLMs)":[4],"like":[5],"ChatGPT":[6],"claim":[7],"that":[8,50,58],"they":[9],"can":[10,23],"assist":[11],"with":[12,46,94],"relevance":[13,43,63,91,116],"judgments":[14,22,44,92,95],"but":[15],"it":[16],"is":[17],"not":[18],"clear":[19],"whether":[20,89],"automated":[21,80],"reliably":[24],"be":[25],"used":[26],"in":[27],"evaluations":[28],"of":[29,78,126],"retrieval":[30],"systems.":[31],"In":[32],"this":[33],"perspectives":[34,107],"paper,":[35],"we":[36,82],"discuss":[37],"possible":[38],"ways":[39],"for~LLMs":[40],"to":[41,60],"support":[42],"along":[45],"concerns":[47],"and":[48,109,118,133],"issues":[49],"arise.":[51],"We":[52,100],"devise":[53],"a":[54,85,119],"human--machine":[55],"collaboration":[56],"spectrum":[57],"allows":[59],"categorize":[61],"different":[62],"judgment":[64],"strategies,":[65],"based":[66],"on":[67,72,88],"how":[68],"much":[69],"humans":[70],"rely":[71],"machines.":[73],"For":[74],"the":[75,102,111,127],"extreme":[76],"point":[77],"'fully":[79],"judgments',":[81],"further":[83],"include":[84],"pilot":[86],"experiment":[87],"LLM-based":[90],"correlate":[93],"from":[96],"trained":[97],"human":[98],"assessors.":[99],"conclude":[101],"paper":[103],"by":[104,123],"providing":[105],"opposing":[106],"for":[108,114],"against":[110],"use":[112],"of~LLMs":[113],"automatic":[115],"judgments,":[117],"compromise":[120],"perspective,":[121],"informed":[122],"our":[124,129,134],"analyses":[125],"literature,":[128],"preliminary":[130],"experimental":[131],"evidence,":[132],"experience":[135],"as":[136],"IR~researchers.":[137]},"counts_by_year":[{"year":2026,"cited_by_count":31},{"year":2025,"cited_by_count":64},{"year":2024,"cited_by_count":65},{"year":2023,"cited_by_count":7}],"updated_date":"2026-07-16T13:24:37.021932","created_date":"2025-10-10T00:00:00"}