{"id":"https://openalex.org/W4410637007","doi":"https://doi.org/10.1145/3701716.3715488","title":"Multi-stage Large Language Model Pipelines Can Outperform GPT-4o in Relevance Assessment","display_name":"Multi-stage Large Language Model Pipelines Can Outperform GPT-4o in Relevance Assessment","publication_year":2025,"publication_date":"2025-05-08","ids":{"openalex":"https://openalex.org/W4410637007","doi":"https://doi.org/10.1145/3701716.3715488"},"language":"en","primary_location":{"id":"doi:10.1145/3701716.3715488","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3701716.3715488","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3701716.3715488","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM on Web Conference 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3701716.3715488","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116039148","display_name":"Julian A. Schnabel","orcid":"https://orcid.org/0009-0001-5461-3254"},"institutions":[{"id":"https://openalex.org/I44260953","display_name":"Heinrich Heine University D\u00fcsseldorf","ror":"https://ror.org/024z2rq82","country_code":"DE","type":"education","lineage":["https://openalex.org/I44260953"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Julian A. Schnabel","raw_affiliation_strings":["Heinrich Heine University, D\u00fcsseldorf, Germany"],"raw_orcid":"https://orcid.org/0009-0001-5461-3254","affiliations":[{"raw_affiliation_string":"Heinrich Heine University, D\u00fcsseldorf, Germany","institution_ids":["https://openalex.org/I44260953"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053547177","display_name":"Johanne R. Trippas","orcid":"https://orcid.org/0000-0002-7801-0239"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Johanne R. Trippas","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0002-7801-0239","affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0001-9094-0810","affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014686840","display_name":"Danula Hettiachchi","orcid":"https://orcid.org/0000-0003-3875-5727"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Danula Hettiachchi","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0003-3875-5727","affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1288","last_page":"1292"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.989799976348877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/stage","display_name":"Stage (stratigraphy)","score":0.7692815065383911},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7556288838386536},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6755355596542358},{"id":"https://openalex.org/keywords/pipeline-transport","display_name":"Pipeline transport","score":0.5725852847099304},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.45356485247612},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4533500075340271},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.36791032552719116},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.1389998495578766},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.11723574995994568},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.05233070254325867}],"concepts":[{"id":"https://openalex.org/C146357865","wikidata":"https://www.wikidata.org/wiki/Q1123245","display_name":"Stage (stratigraphy)","level":2,"score":0.7692815065383911},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7556288838386536},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6755355596542358},{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.5725852847099304},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.45356485247612},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4533500075340271},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36791032552719116},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.1389998495578766},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11723574995994568},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.05233070254325867},{"id":"https://openalex.org/C87717796","wikidata":"https://www.wikidata.org/wiki/Q146326","display_name":"Environmental engineering","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3701716.3715488","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3701716.3715488","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3701716.3715488","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM on Web Conference 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3701716.3715488","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3701716.3715488","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3701716.3715488","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM on Web Conference 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2338326961","display_name":null,"funder_award_id":"CE200100005","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"}],"funders":[{"id":"https://openalex.org/F4320320972","display_name":"RMIT University","ror":"https://ror.org/04ttjf776"},{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4410637007.pdf","grobid_xml":"https://content.openalex.org/works/W4410637007.grobid-xml"},"referenced_works_count":5,"referenced_works":["https://openalex.org/W3175111331","https://openalex.org/W4224316921","https://openalex.org/W4392581042","https://openalex.org/W4396843976","https://openalex.org/W4400526908"],"related_works":["https://openalex.org/W2329386257","https://openalex.org/W2085384747","https://openalex.org/W2088166309","https://openalex.org/W1891216533","https://openalex.org/W2503350049","https://openalex.org/W1995889332","https://openalex.org/W4312133475","https://openalex.org/W4238976562","https://openalex.org/W2276587472","https://openalex.org/W3104163240"],"abstract_inverted_index":{"The":[0],"effectiveness":[1],"of":[2,14,71,82,99,122,132],"search":[3],"systems":[4],"is":[5,28,34],"evaluated":[6],"using":[7],"relevance":[8,23,59,115],"labels":[9,24],"that":[10,56],"indicate":[11],"the":[12,58,119,130,133],"usefulness":[13],"documents":[15],"for":[16,114],"specific":[17],"queries":[18],"and":[19,48,69,74,111],"users.While":[20],"obtaining":[21],"these":[22],"from":[25],"real":[26],"users":[27],"ideal,":[29],"scaling":[30],"such":[31],"data":[32],"collection":[33],"challenging.Consequently,":[35],"third-party":[36],"annotators":[37],"are":[38],"employed,":[39],"but":[40],"their":[41],"inconsistent":[42],"accuracy":[43,89,131],"demands":[44],"costly":[45],"auditing,":[46],"training,":[47],"monitoring.We":[49],"propose":[50],"an":[51,86],"LLM-based":[52],"modular":[53],"classification":[54],"pipeline":[55,127],"divides":[57],"assessment":[60],"task":[61],"into":[62],"multiple":[63],"stages,":[64],"each":[65],"utilising":[66],"different":[67],"prompts":[68],"models":[70],"varying":[72],"sizes":[73],"capabilities.Applied":[75],"to":[76],"TREC":[77],"Deep":[78],"Learning":[79],"(TREC-DL),":[80],"one":[81],"our":[83],"approaches":[84],"showed":[85],"18.4%":[87],"Krippendorff's":[88],"increase":[90],"over":[91],"OpenAI's":[92],"GPT-4o":[93,123,134],"mini":[94],"while":[95],"maintaining":[96],"a":[97,108,126],"cost":[98],"about":[100],"0.2":[101],"USD":[102],"per":[103],"million":[104],"input":[105],"tokens,":[106],"offering":[107],"more":[109],"efficient":[110],"scalable":[112],"solution":[113],"assessment.This":[116],"approach":[117],"beats":[118],"baseline":[120],"performance":[121],"(5":[124],"USD).With":[125],"approach,":[128],"even":[129],"flagship":[135],"model,":[136],"measured":[137],"in":[138],",":[139],"could":[140],"be":[141],"improved":[142],"by":[143],"9.7%.":[144]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}