{"id":"https://openalex.org/W4416017057","doi":"https://doi.org/10.1145/3746252.3761484","title":"RerankArena: A Unified Platform for Evaluating Retrieval, Reranking and RAG with Human and LLM Feedback","display_name":"RerankArena: A Unified Platform for Evaluating Retrieval, Reranking and RAG with Human and LLM Feedback","publication_year":2025,"publication_date":"2025-11-07","ids":{"openalex":"https://openalex.org/W4416017057","doi":"https://doi.org/10.1145/3746252.3761484"},"language":null,"primary_location":{"id":"doi:10.1145/3746252.3761484","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746252.3761484","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3746252.3761484","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064852810","display_name":"Abdelrahman Abdallah","orcid":"https://orcid.org/0000-0001-8747-4927"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":true,"raw_author_name":"Abdelrahman Abdallah","raw_affiliation_strings":["University of Innsbruck, Innsbruck, Tyrol, Austria"],"affiliations":[{"raw_affiliation_string":"University of Innsbruck, Innsbruck, Tyrol, Austria","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114778014","display_name":"Mahmoud Abdalla","orcid":"https://orcid.org/0000-0002-4270-2268"},"institutions":[{"id":"https://openalex.org/I163753206","display_name":"Chungbuk National University","ror":"https://ror.org/02wnxgj78","country_code":"KR","type":"education","lineage":["https://openalex.org/I163753206"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Mahmoud Abdalla","raw_affiliation_strings":["Chungbuk National University, Cheongju-si, Cheongju, Republic of Korea"],"affiliations":[{"raw_affiliation_string":"Chungbuk National University, Cheongju-si, Cheongju, Republic of Korea","institution_ids":["https://openalex.org/I163753206"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007529430","display_name":"Bhawna Piryani","orcid":"https://orcid.org/0009-0005-3578-2393"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Bhawna Piryani","raw_affiliation_strings":["University of Innsbruck, Innsbruck, Tyrol, Austria"],"affiliations":[{"raw_affiliation_string":"University of Innsbruck, Innsbruck, Tyrol, Austria","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001145944","display_name":"Jamshid Mozafari","orcid":"https://orcid.org/0000-0003-4850-9239"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Jamshid Mozafari","raw_affiliation_strings":["University of Innsbruck, Innsbruck, Tyrol, Austria"],"affiliations":[{"raw_affiliation_string":"University of Innsbruck, Innsbruck, Tyrol, Austria","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044054654","display_name":"Mohammed M. Abdelgwad","orcid":"https://orcid.org/0000-0003-3600-0785"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Mohammed Ali","raw_affiliation_strings":["University of Innsbruck, Innsbruck, Tyrol, Austria"],"affiliations":[{"raw_affiliation_string":"University of Innsbruck, Innsbruck, Tyrol, Austria","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079733597","display_name":"Adam Jatowt","orcid":"https://orcid.org/0000-0001-7235-0665"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Adam Jatowt","raw_affiliation_strings":["University of Innsbruck, Innsbruck, Tyrol, Austria"],"affiliations":[{"raw_affiliation_string":"University of Innsbruck, Innsbruck, Tyrol, Austria","institution_ids":["https://openalex.org/I190249584"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5064852810"],"corresponding_institution_ids":["https://openalex.org/I190249584"],"apc_list":null,"apc_paid":null,"fwci":3.1665,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.94160498,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"6593","last_page":"6597"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.4645000100135803,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.4645000100135803,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.25949999690055847,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.13819999992847443,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.756600022315979},{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.6198999881744385},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.6025000214576721},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5310999751091003},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.48840001225471497},{"id":"https://openalex.org/keywords/relevance-feedback","display_name":"Relevance feedback","score":0.4715999960899353}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8256999850273132},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.756600022315979},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.6198999881744385},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.6025000214576721},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5680999755859375},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5310999751091003},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.48840001225471497},{"id":"https://openalex.org/C2779532271","wikidata":"https://www.wikidata.org/wiki/Q445558","display_name":"Relevance feedback","level":4,"score":0.4715999960899353},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4478999972343445},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.4269999861717224},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37439998984336853},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35830000042915344},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.27379998564720154},{"id":"https://openalex.org/C1667742","wikidata":"https://www.wikidata.org/wiki/Q10927554","display_name":"Image retrieval","level":3,"score":0.26820001006126404},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.26170000433921814}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3746252.3761484","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746252.3761484","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3746252.3761484","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3746252.3761484","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W2051167396","https://openalex.org/W2251818205","https://openalex.org/W3021397474","https://openalex.org/W4252076394","https://openalex.org/W4284664419","https://openalex.org/W4384890816","https://openalex.org/W4389523765","https://openalex.org/W4392963986","https://openalex.org/W4402670856","https://openalex.org/W4402671584","https://openalex.org/W4402671832","https://openalex.org/W4404781843","https://openalex.org/W4411113192","https://openalex.org/W4412888159","https://openalex.org/W6911415352"],"related_works":[],"abstract_inverted_index":{"Evaluating":[0],"the":[1,15,34,154],"quality":[2,80,105],"of":[3,17,36],"retrieval-augmented":[4],"generation":[5],"(RAG)":[6],"and":[7,20,32,40,46,76,91,104,118,153],"document":[8,74],"reranking":[9,62],"systems":[10,42],"remains":[11],"challenging":[12],"due":[13],"to":[14,135],"lack":[16],"scalable,":[18],"user-centric,":[19],"multi-perspective":[21],"evaluation":[22,59,129],"tools.":[23],"We":[24],"introduce":[25],"RankArena,":[26],"a":[27],"unified":[28],"platform":[29,108,147],"for":[30,52],"comparing":[31],"analysing":[33],"performance":[35],"retrieval":[37,143],"pipelines,":[38],"rerankers,":[39,137],"RAG":[41,78],"using":[43],"structured":[44,128],"human":[45,68,119],"LLM-based":[47],"feedback":[48,86],"as":[49,51,99,127],"well":[50],"collecting":[53],"such":[54,98],"feedback.":[55],"RankArena":[56],"supports":[57],"multiple":[58],"modes:":[60],"direct":[61],"visualisation,":[63],"blind":[64],"pairwise":[65,89],"comparisons":[66],"with":[67,95],"or":[69,142],"LLM":[70],"voting,":[71],"supervised":[72],"manual":[73],"annotation,":[75],"end-to-end":[77],"answer":[79],"assessment.":[81],"It":[82],"captures":[83],"fine-grained":[84],"relevance":[85],"through":[87],"both":[88],"preferences":[90],"full-list":[92],"annotations,":[93],"along":[94],"auxiliary":[96],"metadata":[97],"movement":[100],"metrics,":[101],"annotation":[102],"time,":[103],"ratings.":[106],"The":[107],"also":[109],"integrates":[110],"LLM-as-a-judge":[111],"evaluation,":[112],"enabling":[113],"comparison":[114],"between":[115],"model-generated":[116],"rankings":[117],"ground":[120],"truth":[121],"annotations.":[122],"All":[123],"interactions":[124],"are":[125],"stored":[126],"datasets":[130],"that":[131],"can":[132],"be":[133],"used":[134],"train":[136],"reward":[138],"models,":[139],"judgment":[140],"agents,":[141],"strategy":[144],"selectors.":[145],"Our":[146],"is":[148,157],"publicly":[149],"available":[150],"at":[151],"https://rankarena.ngrok.io/,":[152],"Demo":[155],"video":[156],"provided.":[158],"https://youtu.be/jIYAP4PaSSI.":[159]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-04-02T15:55:50.835912","created_date":"2025-11-08T00:00:00"}