{"id":"https://openalex.org/W4412377073","doi":"https://doi.org/10.1145/3726302.3730093","title":"The Viability of Crowdsourcing for RAG Evaluation","display_name":"The Viability of Crowdsourcing for RAG Evaluation","publication_year":2025,"publication_date":"2025-07-13","ids":{"openalex":"https://openalex.org/W4412377073","doi":"https://doi.org/10.1145/3726302.3730093"},"language":"en","primary_location":{"id":"doi:10.1145/3726302.3730093","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730093","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730093","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730093","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008775203","display_name":"Lukas Gienapp","orcid":"https://orcid.org/0000-0001-5707-3751"},"institutions":[{"id":"https://openalex.org/I147765834","display_name":"Leipzig University of Applied Sciences","ror":"https://ror.org/03xgcq477","country_code":"DE","type":"education","lineage":["https://openalex.org/I147765834"]},{"id":"https://openalex.org/I926574661","display_name":"Leipzig University","ror":"https://ror.org/03s7gtk40","country_code":"DE","type":"education","lineage":["https://openalex.org/I926574661"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Lukas Gienapp","raw_affiliation_strings":["Leipzig University, Leipzig, Germany and ScaDS.AI, Leipzig, Germany"],"affiliations":[{"raw_affiliation_string":"Leipzig University, Leipzig, Germany and ScaDS.AI, Leipzig, Germany","institution_ids":["https://openalex.org/I147765834","https://openalex.org/I926574661"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086108001","display_name":"Thomas Hagen","orcid":"https://orcid.org/0009-0000-4854-7249"},"institutions":[{"id":"https://openalex.org/I106157433","display_name":"University of Kassel","ror":"https://ror.org/04zc7p361","country_code":"DE","type":"education","lineage":["https://openalex.org/I106157433"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Tim Hagen","raw_affiliation_strings":["University of Kassel, Kassel, Germany and hessian.AI, Kassel, Germany"],"affiliations":[{"raw_affiliation_string":"University of Kassel, Kassel, Germany and hessian.AI, Kassel, Germany","institution_ids":["https://openalex.org/I106157433"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068616222","display_name":"Maik Fr\u00f6be","orcid":"https://orcid.org/0000-0002-1003-981X"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Maik Fr\u00f6be","raw_affiliation_strings":["Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"],"affiliations":[{"raw_affiliation_string":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014322854","display_name":"Matthias Hagen","orcid":"https://orcid.org/0000-0002-9733-2890"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matthias Hagen","raw_affiliation_strings":["Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"],"affiliations":[{"raw_affiliation_string":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027915931","display_name":"Benno Stein","orcid":"https://orcid.org/0000-0001-9033-2217"},"institutions":[{"id":"https://openalex.org/I51441396","display_name":"Bauhaus-Universit\u00e4t Weimar","ror":"https://ror.org/033bb5z47","country_code":"DE","type":"education","lineage":["https://openalex.org/I51441396"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Benno Stein","raw_affiliation_strings":["Bauhaus-Universit\u00e4t Weimar, Weimar, Germany"],"affiliations":[{"raw_affiliation_string":"Bauhaus-Universit\u00e4t Weimar, Weimar, Germany","institution_ids":["https://openalex.org/I51441396"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083712311","display_name":"Martin Potthast","orcid":"https://orcid.org/0000-0003-2451-0665"},"institutions":[{"id":"https://openalex.org/I106157433","display_name":"University of Kassel","ror":"https://ror.org/04zc7p361","country_code":"DE","type":"education","lineage":["https://openalex.org/I106157433"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Martin Potthast","raw_affiliation_strings":["University of Kassel, Kassel, Germany, hessian.AI, Kassel, Germany, and ScaDS.AI, Leipzig, Germany"],"affiliations":[{"raw_affiliation_string":"University of Kassel, Kassel, Germany, hessian.AI, Kassel, Germany, and ScaDS.AI, Leipzig, Germany","institution_ids":["https://openalex.org/I106157433"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075371460","display_name":"Harrisen Scells","orcid":"https://orcid.org/0000-0001-9578-7157"},"institutions":[{"id":"https://openalex.org/I106157433","display_name":"University of Kassel","ror":"https://ror.org/04zc7p361","country_code":"DE","type":"education","lineage":["https://openalex.org/I106157433"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Harrisen Scells","raw_affiliation_strings":["University of Kassel, Kassel, Germany and hessian.AI, Kassel, Germany"],"affiliations":[{"raw_affiliation_string":"University of Kassel, Kassel, Germany and hessian.AI, Kassel, Germany","institution_ids":["https://openalex.org/I106157433"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5008775203"],"corresponding_institution_ids":["https://openalex.org/I147765834","https://openalex.org/I926574661"],"apc_list":null,"apc_paid":null,"fwci":9.3096,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.97579512,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"159","last_page":"169"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9383050799369812},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.45443740487098694},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.13903909921646118}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9383050799369812},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.45443740487098694},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.13903909921646118}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3726302.3730093","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730093","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730093","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2504.15689","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2504.15689","pdf_url":"https://arxiv.org/pdf/2504.15689","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3726302.3730093","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730093","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3730093","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3307378536","display_name":null,"funder_award_id":"Fellow","funder_id":"https://openalex.org/F4320308269","funder_display_name":"Alexander von Humboldt-Stiftung"},{"id":"https://openalex.org/G5082371171","display_name":null,"funder_award_id":"GA 101070014","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G6072843279","display_name":null,"funder_award_id":"101070014","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"},{"id":"https://openalex.org/G6955755495","display_name":null,"funder_award_id":"Germany","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"},{"id":"https://openalex.org/G7504899342","display_name":null,"funder_award_id":"101070014","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8119885652","display_name":null,"funder_award_id":"Germany","funder_id":"https://openalex.org/F4320308269","funder_display_name":"Alexander von Humboldt-Stiftung"}],"funders":[{"id":"https://openalex.org/F4320308269","display_name":"Alexander von Humboldt-Stiftung","ror":"https://ror.org/012kf4317"},{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"},{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412377073.pdf","grobid_xml":"https://content.openalex.org/works/W4412377073.grobid-xml"},"referenced_works_count":46,"referenced_works":["https://openalex.org/W1911845565","https://openalex.org/W1973676054","https://openalex.org/W1980701647","https://openalex.org/W2001575924","https://openalex.org/W2039017419","https://openalex.org/W2328123515","https://openalex.org/W2596512067","https://openalex.org/W2793081411","https://openalex.org/W2794219599","https://openalex.org/W2963532001","https://openalex.org/W3035308065","https://openalex.org/W3102195370","https://openalex.org/W3130759101","https://openalex.org/W3152574220","https://openalex.org/W3174519801","https://openalex.org/W3188970555","https://openalex.org/W4297435087","https://openalex.org/W4384107234","https://openalex.org/W4385474629","https://openalex.org/W4385688511","https://openalex.org/W4388717143","https://openalex.org/W4389984066","https://openalex.org/W4391376072","https://openalex.org/W4391421370","https://openalex.org/W4391494845","https://openalex.org/W4393147129","https://openalex.org/W4396816204","https://openalex.org/W4399555009","https://openalex.org/W4400024847","https://openalex.org/W4400525274","https://openalex.org/W4400526908","https://openalex.org/W4400530685","https://openalex.org/W4401042808","https://openalex.org/W4402951142","https://openalex.org/W4403753183","https://openalex.org/W4403781006","https://openalex.org/W4404407248","https://openalex.org/W4404451336","https://openalex.org/W4404987920","https://openalex.org/W4405253355","https://openalex.org/W4405626034","https://openalex.org/W4405767463","https://openalex.org/W4406139837","https://openalex.org/W4407953136","https://openalex.org/W4411113282","https://openalex.org/W4411117109"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948"],"abstract_inverted_index":{"How":[0],"good":[1],"are":[2,160],"humans":[3],"at":[4],"writing":[5,31,111],"and":[6,32,48,92,103,115,131,143,158],"judging":[7],"responses":[8,51],"in":[9],"retrieval-augmented":[10],"generation":[11],"(RAG)":[12],"scenarios?":[13],"To":[14],"answer":[15],"this":[16],"question,":[17],"we":[18],"investigate":[19],"the":[20,53,57,69,84,116,138],"efficacy":[21],"of":[22,45,56,68,81,118],"crowdsourcing":[23,119],"for":[24,52,113,120,140,148],"RAG":[25,40,114,121],"through":[26],"two":[27],"complementary":[28],"studies:":[29],"response":[30,33,63],"utility":[34,99],"judgment.":[35],"Our":[36,105],"new":[37],"Webis":[38],"Crowd":[39],"Corpus":[41],"2025":[42],"(Webis-CrowdRAG-25)":[43],"consists":[44],"903":[46,49],"human-written":[47,152],"LLM-generated":[50],"301":[54],"topics":[55],"TREC":[58],"2024":[59],"RAG~track,":[60],"with":[61,151],"each":[62],"composed":[64],"according":[65],"to":[66],"one":[67],"three":[70],"discourse":[71],"styles":[72],"'bullet":[73],"list',":[74],"'essay',":[75],"or":[76],"'news'.":[77],"For":[78],"a":[79],"selection":[80],"65":[82],"topics,":[83],"corpus":[85],"further":[86],"contains":[87],"47,320":[88],"pairwise":[89,94,127,142],"human":[90,110,126],"judgments":[91,96,128],"10,556":[93],"LLM":[95],"across":[97],"seven":[98],"dimensions":[100],"(e.g.,":[101],"coverage":[102],"coherence).":[104],"analyses":[106],"give":[107],"insights":[108],"into":[109],"behavior":[112],"viability":[117],"evaluation.":[122],"We":[123],"find":[124],"that":[125],"provide":[129],"reliable":[130],"cost-effective":[132],"results.":[133],"This":[134],"is":[135],"much":[136],"less":[137],"case":[139],"LLM-based":[141],"human/LLM-based":[144],"pointwise":[145],"judgments,":[146],"nor":[147],"automated":[149],"comparisons":[150],"reference":[153],"responses.":[154],"All":[155],"our":[156],"data":[157],"tools":[159],"freely":[161],"available.":[162]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2}],"updated_date":"2026-04-12T07:58:50.170612","created_date":"2025-10-10T00:00:00"}