{"id":"https://openalex.org/W4400530959","doi":"https://doi.org/10.1145/3626772.3657862","title":"Resources for Brewing BEIR: Reproducible Reference Models and Statistical Analyses","display_name":"Resources for Brewing BEIR: Reproducible Reference Models and Statistical Analyses","publication_year":2024,"publication_date":"2024-07-10","ids":{"openalex":"https://openalex.org/W4400530959","doi":"https://doi.org/10.1145/3626772.3657862"},"language":"en","primary_location":{"id":"doi:10.1145/3626772.3657862","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626772.3657862","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043026875","display_name":"Ehsan Kamalloo","orcid":"https://orcid.org/0000-0003-3081-8762"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Ehsan Kamalloo","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0000-0003-3081-8762","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052977545","display_name":"Nandan Thakur","orcid":"https://orcid.org/0000-0001-6107-2460"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Nandan Thakur","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0000-0001-6107-2460","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068114632","display_name":"Carlos Lassance","orcid":"https://orcid.org/0000-0002-7754-6656"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Carlos Lassance","raw_affiliation_strings":["Cohere, Grenoble, France"],"raw_orcid":"https://orcid.org/0000-0002-7754-6656","affiliations":[{"raw_affiliation_string":"Cohere, Grenoble, France","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021623012","display_name":"Xueguang Ma","orcid":"https://orcid.org/0000-0003-3430-4910"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Xueguang Ma","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0000-0003-3430-4910","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014519159","display_name":"Jheng-Hong Yang","orcid":"https://orcid.org/0000-0001-5469-312X"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jheng-Hong Yang","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0000-0001-5469-312X","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082997975","display_name":"Jimmy Lin","orcid":"https://orcid.org/0000-0002-0661-7189"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jimmy Lin","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0000-0002-0661-7189","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5043026875"],"corresponding_institution_ids":["https://openalex.org/I151746483"],"apc_list":null,"apc_paid":null,"fwci":10.444,"has_fulltext":false,"cited_by_count":14,"citation_normalized_percentile":{"value":0.98141287,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1431","last_page":"1440"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.8759999871253967,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.8759999871253967,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.866100013256073,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11106","display_name":"Data Management and Algorithms","score":0.8644999861717224,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5659018754959106},{"id":"https://openalex.org/keywords/brewing","display_name":"Brewing","score":0.4737544655799866},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.1259128749370575},{"id":"https://openalex.org/keywords/food-science","display_name":"Food science","score":0.05407589673995972}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5659018754959106},{"id":"https://openalex.org/C39699697","wikidata":"https://www.wikidata.org/wiki/Q869095","display_name":"Brewing","level":3,"score":0.4737544655799866},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.1259128749370575},{"id":"https://openalex.org/C31903555","wikidata":"https://www.wikidata.org/wiki/Q1637030","display_name":"Food science","level":1,"score":0.05407589673995972},{"id":"https://openalex.org/C100544194","wikidata":"https://www.wikidata.org/wiki/Q41760","display_name":"Fermentation","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3626772.3657862","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626772.3657862","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/2","display_name":"Zero hunger","score":0.7200000286102295}],"awards":[],"funders":[{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W2019509999","https://openalex.org/W2069065514","https://openalex.org/W2470818894","https://openalex.org/W2798329376","https://openalex.org/W2897754576","https://openalex.org/W2898078244","https://openalex.org/W2899154813","https://openalex.org/W2945127593","https://openalex.org/W2998702515","https://openalex.org/W2999905431","https://openalex.org/W3021052948","https://openalex.org/W3021397474","https://openalex.org/W3099446234","https://openalex.org/W3099700870","https://openalex.org/W3134665270","https://openalex.org/W3168875417","https://openalex.org/W3172119680","https://openalex.org/W3174451219","https://openalex.org/W3180230246","https://openalex.org/W3203502727","https://openalex.org/W3206455169","https://openalex.org/W3206770993","https://openalex.org/W4213045215","https://openalex.org/W4221167718","https://openalex.org/W4226208814","https://openalex.org/W4234859835","https://openalex.org/W4248923238","https://openalex.org/W4284664419","https://openalex.org/W4284682639","https://openalex.org/W4295885374","https://openalex.org/W4306317399","https://openalex.org/W4385570324","https://openalex.org/W4400528755","https://openalex.org/W6600655081"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2362949944","https://openalex.org/W2804019727","https://openalex.org/W2351048443","https://openalex.org/W2035936095","https://openalex.org/W2357256928","https://openalex.org/W4310236366","https://openalex.org/W1969858198","https://openalex.org/W2391801439"],"abstract_inverted_index":{"BEIR":[0,56,115],"is":[1,129],"a":[2,125,163],"benchmark":[3,71],"dataset":[4],"originally":[5],"designed":[6,58],"for":[7,95],"zero-shot":[8],"evaluation":[9],"of":[10,27,80,87,165],"retrieval":[11],"models":[12,28,42],"across":[13,143],"18":[14],"different":[15],"domain/task":[16],"combinations.":[17],"In":[18],"recent":[19],"years,":[20],"we":[21,100,136],"have":[22],"witnessed":[23],"the":[24,36,52,70,78,85],"growing":[25],"popularity":[26],"based":[29],"on":[30,114,140],"representation":[31],"learning,":[32],"which":[33],"naturally":[34],"begs":[35],"question:":[37],"How":[38],"effective":[39],"are":[40,116,146],"these":[41],"when":[43],"presented":[44],"with":[45],"queries":[46],"and":[47,84,109],"documents":[48],"that":[49,68,105,128,145],"differ":[50],"from":[51,72,121],"training":[53],"data?":[54],"While":[55],"was":[57],"to":[59,93,131,148],"answer":[60],"this":[61,98],"question,":[62],"our":[63,157],"work":[64,158],"addresses":[65],"two":[66],"shortcomings":[67],"prevent":[69],"achieving":[73],"its":[74],"full":[75],"potential:":[76],"First,":[77],"sophistication":[79],"modern":[81],"neural":[82],"methods":[83],"complexity":[86],"current":[88],"software":[89],"infrastructure":[90],"create":[91],"barriers":[92],"entry":[94],"newcomers.":[96],"To":[97,133],"end,":[99],"provide":[101],"reproducible":[102],"reference":[103],"implementations":[104],"cover":[106],"learned":[107],"dense":[108],"sparse":[110],"models.":[111],"Second,":[112],"comparisons":[113],"performed":[117],"by":[118],"reducing":[119],"scores":[120],"heterogeneous":[122],"datasets":[123,144],"into":[124],"single":[126],"average":[127],"difficult":[130],"interpret.":[132],"remedy":[134],"this,":[135],"present":[137],"meta-analyses":[138],"focusing":[139],"effect":[141],"sizes":[142],"able":[147],"accurately":[149],"quantify":[150],"model":[151],"differences.":[152],"By":[153],"addressing":[154],"both":[155],"shortcomings,":[156],"facilitates":[159],"future":[160],"explorations":[161],"in":[162],"range":[164],"interesting":[166],"research":[167],"questions.":[168]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":3}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}