{"id":"https://openalex.org/W4400528580","doi":"https://doi.org/10.1145/3626772.3657884","title":"CIRAL: A Test Collection for CLIR Evaluations in African Languages","display_name":"CIRAL: A Test Collection for CLIR Evaluations in African Languages","publication_year":2024,"publication_date":"2024-07-10","ids":{"openalex":"https://openalex.org/W4400528580","doi":"https://doi.org/10.1145/3626772.3657884"},"language":"en","primary_location":{"id":"doi:10.1145/3626772.3657884","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626772.3657884","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076302977","display_name":"Mofetoluwa Adeyemi","orcid":"https://orcid.org/0009-0003-2859-7136"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Mofetoluwa Adeyemi","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060546041","display_name":"Akintunde Oladipo","orcid":"https://orcid.org/0009-0000-2630-8167"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Akintunde Oladipo","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100390744","display_name":"Xinyu Zhang","orcid":"https://orcid.org/0009-0009-0756-8110"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Xinyu Zhang","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064431976","display_name":"David Alfonso-Hermelo","orcid":"https://orcid.org/0009-0009-4591-3077"},"institutions":[{"id":"https://openalex.org/I4210115038","display_name":"Huawei Technologies (Canada)","ror":"https://ror.org/026venb53","country_code":"CA","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210115038"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"David Alfonso-Hermelo","raw_affiliation_strings":["Huawei Technologies, Montreal, Canada"],"affiliations":[{"raw_affiliation_string":"Huawei Technologies, Montreal, Canada","institution_ids":["https://openalex.org/I4210115038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028862918","display_name":"Mehdi Rezagholizadeh","orcid":"https://orcid.org/0000-0003-4014-6007"},"institutions":[{"id":"https://openalex.org/I4210115038","display_name":"Huawei Technologies (Canada)","ror":"https://ror.org/026venb53","country_code":"CA","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210115038"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mehdi Rezagholizadeh","raw_affiliation_strings":["Huawei Technologies, Montreal, Canada"],"affiliations":[{"raw_affiliation_string":"Huawei Technologies, Montreal, Canada","institution_ids":["https://openalex.org/I4210115038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019715118","display_name":"Boxing Chen","orcid":"https://orcid.org/0000-0002-3170-4858"},"institutions":[{"id":"https://openalex.org/I4210115038","display_name":"Huawei Technologies (Canada)","ror":"https://ror.org/026venb53","country_code":"CA","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210115038"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Boxing Chen","raw_affiliation_strings":["Huawei Technologies, Montreal, Canada"],"affiliations":[{"raw_affiliation_string":"Huawei Technologies, Montreal, Canada","institution_ids":["https://openalex.org/I4210115038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037687030","display_name":"Abdul-Hakeem Omotayo","orcid":"https://orcid.org/0009-0009-4558-0356"},"institutions":[{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Abdul-Hakeem Omotayo","raw_affiliation_strings":["University of California, Davis, USA"],"affiliations":[{"raw_affiliation_string":"University of California, Davis, USA","institution_ids":["https://openalex.org/I84218800"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054346112","display_name":"Idris Abdulmumin","orcid":"https://orcid.org/0000-0002-3795-8381"},"institutions":[{"id":"https://openalex.org/I69552723","display_name":"University of Pretoria","ror":"https://ror.org/00g0p6g84","country_code":"ZA","type":"education","lineage":["https://openalex.org/I69552723"]}],"countries":["ZA"],"is_corresponding":false,"raw_author_name":"Idris Abdulmumin","raw_affiliation_strings":["University of Pretoria, Pretoria, South Africa"],"affiliations":[{"raw_affiliation_string":"University of Pretoria, Pretoria, South Africa","institution_ids":["https://openalex.org/I69552723"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084495633","display_name":"Naome A. Etori","orcid":"https://orcid.org/0000-0001-7772-1103"},"institutions":[{"id":"https://openalex.org/I130238516","display_name":"University of Minnesota","ror":"https://ror.org/017zqws13","country_code":"US","type":"education","lineage":["https://openalex.org/I130238516"]},{"id":"https://openalex.org/I4210101327","display_name":"Twin Cities Orthopedics","ror":"https://ror.org/01en4s460","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210101327"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Naome A. Etori","raw_affiliation_strings":["University of Minnesota, Twin Cities, USA"],"affiliations":[{"raw_affiliation_string":"University of Minnesota, Twin Cities, USA","institution_ids":["https://openalex.org/I4210101327","https://openalex.org/I130238516"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103308743","display_name":"Toyib Babatunde Musa","orcid":"https://orcid.org/0009-0003-0239-6416"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Toyib Babatunde Musa","raw_affiliation_strings":["Masakhane, Lagos, Nigeria"],"affiliations":[{"raw_affiliation_string":"Masakhane, Lagos, Nigeria","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046036504","display_name":"Samuel Fanijo","orcid":"https://orcid.org/0000-0003-4636-1437"},"institutions":[{"id":"https://openalex.org/I173911158","display_name":"Iowa State University","ror":"https://ror.org/04rswrd78","country_code":"US","type":"education","lineage":["https://openalex.org/I173911158"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Samuel Fanijo","raw_affiliation_strings":["Iowa State University, Ames, USA"],"affiliations":[{"raw_affiliation_string":"Iowa State University, Ames, USA","institution_ids":["https://openalex.org/I173911158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029191755","display_name":"Oluwabusayo Olufunke Awoyomi","orcid":"https://orcid.org/0009-0002-5563-8987"},"institutions":[{"id":"https://openalex.org/I102420622","display_name":"College of Saint Rose","ror":"https://ror.org/00nv9r617","country_code":"US","type":"education","lineage":["https://openalex.org/I102420622"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Oluwabusayo Olufunke Awoyomi","raw_affiliation_strings":["The College of Saint Rose, Albany, USA"],"affiliations":[{"raw_affiliation_string":"The College of Saint Rose, Albany, USA","institution_ids":["https://openalex.org/I102420622"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057217357","display_name":"Saheed Abdullahi Salahudeen","orcid":"https://orcid.org/0000-0002-1458-4057"},"institutions":[{"id":"https://openalex.org/I4210145761","display_name":"Shenzhen Institutes of Advanced Technology","ror":"https://ror.org/04gh4er46","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210145761"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Saheed Abdullahi Salahudeen","raw_affiliation_strings":["Shenzhen Institute of Advanced Technology, CAS, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Shenzhen Institute of Advanced Technology, CAS, Shenzhen, China","institution_ids":["https://openalex.org/I4210145761"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011531690","display_name":"L A Mohammed","orcid":"https://orcid.org/0009-0003-9090-5072"},"institutions":[{"id":"https://openalex.org/I28046988","display_name":"Kwame Nkrumah University of Science and Technology","ror":"https://ror.org/00cb23x68","country_code":"GH","type":"education","lineage":["https://openalex.org/I28046988"]}],"countries":["GH"],"is_corresponding":false,"raw_author_name":"Labaran Adamu Mohammed","raw_affiliation_strings":["Kwame Nkrumah University of Science and Technology, Kumasi, Ghana"],"affiliations":[{"raw_affiliation_string":"Kwame Nkrumah University of Science and Technology, Kumasi, Ghana","institution_ids":["https://openalex.org/I28046988"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093286152","display_name":"Daud Abolade","orcid":"https://orcid.org/0009-0006-0090-9486"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Daud Olamide Abolade","raw_affiliation_strings":["Masakhane, Lagos, Nigeria"],"affiliations":[{"raw_affiliation_string":"Masakhane, Lagos, Nigeria","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050132917","display_name":"Falalu Ibrahim Lawan","orcid":"https://orcid.org/0000-0003-3310-0326"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Falalu Ibrahim Lawan","raw_affiliation_strings":["Masakhane, Kano, Nigeria"],"affiliations":[{"raw_affiliation_string":"Masakhane, Kano, Nigeria","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103325825","display_name":"Maryam Sabo Abubakar","orcid":"https://orcid.org/0009-0001-0379-2256"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maryam Sabo Abubakar","raw_affiliation_strings":["Masakhane, Kano, Nigeria"],"affiliations":[{"raw_affiliation_string":"Masakhane, Kano, Nigeria","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091932373","display_name":"Ruqayya Nasir Iro","orcid":"https://orcid.org/0009-0005-3885-8488"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ruqayya Nasir Iro","raw_affiliation_strings":["Masakhane, Kano, Nigeria"],"affiliations":[{"raw_affiliation_string":"Masakhane, Kano, Nigeria","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051411649","display_name":"Amina Imam Abubakar","orcid":"https://orcid.org/0009-0005-0985-1404"},"institutions":[{"id":"https://openalex.org/I8033065","display_name":"University of Abuja","ror":"https://ror.org/007e69832","country_code":"NG","type":"education","lineage":["https://openalex.org/I8033065"]}],"countries":["NG"],"is_corresponding":false,"raw_author_name":"Amina Imam Abubakar","raw_affiliation_strings":["University of Abuja, Abuja, Nigeria"],"affiliations":[{"raw_affiliation_string":"University of Abuja, Abuja, Nigeria","institution_ids":["https://openalex.org/I8033065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004590250","display_name":"Shafie Abdi Mohamed","orcid":"https://orcid.org/0009-0009-2844-5304"},"institutions":[{"id":"https://openalex.org/I4210110361","display_name":"Mogadishu University","ror":"https://ror.org/025zbpk71","country_code":"SO","type":"education","lineage":["https://openalex.org/I4210110361"]}],"countries":["SO"],"is_corresponding":false,"raw_author_name":"Shafie Abdi Mohamed","raw_affiliation_strings":["Masakhane, Mogadishu, Somalia"],"affiliations":[{"raw_affiliation_string":"Masakhane, Mogadishu, Somalia","institution_ids":["https://openalex.org/I4210110361"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103324571","display_name":"H. Mohamed","orcid":"https://orcid.org/0009-0003-8423-6079"},"institutions":[{"id":"https://openalex.org/I4210110361","display_name":"Mogadishu University","ror":"https://ror.org/025zbpk71","country_code":"SO","type":"education","lineage":["https://openalex.org/I4210110361"]}],"countries":["SO"],"is_corresponding":false,"raw_author_name":"Hanad Mohamud Mohamed","raw_affiliation_strings":["Masakhane, Mogadishu, Somalia"],"affiliations":[{"raw_affiliation_string":"Masakhane, Mogadishu, Somalia","institution_ids":["https://openalex.org/I4210110361"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086490002","display_name":"Tunde Oluwaseyi Ajayi","orcid":"https://orcid.org/0000-0002-1860-609X"},"institutions":[{"id":"https://openalex.org/I188760350","display_name":"Ollscoil na Gaillimhe \u2013 University of Galway","ror":"https://ror.org/03bea9k73","country_code":"IE","type":"education","lineage":["https://openalex.org/I188760350"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Tunde Oluwaseyi Ajayi","raw_affiliation_strings":["University of Galway, Galway, Ireland"],"affiliations":[{"raw_affiliation_string":"University of Galway, Galway, Ireland","institution_ids":["https://openalex.org/I188760350"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082997975","display_name":"Jimmy Lin","orcid":"https://orcid.org/0000-0002-0661-7189"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jimmy Lin","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institutions":[],"countries_distinct_count":8,"institutions_distinct_count":23,"corresponding_author_ids":["https://openalex.org/A5076302977"],"corresponding_institution_ids":["https://openalex.org/I151746483"],"apc_list":null,"apc_paid":null,"fwci":0.683,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.74165209,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"293","last_page":"302"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9657999873161316,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9657999873161316,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9524999856948853,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12377","display_name":"Digital Humanities and Scholarship","score":0.9519000053405762,"subfield":{"id":"https://openalex.org/subfields/1208","display_name":"Literature and Literary Theory"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7433246970176697},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6345850825309753},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.6007779240608215},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5863410234451294},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.36522936820983887}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7433246970176697},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6345850825309753},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.6007779240608215},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5863410234451294},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.36522936820983887},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3626772.3657884","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626772.3657884","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.800000011920929,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W1444168786","https://openalex.org/W1779279021","https://openalex.org/W1983903282","https://openalex.org/W2006832571","https://openalex.org/W2038253946","https://openalex.org/W2053154970","https://openalex.org/W2075893676","https://openalex.org/W2148972377","https://openalex.org/W2164628107","https://openalex.org/W2595551253","https://openalex.org/W2740321901","https://openalex.org/W2740887992","https://openalex.org/W2803620078","https://openalex.org/W2911319979","https://openalex.org/W3045462440","https://openalex.org/W3099700870","https://openalex.org/W3100107515","https://openalex.org/W3101279614","https://openalex.org/W3154280800","https://openalex.org/W3180230246","https://openalex.org/W3213418658","https://openalex.org/W4205103289","https://openalex.org/W4234071252","https://openalex.org/W4252076394","https://openalex.org/W4287855142","https://openalex.org/W4306317212","https://openalex.org/W4327498768","https://openalex.org/W4385574290","https://openalex.org/W4386488973"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Cross-lingual":[0],"information":[1,12],"retrieval":[2,13,67],"(CLIR)":[3],"continues":[4],"to":[5,25,138],"be":[6],"an":[7],"actively":[8],"studied":[9],"topic":[10],"in":[11,21,73,142,166],"(IR),":[14],"and":[15,71,80,90,105,151,158],"there":[16,30],"have":[17],"been":[18],"consistent":[19],"efforts":[20],"curating":[22],"test":[23,63,156],"collections":[24,45],"support":[26],"its":[27],"research.":[28],"However,":[29],"is":[31,173],"a":[32,62,93],"lack":[33],"of":[34,92,95,115,154,170],"high-quality":[35,107],"human-annotated":[36],"CLIR":[37,143],"resources":[38],"for":[39,56,65,133,144],"African":[40,75,88,145],"languages:":[41,76],"the":[42,116,131,149,168],"few":[43],"existing":[44],"are":[46,84],"mostly":[47],"curated":[48],"synthetically":[49],"or":[50],"from":[51,86],"sources":[52],"with":[53,68],"limited":[54],"corpora":[55,83],"these":[57],"languages.":[58,117,146],"We":[59,100,147],"present":[60],"CIRAL,":[61],"collection":[64,157],"cross-lingual":[66],"English":[69],"queries":[70,104],"passages":[72],"four":[74],"Hausa,":[77],"Somali,":[78],"Swahili,":[79],"Yoruba.":[81],"CIRAL's":[82,124,164],"obtained":[85,122],"Indigenous":[87],"websites":[89],"consist":[91],"total":[94],"over":[96,102],"2.5":[97],"million":[98],"passages.":[99],"gathered":[101],"1,600":[103],"30k":[106],"binary":[108],"relevance":[109],"judgments":[110],"annotated":[111],"by":[112],"native":[113],"speakers":[114],"Additional":[118],"pools":[119],"were":[120],"also":[121],"at":[123,130,175],"shared":[125],"task,":[126],"which":[127],"was":[128],"hosted":[129],"Forum":[132],"Information":[134],"Retrieval":[135],"Evaluation":[136],"2023":[137],"encourage":[139],"community":[140],"participation":[141],"describe":[148],"design":[150],"curation":[152],"process":[153],"our":[155],"provide":[159],"reproducible":[160],"baselines":[161],"that":[162],"demonstrate":[163],"utility":[165],"evaluating":[167],"effectiveness":[169],"systems.":[171],"CIRAL":[172],"available":[174],"https://github.com/ciralproject/ciral.":[176]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}