{"id":"https://openalex.org/W4378945316","doi":"https://doi.org/10.1145/3539618.3591888","title":"The Information Retrieval Experiment Platform","display_name":"The Information Retrieval Experiment Platform","publication_year":2023,"publication_date":"2023-07-18","ids":{"openalex":"https://openalex.org/W4378945316","doi":"https://doi.org/10.1145/3539618.3591888"},"language":"en","primary_location":{"id":"doi:10.1145/3539618.3591888","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3539618.3591888","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3591888","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3591888","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068616222","display_name":"Maik Fr\u00f6be","orcid":"https://orcid.org/0000-0002-1003-981X"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Maik Fr\u00f6be","raw_affiliation_strings":["Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"],"raw_orcid":"https://orcid.org/0000-0002-1003-981X","affiliations":[{"raw_affiliation_string":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058247097","display_name":"Jan Heinrich Reimer","orcid":"https://orcid.org/0000-0003-1992-8696"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jan Heinrich Reimer","raw_affiliation_strings":["Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"],"raw_orcid":"https://orcid.org/0000-0003-1992-8696","affiliations":[{"raw_affiliation_string":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014199889","display_name":"Sean MacAvaney","orcid":"https://orcid.org/0000-0002-8914-2659"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Sean MacAvaney","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0002-8914-2659","affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068128049","display_name":"Niklas Deckers","orcid":"https://orcid.org/0000-0001-6803-1223"},"institutions":[{"id":"https://openalex.org/I4401726909","display_name":"Center for Scalable Data Analytics and Artificial Intelligence","ror":"https://ror.org/01t4ttr56","country_code":"DE","type":"education","lineage":["https://openalex.org/I4401726909","https://openalex.org/I78650965","https://openalex.org/I926574661"]},{"id":"https://openalex.org/I926574661","display_name":"Leipzig University","ror":"https://ror.org/03s7gtk40","country_code":"DE","type":"education","lineage":["https://openalex.org/I926574661"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Niklas Deckers","raw_affiliation_strings":["Leipzig University and ScaDS.AI, Leipzig, Germany"],"raw_orcid":"https://orcid.org/0000-0001-6803-1223","affiliations":[{"raw_affiliation_string":"Leipzig University and ScaDS.AI, Leipzig, Germany","institution_ids":["https://openalex.org/I4401726909","https://openalex.org/I926574661"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071918038","display_name":"S. Reich","orcid":"https://orcid.org/0009-0000-3036-126X"},"institutions":[{"id":"https://openalex.org/I926574661","display_name":"Leipzig University","ror":"https://ror.org/03s7gtk40","country_code":"DE","type":"education","lineage":["https://openalex.org/I926574661"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Simon Reich","raw_affiliation_strings":["Leipzig University, Leipzig, Germany"],"raw_orcid":"https://orcid.org/0009-0000-3036-126X","affiliations":[{"raw_affiliation_string":"Leipzig University, Leipzig, Germany","institution_ids":["https://openalex.org/I926574661"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079524270","display_name":"Janek Bevendorff","orcid":"https://orcid.org/0000-0002-3797-0559"},"institutions":[{"id":"https://openalex.org/I51441396","display_name":"Bauhaus-Universit\u00e4t Weimar","ror":"https://ror.org/033bb5z47","country_code":"DE","type":"education","lineage":["https://openalex.org/I51441396"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Janek Bevendorff","raw_affiliation_strings":["Bauhaus-Universit\u00e4t Weimar, Weimar, Germany"],"raw_orcid":"https://orcid.org/0000-0002-3797-0559","affiliations":[{"raw_affiliation_string":"Bauhaus-Universit\u00e4t Weimar, Weimar, Germany","institution_ids":["https://openalex.org/I51441396"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027915931","display_name":"Benno Stein","orcid":"https://orcid.org/0000-0001-9033-2217"},"institutions":[{"id":"https://openalex.org/I51441396","display_name":"Bauhaus-Universit\u00e4t Weimar","ror":"https://ror.org/033bb5z47","country_code":"DE","type":"education","lineage":["https://openalex.org/I51441396"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Benno Stein","raw_affiliation_strings":["Bauhaus-Universit\u00e4t Weimar, Weimar, Germany"],"raw_orcid":"https://orcid.org/0000-0001-9033-2217","affiliations":[{"raw_affiliation_string":"Bauhaus-Universit\u00e4t Weimar, Weimar, Germany","institution_ids":["https://openalex.org/I51441396"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014322854","display_name":"Matthias Hagen","orcid":"https://orcid.org/0000-0002-9733-2890"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matthias Hagen","raw_affiliation_strings":["Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany"],"raw_orcid":"https://orcid.org/0000-0002-9733-2890","affiliations":[{"raw_affiliation_string":"Friedrich-Schiller-Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5083712311","display_name":"Martin Potthast","orcid":"https://orcid.org/0000-0003-2451-0665"},"institutions":[{"id":"https://openalex.org/I4401726909","display_name":"Center for Scalable Data Analytics and Artificial Intelligence","ror":"https://ror.org/01t4ttr56","country_code":"DE","type":"education","lineage":["https://openalex.org/I4401726909","https://openalex.org/I78650965","https://openalex.org/I926574661"]},{"id":"https://openalex.org/I926574661","display_name":"Leipzig University","ror":"https://ror.org/03s7gtk40","country_code":"DE","type":"education","lineage":["https://openalex.org/I926574661"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Martin Potthast","raw_affiliation_strings":["Leipzig University and ScaDS.AI, Leipzig, Germany"],"raw_orcid":"https://orcid.org/0000-0003-2451-0665","affiliations":[{"raw_affiliation_string":"Leipzig University and ScaDS.AI, Leipzig, Germany","institution_ids":["https://openalex.org/I4401726909","https://openalex.org/I926574661"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":54,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2826","last_page":"2836"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9914000034332275,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9869999885559082,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8329415917396545},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.7429496645927429},{"id":"https://openalex.org/keywords/cloud-computing","display_name":"Cloud computing","score":0.7394420504570007},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5456661581993103},{"id":"https://openalex.org/keywords/standardization","display_name":"Standardization","score":0.496797114610672},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.47140100598335266},{"id":"https://openalex.org/keywords/server","display_name":"Server","score":0.43954741954803467},{"id":"https://openalex.org/keywords/sandbox","display_name":"Sandbox (software development)","score":0.42460185289382935},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.39668333530426025},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3592846691608429},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.3078833222389221}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8329415917396545},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.7429496645927429},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.7394420504570007},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5456661581993103},{"id":"https://openalex.org/C188087704","wikidata":"https://www.wikidata.org/wiki/Q369577","display_name":"Standardization","level":2,"score":0.496797114610672},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.47140100598335266},{"id":"https://openalex.org/C93996380","wikidata":"https://www.wikidata.org/wiki/Q44127","display_name":"Server","level":2,"score":0.43954741954803467},{"id":"https://openalex.org/C167981075","wikidata":"https://www.wikidata.org/wiki/Q2667186","display_name":"Sandbox (software development)","level":2,"score":0.42460185289382935},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.39668333530426025},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3592846691608429},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.3078833222389221}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3539618.3591888","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3539618.3591888","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3591888","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2305.18932","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.18932","pdf_url":"https://arxiv.org/pdf/2305.18932","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:eprints.gla.ac.uk:296336","is_oa":true,"landing_page_url":"https://eprints.gla.ac.uk/view/author/60888.html>","pdf_url":"https://eprints.gla.ac.uk/296336/2/296336.pdf","source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference Proceedings"}],"best_oa_location":{"id":"doi:10.1145/3539618.3591888","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3539618.3591888","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3591888","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4378945316.pdf","grobid_xml":"https://content.openalex.org/works/W4378945316.grobid-xml"},"referenced_works_count":118,"referenced_works":["https://openalex.org/W60779023","https://openalex.org/W94637519","https://openalex.org/W95791645","https://openalex.org/W132343450","https://openalex.org/W193708675","https://openalex.org/W299870200","https://openalex.org/W1483624827","https://openalex.org/W1515852583","https://openalex.org/W1546195864","https://openalex.org/W1554394657","https://openalex.org/W1845198550","https://openalex.org/W1965690069","https://openalex.org/W1981208470","https://openalex.org/W2006996936","https://openalex.org/W2052201263","https://openalex.org/W2052739481","https://openalex.org/W2119085739","https://openalex.org/W2168469656","https://openalex.org/W2388783363","https://openalex.org/W2406493898","https://openalex.org/W2468776465","https://openalex.org/W2470818894","https://openalex.org/W2740321901","https://openalex.org/W2752174485","https://openalex.org/W2792505675","https://openalex.org/W2794432940","https://openalex.org/W2883256923","https://openalex.org/W2887394688","https://openalex.org/W2894680593","https://openalex.org/W2910577570","https://openalex.org/W2917056928","https://openalex.org/W2917293077","https://openalex.org/W2917634187","https://openalex.org/W2920151874","https://openalex.org/W2964987975","https://openalex.org/W2965843218","https://openalex.org/W2966427072","https://openalex.org/W2968287414","https://openalex.org/W2969044820","https://openalex.org/W2970641574","https://openalex.org/W2971033966","https://openalex.org/W2982596739","https://openalex.org/W3001344098","https://openalex.org/W3001879867","https://openalex.org/W3013322209","https://openalex.org/W3016194403","https://openalex.org/W3017330897","https://openalex.org/W3020786614","https://openalex.org/W3021397474","https://openalex.org/W3034439313","https://openalex.org/W3034471540","https://openalex.org/W3038572442","https://openalex.org/W3044997488","https://openalex.org/W3045745713","https://openalex.org/W3085914694","https://openalex.org/W3099700870","https://openalex.org/W3100055683","https://openalex.org/W3104894527","https://openalex.org/W3104914963","https://openalex.org/W3118668786","https://openalex.org/W3118831779","https://openalex.org/W3128581554","https://openalex.org/W3130740619","https://openalex.org/W3132995484","https://openalex.org/W3133594109","https://openalex.org/W3142305657","https://openalex.org/W3154670582","https://openalex.org/W3154806249","https://openalex.org/W3156166379","https://openalex.org/W3156817235","https://openalex.org/W3158192345","https://openalex.org/W3172635446","https://openalex.org/W3175111331","https://openalex.org/W3180230246","https://openalex.org/W3200572611","https://openalex.org/W3209981429","https://openalex.org/W3217485291","https://openalex.org/W4221142817","https://openalex.org/W4225557520","https://openalex.org/W4236528112","https://openalex.org/W4284669679","https://openalex.org/W4284669756","https://openalex.org/W4284698816","https://openalex.org/W4284704763","https://openalex.org/W4286969177","https://openalex.org/W4287623114","https://openalex.org/W4287780085","https://openalex.org/W4292939163","https://openalex.org/W4306891354","https://openalex.org/W4307079201","https://openalex.org/W4313002899","https://openalex.org/W4318256943","https://openalex.org/W4327498231","https://openalex.org/W4327498761","https://openalex.org/W4385565351","https://openalex.org/W4385571334","https://openalex.org/W4385571883","https://openalex.org/W6602511761","https://openalex.org/W6603897584","https://openalex.org/W6603899496","https://openalex.org/W6605428294","https://openalex.org/W6628931948","https://openalex.org/W6641564765","https://openalex.org/W6678356393","https://openalex.org/W6753892773","https://openalex.org/W6753895321","https://openalex.org/W6758747439","https://openalex.org/W6759030081","https://openalex.org/W6766663659","https://openalex.org/W6767091586","https://openalex.org/W6775204245","https://openalex.org/W6784940813","https://openalex.org/W6790508278","https://openalex.org/W6805187312","https://openalex.org/W6910653033","https://openalex.org/W7042366687","https://openalex.org/W7054915730","https://openalex.org/W7070546444"],"related_works":["https://openalex.org/W2033352828","https://openalex.org/W2355810117","https://openalex.org/W3098313552","https://openalex.org/W70177500","https://openalex.org/W2546418048","https://openalex.org/W2076427967","https://openalex.org/W3212184609","https://openalex.org/W2499283203","https://openalex.org/W2795849205","https://openalex.org/W2111618996"],"abstract_inverted_index":{"We":[0],"integrate":[1],"irdatasets,":[2],"ir_measures,":[3],"and":[4,21,36,39,48,59,78,108,117],"PyTerrier":[5],"with":[6,46],"TIRA":[7,62,83],"in":[8,71,125],"the":[9,37,100,103,118],"Information":[10],"Retrieval":[11],"Experiment":[12],"Platform":[13],"(TIREx)":[14],"to":[15,122],"promote":[16],"more":[17],"standardized,":[18],"reproducible,":[19],"scalable,":[20],"even":[22],"blinded":[23],"retrieval":[24,31,119],"experiments.":[25],"Standardization":[26],"is":[27,54],"achieved":[28],"when":[29,88],"a":[30,55,72,93,126],"approach":[32],"implements":[33],"PyTerrier's":[34],"interfaces":[35],"input":[38],"output":[40],"of":[41,52,102],"an":[42,89],"experiment":[43,90],"are":[44,111],"compatible":[45],"ir_datasets":[47],"ir_measures.":[49],"However,":[50],"none":[51],"this":[53],"must":[56],"for":[57,85],"reproducibility":[58],"scalability,":[60],"as":[61],"can":[63],"run":[64],"any":[65],"dockerized":[66],"software":[67,120],"locally":[68],"or":[69,96],"remotely":[70],"cloud-native":[73],"execution":[74],"environment.":[75],"Version":[76],"control":[77,101],"caching":[79],"ensure":[80],"efficient":[81],"(re)execution.":[82],"allows":[84],"blind":[86],"evaluation":[87],"runs":[91],"on":[92],"remote":[94],"server":[95],"cloud":[97],"not":[98],"under":[99],"experimenter.":[104],"The":[105],"test":[106],"data":[107,130],"ground":[109],"truth":[110],"then":[112],"hidden":[113],"from":[114],"public":[115],"access,":[116],"has":[121],"process":[123],"them":[124],"sandbox":[127],"that":[128],"prevents":[129],"leaks.":[131]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":15},{"year":2024,"cited_by_count":30},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}