{"id":"https://openalex.org/W4400519911","doi":"https://doi.org/10.1145/3641525.3663619","title":"Toward Evaluating the Reproducibility of Information Retrieval Systems with Simulated Users","display_name":"Toward Evaluating the Reproducibility of Information Retrieval Systems with Simulated Users","publication_year":2024,"publication_date":"2024-06-18","ids":{"openalex":"https://openalex.org/W4400519911","doi":"https://doi.org/10.1145/3641525.3663619"},"language":"en","primary_location":{"id":"doi:10.1145/3641525.3663619","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3641525.3663619","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3641525.3663619?download=true","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3641525.3663619?download=true","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067984355","display_name":"Timo Breuer","orcid":"https://orcid.org/0000-0002-1765-2449"},"institutions":[{"id":"https://openalex.org/I102520234","display_name":"TH K\u00f6ln - University of Applied Sciences","ror":"https://ror.org/014nnvj65","country_code":"DE","type":"education","lineage":["https://openalex.org/I102520234"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Timo Breuer","raw_affiliation_strings":["TH K\u00f6ln (University of Applied Sciences), Germany"],"raw_orcid":"https://orcid.org/0000-0002-1765-2449","affiliations":[{"raw_affiliation_string":"TH K\u00f6ln (University of Applied Sciences), Germany","institution_ids":["https://openalex.org/I102520234"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078639165","display_name":"Maria Maistro","orcid":"https://orcid.org/0000-0002-7001-4817"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Maria Maistro","raw_affiliation_strings":["University of Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0002-7001-4817","affiliations":[{"raw_affiliation_string":"University of Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.0217,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.86338146,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"25","last_page":"29"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.7716957926750183},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7336088418960571},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5426000952720642},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.11853700876235962},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.0997367799282074}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.7716957926750183},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7336088418960571},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5426000952720642},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.11853700876235962},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0997367799282074}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3641525.3663619","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3641525.3663619","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3641525.3663619?download=true","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.atira.dk:publications/0fe81854-27d2-47b8-b156-8266e92e93ff","is_oa":true,"landing_page_url":"https://researchprofiles.ku.dk/da/publications/0fe81854-27d2-47b8-b156-8266e92e93ff","pdf_url":"https://curis.ku.dk/ws/files/412792416/Toward_Evaluating.pdf","source":{"id":"https://openalex.org/S4306401983","display_name":"Research at the University of Copenhagen (University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I124055696","host_organization_name":"University of Copenhagen","host_organization_lineage":["https://openalex.org/I124055696"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Breuer , T & Maistro , M 2024 , Toward Evaluating the Reproducibility of Information Retrieval Systems with Simulated Users . in Proceedings of the 2nd ACM Conference on Reproducibility and Replicability, REP 2024 . Association for Computing Machinery, Inc. , pp. 25-29 , 2nd ACM Conference on Reproducibility and Replicability, REP 2024 , Rennes , France , 18/06/2024 . https://doi.org/10.1145/3641525.3663619","raw_type":"contributionToPeriodical"}],"best_oa_location":{"id":"doi:10.1145/3641525.3663619","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3641525.3663619","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3641525.3663619?download=true","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/5","display_name":"Gender equality","score":0.5799999833106995}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4400519911.pdf"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W1574680346","https://openalex.org/W2038841179","https://openalex.org/W2067833766","https://openalex.org/W2077734850","https://openalex.org/W2090715008","https://openalex.org/W2113640060","https://openalex.org/W2156186232","https://openalex.org/W2326990580","https://openalex.org/W2465191537","https://openalex.org/W2802553787","https://openalex.org/W2891211322","https://openalex.org/W2964020128","https://openalex.org/W3034471540","https://openalex.org/W3045745713","https://openalex.org/W3104914963","https://openalex.org/W3128581554","https://openalex.org/W3133594109","https://openalex.org/W3142305657","https://openalex.org/W3155235146","https://openalex.org/W3156166379","https://openalex.org/W4206396334","https://openalex.org/W4284697843","https://openalex.org/W4306317758","https://openalex.org/W4318256943","https://openalex.org/W4324138256","https://openalex.org/W6911290334"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W2087830269","https://openalex.org/W3106281778","https://openalex.org/W4324138256"],"abstract_inverted_index":{"Reproducibility":[0],"is":[1,90],"a":[2],"fundamental":[3],"part":[4,55,62],"of":[5,31,56,63,79,84,101],"scientific":[6,11],"progress.":[7],"Compared":[8],"to":[9],"other":[10],"fields,":[12],"computational":[13,32],"sciences":[14],"are":[15,45,72],"privileged":[16],"as":[17,54,61],"experimental":[18],"setups":[19],"can":[20],"be":[21,51],"preserved":[22],"with":[23,74],"ease,":[24],"and":[25,77],"regression":[26],"experiments":[27,71],"allow":[28],"the":[29,42,49,98,102],"validation":[30],"results":[33],"by":[34,93],"bitwise":[35],"similarity.":[36],"When":[37],"evaluating":[38],"information":[39],"access":[40],"systems,":[41],"system":[43,89],"users":[44],"often":[46,91],"considered":[47],"in":[48],"experiments,":[50],"it":[52,96],"explicit":[53],"user":[57],"studies":[58],"or":[59],"implicit":[60],"evaluation":[64],"measures.":[65],"Usually,":[66],"system-oriented":[67],"Information":[68,86],"Retrieval":[69,87],"(IR)":[70,88],"evaluated":[73],"effectiveness":[75,100],"measurements":[76],"batches":[78],"multiple":[80],"queries.":[81],"Successful":[82],"reproduction":[83],"an":[85],"determined":[92],"how":[94],"well":[95],"approximates":[97],"averaged":[99],"original":[103],"(reproduced)":[104],"system.":[105]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2025-10-10T00:00:00"}