{"id":"https://openalex.org/W3156135834","doi":"https://doi.org/10.1145/3404835.3463040","title":"Evaluating the Predictivity of IR Experiments","display_name":"Evaluating the Predictivity of IR Experiments","publication_year":2021,"publication_date":"2021-07-11","ids":{"openalex":"https://openalex.org/W3156135834","doi":"https://doi.org/10.1145/3404835.3463040","mag":"3156135834"},"language":"en","primary_location":{"id":"doi:10.1145/3404835.3463040","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463040","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031434805","display_name":"Lida Rashidi","orcid":"https://orcid.org/0000-0002-6189-3274"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Lida Rashidi","raw_affiliation_strings":["The University of Melbourne, Melbourne, VIC, Australia"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Melbourne, VIC, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041495909","display_name":"Justin Zobel","orcid":"https://orcid.org/0000-0001-6622-032X"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Justin Zobel","raw_affiliation_strings":["The University of Melbourne, Melbourne, VIC, Australia"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Melbourne, VIC, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081861848","display_name":"Alistair Moffat","orcid":"https://orcid.org/0000-0002-6638-0232"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Alistair Moffat","raw_affiliation_strings":["The University of Melbourne, Melbourne, VIC, Australia"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne, Melbourne, VIC, Australia","institution_ids":["https://openalex.org/I165779595"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5031434805"],"corresponding_institution_ids":["https://openalex.org/I165779595"],"apc_list":null,"apc_paid":null,"fwci":0.8262,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.7751601,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"1667","last_page":"1671"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9883999824523926,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.984000027179718,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8367017507553101},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.7494434118270874},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.7436066269874573},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6211913228034973},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5567453503608704},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.5388227701187134},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.49356067180633545},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3653295338153839}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8367017507553101},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.7494434118270874},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.7436066269874573},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6211913228034973},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5567453503608704},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.5388227701187134},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.49356067180633545},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3653295338153839},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3404835.3463040","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463040","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7400000095367432,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W1968927634","https://openalex.org/W1983164881","https://openalex.org/W1990190154","https://openalex.org/W1994594731","https://openalex.org/W2015338694","https://openalex.org/W2017292914","https://openalex.org/W2022995284","https://openalex.org/W2068333205","https://openalex.org/W2069870183","https://openalex.org/W2113026607","https://openalex.org/W2336806308","https://openalex.org/W2464171116","https://openalex.org/W2624553223","https://openalex.org/W2742156913","https://openalex.org/W2750746504","https://openalex.org/W2923053417","https://openalex.org/W3094540792"],"related_works":["https://openalex.org/W1585007175","https://openalex.org/W2382521049","https://openalex.org/W2144385241","https://openalex.org/W4300101996","https://openalex.org/W2165950148","https://openalex.org/W4253593777","https://openalex.org/W2951497643","https://openalex.org/W4403053866","https://openalex.org/W2142393343","https://openalex.org/W2772772214"],"abstract_inverted_index":{"Experimental":[0],"evaluation":[1,70,114],"is":[2,16,105,129,146],"regarded":[3],"as":[4,74,143,153],"a":[5,72,75,98],"critical":[6],"element":[7],"of":[8,22,42,52,69,77,111,133],"any":[9,134],"research":[10],"activity":[11],"in":[12,36,50,97,149,155],"Information":[13],"Retrieval,":[14],"and":[15,57,116,126],"typically":[17],"used":[18,56],"to":[19,45,107,117,161],"support":[20],"assertions":[21],"the":[23,40,43,47,54,58,67,109,120,144],"form":[24],"\"Technique":[25],"A":[26],"provides":[27],"better":[28,118],"retrieval":[29],"effectiveness":[30],"than":[31],"does":[32],"Technique":[33],"B\".":[34],"Implicit":[35],"such":[37,152],"claims":[38],"are":[39],"characteristics":[41],"data":[44],"which":[46],"results":[48,138],"apply,":[49],"terms":[51],"both":[53],"queries":[55],"documents":[59],"they":[60],"were":[61],"applied":[62],"to.":[63],"Here":[64],"we":[65,101],"explore":[66,108],"role":[68],"on":[71,80],"collection":[73,145],"prediction":[76],"relative":[78],"performance":[79],"collections":[81,91],"that":[82,92,103,128,140],"have":[83],"different":[84],"characteristics.":[85],"In":[86],"particular,":[87],"by":[88],"synthesizing":[89],"new":[90],"vary":[93],"from":[94,157],"each":[95],"other":[96],"controlled":[99],"way,":[100],"show":[102,139],"it":[104],"possible":[106],"reliability":[110],"an":[112,130],"IR":[113],"pipeline,":[115],"understand":[119],"complex":[121],"interrelationship":[122],"between":[123],"documents,":[124],"queries,":[125],"metrics":[127],"important":[131],"part":[132],"experimental":[135],"validation.":[136],"Our":[137],"predictivity":[141],"declines":[142],"varied,":[147],"even":[148],"simple":[150],"ways":[151],"shifting":[154],"focus":[156],"one":[158],"document":[159],"source":[160],"another":[162],"similar":[163],"source.":[164]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}