{"id":"https://openalex.org/W115689999","doi":null,"title":"Repeatable evaluation of information retrieval effectiveness in dynamic environments","display_name":"Repeatable evaluation of information retrieval effectiveness in dynamic environments","publication_year":2006,"publication_date":"2006-01-01","ids":{"openalex":"https://openalex.org/W115689999","mag":"115689999"},"language":"en","primary_location":{"id":"mag:115689999","doi":null,"is_oa":false,"landing_page_url":"https://dl.acm.org/citation.cfm?id=1195576","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062591304","display_name":"Ophir Frieder","orcid":"https://orcid.org/0000-0001-5076-8171","observed_orcids":["https://orcid.org/0000-0001-5076-8171"]},"institutions":[{"id":"https://openalex.org/I180949307","display_name":"Illinois Institute of Technology","ror":"https://ror.org/037t3ry66","country_code":"US","type":"education","lineage":["https://openalex.org/I180949307"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ophir Frieder","raw_affiliation_strings":["Illinois Institute of Technology"],"raw_orcid":"https://orcid.org/0000-0001-5076-8171","affiliations":[{"raw_affiliation_string":"Illinois Institute of Technology","institution_ids":["https://openalex.org/I180949307"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064265420","display_name":"Eric C. Jensen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I180949307","display_name":"Illinois Institute of Technology","ror":"https://ror.org/037t3ry66","country_code":"US","type":"education","lineage":["https://openalex.org/I180949307"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eric C. Jensen","raw_affiliation_strings":["Illinois Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Illinois Institute of Technology","institution_ids":["https://openalex.org/I180949307"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I180949307"],"apc_list":null,"apc_paid":null,"fwci":2.4823,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.89994223,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":93},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/information-retrieval-evaluation","display_name":"information retrieval evaluation","score":0.7519999742507935},{"id":"https://openalex.org/keywords/dynamic-environments","display_name":"dynamic environments","score":0.6729999780654907},{"id":"https://openalex.org/keywords/search-engines","display_name":"search engines","score":0.7459999918937683},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.5699999928474426},{"id":"https://openalex.org/keywords/bootstrap-method","display_name":"bootstrapping","score":0.46000000834465027}],"concepts":[{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.8281784057617188},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7963228225708008},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6919286847114563},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6840956807136536},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5867080688476562},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.478372722864151},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.47122088074684143},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.46143481135368347},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.423525333404541},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.41529107093811035},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.25560086965560913},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.0},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:115689999","doi":null,"is_oa":false,"landing_page_url":"https://dl.acm.org/citation.cfm?id=1195576","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":75,"referenced_works":["https://openalex.org/W60779023","https://openalex.org/W75771211","https://openalex.org/W173028006","https://openalex.org/W1488449674","https://openalex.org/W1495008003","https://openalex.org/W1498152603","https://openalex.org/W1553330854","https://openalex.org/W1596250955","https://openalex.org/W1598759141","https://openalex.org/W1859938857","https://openalex.org/W1964038241","https://openalex.org/W1982858363","https://openalex.org/W1982889956","https://openalex.org/W1983595289","https://openalex.org/W1987690005","https://openalex.org/W1988121588","https://openalex.org/W1988520084","https://openalex.org/W1989626928","https://openalex.org/W1989771757","https://openalex.org/W2004758472","https://openalex.org/W2009155701","https://openalex.org/W2015338694","https://openalex.org/W2015629112","https://openalex.org/W2017627972","https://openalex.org/W2020686607","https://openalex.org/W2022122406","https://openalex.org/W2029071516","https://openalex.org/W2029341294","https://openalex.org/W2031088642","https://openalex.org/W2037150236","https://openalex.org/W2045998703","https://openalex.org/W2048987619","https://openalex.org/W2051987487","https://openalex.org/W2053584122","https://openalex.org/W2057970600","https://openalex.org/W2060487963","https://openalex.org/W2064522604","https://openalex.org/W2067472300","https://openalex.org/W2069523442","https://openalex.org/W2071098104","https://openalex.org/W2073392286","https://openalex.org/W2073722401","https://openalex.org/W2073913493","https://openalex.org/W2075893676","https://openalex.org/W2077046902","https://openalex.org/W2086446587","https://openalex.org/W2087486280","https://openalex.org/W2090715008","https://openalex.org/W2100240666","https://openalex.org/W2104679989","https://openalex.org/W2109244020","https://openalex.org/W2109890018","https://openalex.org/W2122030147","https://openalex.org/W2125125501","https://openalex.org/W2126843316","https://openalex.org/W2130145422","https://openalex.org/W2133986984","https://openalex.org/W2136634361","https://openalex.org/W2144200917","https://openalex.org/W2147164982","https://openalex.org/W2151005602","https://openalex.org/W2152314154","https://openalex.org/W2154724067","https://openalex.org/W2155063211","https://openalex.org/W2156037541","https://openalex.org/W2157748587","https://openalex.org/W2158676623","https://openalex.org/W2160892561","https://openalex.org/W2162495634","https://openalex.org/W2164380650","https://openalex.org/W2165387257","https://openalex.org/W2170188121","https://openalex.org/W2170714116","https://openalex.org/W2917442971","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W2077620588","https://openalex.org/W7095292768","https://openalex.org/W2109890018","https://openalex.org/W2894728481","https://openalex.org/W2125125501","https://openalex.org/W7095116628","https://openalex.org/W2132232498","https://openalex.org/W7097590176","https://openalex.org/W1500609524","https://openalex.org/W7096210818"],"abstract_inverted_index":{"In":[0,50],"dynamic":[1],"environments,":[2],"such":[3,32,85],"as":[4,33,40],"the":[5,93,98,144,175],"World":[6],"Wide":[7],"Web,":[8],"a":[9,122,140,179],"changing":[10],"document":[11],"collection,":[12],"query":[13,80],"population,":[14],"and":[15,158,183],"set":[16],"of":[17,23,62,97,103,143,148,177,185],"search":[18,24,151],"services":[19],"demands":[20],"frequent":[21],"repetition":[22],"effectiveness":[25],"(relevance)":[26],"evaluations.":[27],"Reconstructing":[28],"static":[29,118],"test":[30,77],"collections,":[31],"in":[34,84,156],"TREC,":[35],"requires":[36],"considerable":[37],"human":[38],"effort,":[39],"large":[41],"collection":[42],"sizes":[43,82],"demand":[44],"judgments":[45,163,166],"deep":[46],"into":[47],"retrieved":[48],"pools.":[49],"practice":[51],"it":[52],"is":[53],"common":[54],"to":[55,74,126],"perform":[56],"shallow":[57],"evaluations":[58,86],"over":[59],"small":[60],"numbers":[61],"conditions":[63],"(often":[64],"binary,":[65],"A":[66],"vs.":[67],"B)":[68],"without":[69],"system":[70],"pooling,":[71],"not":[72],"intending":[73],"construct":[75],"reusable":[76],"collections.":[78,119],"The":[79],"sample":[81],"required":[83],"can":[87],"be":[88],"reliably":[89],"estimated":[90],"by":[91,130,190],"leveraging":[92],"simple":[94],"bootstrap":[95],"estimate":[96],"reproducibility":[99],"probability":[100],"(observed":[101],"power)":[102],"hypothesis":[104],"tests.":[105],"However,":[106],"they":[107],"are":[108,115],"typically":[109],"much":[110],"larger":[111],"than":[112],"those":[113,184],"that":[114],"needed":[116],"for":[117],"We":[120,135],"propose":[121],"semiautomatic":[123],"evaluation":[124,133,142],"framework":[125,138],"reduce":[127],"this":[128,137],"effort":[129],"enabling":[131],"intelligent":[132],"strategies.":[134],"validate":[136],"against":[139],"manual":[141,162],"top":[145],"ten":[146,149],"results":[147],"web":[150,171],"engines":[152],"across":[153],"896":[154],"queries":[155],"navigational":[157],"informational":[159],"tasks.":[160],"Augmenting":[161],"with":[164],"pseudo-relevance":[165],"mined,":[167],"even":[168],"naively,":[169],"from":[170],"taxonomies":[172],"reduces":[173],"both":[174],"chances":[176],"missing":[178],"correct":[180],"binary":[181],"conclusion":[182,189],"finding":[186],"an":[187],"errant":[188],"approximately":[191],"half.":[192]},"counts_by_year":[{"year":2015,"cited_by_count":1},{"year":2013,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2016-06-24T00:00:00"}