{"id":"https://openalex.org/W6991850254","doi":null,"title":"Information retrieval system evaluation: Effort, sensitivity, and reliability","display_name":"Information retrieval system evaluation: Effort, sensitivity, and reliability","alternate_titles":[],"publication_year":2005,"publication_date":"2005-01-01","ids":{"openalex":"https://openalex.org/W6991850254"},"language":"en","primary_location":{"id":"pmh:oai:researchbank.rmit.edu.au:rmit:1575","doi":null,"is_oa":false,"landing_page_url":"http://researchbank.rmit.edu.au/view/rmit:1575","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"},"parent_works":[],"lineage":["https://openalex.org/W6991850254"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Mark Sanderson","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mark Sanderson","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5041495909","display_name":"Justin Zobel","orcid":"https://orcid.org/0000-0001-6622-032X","observed_orcids":["https://orcid.org/0000-0001-6622-032X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Justin Zobel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.1422,"has_fulltext":false,"cited_by_count":23,"citation_normalized_percentile":{"value":0.76331506,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":86,"max":96},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2614","display_name":"Theoretical Computer Science"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/information-retrieval-evaluation","display_name":"information retrieval evaluation","score":0.8550000190734863},{"id":"https://openalex.org/keywords/significance-testing","display_name":"significance testing","score":0.8600000143051147},{"id":"https://openalex.org/keywords/t-test","display_name":"t-test","score":0.8410000205039978},{"id":"https://openalex.org/keywords/wilcoxon-test","display_name":"Wilcoxon test","score":0.7379999756813049},{"id":"https://openalex.org/keywords/sign-test","display_name":"sign test","score":0.9449999928474426},{"id":"https://openalex.org/keywords/mean-average-precision","display_name":"mean average precision","score":0.9980000257492065},{"id":"https://openalex.org/keywords/assessment-reliability","display_name":"assessment reliability","score":0.45500001311302185},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.35100001096725464}],"concepts":[{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7815999984741211},{"id":"https://openalex.org/C206041023","wikidata":"https://www.wikidata.org/wiki/Q1751970","display_name":"Wilcoxon signed-rank test","level":3,"score":0.7487000226974487},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6092000007629395},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5647000074386597},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5616000294685364},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.49619999527931213},{"id":"https://openalex.org/C139676723","wikidata":"https://www.wikidata.org/wiki/Q1193832","display_name":"Sign (mathematics)","level":2,"score":0.44999998807907104},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.44859999418258667},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.4334999918937683},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.42570000886917114},{"id":"https://openalex.org/C165793278","wikidata":"https://www.wikidata.org/wiki/Q2301904","display_name":"Sign test","level":4,"score":0.4065999984741211},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.38420000672340393},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.37540000677108765},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.35350000858306885},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.35040000081062317},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3393000066280365},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.33340001106262207},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.3107999861240387},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3025999963283539},{"id":"https://openalex.org/C183905921","wikidata":"https://www.wikidata.org/wiki/Q1038757","display_name":"Multiple comparisons problem","level":2,"score":0.2831000089645386},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.28139999508857727},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27970001101493835},{"id":"https://openalex.org/C161156560","wikidata":"https://www.wikidata.org/wiki/Q1638872","display_name":"Document retrieval","level":2,"score":0.2759000062942505}],"mesh":[],"locations_count":5,"locations":[{"id":"pmh:oai:researchbank.rmit.edu.au:rmit:1575","doi":null,"is_oa":false,"landing_page_url":"http://researchbank.rmit.edu.au/view/rmit:1575","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"},{"id":"pmh:oai:eprints.whiterose.ac.uk:4595","doi":null,"is_oa":false,"landing_page_url":"http://eprints.whiterose.ac.uk/4595/","pdf_url":null,"source":{"id":"https://openalex.org/S4306400854","display_name":"White Rose Research Online (University of Leeds, The University of Sheffield, University of York)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2800616092","host_organization_name":"White Rose University Consortium","host_organization_lineage":["https://openalex.org/I2800616092"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Proceedings Paper"},{"id":"pmh:oai:figshare.com:article/27335565","doi":null,"is_oa":false,"landing_page_url":"https://figshare.com/articles/conference_contribution/Information_retrieval_system_evaluation_Effort_sensitivity_and_reliability/27335565","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference contribution"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.492.6958","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.492.6958","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://itls.usu.edu/~beijxu/articles/effortSensibilityReliability.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.63.2383","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.63.2383","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://dis.shef.ac.uk/mark/cv/publications/papers/my_papers/SIGIR2005.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2015338694","https://openalex.org/W1985382447","https://openalex.org/W2914164530","https://openalex.org/W7097911801","https://openalex.org/W2052989395","https://openalex.org/W2064459117","https://openalex.org/W2138728775","https://openalex.org/W7097590176","https://openalex.org/W7096448026","https://openalex.org/W2424687048"],"abstract_inverted_index":{"The":[0],"effectiveness":[1,101],"of":[2,15,28,65,119,129],"information":[3],"retrieval":[4],"systems":[5],"is":[6,77,89],"measured":[7],"by":[8],"comparing":[9],"performance":[10],"on":[11,113],"a":[12,96],"common":[13],"set":[14],"queries":[16],"and":[17,88,134],"documents.":[18],"Significance":[19],"tests":[20,68,115],"are":[21],"often":[22],"used":[23],"to":[24,149],"evaluate":[25],"the":[26,52,63,75,83,117,127,145],"reliability":[27,128],"such":[29,35,67,120,151],"comparisons.":[30],"Previous":[31],"work":[32,44,112],"has":[33],"examined":[34],"tests,":[36],"but":[37,51],"produced":[38],"results":[39,107],"with":[40,166],"limited":[41],"application.":[42],"Other":[43],"established":[45],"an":[46],"alternative":[47],"benchmark":[48],"for":[49],"significance,":[50],"resulting":[53],"test":[54,164],"was":[55],"too":[56],"stringent.":[57],"In":[58],"this":[59],"paper,":[60],"we":[61],"revisit":[62],"question":[64],"how":[66],"should":[69],"be":[70,160],"used.":[71],"We":[72,122],"find":[73],"that":[74,109,139,156],"t-test":[76],"highly":[78],"reliable":[79,92],"(more":[80],"so":[81],"than":[82,93],"sign":[84],"or":[85],"Wilcoxon":[86],"test),":[87],"far":[90],"more":[91,167],"simply":[94],"showing":[95],"large":[97],"percentage":[98],"difference":[99],"in":[100,171],"measures":[102],"between":[103,126],"IR":[104],"systems.":[105],"Our":[106],"show":[108],"past":[110,140],"empirical":[111],"significance":[114],"over-estimated":[116],"error":[118],"tests.":[121],"also":[123],"re-consider":[124],"comparisons":[125,141],"precision":[130],"at":[131],"rank":[132],"10":[133],"mean":[135],"average":[136],"precision,":[137],"arguing":[138],"did":[142],"not":[143],"consider":[144],"assessor":[146,157],"effort":[147,158],"required":[148],"compute":[150],"measures.":[152],"This":[153],"investigation":[154],"shows":[155],"would":[159],"better":[161],"spent":[162],"building":[163],"collections":[165],"topics,":[168],"each":[169],"assessed":[170],"less":[172],"detail.":[173]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W6991850254","counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":1},{"year":2014,"cited_by_count":1},{"year":2013,"cited_by_count":3},{"year":2012,"cited_by_count":1}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2025-10-10T00:00:00"}
