{"id":"https://openalex.org/W2739937203","doi":"https://doi.org/10.1145/3077136.3080766","title":"The Probability that Your Hypothesis Is Correct, Credible Intervals, and Effect Sizes for IR Evaluation","display_name":"The Probability that Your Hypothesis Is Correct, Credible Intervals, and Effect Sizes for IR Evaluation","publication_year":2017,"publication_date":"2017-07-28","ids":{"openalex":"https://openalex.org/W2739937203","doi":"https://doi.org/10.1145/3077136.3080766","mag":"2739937203"},"language":"en","primary_location":{"id":"doi:10.1145/3077136.3080766","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3080766","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["Waseda University, Tokyo, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Waseda University, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5023595778"],"corresponding_institution_ids":["https://openalex.org/I150744194"],"apc_list":null,"apc_paid":null,"fwci":1.2458,"has_fulltext":false,"cited_by_count":14,"citation_normalized_percentile":{"value":0.84577016,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"25","last_page":"34"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9789999723434448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9789999723434448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.9639999866485596,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12157","display_name":"Geochemistry and Geologic Mapping","score":0.9584000110626221,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.642270565032959},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.6336965560913086},{"id":"https://openalex.org/keywords/confidence-interval","display_name":"Confidence interval","score":0.6240006685256958},{"id":"https://openalex.org/keywords/credible-interval","display_name":"Credible interval","score":0.5842827558517456},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.539764404296875},{"id":"https://openalex.org/keywords/markov-chain-monte-carlo","display_name":"Markov chain Monte Carlo","score":0.5343496203422546},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.5147490501403809},{"id":"https://openalex.org/keywords/p-value","display_name":"p-value","score":0.5141599178314209},{"id":"https://openalex.org/keywords/test-statistic","display_name":"Test statistic","score":0.498779296875},{"id":"https://openalex.org/keywords/statistic","display_name":"Statistic","score":0.49451062083244324},{"id":"https://openalex.org/keywords/frequentist-probability","display_name":"Frequentist probability","score":0.4569679796695709},{"id":"https://openalex.org/keywords/empirical-probability","display_name":"Empirical probability","score":0.41959476470947266},{"id":"https://openalex.org/keywords/posterior-probability","display_name":"Posterior probability","score":0.3881579339504242}],"concepts":[{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.642270565032959},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.6336965560913086},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.6240006685256958},{"id":"https://openalex.org/C121117317","wikidata":"https://www.wikidata.org/wiki/Q4439221","display_name":"Credible interval","level":3,"score":0.5842827558517456},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.539764404296875},{"id":"https://openalex.org/C111350023","wikidata":"https://www.wikidata.org/wiki/Q1191869","display_name":"Markov chain Monte Carlo","level":3,"score":0.5343496203422546},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5147490501403809},{"id":"https://openalex.org/C44036329","wikidata":"https://www.wikidata.org/wiki/Q253255","display_name":"p-value","level":3,"score":0.5141599178314209},{"id":"https://openalex.org/C169857963","wikidata":"https://www.wikidata.org/wiki/Q1461038","display_name":"Test statistic","level":3,"score":0.498779296875},{"id":"https://openalex.org/C89128539","wikidata":"https://www.wikidata.org/wiki/Q1949963","display_name":"Statistic","level":2,"score":0.49451062083244324},{"id":"https://openalex.org/C75455068","wikidata":"https://www.wikidata.org/wiki/Q1455566","display_name":"Frequentist probability","level":3,"score":0.4569679796695709},{"id":"https://openalex.org/C97933134","wikidata":"https://www.wikidata.org/wiki/Q5374249","display_name":"Empirical probability","level":4,"score":0.41959476470947266},{"id":"https://openalex.org/C57830394","wikidata":"https://www.wikidata.org/wiki/Q278079","display_name":"Posterior probability","level":3,"score":0.3881579339504242}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3077136.3080766","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3080766","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"No poverty","id":"https://metadata.un.org/sdg/1","score":0.7799999713897705}],"awards":[{"id":"https://openalex.org/G1218873234","display_name":"Exploratory Search Considering the User's Situation","funder_award_id":"16H01756","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"}],"funders":[{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W39572901","https://openalex.org/W137875067","https://openalex.org/W1507711745","https://openalex.org/W1533179050","https://openalex.org/W1838260653","https://openalex.org/W1990190154","https://openalex.org/W2017292914","https://openalex.org/W2026430213","https://openalex.org/W2052569738","https://openalex.org/W2058624977","https://openalex.org/W2058896506","https://openalex.org/W2064459117","https://openalex.org/W2064604759","https://openalex.org/W2074466695","https://openalex.org/W2094779818","https://openalex.org/W2169713291","https://openalex.org/W2293040502","https://openalex.org/W2294145134","https://openalex.org/W2337811542","https://openalex.org/W2404188968","https://openalex.org/W2463093044","https://openalex.org/W2593269848","https://openalex.org/W2949068269","https://openalex.org/W2954040150","https://openalex.org/W2963977107","https://openalex.org/W3013531578","https://openalex.org/W3021094736","https://openalex.org/W3083113686","https://openalex.org/W4205110562","https://openalex.org/W4230900352","https://openalex.org/W4300031860"],"related_works":["https://openalex.org/W3098349241","https://openalex.org/W4281640962","https://openalex.org/W2068872878","https://openalex.org/W2320870285","https://openalex.org/W3037092793","https://openalex.org/W2354548288","https://openalex.org/W4285287907","https://openalex.org/W4240299387","https://openalex.org/W2330406685","https://openalex.org/W2739937203"],"abstract_inverted_index":{"Using":[0,177],"classical":[1,80,201,234],"statistical":[2],"significance":[3,235],"tests,":[4,127,264],"researchers":[5],"can":[6,112,129,212,237,251],"only":[7,285],"discuss":[8],"P(D+|H),":[9],"the":[10,14,24,27,33,42,50,79,94,101,107,122,162,241,249,268,272,274,278,287,295],"probability":[11,43,279],"of":[12,78,138,148,180,218,255,280,300],"observing":[13],"data":[15],"D":[16],"at":[17],"hand":[18],"or":[19],"something":[20],"more":[21],"extreme,":[22],"under":[23],"assumption":[25],"that":[26,44,93,100,151,206,248,267],"hypothesis":[28,46,95,281],"H":[29],"is":[30,40,47,70,96,109],"true":[31,102],"(i.e.,":[32],"p-value).":[34],"But":[35],"what":[36,219],"we":[37,53,111,128,152,185,220,265],"usually":[38],"want":[39],"P(H|D),":[41],"a":[45,73,178],"true,":[48,283],"given":[49],"data.":[51],"If":[52],"use":[54],"Bayesian":[55,126,171,196,242],"statistics":[56],"with":[57,125,188,192,200],"state-of-the-art":[58],"Markov":[59],"Chain":[60],"Monte":[61],"Carlo":[62],"(MCMC)":[63],"methods":[64],"for":[65,173,286,294],"obtaining":[66],"posterior":[67],"distributions,":[68],"this":[69],"no":[71],"longer":[72],"problem.":[74],"That":[75],"is,":[76],"instead":[77],"p-values":[81,208],"and":[82,98,116,140,169,182,195,209,225,228,262,277],"95%":[83],"confidence":[84,193,210],"intervals,":[85,194],"which":[86,119,246],"are":[87,153],"often":[88],"misinterpreted":[89],"respectively":[90,213],"as":[91,216],"\"probability":[92,99],"(in)correct\"":[97],"parameter":[103],"value":[104,147],"drops":[105],"within":[106],"interval":[108],"95%,\"":[110],"easily":[113,130],"obtain":[114,141],"P(H|D)":[115,187,224],"credible":[117,190,226,275],"intervals":[118,191,211],"represent":[120],"exactly":[121],"above.":[123],"Moreover,":[124],"handle":[131],"virtually":[132],"any":[133,149],"hypothesis,":[134],"not":[135,284],"just":[136],"\"equality":[137],"means,\"":[139],"an":[142],"Expected":[143],"A":[144],"Posteriori":[145],"(EAP)":[146],"statistic":[150],"interested":[154],"in.":[155],"We":[156],"provide":[157],"simple":[158],"tools":[159],"to":[160,165],"encourage":[161],"IR":[163,269],"community":[164,270],"take":[166],"up":[167],"paired":[168,261],"unpaired":[170,263],"tests":[172,236],"comparing":[174],"two":[175],"systems.":[176],"variety":[179],"TREC":[181],"NTCIR":[183],"data,":[184],"compare":[186],"p-values,":[189],"EAP":[197,243],"effect":[198,231,244,257,296],"sizes":[199,232],"ones.":[202],"Our":[203],"results":[204],"show":[205],"(a)":[207],"be":[214,252],"regarded":[215],"approximations":[217],"really":[221],"want,":[222],"namely,":[223],"intervals;":[227],"(b)":[229],"sample":[230],"from":[233,240],"differ":[238],"considerably":[239],"sizes,":[245],"suggests":[247],"former":[250],"poor":[253],"estimates":[254],"population":[256],"sizes.":[258],"For":[259],"both":[260],"propose":[266],"report":[271],"EAP,":[273],"interval,":[276],"being":[282],"raw":[288],"difference":[289],"in":[290,298],"means":[291],"but":[292],"also":[293],"size":[297],"terms":[299],"Glass's":[301],"\u0394.":[302]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}