{"id":"https://openalex.org/W3034360475","doi":"https://doi.org/10.18653/v1/2020.acl-main.506","title":"Not All Claims are Created Equal: Choosing the Right Statistical Approach to Assess Hypotheses","display_name":"Not All Claims are Created Equal: Choosing the Right Statistical Approach to Assess Hypotheses","publication_year":2020,"publication_date":"2020-01-01","ids":{"openalex":"https://openalex.org/W3034360475","doi":"https://doi.org/10.18653/v1/2020.acl-main.506","mag":"3034360475"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2020.acl-main.506","doi":"https://doi.org/10.18653/v1/2020.acl-main.506","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.acl-main.506","pdf_url":"https://www.aclweb.org/anthology/2020.acl-main.506.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://www.aclweb.org/anthology/2020.acl-main.506.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000077084","display_name":"Erfan Sadeqi Azer","orcid":"https://orcid.org/0000-0002-4708-9351","observed_orcids":["https://orcid.org/0000-0002-4708-9351"]},"institutions":[{"id":"https://openalex.org/I592451","display_name":"Indiana University","ror":"https://ror.org/01kg8sb98","country_code":"US","type":"education","lineage":["https://openalex.org/I592451"]},{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Erfan Sadeqi Azer","raw_affiliation_strings":["Indiana University","University of Pennsylvania"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indiana University","institution_ids":["https://openalex.org/I592451"]},{"raw_affiliation_string":"University of Pennsylvania","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043628255","display_name":"Daniel Khashabi","orcid":"https://orcid.org/0009-0009-7664-2230","observed_orcids":["https://orcid.org/0009-0009-7664-2230"]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel Khashabi","raw_affiliation_strings":["Allen Institute for Artificial Intelligence"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for Artificial Intelligence","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077726785","display_name":"Ashish Sabharwal","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ashish Sabharwal","raw_affiliation_strings":["Allen Institute for Artificial Intelligence"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for Artificial Intelligence","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023802054","display_name":"Dan Roth","orcid":"https://orcid.org/0009-0002-1447-5173","observed_orcids":["https://orcid.org/0009-0002-1447-5173"]},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dan Roth","raw_affiliation_strings":["University of Pennsylvania"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pennsylvania","institution_ids":["https://openalex.org/I79576946"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5077726785"],"corresponding_institution_ids":["https://openalex.org/I4210156221"],"apc_list":null,"apc_paid":null,"fwci":8.2174,"has_fulltext":true,"cited_by_count":18,"citation_normalized_percentile":{"value":0.9820442,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"5715","last_page":"5725"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10243","display_name":"Statistical Methods and Bayesian Inference","score":0.5785999894142151,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10243","display_name":"Statistical Methods and Bayesian Inference","score":0.5785999894142151,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.08749999850988388,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.05290000140666962,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/statistical-significance-testing","display_name":"statistical significance testing","score":0.734000027179718},{"id":"https://openalex.org/keywords/p-values","display_name":"p-values","score":0.9300000071525574},{"id":"https://openalex.org/keywords/bayesian-hypothesis-testing","display_name":"Bayesian hypothesis testing","score":0.7919999957084656},{"id":"https://openalex.org/keywords/research-methodology","display_name":"research methodology","score":0.35199999809265137},{"id":"https://openalex.org/keywords/misinterpretation-of-statistical-results","display_name":"misinterpretation of statistical results","score":0.6110000014305115}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6660531759262085},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.6182003021240234},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5838358402252197},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.562275767326355},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.5506691932678223},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.5271468162536621},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5248710513114929},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.49902892112731934},{"id":"https://openalex.org/C2779377595","wikidata":"https://www.wikidata.org/wiki/Q21045424","display_name":"Approximate Bayesian computation","level":3,"score":0.4932247996330261},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.47694435715675354},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.4124924838542938},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3993515372276306},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3327784240245819},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.16200563311576843},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1348603069782257},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.18653/v1/2020.acl-main.506","doi":"https://doi.org/10.18653/v1/2020.acl-main.506","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.acl-main.506","pdf_url":"https://www.aclweb.org/anthology/2020.acl-main.506.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1911.03850","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1911.03850","pdf_url":"https://arxiv.org/pdf/1911.03850","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2984743581","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/1911.03850","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1911.03850","doi":"https://doi.org/10.48550/arxiv.1911.03850","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1911.03850","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2020.acl-main.506","doi":"https://doi.org/10.18653/v1/2020.acl-main.506","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.acl-main.506","pdf_url":"https://www.aclweb.org/anthology/2020.acl-main.506.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1472423617","display_name":null,"funder_award_id":"FA8750-19-2-1004","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G4713059963","display_name":null,"funder_award_id":"FA8750","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320309370","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10"},{"id":"https://openalex.org/F4320317052","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3034360475.pdf","grobid_xml":"https://content.openalex.org/works/W3034360475.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W88791450","https://openalex.org/W222053410","https://openalex.org/W307744830","https://openalex.org/W759515131","https://openalex.org/W850575977","https://openalex.org/W1541898988","https://openalex.org/W1977234485","https://openalex.org/W1991903527","https://openalex.org/W1993725118","https://openalex.org/W1994560811","https://openalex.org/W2006011720","https://openalex.org/W2027873737","https://openalex.org/W2045656233","https://openalex.org/W2049882758","https://openalex.org/W2056760934","https://openalex.org/W2095720871","https://openalex.org/W2103638447","https://openalex.org/W2105713027","https://openalex.org/W2108301111","https://openalex.org/W2111216449","https://openalex.org/W2118593414","https://openalex.org/W2160331326","https://openalex.org/W2164167147","https://openalex.org/W2171421863","https://openalex.org/W2476329375","https://openalex.org/W2521110250","https://openalex.org/W2610297457","https://openalex.org/W2794325560","https://openalex.org/W2798935874","https://openalex.org/W2801776818","https://openalex.org/W2891424260","https://openalex.org/W2896457183","https://openalex.org/W2899369816","https://openalex.org/W2963341956","https://openalex.org/W3124625789","https://openalex.org/W3133603537","https://openalex.org/W4247198512","https://openalex.org/W4247880210","https://openalex.org/W4251721956","https://openalex.org/W4394331269"],"related_works":["https://openalex.org/W2984743581","https://openalex.org/W2798935874","https://openalex.org/W4285717708","https://openalex.org/W4293118276","https://openalex.org/W2914098476","https://openalex.org/W4413963302","https://openalex.org/W2970363796","https://openalex.org/W3109301198","https://openalex.org/W1010449144","https://openalex.org/W2111216449"],"abstract_inverted_index":{"Empirical":[0],"research":[1,138],"in":[2,35,62,76,135],"Natural":[3],"Language":[4],"Processing":[5],"(NLP)":[6],"has":[7],"adopted":[8,34],"a":[9,112,140],"narrow":[10],"set":[11],"of":[12,164],"principles":[13],"for":[14,161],"assessing":[15],"hypotheses,":[16,165],"relying":[17],"mainly":[18],"on":[19,69],"p-value":[20],"computation,":[21],"which":[22],"suffers":[23],"from":[24,111],"several":[25],"known":[26],"issues.While":[27],"alternative":[28],"proposals":[29],"have":[30],"been":[31],"well-debated":[32],"and":[33,103,147],"other":[36],"fields,":[37],"they":[38,79],"remain":[39],"rarely":[40],"discussed":[41],"or":[42],"used":[43,61,125],"within":[44],"the":[45,63,77,123,136],"NLP":[46,137,151],"community.We":[47],"address":[48],"this":[49],"gap":[50],"by":[51],"contrasting":[52],"various":[53],"hypothesis":[54,91,106],"assessment":[55,95,107,163],"techniques,":[56],"especially":[57],"those":[58],"not":[59],"commonly":[60],"field":[64],"(such":[65],"as":[66,153,155],"evaluations":[67],"based":[68],"Bayesian":[70,162],"inference).Since":[71],"these":[72,131],"statistical":[73],"techniques":[74],"differ":[75],"hypotheses":[78],"can":[80],"support,":[81],"we":[82,143],"argue":[83],"that":[84,130],"practitioners":[85],"should":[86],"first":[87],"decide":[88],"their":[89],"target":[90],"before":[92],"choosing":[93],"an":[94,156],"method.This":[96],"is":[97],"crucial":[98],"because":[99],"common":[100],"fallacies,":[101],"misconceptions,":[102],"misinterpretation":[104],"surrounding":[105],"methods":[108],"often":[109],"stem":[110],"discrepancy":[113],"between":[114],"what":[115,122],"one":[116],"would":[117],"like":[118],"to":[119],"claim":[120],"versus":[121],"method":[124],"actually":[126],"assesses.Our":[127],"survey":[128],"reveals":[129],"issues":[132],"are":[133],"omnipresent":[134],"community.As":[139],"step":[141],"forward,":[142],"provide":[144],"best":[145],"practices":[146],"guidelines":[148],"tailored":[149],"towards":[150],"research,":[152],"well":[154],"easy-to-use":[157],"package":[158],"called":[159],"HyBayes":[160],"1":[166],"complementing":[167],"existing":[168],"tools.":[169]},"counts_by_year":[{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2020-06-19T00:00:00"}