{"id":"https://openalex.org/W7140119748","doi":"https://doi.org/10.18653/v1/2026.findings-eacl.223","title":"How Many Ratings per Item are Necessary for Reliable Significance Testing?","display_name":"How Many Ratings per Item are Necessary for Reliable Significance Testing?","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7140119748","doi":"https://doi.org/10.18653/v1/2026.findings-eacl.223"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.findings-eacl.223","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-eacl.223","pdf_url":"https://aclanthology.org/2026.findings-eacl.223.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2026","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.findings-eacl.223.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Christopher M Homan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Christopher M Homan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Flip Korn","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Flip Korn","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Deepak Pandita","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deepak Pandita","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Chris Welty","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chris Welty","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.28051948,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4258","last_page":"4273"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.6079999804496765,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.6079999804496765,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.014800000004470348,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11032","display_name":"VLSI and Analog Circuit Testing","score":0.006099999882280827,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.35429999232292175},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.3325999975204468},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.25529998540878296},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.22789999842643738},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.22349999845027924}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4043000042438507},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.35429999232292175},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.3325999975204468},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2644999921321869},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.26440000534057617},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.25529998540878296},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24899999797344208},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.22789999842643738},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.22349999845027924},{"id":"https://openalex.org/C2986587452","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical analysis","level":2,"score":0.22310000658035278}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.findings-eacl.223","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-eacl.223","pdf_url":"https://aclanthology.org/2026.findings-eacl.223.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2026","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.findings-eacl.223","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-eacl.223","pdf_url":"https://aclanthology.org/2026.findings-eacl.223.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7140119748.pdf","grobid_xml":"https://content.openalex.org/works/W7140119748.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"A":[0],"cornerstone":[1],"of":[2,55,71,96,142,153],"machine":[3,102],"learning":[4,103],"evaluation":[5],"is":[6,145],"the":[7,47,94,154],"(often":[8],"hidden)":[9],"assumption":[10],"that":[11,66,170],"model":[12,72,139],"and":[13,35,99,140,168],"human":[14,143],"responses":[15,83,115,134,165,176],"are":[16,68,74],"reliable":[17,120],"enough":[18,114,175],"to":[19,42,92,105,118,151,192],"evaluate":[20,93],"models":[21],"against":[22],"unitary,":[23],"authoritative,":[24],"\"gold":[25],"standard\"":[26],"data,":[27],"via":[28],"simple":[29],"metrics":[30,98],"such":[31],"as":[32],"accuracy,":[33],"precision,":[34],"recall.The":[36],"generative":[37],"AI":[38,185,196],"revolution":[39],"would":[40],"seem":[41],"explode":[43],"this":[44],"assumption,":[45],"given":[46],"critical":[48],"role":[49],"stochastic":[50],"inference":[51],"plays.Yet,":[52],"in":[53,61],"spite":[54],"public":[56],"demand":[57],"for":[58,101,127,195],"more":[59],"transparency":[60],"AI-along":[62],"with":[63,162],"strong":[64],"evidence":[65],"humans":[67],"unreliable":[69],"judgesestimates":[70],"reliability":[73,95],"conventionally":[75],"based":[76],"on,":[77],"at":[78],"most,":[79],"a":[80,88],"few":[81,156],"output":[82],"per":[84,116,135,166,177],"input":[85],"item.We":[86,178],"adapt":[87],"method,":[89],"previously":[90],"used":[91],"various":[97],"estimators":[100],"evaluation,":[104],"determine":[106],"whether":[107],"an":[108],"(existing":[109],"or":[110],"planned)":[111],"dataset":[112],"has":[113],"item":[117,136,167],"assure":[119],"null":[121],"hypothesis":[122],"statistical":[123],"testing.We":[124],"show":[125,169,179],"that,":[126],"many":[128],"common":[129],"metrics,":[130],"collecting":[131],"even":[132,171],"5-10":[133],"(from":[137],"each":[138],"team":[141],"evaluators)":[144],"not":[146],"sufficient.We":[147],"apply":[148],"our":[149,181],"methods":[150,182],"several":[152],"very":[155],"extant":[157],"gold":[158],"standard":[159],"test":[160],"sets":[161],"multiple":[163],"disaggregated":[164],"these":[172],"datasets":[173],"lack":[174],"how":[180,191],"can":[183],"help":[184],"researchers":[186],"make":[187],"better":[188],"decisions":[189],"about":[190],"collect":[193],"data":[194],"evaluation.":[197]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-02-02T00:00:00"}