{"id":"https://openalex.org/W7161108805","doi":"https://doi.org/10.48550/arxiv.2605.13801","title":"Improving Reproducibility in Evaluation through Multi-Level Annotator Modeling","display_name":"Improving Reproducibility in Evaluation through Multi-Level Annotator Modeling","publication_year":2026,"publication_date":"2026-05-13","ids":{"openalex":"https://openalex.org/W7161108805","doi":"https://doi.org/10.48550/arxiv.2605.13801"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.13801","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13801","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.13801","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011457544","display_name":"Deepak Pandita","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pandita, Deepak","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136104098","display_name":"Flip Korn","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Korn, Flip","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106085231","display_name":"Chris Welty","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Welty, Chris","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136137576","display_name":"Christopher M. Homan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Homan, Christopher M.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.39570000767707825,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.39570000767707825,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.11249999701976776,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.057100001722574234,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.660099983215332},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.642799973487854},{"id":"https://openalex.org/keywords/bootstrapping","display_name":"Bootstrapping (finance)","score":0.6327000260353088},{"id":"https://openalex.org/keywords/spurious-relationship","display_name":"Spurious relationship","score":0.43799999356269836},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.43720000982284546},{"id":"https://openalex.org/keywords/statistical-model","display_name":"Statistical model","score":0.40790000557899475},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.3982999920845032},{"id":"https://openalex.org/keywords/repeatability","display_name":"Repeatability","score":0.3686999976634979},{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.36239999532699585}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7545999884605408},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.660099983215332},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.642799973487854},{"id":"https://openalex.org/C207609745","wikidata":"https://www.wikidata.org/wiki/Q4944086","display_name":"Bootstrapping (finance)","level":2,"score":0.6327000260353088},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5307000279426575},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5087000131607056},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.43799999356269836},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.43720000982284546},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.40790000557899475},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.3982999920845032},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.38359999656677246},{"id":"https://openalex.org/C154020017","wikidata":"https://www.wikidata.org/wiki/Q520171","display_name":"Repeatability","level":2,"score":0.3686999976634979},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.36239999532699585},{"id":"https://openalex.org/C37381756","wikidata":"https://www.wikidata.org/wiki/Q20203288","display_name":"Representativeness heuristic","level":2,"score":0.34040001034736633},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.3393000066280365},{"id":"https://openalex.org/C154504017","wikidata":"https://www.wikidata.org/wiki/Q853614","display_name":"Identifier","level":2,"score":0.3361000120639801},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.3319999873638153},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3163999915122986},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.30809998512268066},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.29919999837875366},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.29910001158714294},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.28060001134872437},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.27090001106262207},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.2696000039577484},{"id":"https://openalex.org/C183905921","wikidata":"https://www.wikidata.org/wiki/Q1038757","display_name":"Multiple comparisons problem","level":2,"score":0.26170000433921814},{"id":"https://openalex.org/C2780224610","wikidata":"https://www.wikidata.org/wiki/Q1530061","display_name":"Credibility","level":2,"score":0.25839999318122864},{"id":"https://openalex.org/C169345407","wikidata":"https://www.wikidata.org/wiki/Q8216221","display_name":"Uncorrelated","level":2,"score":0.257999986410141},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.25760000944137573},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.2540000081062317},{"id":"https://openalex.org/C3018076075","wikidata":"https://www.wikidata.org/wiki/Q1826427","display_name":"Variance components","level":2,"score":0.2529999911785126}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.13801","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13801","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.13801","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13801","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"generative":[1],"AI":[2,26],"models":[3,8,49],"such":[4],"as":[5,82],"large":[6,135],"language":[7],"(LLMs)":[9],"become":[10],"more":[11],"pervasive,":[12],"ensuring":[13],"the":[14,83,106,145,148,154],"safety,":[15,53],"robustness,":[16],"and":[17,37,52,58,104,139,153],"overall":[18],"trustworthiness":[19],"of":[20,96,137,150,156],"these":[21],"systems":[22],"is":[23,27,67],"paramount.":[24],"However,":[25],"currently":[28],"facing":[29],"a":[30,93,122,134],"reproducibility":[31],"crisis":[32],"driven":[33],"by":[34],"unreliable":[35],"evaluations":[36],"unrepeatable":[38],"experimental":[39,78],"results.":[40],"While":[41],"human":[42],"raters":[43],"are":[44],"often":[45],"used":[46],"to":[47,75,102,111,126,162],"assess":[48],"for":[50],"utility":[51],"they":[54],"introduce":[55,121],"divergent":[56],"biases":[57],"subjective":[59],"opinions":[60],"into":[61],"their":[62],"annotations.":[63],"Overcoming":[64],"this":[65,118],"variance":[66,114],"exceptionally":[68],"challenging":[69],"because":[70],"very":[71],"little":[72],"data":[73],"exists":[74],"study":[76],"how":[77],"repeatability":[79],"actually":[80],"improves":[81],"annotator":[84,128],"pool":[85],"grows.":[86],"Standard":[87],"evaluation":[88],"practices":[89],"typically":[90],"rely":[91],"on":[92],"small":[94],"number":[95,136,149,155],"annotations":[97],"per":[98,158],"item":[99,159],"(often":[100],"3":[101],"5)":[103],"lack":[105],"persistent":[107,140],"rater":[108,141],"identifiers":[109],"necessary":[110],"model":[112,127],"individual":[113],"across":[115],"items.":[116],"In":[117],"work,":[119],"we":[120,143],"multi-level":[123],"bootstrapping":[124],"approach":[125],"behavior":[129],"realistically.":[130],"Leveraging":[131],"datasets":[132],"with":[133],"ratings":[138],"identifiers,":[142],"analyze":[144],"tradeoffs":[146],"between":[147],"items":[151],"($N$)":[152],"responses":[157],"($K$)":[160],"required":[161],"achieve":[163],"statistical":[164],"significance.":[165]},"counts_by_year":[],"updated_date":"2026-08-06T08:24:18.245995","created_date":"2026-05-15T00:00:00"}