{"id":"https://openalex.org/W7152991919","doi":"https://doi.org/10.48550/arxiv.2604.07591","title":"From Ground Truth to Measurement: A Statistical Framework for Human Labeling","display_name":"From Ground Truth to Measurement: A Statistical Framework for Human Labeling","publication_year":2026,"publication_date":"2026-04-08","ids":{"openalex":"https://openalex.org/W7152991919","doi":"https://doi.org/10.48550/arxiv.2604.07591"},"language":"en","primary_location":{"id":"pmh:oai:epub.ub.uni-muenchen.de:135761","is_oa":false,"landing_page_url":"http://nbn-resolving.de/urn:nbn:de:bvb:19-epub-135761-1","pdf_url":null,"source":{"id":"https://openalex.org/S4306401845","display_name":"Open access LMU (Ludwid Maxmilian's Universitat Munchen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I8204097","host_organization_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","host_organization_lineage":["https://openalex.org/I8204097"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"NonPeerReviewed"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.07591","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078399314","display_name":"Robert Chew","orcid":"https://orcid.org/0000-0002-6979-1766"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chew, Robert","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085081313","display_name":"Stephanie Eckman","orcid":"https://orcid.org/0000-0002-8577-4085"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Eckman, Stephanie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066229916","display_name":"Christoph Kern","orcid":"https://orcid.org/0000-0001-7363-4299"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kern, Christoph","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133340821","display_name":"Frauke Kreuter","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kreuter, Frauke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.26930001378059387,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.26930001378059387,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.13840000331401825,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.13199999928474426,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/variation","display_name":"Variation (astronomy)","score":0.5964000225067139},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5914000272750854},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5217000246047974},{"id":"https://openalex.org/keywords/statistical-model","display_name":"Statistical model","score":0.4101000130176544},{"id":"https://openalex.org/keywords/statistical-inference","display_name":"Statistical inference","score":0.4081999957561493},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.40450000762939453},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.37610000371932983},{"id":"https://openalex.org/keywords/supervised-learning","display_name":"Supervised learning","score":0.3564999997615814}],"concepts":[{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6862999796867371},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6776000261306763},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.5964000225067139},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5914000272750854},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5651000142097473},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5217000246047974},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5047000050544739},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.4101000130176544},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.4081999957561493},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.40450000762939453},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.37610000371932983},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.3564999997615814},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3515999913215637},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.32260000705718994},{"id":"https://openalex.org/C32254414","wikidata":"https://www.wikidata.org/wiki/Q4724364","display_name":"Algorithmic learning theory","level":3,"score":0.31619998812675476},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.31200000643730164},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.30630001425743103},{"id":"https://openalex.org/C2982736386","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Statistical learning","level":2,"score":0.298799991607666},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.2930000126361847},{"id":"https://openalex.org/C177877439","wikidata":"https://www.wikidata.org/wiki/Q7604413","display_name":"Statistical relational learning","level":3,"score":0.2849000096321106},{"id":"https://openalex.org/C9114305","wikidata":"https://www.wikidata.org/wiki/Q1428317","display_name":"Situational ethics","level":2,"score":0.28360000252723694},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.2809999883174896},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.2565000057220459},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.2547000050544739}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:epub.ub.uni-muenchen.de:135761","is_oa":false,"landing_page_url":"http://nbn-resolving.de/urn:nbn:de:bvb:19-epub-135761-1","pdf_url":null,"source":{"id":"https://openalex.org/S4306401845","display_name":"Open access LMU (Ludwid Maxmilian's Universitat Munchen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I8204097","host_organization_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","host_organization_lineage":["https://openalex.org/I8204097"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"NonPeerReviewed"},{"id":"doi:10.48550/arxiv.2604.07591","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.07591","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.07591","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.07591","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5830948948860168}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Supervised":[0],"machine":[1,158],"learning":[2,36,159],"assumes":[3],"that":[4],"labeled":[5],"data":[6],"provide":[7],"accurate":[8],"measurements":[9],"of":[10,52,77,101,110,149,169,174],"the":[11,126,147,167],"concepts":[12],"models":[13,54,93],"are":[14],"meant":[15],"to":[16,94,129],"learn.":[17,56],"Yet":[18],"in":[19],"practice,":[20],"human":[21,106],"labeling":[22,72],"introduces":[23,66],"systematic":[24,172],"variation":[25,108],"arising":[26],"from":[27],"ambiguous":[28],"items,":[29],"divergent":[30],"interpretations,":[31],"and":[32,48,65,85,98,105,112,145,160],"simple":[33],"mistakes.":[34],"Machine":[35],"research":[37],"commonly":[38],"treats":[39],"all":[40,141],"disagreement":[41],"as":[42,61],"noise,":[43,84],"which":[44,118],"obscures":[45],"these":[46],"distinctions":[47],"limits":[49],"our":[50,150],"understanding":[51],"what":[53],"actually":[55],"This":[57],"paper":[58],"reframes":[59],"annotation":[60],"a":[62,67,114,122,130,170],"measurement":[63],"process":[64],"statistical":[68],"framework":[69,89],"for":[70,116,140,156],"decomposing":[71],"outcomes":[73],"into":[74],"interpretable":[75],"sources":[76],"variation:":[78],"instance":[79],"difficulty,":[80],"annotator":[81],"bias,":[82],"situational":[83],"relational":[86],"alignment.":[87],"The":[88],"extends":[90],"classical":[91],"measurement-error":[92],"accommodate":[95],"both":[96],"shared":[97],"individualized":[99],"notions":[100],"truth,":[102],"reflecting":[103],"traditional":[104],"label":[107],"interpretations":[109],"error,":[111],"provides":[113],"diagnostic":[115],"assessing":[117],"regime":[119],"better":[120],"characterizes":[121],"given":[123],"task.":[124],"Applying":[125],"proposed":[127],"model":[128],"multi-annotator":[131],"natural":[132],"language":[133],"inference":[134],"dataset,":[135],"we":[136],"find":[137],"empirical":[138],"evidence":[139],"four":[142],"theorized":[143],"components":[144],"demonstrate":[146],"effectiveness":[148],"approach.":[151],"We":[152],"conclude":[153],"with":[154],"implications":[155],"data-centric":[157],"outline":[161],"how":[162],"this":[163],"approach":[164],"can":[165],"guide":[166],"development":[168],"more":[171],"science":[173],"labeling.":[175]},"counts_by_year":[],"updated_date":"2026-06-14T07:44:22.658603","created_date":"2026-04-11T00:00:00"}