{"id":"https://openalex.org/W4402870542","doi":"https://doi.org/10.1145/3688394","title":"Generalizable Error Modeling for Human Data Annotation: Evidence From an Industry-Scale Search Data Annotation Program","display_name":"Generalizable Error Modeling for Human Data Annotation: Evidence From an Industry-Scale Search Data Annotation Program","publication_year":2024,"publication_date":"2024-09-26","ids":{"openalex":"https://openalex.org/W4402870542","doi":"https://doi.org/10.1145/3688394"},"language":"en","primary_location":{"id":"doi:10.1145/3688394","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3688394","pdf_url":null,"source":{"id":"https://openalex.org/S110189822","display_name":"Journal of Data and Information Quality","issn_l":"1936-1955","issn":["1936-1955","1936-1963"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Quality","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016086371","display_name":"Heinrich Peters","orcid":"https://orcid.org/0000-0002-0571-6388"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Heinrich Peters","raw_affiliation_strings":["Apple Inc, New York, United States","Apple Inc,  New York, United States"],"raw_orcid":"https://orcid.org/0000-0002-0571-6388","affiliations":[{"raw_affiliation_string":"Apple Inc, New York, United States","institution_ids":["https://openalex.org/I4210153776"]},{"raw_affiliation_string":"Apple Inc,  New York, United States","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101797049","display_name":"Alireza Hashemi","orcid":"https://orcid.org/0000-0003-0561-6130"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]},{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alireza Hashemi","raw_affiliation_strings":["Apple Inc, Seattle, United States","Apple Inc,  Seattle, United States"],"raw_orcid":"https://orcid.org/0000-0003-0561-6130","affiliations":[{"raw_affiliation_string":"Apple Inc, Seattle, United States","institution_ids":["https://openalex.org/I4210153776"]},{"raw_affiliation_string":"Apple Inc,  Seattle, United States","institution_ids":["https://openalex.org/I4210153776","https://openalex.org/I58610484"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034400698","display_name":"James DeShaw Rae","orcid":null},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]},{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James Rae","raw_affiliation_strings":["Apple Inc, Seattle, United States","Apple Inc,  Seattle, United States"],"raw_orcid":"https://orcid.org/0009-0001-3828-4972","affiliations":[{"raw_affiliation_string":"Apple Inc, Seattle, United States","institution_ids":["https://openalex.org/I4210153776"]},{"raw_affiliation_string":"Apple Inc,  Seattle, United States","institution_ids":["https://openalex.org/I4210153776","https://openalex.org/I58610484"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.5557,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.72078615,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":"16","issue":"3","first_page":"1","last_page":"15"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9919000267982483,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.8091597557067871},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.792534351348877},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5599554181098938},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4408726394176483},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.42196139693260193},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.40559521317481995},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2847294211387634}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8091597557067871},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.792534351348877},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5599554181098938},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4408726394176483},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.42196139693260193},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40559521317481995},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2847294211387634},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3688394","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3688394","pdf_url":null,"source":{"id":"https://openalex.org/S110189822","display_name":"Journal of Data and Information Quality","issn_l":"1936-1955","issn":["1936-1955","1936-1963"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Quality","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W1585854823","https://openalex.org/W1962580118","https://openalex.org/W2021664962","https://openalex.org/W2108598243","https://openalex.org/W2137406659","https://openalex.org/W2295598076","https://openalex.org/W2531638282","https://openalex.org/W2549086141","https://openalex.org/W2558888286","https://openalex.org/W2585226541","https://openalex.org/W2804447254","https://openalex.org/W2892181857","https://openalex.org/W2946085671","https://openalex.org/W2972573244","https://openalex.org/W3034891697","https://openalex.org/W3041011740","https://openalex.org/W3088911247","https://openalex.org/W3113657720","https://openalex.org/W3118813946","https://openalex.org/W4214938700","https://openalex.org/W4226278401","https://openalex.org/W4249502209","https://openalex.org/W4287124503","https://openalex.org/W4287854312"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W2625833328","https://openalex.org/W1533177136","https://openalex.org/W4380994516","https://openalex.org/W2251519152"],"abstract_inverted_index":{"Machine":[0],"learning":[1],"(ML)":[2],"and":[3,15,62,87,136,205],"artificial":[4],"intelligence":[5],"(AI)":[6],"systems":[7],"rely":[8],"heavily":[9],"on":[10,66,101,115],"human-annotated":[11],"data":[12,67,208,222],"for":[13,53,185],"training":[14],"evaluation.":[16],"A":[17],"major":[18],"challenge":[19],"in":[20,48,144,160,202,220],"this":[21],"context":[22,162],"is":[23,124],"the":[24,141,155,158,161,175,186,203,221,228],"occurrence":[25],"of":[26,133,151,157,163,177,207,231],"annotation":[27,51,73,117,142,179,209,223],"errors,":[28],"as":[29],"their":[30],"effects":[31],"can":[32,79,198],"degrade":[33],"model":[34,42,84,89,99,123,159],"performance.":[35],"This":[36],"paper":[37],"presents":[38],"a":[39,96,131,148],"predictive":[40],"error":[41,171,195,218],"trained":[43,125],"to":[44,108,126,146,227],"detect":[45],"potential":[46],"errors":[47,78,128,180],"search":[49,71],"relevance":[50,72],"tasks":[52,167],"three":[54],"industry-scale":[55],"ML":[56],"applications":[57,94],"(music":[58],"streaming,":[59,61],"video":[60],"mobile":[63],"apps).":[64],"Drawing":[65],"from":[68,119,130,140],"an":[69],"extensive":[70],"program,":[74],"we":[75],"demonstrate":[76,154],"that":[77,88,193],"be":[80],"predicted":[81,170],"with":[82,103,168],"moderate":[83],"performance":[85,90],"(AUC=0.65-0.75)":[86],"generalizes":[91],"well":[92],"across":[93],"(i.e.,":[95],"global,":[97],"task-agnostic":[98],"performs":[100],"par":[102],"task-specific":[104,120],"models).":[105],"In":[106],"contrast":[107],"past":[109],"research,":[110],"which":[111],"has":[112],"often":[113],"focused":[114],"predicting":[116],"labels":[118],"features,":[121],"our":[122],"predict":[127],"directly":[129],"combination":[132],"task":[134],"features":[135,138],"behavioral":[137,194],"derived":[139],"process,":[143,224],"order":[145],"achieve":[147],"high":[149,169],"degree":[150],"generalizability.":[152],"We":[153],"usefulness":[156],"auditing,":[164],"where":[165],"prioritizing":[166],"probabilities":[172],"considerably":[173],"increases":[174],"amount":[176],"corrected":[178],"(e.g.,":[181],"40%":[182],"efficiency":[183,204],"gains":[184],"music":[187],"streaming":[188],"application).":[189],"These":[190],"results":[191],"highlight":[192],"detection":[196],"models":[197],"yield":[199],"considerable":[200],"improvements":[201],"quality":[206],"processes.":[210],"Our":[211],"findings":[212],"reveal":[213],"critical":[214],"insights":[215],"into":[216],"effective":[217],"management":[219],"thereby":[225],"contributing":[226],"broader":[229],"field":[230],"human-in-the-loop":[232],"ML.":[233]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}