{"id":"https://openalex.org/W3197578004","doi":"https://doi.org/10.25300/misq/2021/16564","title":"Is AI ground truth really true? The dangers of training and evaluating AI tools based on experts\u2019 know-what
\\n","display_name":"Is AI ground truth really true? The dangers of training and evaluating AI tools based on experts\u2019 know-what
\\n","publication_year":2021,"publication_date":"2021-09-01","ids":{"openalex":"https://openalex.org/W3197578004","doi":"https://doi.org/10.25300/misq/2021/16564","mag":"3197578004"},"language":"en","primary_location":{"id":"pmh:oai:wrap.warwick.ac.uk:155579","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306400665","display_name":"Warwick Research Archive Portal (University of Warwick)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I39555362","host_organization_name":"University of Warwick","host_organization_lineage":["https://openalex.org/I39555362"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Journal Article"},"type":"article","indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066366951","display_name":"Sarah Lebovitz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lebovitz, S.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Levina, Natalie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Levina, Natalie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5108760893","display_name":"Hila Lifshitz-Assa","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lifshitz-Assaf, Hila","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":32.8817,"has_fulltext":false,"cited_by_count":318,"citation_normalized_percentile":{"value":0.99834879,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.9830999970436096,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.9830999970436096,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9280999898910522,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11918","display_name":"Forecasting Techniques and Applications","score":0.9121999740600586,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.7086689472198486},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5715909600257874},{"id":"https://openalex.org/keywords/common-ground","display_name":"Common ground","score":0.5689634084701538},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5686869621276855},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5532358884811401},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.4897420406341553},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.4696953296661377},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.4451349675655365},{"id":"https://openalex.org/keywords/need-to-know","display_name":"Need to know","score":0.4440310001373291},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.3713842034339905},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.32521724700927734},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.2147316038608551},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.13946717977523804},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10499715805053711},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.1026470959186554},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.08551323413848877}],"concepts":[{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.7086689472198486},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5715909600257874},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.5689634084701538},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5686869621276855},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5532358884811401},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4897420406341553},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4696953296661377},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.4451349675655365},{"id":"https://openalex.org/C2778519782","wikidata":"https://www.wikidata.org/wiki/Q1974060","display_name":"Need to know","level":2,"score":0.4440310001373291},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3713842034339905},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32521724700927734},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2147316038608551},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.13946717977523804},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10499715805053711},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.1026470959186554},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.08551323413848877},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:wrap.warwick.ac.uk:155579","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306400665","display_name":"Warwick Research Archive Portal (University of Warwick)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I39555362","host_organization_name":"University of Warwick","host_organization_lineage":["https://openalex.org/I39555362"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Journal Article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.5899999737739563}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":74,"referenced_works":["https://openalex.org/W1562773932","https://openalex.org/W1608168306","https://openalex.org/W1821977771","https://openalex.org/W1872016866","https://openalex.org/W1978603569","https://openalex.org/W1985834652","https://openalex.org/W1991288524","https://openalex.org/W1993882792","https://openalex.org/W2009638102","https://openalex.org/W2050250537","https://openalex.org/W2052973171","https://openalex.org/W2055673921","https://openalex.org/W2096942889","https://openalex.org/W2097036415","https://openalex.org/W2098319730","https://openalex.org/W2101419153","https://openalex.org/W2107872096","https://openalex.org/W2115925287","https://openalex.org/W2127065705","https://openalex.org/W2131092863","https://openalex.org/W2143712959","https://openalex.org/W2150587481","https://openalex.org/W2160794623","https://openalex.org/W2165040403","https://openalex.org/W2168199261","https://openalex.org/W2168661650","https://openalex.org/W2171196249","https://openalex.org/W2172157658","https://openalex.org/W2174769915","https://openalex.org/W2179492332","https://openalex.org/W2526781987","https://openalex.org/W2556679002","https://openalex.org/W2560617851","https://openalex.org/W2745394387","https://openalex.org/W2792453939","https://openalex.org/W2804305854","https://openalex.org/W2805440323","https://openalex.org/W2893253758","https://openalex.org/W2901789318","https://openalex.org/W2903759629","https://openalex.org/W2946185430","https://openalex.org/W2962772482","https://openalex.org/W2966665347","https://openalex.org/W2981731882","https://openalex.org/W2996055412","https://openalex.org/W3001400233","https://openalex.org/W3023242059","https://openalex.org/W3044166536","https://openalex.org/W3045613516","https://openalex.org/W3049046209","https://openalex.org/W3119527628","https://openalex.org/W3131274985","https://openalex.org/W3134081764","https://openalex.org/W3157172840","https://openalex.org/W4207081475","https://openalex.org/W4211196262","https://openalex.org/W4236544382","https://openalex.org/W4239674353","https://openalex.org/W4240671868","https://openalex.org/W4251036056","https://openalex.org/W4298286109","https://openalex.org/W4299701692","https://openalex.org/W6667462837","https://openalex.org/W6676297131","https://openalex.org/W6678964861","https://openalex.org/W6679470589","https://openalex.org/W6681400661","https://openalex.org/W6754725728","https://openalex.org/W6762444658","https://openalex.org/W6762983504","https://openalex.org/W6770917826","https://openalex.org/W6775860266","https://openalex.org/W6809680140","https://openalex.org/W7043098716"],"related_works":["https://openalex.org/W4385274231","https://openalex.org/W2337773048","https://openalex.org/W2325616849","https://openalex.org/W4392669623","https://openalex.org/W2516476274","https://openalex.org/W1785988750","https://openalex.org/W2917919721","https://openalex.org/W1546315632","https://openalex.org/W4307535225","https://openalex.org/W2905544777"],"abstract_inverted_index":{"Organizational":[0],"decision-makers":[1],"need":[2],"to":[3,32,69,118,155],"evaluate":[4,33],"AI":[5,34,61,71,196],"tools":[6,14,87],"in":[7,36,89,113,175],"light":[8],"of":[9,22,30,47,95,108,162,169,188],"increasing":[10],"claims":[11],"that":[12,93],"such":[13,37],"outperform":[15],"human":[16],"experts.":[17,84],"Yet,":[18],"measuring":[19],"the":[20,28,105,145,158,180],"quality":[21],"knowledge":[23,111,182,198],"work":[24],"is":[25,183],"challenging,":[26],"raising":[27],"question":[29,42],"how":[31,53],"performance":[35,67],"contexts.":[38],"We":[39,185],"investigate":[40],"this":[41,128],"through":[43],"a":[44,48],"field":[45],"study":[46,166,190],"major":[49],"U.S.":[50],"hospital,":[51],"observing":[52],"managers":[54,102,154],"evaluated":[55],"five":[56],"different":[57],"machine-learning":[58],"(ML)":[59],"based":[60,76],"tools.":[62,143],"Each":[63],"tool":[64],"reported":[65],"high":[66,106],"according":[68],"standard":[70],"accuracy":[72],"measures,":[73],"which":[74,136],"were":[75,137],"on":[77,132],"ground":[78,114,171],"truth":[79,115,172],"labels":[80,116,173],"provided":[81],"by":[82,130],"qualified":[83],"Trying":[85],"these":[86,141],"out":[88],"practice,":[90,125],"however,":[91],"revealed":[92],"none":[94],"them":[96],"met":[97],"expectations.":[98],"Searching":[99],"for":[100,191,197],"explanations,":[101],"began":[103],"confronting":[104],"uncertainty":[107,129],"experts\u2019":[109,151],"know-what":[110,149],"captured":[112],"used":[117,174],"train":[119],"and":[120,150,160,194],"validate":[121],"ML":[122,176],"models.":[123],"In":[124],"experts":[126],"address":[127],"drawing":[131],"rich":[133],"know-how":[134,152],"practices,":[135],"not":[138],"incorporated":[139],"into":[140],"ML-based":[142],"Discovering":[144],"disconnect":[146],"between":[147],"AI\u2019s":[148],"enabled":[153],"better":[156],"understand":[157],"risks":[159],"benefits":[161],"each":[163],"tool.":[164],"This":[165],"shows":[167],"dangers":[168],"treating":[170],"models":[177],"objectively":[178],"when":[179],"underlying":[181],"uncertain.":[184],"outline":[186],"implications":[187],"our":[189],"developing,":[192],"training,":[193],"evaluating":[195],"work.
\\n":[199]},"counts_by_year":[{"year":2026,"cited_by_count":53},{"year":2025,"cited_by_count":95},{"year":2024,"cited_by_count":83},{"year":2023,"cited_by_count":49},{"year":2022,"cited_by_count":30},{"year":2021,"cited_by_count":6},{"year":2019,"cited_by_count":2}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}