{"id":"https://openalex.org/W3168956186","doi":"https://doi.org/10.25300/misq/2021/16564","title":"Is AI Ground Truth Really True? The Dangers of Training and Evaluating AI Tools Based on Experts\u2019 Know-What","display_name":"Is AI Ground Truth Really True? The Dangers of Training and Evaluating AI Tools Based on Experts\u2019 Know-What","publication_year":2021,"publication_date":"2021-09-01","ids":{"openalex":"https://openalex.org/W3168956186","doi":"https://doi.org/10.25300/misq/2021/16564","mag":"3168956186"},"language":"en","primary_location":{"id":"doi:10.25300/misq/2021/16564","is_oa":false,"landing_page_url":"https://doi.org/10.25300/misq/2021/16564","pdf_url":null,"source":{"id":"https://openalex.org/S57293258","display_name":"MIS Quarterly","issn_l":"0276-7783","issn":["0276-7783","2162-9730"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4327875293","host_organization_name":"MIS Quarterly","host_organization_lineage":["https://openalex.org/P4327875293"],"host_organization_lineage_names":["MIS Quarterly"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"MIS Quarterly","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066366951","display_name":"Sarah Lebovitz","orcid":null},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sarah Lebovitz","raw_affiliation_strings":["University of Virginia, McIntire School of Commerce, Charlottesville, VA 22904 U.S.A","University of Virginia\u2014McIntire School of Commerce"],"affiliations":[{"raw_affiliation_string":"University of Virginia, McIntire School of Commerce, Charlottesville, VA 22904 U.S.A","institution_ids":[]},{"raw_affiliation_string":"University of Virginia\u2014McIntire School of Commerce","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012518928","display_name":"Natalia Levina","orcid":"https://orcid.org/0000-0002-2858-0233"},"institutions":[{"id":"https://openalex.org/I4210102264","display_name":"Shandong University of Political Science and Law","ror":"https://ror.org/01b2j5886","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210102264"]},{"id":"https://openalex.org/I102475099","display_name":"Sup\u00e9lec","ror":"https://ror.org/00n7gwn90","country_code":"FR","type":"education","lineage":["https://openalex.org/I102475099"]},{"id":"https://openalex.org/I15196421","display_name":"University of Applied Sciences and Arts of Southern Switzerland","ror":"https://ror.org/05ep8g269","country_code":"CH","type":"education","lineage":["https://openalex.org/I15196421"]}],"countries":["CH","CN","FR"],"is_corresponding":false,"raw_author_name":"Natalia Levina","raw_affiliation_strings":["New York University, Stern School of Business, New York, NY 10003 U.S.A","New York University<sup>**</sup>"],"affiliations":[{"raw_affiliation_string":"New York University, Stern School of Business, New York, NY 10003 U.S.A","institution_ids":[]},{"raw_affiliation_string":"New York University<sup>**</sup>","institution_ids":["https://openalex.org/I15196421","https://openalex.org/I4210102264","https://openalex.org/I102475099"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036969088","display_name":"Hila Lifshitz\u2010Assaf","orcid":"https://orcid.org/0000-0002-3461-003X"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]},{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hila Lifshitz-Assaf","raw_affiliation_strings":["New York University, Stern School of Business, New York, NY 10003 U.S.A","Harvard University - Business School (HBS)","New York University (NYU) - Leonard N. Stern School of Business"],"affiliations":[{"raw_affiliation_string":"New York University, Stern School of Business, New York, NY 10003 U.S.A","institution_ids":[]},{"raw_affiliation_string":"Harvard University - Business School (HBS)","institution_ids":["https://openalex.org/I136199984"]},{"raw_affiliation_string":"New York University (NYU) - Leonard N. Stern School of Business","institution_ids":["https://openalex.org/I57206974"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5066366951"],"corresponding_institution_ids":["https://openalex.org/I51556381"],"apc_list":null,"apc_paid":null,"fwci":8.1489,"has_fulltext":false,"cited_by_count":49,"citation_normalized_percentile":{"value":0.9774615,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"45","issue":"3","first_page":"1501","last_page":"1526"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.9836000204086304,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.9836000204086304,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9417999982833862,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.6733506917953491},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5596945285797119},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5385981202125549},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5336214900016785},{"id":"https://openalex.org/keywords/common-ground","display_name":"Common ground","score":0.5129653811454773},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.4931411147117615},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.46283984184265137},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.42123350501060486},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.39846473932266235},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.2274627983570099},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.17978200316429138},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10400506854057312},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.10116598010063171},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.09063655138015747}],"concepts":[{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.6733506917953491},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5596945285797119},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5385981202125549},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5336214900016785},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.5129653811454773},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4931411147117615},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.46283984184265137},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.42123350501060486},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.39846473932266235},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2274627983570099},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.17978200316429138},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10400506854057312},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.10116598010063171},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.09063655138015747},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.25300/misq/2021/16564","is_oa":false,"landing_page_url":"https://doi.org/10.25300/misq/2021/16564","pdf_url":null,"source":{"id":"https://openalex.org/S57293258","display_name":"MIS Quarterly","issn_l":"0276-7783","issn":["0276-7783","2162-9730"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4327875293","host_organization_name":"MIS Quarterly","host_organization_lineage":["https://openalex.org/P4327875293"],"host_organization_lineage_names":["MIS Quarterly"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"MIS Quarterly","raw_type":"journal-article"},{"id":"pmh:oai:wrap.warwick.ac.uk:169972","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306400665","display_name":"Warwick Research Archive Portal (University of Warwick)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I39555362","host_organization_name":"University of Warwick","host_organization_lineage":["https://openalex.org/I39555362"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Journal Article"},{"id":"pmh:oai:aisel.aisnet.org:misq-3666","is_oa":false,"landing_page_url":"https://aisel.aisnet.org/cgi/viewcontent.cgi?article=3666&context=misq","pdf_url":null,"source":{"id":"https://openalex.org/S30879505","display_name":"Journal of the Association for Information Systems","issn_l":"1536-9323","issn":["1536-9323"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310321080","host_organization_name":"Association for Information Systems","host_organization_lineage":["https://openalex.org/P4310321080"],"host_organization_lineage_names":["Association for Information Systems"],"type":"journal"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Management Information Systems Quarterly","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.5799999833106995}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":59,"referenced_works":["https://openalex.org/W1562773932","https://openalex.org/W1608168306","https://openalex.org/W1821977771","https://openalex.org/W1872016866","https://openalex.org/W1978603569","https://openalex.org/W1985834652","https://openalex.org/W1991288524","https://openalex.org/W1993882792","https://openalex.org/W2009638102","https://openalex.org/W2050250537","https://openalex.org/W2052973171","https://openalex.org/W2055673921","https://openalex.org/W2096942889","https://openalex.org/W2097036415","https://openalex.org/W2098319730","https://openalex.org/W2101419153","https://openalex.org/W2107872096","https://openalex.org/W2115925287","https://openalex.org/W2127065705","https://openalex.org/W2131092863","https://openalex.org/W2143712959","https://openalex.org/W2150587481","https://openalex.org/W2160794623","https://openalex.org/W2165040403","https://openalex.org/W2168199261","https://openalex.org/W2168661650","https://openalex.org/W2171196249","https://openalex.org/W2172157658","https://openalex.org/W2174769915","https://openalex.org/W2179492332","https://openalex.org/W2526781987","https://openalex.org/W2556679002","https://openalex.org/W2560617851","https://openalex.org/W2745394387","https://openalex.org/W2792453939","https://openalex.org/W2804305854","https://openalex.org/W2805440323","https://openalex.org/W2893253758","https://openalex.org/W2901789318","https://openalex.org/W2903759629","https://openalex.org/W2946185430","https://openalex.org/W2962772482","https://openalex.org/W2966665347","https://openalex.org/W2981731882","https://openalex.org/W2996055412","https://openalex.org/W3023242059","https://openalex.org/W3044166536","https://openalex.org/W3049046209","https://openalex.org/W3119527628","https://openalex.org/W3131274985","https://openalex.org/W3134081764","https://openalex.org/W3157172840","https://openalex.org/W4207081475","https://openalex.org/W4211196262","https://openalex.org/W4236544382","https://openalex.org/W4239674353","https://openalex.org/W4251036056","https://openalex.org/W4298286109","https://openalex.org/W4299701692"],"related_works":["https://openalex.org/W3197578004","https://openalex.org/W3209121796","https://openalex.org/W804613137","https://openalex.org/W3181262202","https://openalex.org/W2146246854","https://openalex.org/W3085720154","https://openalex.org/W2136666237","https://openalex.org/W2090832897","https://openalex.org/W3089162373","https://openalex.org/W3095475933","https://openalex.org/W66800236","https://openalex.org/W3027437821","https://openalex.org/W3116663657","https://openalex.org/W2089687035","https://openalex.org/W3170010439","https://openalex.org/W13258373","https://openalex.org/W2062320164","https://openalex.org/W1981352438","https://openalex.org/W2768909403","https://openalex.org/W2019727497"],"abstract_inverted_index":{"Organizational":[0],"decision-makers":[1],"need":[2],"to":[3,32,69,118,155],"evaluate":[4,33],"AI":[5,34,61,71,196],"tools":[6,14,87],"in":[7,36,89,113,175],"light":[8],"of":[9,22,30,47,95,108,162,169,188],"increasing":[10],"claims":[11],"that":[12,93],"such":[13,37],"outperform":[15],"human":[16],"experts.":[17,84],"Yet,":[18],"measuring":[19],"the":[20,28,105,145,158,180],"quality":[21],"knowledge":[23,111,182,198],"work":[24],"is":[25,183],"challenging,":[26],"raising":[27],"question":[29,42],"how":[31,53],"performance":[35,67],"contexts.":[38],"We":[39,185],"investigate":[40],"this":[41,128],"through":[43],"a":[44,48],"field":[45],"study":[46,166,190],"major":[49],"U.S.":[50],"hospital,":[51],"observing":[52],"managers":[54,102,154],"evaluated":[55],"five":[56],"different":[57],"machine-learning":[58],"(ML)":[59],"based":[60,76],"tools.":[62,143],"Each":[63],"tool":[64],"reported":[65],"high":[66,106],"according":[68],"standard":[70],"accuracy":[72],"measures,":[73],"which":[74,136],"were":[75,137],"on":[77,132],"ground":[78,114,171],"truth":[79,115,172],"labels":[80,116,173],"provided":[81],"by":[82,130],"qualified":[83],"Trying":[85],"these":[86,141],"out":[88],"practice,":[90,125],"however,":[91],"revealed":[92],"none":[94],"them":[96],"met":[97],"expectations.":[98],"Searching":[99],"for":[100,191,197],"explanations,":[101],"began":[103],"confronting":[104],"uncertainty":[107,129],"experts\u2019":[109,151],"know-what":[110,149],"captured":[112],"used":[117,174],"train":[119],"and":[120,150,160,194],"validate":[121],"ML":[122,176],"models.":[123],"In":[124],"experts":[126],"address":[127],"drawing":[131],"rich":[133],"know-how":[134,152],"practices,":[135],"not":[138],"incorporated":[139],"into":[140],"ML-based":[142],"Discovering":[144],"disconnect":[146],"between":[147],"AI\u2019s":[148],"enabled":[153],"better":[156],"understand":[157],"risks":[159],"benefits":[161],"each":[163],"tool.":[164],"This":[165],"shows":[167],"dangers":[168],"treating":[170],"models":[177],"objectively":[178],"when":[179],"underlying":[181],"uncertain.":[184],"outline":[186],"implications":[187],"our":[189],"developing,":[192],"training,":[193],"evaluating":[195],"work.":[199]},"counts_by_year":[{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":19},{"year":2023,"cited_by_count":14},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}