{"id":"https://openalex.org/W4386249192","doi":"https://doi.org/10.1145/3600211.3604766","title":"Ground Truth Or Dare: Factors Affecting The Creation Of Medical Datasets For Training AI","display_name":"Ground Truth Or Dare: Factors Affecting The Creation Of Medical Datasets For Training AI","publication_year":2023,"publication_date":"2023-08-08","ids":{"openalex":"https://openalex.org/W4386249192","doi":"https://doi.org/10.1145/3600211.3604766"},"language":"en","primary_location":{"id":"doi:10.1145/3600211.3604766","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600211.3604766","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3600211.3604766","source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3600211.3604766","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069554200","display_name":"Hubert Dariusz Zaj\u0105c","orcid":"https://orcid.org/0000-0003-0689-6912"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":true,"raw_author_name":"Hubert Dariusz Zaj\u0105c","raw_affiliation_strings":["Department of Computer Science, University of Copenhagen, Denmark"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057778323","display_name":"Natalia-Rozalia Avlona","orcid":"https://orcid.org/0000-0002-1009-1810"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Natalia Rozalia Avlona","raw_affiliation_strings":["Department of Computer Science, University of Copenhagen, Denmark"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089084748","display_name":"Finn Kensing","orcid":"https://orcid.org/0000-0002-1392-5999"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Finn Kensing","raw_affiliation_strings":["Department of Computer Science, University of Copenhagen, Denmark"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071322830","display_name":"Tariq Osman Andersen","orcid":"https://orcid.org/0000-0002-9342-5520"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Tariq Osman Andersen","raw_affiliation_strings":["Department of Computer Science, University of Copenhagen, Denmark"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070236501","display_name":"Irina Shklovski","orcid":"https://orcid.org/0000-0003-1874-0958"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Irina Shklovski","raw_affiliation_strings":["Department of Computer Science, University of Copenhagen, Denmark"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5069554200"],"corresponding_institution_ids":["https://openalex.org/I124055696"],"apc_list":null,"apc_paid":null,"fwci":3.6918,"has_fulltext":true,"cited_by_count":21,"citation_normalized_percentile":{"value":0.94490756,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"351","last_page":"362"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/schema","display_name":"Schema (genetic algorithms)","score":0.8397706747055054},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7192222476005554},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.7168684601783752},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5943020582199097},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.486905962228775},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.48572975397109985},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.44688788056373596},{"id":"https://openalex.org/keywords/common-ground","display_name":"Common ground","score":0.44468533992767334},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.382139652967453},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3303844928741455},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2897067964076996},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.08221116662025452}],"concepts":[{"id":"https://openalex.org/C52146309","wikidata":"https://www.wikidata.org/wiki/Q7431116","display_name":"Schema (genetic algorithms)","level":2,"score":0.8397706747055054},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7192222476005554},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.7168684601783752},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5943020582199097},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.486905962228775},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.48572975397109985},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.44688788056373596},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.44468533992767334},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.382139652967453},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3303844928741455},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2897067964076996},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.08221116662025452},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3600211.3604766","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600211.3604766","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3600211.3604766","source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2309.12327","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.12327","pdf_url":"https://arxiv.org/pdf/2309.12327","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/6a371d22-e90f-4e6b-b7c3-94db4511641d","is_oa":true,"landing_page_url":"https://researchprofiles.ku.dk/da/publications/6a371d22-e90f-4e6b-b7c3-94db4511641d","pdf_url":"https://curis.ku.dk/ws/files/375721803/Ground_Truth_Or_Dare.pdf","source":{"id":"https://openalex.org/S4306401983","display_name":"Research at the University of Copenhagen (University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I124055696","host_organization_name":"University of Copenhagen","host_organization_lineage":["https://openalex.org/I124055696"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Zajac , H D , Avlona , R N , Andersen , T O , Kensing , F & Shklovski , I 2023 , Ground Truth Or Dare : Factors Affecting The Creation Of Medical Datasets For Training AI . in AIES \u201923, August 8\u201310, 2023, Montr\u00e9al, QC, Canada . Association for Computing Machinery , pp. 351\u2013362 , 2023 AAAI/ACM Conference on AI, Ethics, and Society - AIES '23 , Montreal , Canada , 08/08/2023 . https://doi.org/10.1145/3600211.3604766","raw_type":"contributionToPeriodical"}],"best_oa_location":{"id":"doi:10.1145/3600211.3604766","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600211.3604766","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3600211.3604766","source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals","score":0.41999998688697815}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386249192.pdf","grobid_xml":"https://content.openalex.org/works/W4386249192.grobid-xml"},"referenced_works_count":70,"referenced_works":["https://openalex.org/W409344806","https://openalex.org/W1625764576","https://openalex.org/W1962580118","https://openalex.org/W2021370805","https://openalex.org/W2053486405","https://openalex.org/W2059239014","https://openalex.org/W2061253660","https://openalex.org/W2091950237","https://openalex.org/W2120096569","https://openalex.org/W2152772232","https://openalex.org/W2552408584","https://openalex.org/W2604748391","https://openalex.org/W2611789916","https://openalex.org/W2784241156","https://openalex.org/W2897042519","https://openalex.org/W2901013729","https://openalex.org/W2911227954","https://openalex.org/W2912190266","https://openalex.org/W2912664121","https://openalex.org/W2922234936","https://openalex.org/W2935719450","https://openalex.org/W2936086693","https://openalex.org/W2941766203","https://openalex.org/W2946670765","https://openalex.org/W2963466845","https://openalex.org/W2977613223","https://openalex.org/W2980482960","https://openalex.org/W2981731882","https://openalex.org/W2986571455","https://openalex.org/W3011721937","https://openalex.org/W3022967462","https://openalex.org/W3029504795","https://openalex.org/W3041914373","https://openalex.org/W3093541323","https://openalex.org/W3095415574","https://openalex.org/W3100279624","https://openalex.org/W3118813946","https://openalex.org/W3133631714","https://openalex.org/W3135371071","https://openalex.org/W3157831956","https://openalex.org/W3159579291","https://openalex.org/W3160343688","https://openalex.org/W3163469193","https://openalex.org/W3164652469","https://openalex.org/W3173669456","https://openalex.org/W3177746823","https://openalex.org/W3183266055","https://openalex.org/W3188289380","https://openalex.org/W3189849087","https://openalex.org/W3192254170","https://openalex.org/W3205020946","https://openalex.org/W3207830467","https://openalex.org/W3212368439","https://openalex.org/W4200632307","https://openalex.org/W4205570006","https://openalex.org/W4224991249","https://openalex.org/W4239571720","https://openalex.org/W4287243651","https://openalex.org/W4288058238","https://openalex.org/W4288086169","https://openalex.org/W4294152847","https://openalex.org/W4294214983","https://openalex.org/W4296186062","https://openalex.org/W4297915576","https://openalex.org/W4302072479","https://openalex.org/W4308789076","https://openalex.org/W4309619003","https://openalex.org/W4312126565","https://openalex.org/W4394659038","https://openalex.org/W6947959814"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W1546315632","https://openalex.org/W2625833328","https://openalex.org/W4307535225","https://openalex.org/W2905544777"],"abstract_inverted_index":{"One":[0],"of":[1,5,21,28,51,63,104,134,141,147],"the":[2,19,22,26,38,49,57,61,66,102,121,139,142,157],"core":[3],"goals":[4],"responsible":[6,171],"AI":[7,172],"development":[8],"is":[9,164],"ensuring":[10],"high-quality":[11,29,150],"training":[12],"datasets.":[13],"Many":[14],"researchers":[15],"have":[16],"pointed":[17],"to":[18,37,153,156,166,169],"importance":[20],"annotation":[23],"step":[24],"in":[25,60,65,79],"creation":[27,62,94,105],"data,":[30],"but":[31],"less":[32],"attention":[33,155],"has":[34],"been":[35],"paid":[36],"work":[39,47,78],"that":[40,90,159],"enables":[41],"data":[42,117,151],"annotation.":[43],"We":[44],"define":[45],"this":[46],"as":[48],"design":[50,140],"ground":[52,122,143],"truth":[53,123,144],"schema":[54,124],"and":[55,87,106,108,110,119,132,161],"explore":[56],"challenges":[58],"involved":[59],"datasets":[64],"medical":[67,92,116],"domain":[68],"even":[69],"before":[70],"any":[71],"annotations":[72],"are":[73],"made.":[74],"Based":[75],"on":[76],"extensive":[77],"three":[80],"health-tech":[81],"organisations,":[82],"we":[83],"describe":[84],"five":[85],"external":[86,97],"internal":[88,127],"factors":[89,98,114,128,158],"condition":[91,115],"dataset":[93],"processes.":[95],"Three":[96],"include":[99,129],"regulatory":[100],"constraints,":[101],"context":[103],"use,":[107],"commercial":[109],"operational":[111],"pressures.":[112],"These":[113,136],"collection":[118],"shape":[120,138,160],"design.":[125,173],"Two":[126],"epistemic":[130],"differences":[131],"limits":[133],"labelling.":[135],"directly":[137],"schema.":[145],"Discussions":[146],"what":[148,163],"constitutes":[149],"need":[152],"pay":[154],"constrain":[162],"possible":[165],"be":[167],"created,":[168],"ensure":[170]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":6}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2023-08-30T00:00:00"}