{"id":"https://openalex.org/W3085682242","doi":"https://doi.org/10.1186/s12911-020-01224-9","title":"As if sand were stone. New concepts and metrics to probe the ground on which to build trustable AI","display_name":"As if sand were stone. New concepts and metrics to probe the ground on which to build trustable AI","alternate_titles":[],"publication_year":2020,"publication_date":"2020-09-11","ids":{"openalex":"https://openalex.org/W3085682242","doi":"https://doi.org/10.1186/s12911-020-01224-9","mag":"3085682242","pmid":"https://pubmed.ncbi.nlm.nih.gov/32917183","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/7488864"},"language":"en","primary_location":{"id":"doi:10.1186/s12911-020-01224-9","doi":"https://doi.org/10.1186/s12911-020-01224-9","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s12911-020-01224-9","pdf_url":"https://bmcmedinformdecismak.biomedcentral.com/track/pdf/10.1186/s12911-020-01224-9","source":{"id":"https://openalex.org/S107516304","display_name":"BMC Medical Informatics and Decision Making","issn_l":"1472-6947","issn":["1472-6947"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-1","oarl","poland-70","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Medical Informatics and Decision Making","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W3085682242"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://bmcmedinformdecismak.biomedcentral.com/track/pdf/10.1186/s12911-020-01224-9","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023849668","display_name":"Federico Cabitza","orcid":"https://orcid.org/0000-0002-4065-3415","observed_orcids":["https://orcid.org/0000-0002-4065-3415"]},"institutions":[{"id":"https://openalex.org/I66752286","display_name":"University of Milano-Bicocca","ror":"https://ror.org/01ynf4891","country_code":"IT","type":"education","lineage":["https://openalex.org/I66752286"]}],"countries":["IT"],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Federico Cabitza","raw_affiliation_strings":["Dipartimento di Informatica, Sistemistica e Comunicazione, Universit\u00e1 degli Studi di Milano-Bicocca, Viale Sarca, 336, Milan, 20125, Italy. federico.cabitza@unimib.it","Dipartimento di Informatica, Sistemistica e Comunicazione, Universit\u00e1 degli Studi di Milano-Bicocca, Viale Sarca, 336, Milan, 20125, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dipartimento di Informatica, Sistemistica e Comunicazione, Universit\u00e1 degli Studi di Milano-Bicocca, Viale Sarca, 336, Milan, 20125, Italy. federico.cabitza@unimib.it","institution_ids":["https://openalex.org/I66752286"]},{"raw_affiliation_string":"Dipartimento di Informatica, Sistemistica e Comunicazione, Universit\u00e1 degli Studi di Milano-Bicocca, Viale Sarca, 336, Milan, 20125, Italy","institution_ids":["https://openalex.org/I66752286"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045931370","display_name":"Andrea Campagner","orcid":"https://orcid.org/0000-0002-0027-5157","observed_orcids":["https://orcid.org/0000-0002-0027-5157"]},"institutions":[{"id":"https://openalex.org/I4210113215","display_name":"IRCCS Ospedale Galeazzi - Sant'Ambrogio","ror":"https://ror.org/01vyrje42","country_code":"IT","type":"healthcare","lineage":["https://openalex.org/I4210113215","https://openalex.org/I4210153126"]}],"countries":["IT"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Andrea Campagner","raw_affiliation_strings":["IRCCS Istituto Ortopedico Galeazzi, Via Riccardo Galeazzi 4, Milan, 20161, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IRCCS Istituto Ortopedico Galeazzi, Via Riccardo Galeazzi 4, Milan, 20161, Italy","institution_ids":["https://openalex.org/I4210113215"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079824065","display_name":"Luca Maria Sconfienza","orcid":"https://orcid.org/0000-0003-0759-8431","observed_orcids":["https://orcid.org/0000-0003-0759-8431"]},"institutions":[{"id":"https://openalex.org/I189158943","display_name":"University of Milan","ror":"https://ror.org/00wjc7c48","country_code":"IT","type":"education","lineage":["https://openalex.org/I189158943"]},{"id":"https://openalex.org/I4210113215","display_name":"IRCCS Ospedale Galeazzi - Sant'Ambrogio","ror":"https://ror.org/01vyrje42","country_code":"IT","type":"healthcare","lineage":["https://openalex.org/I4210113215","https://openalex.org/I4210153126"]}],"countries":["IT"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Luca Maria Sconfienza","raw_affiliation_strings":["Department of Biomedical Sciences for Health, Universit\u00e0 degli Studi di Milano, Via Mangiagalli 31, Milan, 20133, Italy","IRCCS Istituto Ortopedico Galeazzi, Via Riccardo Galeazzi 4, Milan, 20161, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Biomedical Sciences for Health, Universit\u00e0 degli Studi di Milano, Via Mangiagalli 31, Milan, 20133, Italy","institution_ids":["https://openalex.org/I189158943"]},{"raw_affiliation_string":"IRCCS Istituto Ortopedico Galeazzi, Via Riccardo Galeazzi 4, Milan, 20161, Italy","institution_ids":["https://openalex.org/I4210113215"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5023849668"],"corresponding_institution_ids":["https://openalex.org/I66752286"],"apc_list":{"value":2380,"currency":"USD","value_usd":2380},"apc_paid":{"value":2130,"currency":"EUR","value_usd":2297},"fwci":2.1848,"has_fulltext":true,"cited_by_count":57,"citation_normalized_percentile":{"value":0.89395114,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":86,"max":99},"biblio":{"volume":"20","issue":"1","first_page":"219","last_page":"219"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.6312999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.6312999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.11680000275373459,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.08550000190734863,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/gold-standard","display_name":"gold standard","score":0.6190000176429749},{"id":"https://openalex.org/keywords/representativeness","display_name":"representativeness","score":0.7799999713897705},{"id":"https://openalex.org/keywords/rater-reliability","display_name":"rater reliability","score":0.7360000014305115},{"id":"https://openalex.org/keywords/annotation-accuracy","display_name":"annotation accuracy","score":0.9430000185966492},{"id":"https://openalex.org/keywords/explainable-ai","display_name":"explainable AI","score":0.8460000157356262},{"id":"https://openalex.org/keywords/artificial-intelligence-in-healthcare","display_name":"artificial intelligence in healthcare","score":0.949999988079071},{"id":"https://openalex.org/keywords/radiologist-annotation","display_name":"radiologist annotation","score":0.6639999747276306},{"id":"https://openalex.org/keywords/human-decision-support","display_name":"human decision support","score":0.3970000147819519}],"concepts":[{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7213481068611145},{"id":"https://openalex.org/C37381756","wikidata":"https://www.wikidata.org/wiki/Q20203288","display_name":"Representativeness heuristic","level":2,"score":0.6897984743118286},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6585978269577026},{"id":"https://openalex.org/C2775997480","wikidata":"https://www.wikidata.org/wiki/Q586277","display_name":"Degree (music)","level":2,"score":0.6117464900016785},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5889962315559387},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5764439105987549},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5479793548583984},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.46749961376190186},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.45978620648384094},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4502581059932709},{"id":"https://openalex.org/C160798450","wikidata":"https://www.wikidata.org/wiki/Q4230870","display_name":"Concordance","level":2,"score":0.4196372628211975},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.37831151485443115},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36628732085227966},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34986239671707153},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2908463180065155},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.22486543655395508},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D019985","descriptor_name":"Benchmarking","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":7,"locations":[{"id":"doi:10.1186/s12911-020-01224-9","doi":"https://doi.org/10.1186/s12911-020-01224-9","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s12911-020-01224-9","pdf_url":"https://bmcmedinformdecismak.biomedcentral.com/track/pdf/10.1186/s12911-020-01224-9","source":{"id":"https://openalex.org/S107516304","display_name":"BMC Medical Informatics and Decision Making","issn_l":"1472-6947","issn":["1472-6947"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-1","oarl","poland-70","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Medical Informatics and Decision Making","raw_type":"journal-article"},{"id":"pmh:oai:pubmedcentral.nih.gov:7488864","doi":null,"is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC7488864/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Med Inform Decis Mak","raw_type":"Text"},{"id":"pmid:32917183","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/32917183","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC medical informatics and decision making","raw_type":"Journal Article"},{"id":"pmh:oai:air.unimi.it:2434/764720","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/2434/764720","pdf_url":"https://air.unimi.it/bitstream/2434/764720/2/s12911-020-01224-9.pdf","source":{"id":"https://openalex.org/S4306400516","display_name":"Archivio Istituzionale della Ricerca (Universita Degli Studi Di Milano)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I189158943","host_organization_name":"University of Milan","host_organization_lineage":["https://openalex.org/I189158943"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:boa.unimib.it:10281/292798","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/10281/292798","pdf_url":null,"source":{"id":"https://openalex.org/S4306401259","display_name":"BOA (University of Milano-Bicocca)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I66752286","host_organization_name":"University of Milano-Bicocca","host_organization_lineage":["https://openalex.org/I66752286"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:doaj.org/article:a3c7b490f6a141ec98e8ac3a86122b34","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/a3c7b490f6a141ec98e8ac3a86122b34","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"BMC Medical Informatics and Decision Making, Vol 20, Iss 1, Pp 1-21 (2020)","raw_type":"article"},{"id":"pmh:oai:zenodo.org:4562597","doi":null,"is_oa":false,"landing_page_url":"https://zenodo.org/record/4562597","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"BMC Medical Informatics and Decision Making, 20(219), (2020-09-11)","raw_type":"info:eu-repo/semantics/other"}],"best_oa_location":{"id":"doi:10.1186/s12911-020-01224-9","doi":"https://doi.org/10.1186/s12911-020-01224-9","is_oa":true,"landing_page_url":"https://doi.org/10.1186/s12911-020-01224-9","pdf_url":"https://bmcmedinformdecismak.biomedcentral.com/track/pdf/10.1186/s12911-020-01224-9","source":{"id":"https://openalex.org/S107516304","display_name":"BMC Medical Informatics and Decision Making","issn_l":"1472-6947","issn":["1472-6947"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-1","oarl","poland-70","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Medical Informatics and Decision Making","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3085682242.pdf","grobid_xml":"https://content.openalex.org/works/W3085682242.grobid-xml"},"referenced_works_count":81,"referenced_works":["https://openalex.org/W52593152","https://openalex.org/W112114804","https://openalex.org/W114565888","https://openalex.org/W147565337","https://openalex.org/W160583832","https://openalex.org/W383308755","https://openalex.org/W592737779","https://openalex.org/W603636824","https://openalex.org/W613277099","https://openalex.org/W630584265","https://openalex.org/W1524012148","https://openalex.org/W1600834507","https://openalex.org/W1762172548","https://openalex.org/W1921594208","https://openalex.org/W1972571116","https://openalex.org/W1975971445","https://openalex.org/W1981276685","https://openalex.org/W2003052370","https://openalex.org/W2010067099","https://openalex.org/W2019363670","https://openalex.org/W2026653933","https://openalex.org/W2029538739","https://openalex.org/W2061402733","https://openalex.org/W2061504941","https://openalex.org/W2106412328","https://openalex.org/W2107189314","https://openalex.org/W2117079761","https://openalex.org/W2125865219","https://openalex.org/W2127374659","https://openalex.org/W2132131059","https://openalex.org/W2132138475","https://openalex.org/W2132481658","https://openalex.org/W2140080602","https://openalex.org/W2144372981","https://openalex.org/W2146772657","https://openalex.org/W2148215741","https://openalex.org/W2153130422","https://openalex.org/W2153222072","https://openalex.org/W2157510479","https://openalex.org/W2164777277","https://openalex.org/W2165990543","https://openalex.org/W2166944917","https://openalex.org/W2293246850","https://openalex.org/W2406470551","https://openalex.org/W2612708909","https://openalex.org/W2721145261","https://openalex.org/W2783108014","https://openalex.org/W2797148637","https://openalex.org/W2800117778","https://openalex.org/W2804071063","https://openalex.org/W2881747041","https://openalex.org/W2896215772","https://openalex.org/W2902874468","https://openalex.org/W2903995489","https://openalex.org/W2909334458","https://openalex.org/W2913345658","https://openalex.org/W2913402480","https://openalex.org/W2921675416","https://openalex.org/W2941140577","https://openalex.org/W2942522480","https://openalex.org/W2950181023","https://openalex.org/W2964340499","https://openalex.org/W2969866567","https://openalex.org/W2970526411","https://openalex.org/W2975035192","https://openalex.org/W2993404698","https://openalex.org/W2994218692","https://openalex.org/W2999905431","https://openalex.org/W3007464329","https://openalex.org/W3013578857","https://openalex.org/W3021916629","https://openalex.org/W3035203924","https://openalex.org/W3192862352","https://openalex.org/W4230388840","https://openalex.org/W4238284510","https://openalex.org/W4238893454","https://openalex.org/W4241623321","https://openalex.org/W4252793191","https://openalex.org/W4292414119","https://openalex.org/W4301347335","https://openalex.org/W6606250760"],"related_works":["https://openalex.org/W3035203924","https://openalex.org/W2969866567","https://openalex.org/W3036069104","https://openalex.org/W6922535900","https://openalex.org/W7218392741","https://openalex.org/W3174629909","https://openalex.org/W7127969741","https://openalex.org/W7167600653","https://openalex.org/W4404622590","https://openalex.org/W7164888676"],"abstract_inverted_index":{"BACKGROUND:":[0],"We":[1,66,130,145,163,181,271,295],"focus":[2],"on":[3],"the":[4,8,11,15,22,46,53,78,88,100,113,118,125,172,183,204,212,220,223,228,235,252,255,263,280,283,298,309],"importance":[5],"of":[6,10,17,24,29,55,70,82,115,120,127,174,185,214,219,222,237,254,265,282,312],"interpreting":[7],"quality":[9,71,281],"labeling":[12,48],"used":[13,169,285],"as":[14,37,216],"input":[16],"predictive":[18,288],"models":[19,289],"to":[20,170,243,262,278,286],"understand":[21],"reliability":[23,86,173,196],"their":[25,95,200],"output":[26],"in":[27,33,94,136,160,305],"support":[28],"human":[30],"decision-making,":[31],"especially":[32],"critical":[34],"domains,":[35],"such":[36],"medicine.":[38],"METHODS:":[39],"Accordingly,":[40],"we":[41,210,241,250],"propose":[42,272],"a":[43,68,103,137],"framework":[44],"distinguishing":[45],"reference":[47,84],"(or":[49],"Gold":[50],"Standard)":[51],"from":[52,57],"set":[54,69],"annotations":[56,102,225],"which":[58,240],"it":[59,193],"is":[60],"usually":[61],"derived":[62],"(the":[63],"Diamond":[64],"Standard).":[65],"define":[67],"dimensions":[72,110,257,274],"and":[73,97,124,132,191,203,226,234,258,275,290,314],"related":[74,276],"metrics:":[75],"representativeness":[76],"(are":[77,99],"available":[79],"data":[80],"representative":[81],"its":[83],"population?);":[85],"(do":[87],"raters":[89],"agree":[90],"with":[91,194,260],"each":[92],"other":[93],"ratings?);":[96],"accuracy":[98,221],"raters'":[101],"true":[104],"representation?).":[105],"The":[106],"metrics":[107,135,154,259,277,300],"for":[108,178,187,303,308],"these":[109,134],"are,":[111],"respectively,":[112],"degree":[114,119,126,213,233,236],"correspondence,":[116],"\u03a8,":[117],"weighted":[121,238],"concordance":[122],"\u03f1,":[123],"fineness,":[128],"\u03a6.":[129],"apply":[131],"evaluate":[133,146,157],"diagnostic":[138],"user":[139],"study":[140,190],"involving":[141],"13":[142],"radiologists.":[143],"RESULTS:":[144],"\u03a8":[147,166],"against":[148],"hypothesis-testing":[149],"techniques,":[150],"highlighting":[151,198],"that":[152,206,297],"our":[153,188],"can":[155],"better":[156],"distribution":[158],"similarity":[159],"high-dimensional":[161],"spaces.":[162],"discuss":[164,227,251],"how":[165],"could":[167],"be":[168,244],"assess":[171,279],"new":[175],"predictions":[176],"or":[177],"train-test":[179],"selection.":[180],"report":[182,211],"value":[184],"\u03f1":[186],"case":[189],"compare":[192],"traditional":[195],"metrics,":[197],"both":[199],"theoretical":[201],"properties":[202],"reasons":[205],"they":[207],"differ.":[208],"Then,":[209],"fineness":[215],"an":[217],"estimate":[218],"collected":[224],"relationship":[229],"between":[230],"this":[231],"latter":[232],"concordance,":[239],"find":[242],"moderately":[245],"but":[246],"significantly":[247],"correlated.":[248],"Finally,":[249],"implications":[253],"proposed":[256,299],"respect":[261],"context":[264],"Explainable":[266],"Artificial":[267,292],"Intelligence":[268,293],"(XAI).":[269],"CONCLUSION:":[270],"different":[273],"datasets":[284],"build":[287],"Medical":[291],"(MAI).":[294],"argue":[296],"are":[301],"feasible":[302],"application":[304],"real-world":[306],"settings":[307],"continuous":[310],"development":[311],"trustable":[313],"interpretable":[315],"MAI":[316],"systems.":[317]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3085682242","counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":11},{"year":2022,"cited_by_count":12},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":1}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2020-09-21T00:00:00"}
