{"id":"https://openalex.org/W4312536611","doi":"https://doi.org/10.1007/978-3-031-17721-7_10","title":"Was that\u00a0so\u00a0Hard? Estimating Human Classification Difficulty","display_name":"Was that\u00a0so\u00a0Hard? Estimating Human Classification Difficulty","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4312536611","doi":"https://doi.org/10.1007/978-3-031-17721-7_10"},"language":"en","primary_location":{"id":"doi:10.1007/978-3-031-17721-7_10","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-031-17721-7_10","pdf_url":"https://link.springer.com/content/pdf/10.1007/978-3-031-17721-7_10.pdf","source":{"id":"https://openalex.org/S106296714","display_name":"Lecture notes in computer science","issn_l":"0302-9743","issn":["0302-9743","1611-3349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lecture Notes in Computer Science","raw_type":"book-chapter"},"type":"book-chapter","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/978-3-031-17721-7_10.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058264848","display_name":"Morten Rieger Hannemose","orcid":"https://orcid.org/0000-0002-9956-9226"},"institutions":[{"id":"https://openalex.org/I96673099","display_name":"Technical University of Denmark","ror":"https://ror.org/04qtj9h94","country_code":"DK","type":"education","lineage":["https://openalex.org/I96673099"]}],"countries":["DK"],"is_corresponding":true,"raw_author_name":"Morten Rieger Hannemose","raw_affiliation_strings":["Department of Applied Mathematics and Computer Science, Technical\u00a0University of Denmark, Kgs. Lyngby, Denmark"],"affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics and Computer Science, Technical\u00a0University of Denmark, Kgs. Lyngby, Denmark","institution_ids":["https://openalex.org/I96673099"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034724113","display_name":"Josefine Vilsb\u00f8ll Sundgaard","orcid":"https://orcid.org/0000-0003-2872-4660"},"institutions":[{"id":"https://openalex.org/I96673099","display_name":"Technical University of Denmark","ror":"https://ror.org/04qtj9h94","country_code":"DK","type":"education","lineage":["https://openalex.org/I96673099"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Josefine Vilsb\u00f8ll Sundgaard","raw_affiliation_strings":["Department of Applied Mathematics and Computer Science, Technical\u00a0University of Denmark, Kgs. Lyngby, Denmark"],"affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics and Computer Science, Technical\u00a0University of Denmark, Kgs. Lyngby, Denmark","institution_ids":["https://openalex.org/I96673099"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022145682","display_name":"Niels Kvorning Ternov","orcid":"https://orcid.org/0000-0002-8034-6727"},"institutions":[{"id":"https://openalex.org/I2802582725","display_name":"Gentofte Hospital","ror":"https://ror.org/051dzw862","country_code":"DK","type":"healthcare","lineage":["https://openalex.org/I2802582725"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Niels Kvorning Ternov","raw_affiliation_strings":["Department of Plastic Surgery, Copenhagen University, Herlev and Gentofte Hospital, Copenhagen, Denmark"],"affiliations":[{"raw_affiliation_string":"Department of Plastic Surgery, Copenhagen University, Herlev and Gentofte Hospital, Copenhagen, Denmark","institution_ids":["https://openalex.org/I2802582725"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036047595","display_name":"Rasmus R. Paulsen","orcid":"https://orcid.org/0000-0003-0647-3215"},"institutions":[{"id":"https://openalex.org/I96673099","display_name":"Technical University of Denmark","ror":"https://ror.org/04qtj9h94","country_code":"DK","type":"education","lineage":["https://openalex.org/I96673099"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Rasmus R. Paulsen","raw_affiliation_strings":["Department of Applied Mathematics and Computer Science, Technical\u00a0University of Denmark, Kgs. Lyngby, Denmark"],"affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics and Computer Science, Technical\u00a0University of Denmark, Kgs. Lyngby, Denmark","institution_ids":["https://openalex.org/I96673099"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004687651","display_name":"Anders Nymark Christensen","orcid":"https://orcid.org/0000-0002-3668-3128"},"institutions":[{"id":"https://openalex.org/I96673099","display_name":"Technical University of Denmark","ror":"https://ror.org/04qtj9h94","country_code":"DK","type":"education","lineage":["https://openalex.org/I96673099"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Anders Nymark Christensen","raw_affiliation_strings":["Department of Applied Mathematics and Computer Science, Technical\u00a0University of Denmark, Kgs. Lyngby, Denmark"],"affiliations":[{"raw_affiliation_string":"Department of Applied Mathematics and Computer Science, Technical\u00a0University of Denmark, Kgs. Lyngby, Denmark","institution_ids":["https://openalex.org/I96673099"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5058264848"],"corresponding_institution_ids":["https://openalex.org/I96673099"],"apc_list":{"value":5000,"currency":"EUR","value_usd":5392},"apc_paid":{"value":5000,"currency":"EUR","value_usd":5392},"fwci":1.8474,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.87670921,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"88","last_page":"97"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11775","display_name":"COVID-19 diagnosis using AI","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11775","display_name":"COVID-19 diagnosis using AI","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9929999709129333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9883999824523926,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8201732635498047},{"id":"https://openalex.org/keywords/margin","display_name":"Margin (machine learning)","score":0.7990033030509949},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.720656156539917},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6941241025924683},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5831486582756042},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.5475500822067261},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.5071027278900146},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4835664927959442},{"id":"https://openalex.org/keywords/certainty","display_name":"Certainty","score":0.45427072048187256},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.375556617975235},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3273320198059082},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1074627935886383}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8201732635498047},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.7990033030509949},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.720656156539917},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6941241025924683},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5831486582756042},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5475500822067261},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.5071027278900146},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4835664927959442},{"id":"https://openalex.org/C7493553","wikidata":"https://www.wikidata.org/wiki/Q1520777","display_name":"Certainty","level":2,"score":0.45427072048187256},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.375556617975235},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3273320198059082},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1074627935886383},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/978-3-031-17721-7_10","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-031-17721-7_10","pdf_url":"https://link.springer.com/content/pdf/10.1007/978-3-031-17721-7_10.pdf","source":{"id":"https://openalex.org/S106296714","display_name":"Lecture notes in computer science","issn_l":"0302-9743","issn":["0302-9743","1611-3349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lecture Notes in Computer Science","raw_type":"book-chapter"},{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/83d8ad79-2771-4281-96c3-76839c04a544","is_oa":true,"landing_page_url":"https://researchprofiles.ku.dk/da/publications/83d8ad79-2771-4281-96c3-76839c04a544","pdf_url":null,"source":{"id":"https://openalex.org/S4306401983","display_name":"Research at the University of Copenhagen (University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I124055696","host_organization_name":"University of Copenhagen","host_organization_lineage":["https://openalex.org/I124055696"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Hannemose , M R , Sundgaard , J V , Ternov , N K , Paulsen , R R & Christensen , A N 2022 , Was that\u00a0so\u00a0Hard? Estimating Human Classification Difficulty . in S Wu , B Shabestari & L Xing (eds) , Applications of Medical Artificial Intelligence : 1st International Workshop, AMAI 2022, Held in Conjunction with MICCAI 2022, Proceedings . Springer , Lecture Notes in Computer Science , vol. 13540 LNCS , pp. 88-97 , 1st International Workshop on Applications of Medical Artificial Intelligence, AMAI 2022, held in conjunction with the 25th International Conference on Medical Image Computing and Computer Assisted Intervention, MICCAI 2022 , Virtual, Online , 18/09/2022 . https://doi.org/10.1007/978-3-031-17721-7_10","raw_type":"contributionToPeriodical"}],"best_oa_location":{"id":"doi:10.1007/978-3-031-17721-7_10","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-031-17721-7_10","pdf_url":"https://link.springer.com/content/pdf/10.1007/978-3-031-17721-7_10.pdf","source":{"id":"https://openalex.org/S106296714","display_name":"Lecture notes in computer science","issn_l":"0302-9743","issn":["0302-9743","1611-3349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lecture Notes in Computer Science","raw_type":"book-chapter"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.5699999928474426,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4312536611.pdf","grobid_xml":"https://content.openalex.org/works/W4312536611.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W1502895115","https://openalex.org/W1553262910","https://openalex.org/W2056132907","https://openalex.org/W2091150000","https://openalex.org/W2101825085","https://openalex.org/W2107131609","https://openalex.org/W2108598243","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2208417422","https://openalex.org/W2296073425","https://openalex.org/W2528287124","https://openalex.org/W2794825826","https://openalex.org/W2893892086","https://openalex.org/W2948077755","https://openalex.org/W2951911250","https://openalex.org/W2955214356","https://openalex.org/W2963946669","https://openalex.org/W2964271799","https://openalex.org/W3014249243","https://openalex.org/W3026010420","https://openalex.org/W3045677590","https://openalex.org/W3102785203","https://openalex.org/W3136619793","https://openalex.org/W4206766178","https://openalex.org/W6675354045"],"related_works":["https://openalex.org/W2418227039","https://openalex.org/W2387971622","https://openalex.org/W1559214593","https://openalex.org/W2141935323","https://openalex.org/W2418299857","https://openalex.org/W2466655349","https://openalex.org/W2348150032","https://openalex.org/W2266786133","https://openalex.org/W2997457842","https://openalex.org/W1997431798"],"abstract_inverted_index":{"When":[0],"doctors":[1,33],"are":[2,68,75,79],"trained":[3],"to":[4,34,54,114],"diagnose":[5,55],"a":[6,36,52,56,60,91,105,134],"specific":[7],"disease,":[8],"they":[9,74],"learn":[10],"faster":[11],"when":[12,64,73],"presented":[13],"with":[14,84],"cases":[15],"in":[16,104],"order":[17],"of":[18],"increasing":[19],"difficulty.":[20],"This":[21],"creates":[22],"the":[23],"need":[24],"for":[25,32,45,51,70,94,100],"automatically":[26],"estimating":[27,46],"how":[28,47],"difficult":[29],"it":[30,49],"is":[31,50],"classify":[35],"given":[37],"case.":[38],"In":[39],"this":[40],"paper,":[41],"we":[42,89,129],"introduce":[43,90],"methods":[44,78,113,132],"hard":[48],"doctor":[53],"case":[57,103],"represented":[58],"by":[59,133],"medical":[61,117],"image,":[62],"both":[63],"ground":[65,96],"truth":[66,97],"difficulties":[67],"available":[69],"training,":[71],"and":[72,140],"not.":[76],"Our":[77],"based":[80],"on":[81,125,137],"embeddings":[82],"obtained":[83],"deep":[85],"metric":[86],"learning.":[87],"Additionally,":[88],"practical":[92],"method":[93],"obtaining":[95],"human":[98],"difficulty":[99],"each":[101],"image":[102],"dataset":[106],"using":[107],"self-assessed":[108],"certainty.":[109],"We":[110],"apply":[111],"our":[112,138],"two":[115],"different":[116],"datasets,":[118],"achieving":[119],"high":[120],"Kendall":[121],"rank":[122],"correlation":[123],"coefficients":[124],"both,":[126],"showing":[127],"that":[128],"outperform":[130],"existing":[131],"large":[135],"margin":[136],"problem":[139],"data.":[141]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":3}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}