{"id":"https://openalex.org/W3206494219","doi":"https://doi.org/10.1145/3462244.3479879","title":"Towards Sonification in Multimodal and User-friendlyExplainable Artificial Intelligence","display_name":"Towards Sonification in Multimodal and User-friendlyExplainable Artificial Intelligence","publication_year":2021,"publication_date":"2021-10-15","ids":{"openalex":"https://openalex.org/W3206494219","doi":"https://doi.org/10.1145/3462244.3479879","mag":"3206494219"},"language":"en","primary_location":{"id":"doi:10.1145/3462244.3479879","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3462244.3479879","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 International Conference on Multimodal Interaction","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://opus.bibliothek.uni-augsburg.de/opus4/files/91495/91495.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043060302","display_name":"Bj\u00f6rn W. Schuller","orcid":"https://orcid.org/0000-0002-6478-8699"},"institutions":[{"id":"https://openalex.org/I179225836","display_name":"University of Augsburg","ror":"https://ror.org/03p14d497","country_code":"DE","type":"education","lineage":["https://openalex.org/I179225836"]},{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["DE","GB"],"is_corresponding":true,"raw_author_name":"Bj\u00f6rn W. Schuller","raw_affiliation_strings":["Chair of Embedded Intelligence for Health Care & Wellbeing, University of Augsburg, Germany and GLAM -- Group on Language, Audio, & Music, Imperial College London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Chair of Embedded Intelligence for Health Care & Wellbeing, University of Augsburg, Germany and GLAM -- Group on Language, Audio, & Music, Imperial College London, United Kingdom","institution_ids":["https://openalex.org/I179225836","https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049691461","display_name":"Tuomas Virtanen","orcid":"https://orcid.org/0000-0002-4604-9729"},"institutions":[{"id":"https://openalex.org/I166825849","display_name":"Tampere University","ror":"https://ror.org/033003e23","country_code":"FI","type":"education","lineage":["https://openalex.org/I166825849"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Tuomas Virtanen","raw_affiliation_strings":["Audio Research Group, Tampere University, Finland"],"affiliations":[{"raw_affiliation_string":"Audio Research Group, Tampere University, Finland","institution_ids":["https://openalex.org/I166825849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041841729","display_name":"Maria Riveiro","orcid":"https://orcid.org/0000-0003-2900-9335"},"institutions":[{"id":"https://openalex.org/I94616838","display_name":"J\u00f6nk\u00f6ping University","ror":"https://ror.org/03t54am93","country_code":"SE","type":"education","lineage":["https://openalex.org/I94616838"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Maria Riveiro","raw_affiliation_strings":["Department of Computing, J\u00f6nk\u00f6ping University, Sweden"],"affiliations":[{"raw_affiliation_string":"Department of Computing, J\u00f6nk\u00f6ping University, Sweden","institution_ids":["https://openalex.org/I94616838"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054300849","display_name":"Georgios Rizos","orcid":"https://orcid.org/0000-0003-2483-5574"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Georgios Rizos","raw_affiliation_strings":["GLAM -- Group on Language, Audio, & Music, Imperial College London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"GLAM -- Group on Language, Audio, & Music, Imperial College London, United Kingdom","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069921704","display_name":"Jing Han","orcid":"https://orcid.org/0000-0001-5776-6849"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jing Han","raw_affiliation_strings":["Department of Computer Science and Technology, University of Cambridge, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, University of Cambridge, United Kingdom","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079981416","display_name":"Annamaria Mesaros","orcid":"https://orcid.org/0000-0002-6640-9752"},"institutions":[{"id":"https://openalex.org/I166825849","display_name":"Tampere University","ror":"https://ror.org/033003e23","country_code":"FI","type":"education","lineage":["https://openalex.org/I166825849"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Annamaria Mesaros","raw_affiliation_strings":["Audio Research Group, Tampere University, Finland"],"affiliations":[{"raw_affiliation_string":"Audio Research Group, Tampere University, Finland","institution_ids":["https://openalex.org/I166825849"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108358814","display_name":"Konstantinos Drossos","orcid":null},"institutions":[{"id":"https://openalex.org/I166825849","display_name":"Tampere University","ror":"https://ror.org/033003e23","country_code":"FI","type":"education","lineage":["https://openalex.org/I166825849"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Konstantinos Drossos","raw_affiliation_strings":["Audio Research Group, Tampere University, Finland"],"affiliations":[{"raw_affiliation_string":"Audio Research Group, Tampere University, Finland","institution_ids":["https://openalex.org/I166825849"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5043060302"],"corresponding_institution_ids":["https://openalex.org/I179225836","https://openalex.org/I47508984"],"apc_list":null,"apc_paid":null,"fwci":2.0063,"has_fulltext":true,"cited_by_count":19,"citation_normalized_percentile":{"value":0.87492479,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"788","last_page":"792"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10914","display_name":"Tactile and Sensory Interactions","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sonification","display_name":"Sonification","score":0.8786188960075378},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7624318599700928},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6665743589401245},{"id":"https://openalex.org/keywords/salient","display_name":"Salient","score":0.5990855097770691},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5530359745025635},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4916849732398987},{"id":"https://openalex.org/keywords/preprocessor","display_name":"Preprocessor","score":0.476202130317688},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4752599895000458},{"id":"https://openalex.org/keywords/visual-analytics","display_name":"Visual analytics","score":0.41439494490623474},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.3480376601219177},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3393675684928894}],"concepts":[{"id":"https://openalex.org/C91607612","wikidata":"https://www.wikidata.org/wiki/Q1416058","display_name":"Sonification","level":2,"score":0.8786188960075378},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7624318599700928},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6665743589401245},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.5990855097770691},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5530359745025635},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4916849732398987},{"id":"https://openalex.org/C34736171","wikidata":"https://www.wikidata.org/wiki/Q918333","display_name":"Preprocessor","level":2,"score":0.476202130317688},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4752599895000458},{"id":"https://openalex.org/C59732488","wikidata":"https://www.wikidata.org/wiki/Q2528440","display_name":"Visual analytics","level":3,"score":0.41439494490623474},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.3480376601219177},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3393675684928894},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3462244.3479879","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3462244.3479879","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 International Conference on Multimodal Interaction","raw_type":"proceedings-article"},{"id":"pmh:oai:uni-augsburg.opus-bayern.de:91495","is_oa":true,"landing_page_url":"https://nbn-resolving.org/urn:nbn:de:bvb:384-opus4-914954","pdf_url":"https://opus.bibliothek.uni-augsburg.de/opus4/files/91495/91495.pdf","source":{"id":"https://openalex.org/S4306400930","display_name":"OPUS (Augsburg University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I119916105","host_organization_name":"Augsburg University","host_organization_lineage":["https://openalex.org/I119916105"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"bookpart"},{"id":"pmh:oai:trepo.tuni.fi:10024/225639","is_oa":false,"landing_page_url":"https://trepo.tuni.fi/handle/10024/225639","pdf_url":null,"source":{"id":"https://openalex.org/S7407055260","display_name":"Trepo - Institutional Repository of Tampere University","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference"}],"best_oa_location":{"id":"pmh:oai:uni-augsburg.opus-bayern.de:91495","is_oa":true,"landing_page_url":"https://nbn-resolving.org/urn:nbn:de:bvb:384-opus4-914954","pdf_url":"https://opus.bibliothek.uni-augsburg.de/opus4/files/91495/91495.pdf","source":{"id":"https://openalex.org/S4306400930","display_name":"OPUS (Augsburg University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I119916105","host_organization_name":"Augsburg University","host_organization_lineage":["https://openalex.org/I119916105"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"bookpart"},"sustainable_development_goals":[{"score":0.7400000095367432,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[{"id":"https://openalex.org/G2136136281","display_name":null,"funder_award_id":"826506","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G4563061561","display_name":null,"funder_award_id":"826506","funder_id":"https://openalex.org/F4320332999","funder_display_name":"Horizon 2020 Framework Programme"},{"id":"https://openalex.org/G4956428346","display_name":null,"funder_award_id":"Horizon 2020 research and innovatio","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5036817778","display_name":null,"funder_award_id":"European Union's Horizon 2020 research and innov","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5399556803","display_name":null,"funder_award_id":"This project has received funding from the Europea","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8318064016","display_name":null,"funder_award_id":"Horizon","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8633428685","display_name":null,"funder_award_id":"European Union's Horizon 2020 research and innovat","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320332999","display_name":"Horizon 2020 Framework Programme","ror":"https://ror.org/00k4n6c32"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3206494219.pdf","grobid_xml":"https://content.openalex.org/works/W3206494219.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W2032152873","https://openalex.org/W2099471712","https://openalex.org/W2112285779","https://openalex.org/W2150415460","https://openalex.org/W2221409856","https://openalex.org/W2282821441","https://openalex.org/W2789070602","https://openalex.org/W2885138528","https://openalex.org/W2886947409","https://openalex.org/W2891503716","https://openalex.org/W2952218014","https://openalex.org/W2952529067","https://openalex.org/W2962772482","https://openalex.org/W2963095307","https://openalex.org/W2963939953","https://openalex.org/W2992923261","https://openalex.org/W3000346474","https://openalex.org/W3090395639","https://openalex.org/W3094535122","https://openalex.org/W3162917476","https://openalex.org/W4404783772"],"related_works":["https://openalex.org/W4251837693","https://openalex.org/W4304099409","https://openalex.org/W2131473689","https://openalex.org/W1521847617","https://openalex.org/W2362367986","https://openalex.org/W4281696028","https://openalex.org/W348707231","https://openalex.org/W2064719069","https://openalex.org/W3182718252","https://openalex.org/W3041760129"],"abstract_inverted_index":{"We":[0],"are":[1,13,53,204,343],"largely":[2,39],"used":[3],"to":[4,19,78,161,177,183,190,216,238,255,313,318,329,331],"hearing":[5,340],"explanations.":[6],"For":[7],"example,":[8,126],"if":[9,35],"someone":[10],"thinks":[11],"you":[12,24],"sad":[14],"today,":[15],"they":[16,70],"might":[17],"reply":[18],"your":[20],"\u201cwhy?\u201d":[21],"with":[22,121],"\u201cbecause":[23],"were":[25,262],"so":[26],"Hmmmmm-mmm-mmm\u201d.":[27],"Today\u2019s":[28],"Artificial":[29],"Intelligence":[30],"(AI),":[31],"however,":[32],"is":[33,85,95,279],"\u2013":[34,38,124],"at":[36,165,213],"all":[37,303],"providing":[40],"explanations":[41,100,142,235,310],"of":[42,67,137,143,180,194,221,224,243,253,290,302,321,337,365,367],"decisions":[43,145,171,342],"in":[44,62,117,226,259,305,335],"a":[45,138,147,218,285,288,332],"visual":[46,58,99,132,293],"or":[47,65,134],"textual":[48,295],"manner.":[49],"While":[50],"such":[51,60],"approaches":[52],"good":[54],"for":[55,113,125,170,206,241,264,325],"communication":[56],"via":[57,339],"media":[59],"as":[61,281,297,299],"research":[63],"papers":[64],"screens":[66],"intelligent":[68],"devices,":[69],"may":[71],"not":[72,86,130],"always":[73],"be":[74],"the":[75,82,92,166,181,192,195,207,214,222,244,260,265,322,356],"best":[76],"way":[77],"explain;":[79],"especially":[80],"when":[81,91],"end":[83],"user":[84,306,316,327],"an":[87,174],"expert.":[88],"In":[89],"particular,":[90],"AI\u2019s":[93,349],"task":[94],"about":[96],"Audio":[97],"Intelligence,":[98],"appear":[101],"less":[102],"intuitive":[103],"than":[104],"audible,":[105],"sonified":[106,141,277],"ones.":[107],"Sonification":[108],"has":[109],"also":[110],"great":[111],"potential":[112],"explainable":[114],"AI":[115,144,234,309,338],"(XAI)":[116],"systems":[118],"that":[119,203,247],"deal":[120],"non-audio":[122],"data":[123,168,182],"because":[127],"it":[128,210,282],"does":[129],"require":[131,284],"contact":[133],"active":[135],"attention":[136,201,269],"user.":[139],"Hence,":[140],"face":[146],"challenging,":[148,280],"yet":[149],"highly":[150],"promising":[151],"and":[152,176,199,251,267,271,294,354,359,370],"pioneering":[153],"task.":[154],"That":[155],"involves":[156],"incorporating":[157],"innovative":[158],"XAI":[159,278,369],"algorithms":[160],"allow":[162,319],"pointing":[163],"back":[164],"learning":[167],"responsible":[169,205,263],"made":[172],"by":[173],"AI,":[175],"include":[178],"decomposition":[179,250],"identify":[184,191],"salient":[185,272],"aspects.":[186],"It":[187],"further":[188],"aims":[189],"components":[193],"preprocessing,":[196],"feature":[197,273],"representation,":[198],"learnt":[200],"patterns":[202,270],"decisions.":[208],"Finally,":[209],"targets":[211],"decision-making":[212],"model-level,":[215],"provide":[217,360],"holistic":[219],"explanation":[220],"chain":[223],"processing":[225],"typical":[227],"pattern":[228],"recognition":[229],"problems":[230],"from":[231],"end-to-end.":[232],"Sonified":[233,308],"will":[236,283,311],"need":[237,312],"unite":[239],"methods":[240],"sonification":[242,323],"identified":[245],"aspects":[246],"benefit":[248],"decisions,":[249],"recomposition":[252],"audio":[254,261],"sonify":[256],"which":[257],"parts":[258],"decision,":[266],"rendering":[268],"representations":[274],"audible.":[275],"Benchmarking":[276],"comparison":[286],"against":[287],"backdrop":[289],"existing,":[291],"state-of-the-art":[292],"alternatives,":[296],"well":[298],"synergistic":[300],"complementation":[301],"modalities":[304],"evaluations.":[307],"target":[314],"different":[315,326],"groups":[317],"personalisation":[320],"experience":[324],"needs,":[328],"lead":[330],"major":[333],"breakthrough":[334],"comprehensibility":[336],"how":[341],"made,":[344],"hence":[345],"supporting":[346],"tomorrow\u2019s":[347],"humane":[348],"trustability.":[350],"Here,":[351],"we":[352],"introduce":[353],"motivate":[355],"general":[357],"idea,":[358],"accompanying":[361],"considerations":[362],"including":[363],"milestones":[364],"realisation":[366],"sonifed":[368],"foreseeable":[371],"risks.":[372]},"counts_by_year":[{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}