{"id":"https://openalex.org/W7138833740","doi":"https://doi.org/10.1609/aaai.v40i41.40746","title":"WhisperDiari: A Whisper-Based Speaker Diarization Framework in Token Space Leveraging Semantic and Speaker Information for Better Text Adaptability","display_name":"WhisperDiari: A Whisper-Based Speaker Diarization Framework in Token Space Leveraging Semantic and Speaker Information for Better Text Adaptability","publication_year":2026,"publication_date":"2026-03-14","ids":{"openalex":"https://openalex.org/W7138833740","doi":"https://doi.org/10.1609/aaai.v40i41.40746"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v40i41.40746","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i41.40746","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40746/44707","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40746/44707","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100580962","display_name":"Yongkang Yin","orcid":null},"institutions":[{"id":"https://openalex.org/I4210128628","display_name":"Peking University Shenzhen Hospital","ror":"https://ror.org/03kkjyb15","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210128628"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongkang Yin","raw_affiliation_strings":["Guangdong Provincial Key Laboratory of Ultra High Definition lmmersive Media Technology, Shenzhen Graduate School, Peking University\nADSPLAB, School of ECE, Peking University, Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Guangdong Provincial Key Laboratory of Ultra High Definition lmmersive Media Technology, Shenzhen Graduate School, Peking University\nADSPLAB, School of ECE, Peking University, Shenzhen, China","institution_ids":["https://openalex.org/I4210128628"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034401157","display_name":"Yuexian Zou","orcid":"https://orcid.org/0000-0002-0144-1794"},"institutions":[{"id":"https://openalex.org/I4210128628","display_name":"Peking University Shenzhen Hospital","ror":"https://ror.org/03kkjyb15","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210128628"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuexian Zou","raw_affiliation_strings":["Guangdong Provincial Key Laboratory of Ultra High Definition lmmersive Media Technology, Shenzhen Graduate School, Peking University\nADSPLAB, School of ECE, Peking University, Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Guangdong Provincial Key Laboratory of Ultra High Definition lmmersive Media Technology, Shenzhen Graduate School, Peking University\nADSPLAB, School of ECE, Peking University, Shenzhen, China","institution_ids":["https://openalex.org/I4210128628"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210128628"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"40","issue":"41","first_page":"34477","last_page":"34485"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9345999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9345999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.03779999911785126,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.0024999999441206455,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/speaker-diarisation","display_name":"Speaker diarisation","score":0.9373999834060669},{"id":"https://openalex.org/keywords/transcription","display_name":"Transcription (linguistics)","score":0.6233999729156494},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.5011000037193298},{"id":"https://openalex.org/keywords/speaker-recognition","display_name":"Speaker recognition","score":0.46970000863075256},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.46470001339912415},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.461899995803833},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4465000033378601},{"id":"https://openalex.org/keywords/adaptability","display_name":"Adaptability","score":0.4074000120162964},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.40450000762939453}],"concepts":[{"id":"https://openalex.org/C149838564","wikidata":"https://www.wikidata.org/wiki/Q7574248","display_name":"Speaker diarisation","level":3,"score":0.9373999834060669},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8393999934196472},{"id":"https://openalex.org/C179926584","wikidata":"https://www.wikidata.org/wiki/Q207714","display_name":"Transcription (linguistics)","level":2,"score":0.6233999729156494},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5891000032424927},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5011000037193298},{"id":"https://openalex.org/C133892786","wikidata":"https://www.wikidata.org/wiki/Q1145189","display_name":"Speaker recognition","level":2,"score":0.46970000863075256},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.46470001339912415},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.461899995803833},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4465000033378601},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4154999852180481},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4115000069141388},{"id":"https://openalex.org/C177606310","wikidata":"https://www.wikidata.org/wiki/Q5674297","display_name":"Adaptability","level":2,"score":0.4074000120162964},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.40450000762939453},{"id":"https://openalex.org/C113954288","wikidata":"https://www.wikidata.org/wiki/Q186885","display_name":"Timestamp","level":2,"score":0.3874000012874603},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.38350000977516174},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.36070001125335693},{"id":"https://openalex.org/C204201278","wikidata":"https://www.wikidata.org/wiki/Q1332614","display_name":"Voice activity detection","level":3,"score":0.35040000081062317},{"id":"https://openalex.org/C61328038","wikidata":"https://www.wikidata.org/wiki/Q3358061","display_name":"Speech processing","level":2,"score":0.34060001373291016},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.32499998807907104},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.32120001316070557},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.3066999912261963},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.29089999198913574},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C91863865","wikidata":"https://www.wikidata.org/wiki/Q4349497","display_name":"Speech corpus","level":3,"score":0.26820001006126404},{"id":"https://openalex.org/C11693617","wikidata":"https://www.wikidata.org/wiki/Q181839","display_name":"Pragmatics","level":2,"score":0.2565999925136566},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.25369998812675476}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aaai.v40i41.40746","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i41.40746","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40746/44707","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/40746","is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/40746","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aaai.v40i41.40746","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v40i41.40746","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/40746/44707","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6078603863716125}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7138833740.pdf","grobid_xml":"https://content.openalex.org/works/W7138833740.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Speaker":[0,119],"diarization":[1,86,152],"is":[2],"a":[3,51,81,93,129],"fundamental":[4],"task":[5],"in":[6],"speech":[7],"processing":[8],"aims":[9],"to":[10,123],"determine":[11],"'who":[12,159],"speaks":[13,160],"when'.":[14],"When":[15],"combined":[16],"with":[17,23,57,104,136,153],"ASR,":[18],"it":[19,176],"enables":[20],"speaker-labeled":[21],"transcription":[22],"broad":[24],"practical":[25],"value.":[26],"Most":[27],"existing":[28],"methods":[29],"rely":[30],"on":[31,111,172,177],"frame-level":[32],"classification,":[33],"but":[34],"the":[35,43,112,170,181],"high":[36],"cost":[37],"of":[38,45],"annotating":[39],"mixed-speaker":[40],"audio":[41,102],"limits":[42],"availability":[44],"large-scale,":[46],"accurately":[47],"labeled":[48],"datasets.":[49],"As":[50],"result,":[52],"even":[53],"state-of-the-art":[54,192],"models":[55,158],"struggle":[56],"imprecise":[58],"speaker":[59,85,100,107,116,125,131,134],"boundary":[60],"detection":[61],"and":[62,70,87,106,118,156,162,174,180],"semantic":[63,154],"segmentation":[64,149],"errors,":[65],"which":[66],"degrade":[67],"timestamp":[68],"accuracy":[69],"downstream":[71],"ASR":[72],"performance.":[73],"To":[74],"address":[75],"these":[76],"challenges,":[77],"we":[78],"propose":[79],"WhisperDiari,":[80],"unified":[82],"framework":[83],"for":[84],"ASR.":[88],"We":[89,168],"first":[90],"construct":[91],"LibriDiari,":[92],"dataset":[94],"derived":[95],"from":[96,139],"LibriSpeech,":[97],"containing":[98],"2\u20134":[99],"mixed":[101],"annotated":[103],"transcripts":[105],"labels.":[108],"WhisperDiari":[109,189],"builds":[110],"Whisper":[113],"model,":[114],"incorporating":[115],"adapters":[117],"Similarity":[120],"Matrix":[121],"Supervision":[122],"enhance":[124],"representation.":[126],"In":[127],"addition,":[128],"dedicated":[130],"decoder":[132],"fuses":[133],"embeddings":[135],"contextual":[137],"semantics":[138],"Whisper's":[140],"decoder,":[141],"enabling":[142],"token-level":[143],"diarization.":[144],"This":[145],"design":[146],"effectively":[147],"resolves":[148],"ambiguity,":[150],"aligns":[151],"units,":[155],"jointly":[157],"what":[161],"when',":[163],"producing":[164],"accurate,":[165],"timestamped":[166],"transcripts.":[167],"train":[169],"model":[171],"LibriDiari":[173,179],"evaluate":[175],"both":[178],"real-world":[182],"AMI":[183],"corpus.":[184],"Experimental":[185],"results":[186],"demonstrate":[187],"that":[188],"consistently":[190],"outperforms":[191],"open-source":[193],"baselines.":[194]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-03-20T00:00:00"}